From 06cde2e7a6000ea960b7873baf74c7963de5900d Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Thu, 20 Aug 2026 11:42:52 +0200 Subject: [PATCH 1/9] chore(transform): move specific methods to specialised mappers - return typed dictionaries to help with data flow - avoid unnnecessary inheritance - separate data checking from data building (Entry vs. Transform class) # Conflicts: # cds_migrator_kit/rdm/records/load/load.py # cds_migrator_kit/rdm/records/transform/transform.py --- .../rdm/records/load/ep_approval_entry.py | 15 +- cds_migrator_kit/rdm/records/load/load.py | 44 +- .../rdm/records/transform/entry_types.py | 182 +++++ .../rdm/records/transform/mappers/__init__.py | 13 + .../rdm/records/transform/mappers/base.py | 98 +++ .../records/transform/mappers/contributors.py | 194 +++++ .../transform/mappers/custom_fields.py | 194 +++++ .../rdm/records/transform/mappers/metadata.py | 173 +++++ .../rdm/records/transform/mappers/record.py | 23 + .../rdm/records/transform/mappers/registry.py | 81 ++ .../records/transform/mappers/vocabulary.py | 81 ++ .../rdm/records/transform/transform.py | 729 ++++-------------- .../transform/xml_processing/rules/base.py | 4 +- 13 files changed, 1217 insertions(+), 614 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/entry_types.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/__init__.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/base.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/contributors.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/metadata.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/record.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/registry.py create mode 100644 cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py b/cds_migrator_kit/rdm/records/load/ep_approval_entry.py index ffbc7150..ddca2423 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py +++ b/cds_migrator_kit/rdm/records/load/ep_approval_entry.py @@ -9,10 +9,15 @@ import re from collections import OrderedDict from copy import deepcopy +from typing import Dict from flask import current_app from cds_migrator_kit.errors import UnexpectedValue +from cds_migrator_kit.rdm.records.transform.entry_types import ( + MigrationEntry, + VersionEntry, +) EPPHAPP_FILE_TYPE = "EPPHAPP_FILE" EP_APPROVAL_REPORT_NUMBER_PREFIX = "CERN-EP" @@ -35,7 +40,7 @@ def _cern_scientific_community_id(): class MetadataEntry: """Build a load entry for the public or restricted EP approval split.""" - def __init__(self, entry, approval_request, migration_logger): + def __init__(self, entry: MigrationEntry, approval_request, migration_logger): self.entry = entry self.approval_request = approval_request self.migration_logger = migration_logger @@ -44,7 +49,7 @@ def identifiers(self, identifiers): """Return identifiers for this split.""" raise NotImplementedError - def build(self): + def build(self) -> MigrationEntry: """Return a load entry with split files and modified metadata.""" split = deepcopy(self.entry) split["record"].pop("ep_approval", None) @@ -91,7 +96,7 @@ def _remove_doi_pid(self, split): """Remove DOI PID from record.""" pass - def _build_versions(self, split): + def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: """Return versioned files for this split; override in subclasses.""" raise NotImplementedError @@ -115,7 +120,7 @@ def _version_signature(versioned_files): class PublicEntry(MetadataEntry): """Build the public EP approval split entry.""" - def _build_versions(self, split): + def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: new_versions = OrderedDict() versioned_files = OrderedDict() previous_signature = None @@ -239,7 +244,7 @@ def _has_restricted_files(self, split): for file_data in version_data.get("files", {}).values() ) - def _build_versions(self, split): + def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: new_versions = OrderedDict() versioned_files = OrderedDict() previous_signature = None diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 9d1c0fff..0a9829b2 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -12,6 +12,7 @@ import os import re from copy import deepcopy +from typing import Dict import arrow from cds_rdm.clc_sync.models import CDSToCLCSyncModel @@ -49,6 +50,11 @@ RecordFlaggedCuration, UnexpectedValue, ) +from cds_migrator_kit.rdm.records.transform.entry_types import ( + MigrationEntry, + VersionAccess, + VersionFileEntry, +) def import_legacy_files(filepath): @@ -100,7 +106,13 @@ def _prepare(self, entry): """Prepare the record.""" pass - def _load_files(self, draft, entry, version_files, uow=None): + def _load_files( + self, + draft, + entry: MigrationEntry, + version_files: Dict[str, VersionFileEntry], + uow=None, + ): """Load files to draft.""" recid = entry.get("record", {}).get("recid", {}) identity = system_identity # Should we create an identity for the migration? @@ -182,7 +194,7 @@ def _load_files(self, draft, entry, version_files, uow=None): self.migration_logger.add_log(exc, record=entry) raise e - def _load_parent_access_and_communities(self, draft, entry): + def _load_parent_access_and_communities(self, draft, entry: MigrationEntry): """Load access rights and communities in a single parent commit.""" parent = draft._record.parent parent.access = entry["parent"]["json"]["access"] @@ -192,7 +204,7 @@ def _load_parent_access_and_communities(self, draft, entry): parent.communities.default = entry["parent"]["json"]["communities"]["default"] parent.commit() - def _load_record_access(self, draft, access_dict): + def _load_record_access(self, draft, access_dict: VersionAccess): record = draft._record record.access = access_dict["access_obj"] record.commit() @@ -232,7 +244,9 @@ def _after_publish_update_dois(self, identity, record, entry, uow): ) return record - def _after_publish_load_parent_access_grants(self, draft, version, entry): + def _after_publish_load_parent_access_grants( + self, draft, version, entry: MigrationEntry + ): """Load access grants from metadata and record grants efficiently.""" def _normalize_group_name(subject): @@ -406,7 +420,7 @@ def _create_grant(subject_type, subject_id, permission): parent.commit() - def _after_publish_update_created(self, record, entry, version): + def _after_publish_update_created(self, record, entry: MigrationEntry, version): """Update created timestamp post publish. Ensures that the `created` timestamp is correctly set, preferring: @@ -431,7 +445,7 @@ def _after_publish_update_created(self, record, entry, version): record._record.model.created = creation_date db.session.add(record._record.model) - def _after_publish_mint_recid(self, record, entry, version): + def _after_publish_mint_recid(self, record, entry: MigrationEntry, version): """Mint legacy ids for redirections assigned to the parent.""" if not self._is_final_record: return @@ -515,7 +529,9 @@ def create_event(request_model, payload, event_type, user): uow.register(RecordCommitOp(request, indexer=current_requests_service.indexer)) - def _after_publish_update_files_created(self, record, entry, version): + def _after_publish_update_files_created( + self, record, entry: MigrationEntry, version + ): """Update the created date of the files post publish.""" # Fix the `created` timestamp forcing the one from the legacy system # Force the created date. This can be done after publish as the service @@ -580,7 +596,9 @@ def _after_publish_set_committee_approval(self, published_record, entry, uow): except PIDAlreadyExists: pass # already minted on a previous run — idempotent - def _after_publish(self, identity, published_record, entry, version, uow): + def _after_publish( + self, identity, published_record, entry: MigrationEntry, version, uow + ): """Run fixes after record publish.""" record = self._after_publish_update_dois(identity, published_record, entry, uow) if record: @@ -639,7 +657,7 @@ def _assign_rep_numbers(self, draft): f"Report number {report_number} already exists." ) - def _pre_publish(self, identity, entry, version, draft, uow): + def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): """Create and process draft before publish.""" versions = entry["versions"] files = versions[version]["files"] @@ -699,7 +717,7 @@ def _pre_publish(self, identity, entry, version, draft, uow): return draft - def _load_versions(self, entry, uow): + def _load_versions(self, entry: MigrationEntry, uow): """Load other versions of the record.""" versions = entry["versions"] legacy_recid = entry["record"]["recid"] @@ -730,7 +748,7 @@ def _load_versions(self, entry, uow): self.record_state_logger.add_record_state(record_state_context) return record_state_context - def _dry_load(self, entry): + def _dry_load(self, entry: MigrationEntry): current_rdm_records_service.schema.load( entry["record"]["json"], context=dict( @@ -812,7 +830,7 @@ def extract_record_version(record): recid_state["latest_version_object_uuid"] = str(rec.id) return recid_state - def _save_original_dumped_record(self, entry, recid_state): + def _save_original_dumped_record(self, entry: MigrationEntry, recid_state): """Save the original dumped record. This is the originally extracted record before any transformation. @@ -854,7 +872,7 @@ def _after_load_clc_sync(self, record_state): ) db.session.add(sync) - def _load(self, entry, uow=None): + def _load(self, entry: MigrationEntry, uow=None): """Use the services to load the entries. If ``uow`` is provided, operations are registered on it without diff --git a/cds_migrator_kit/rdm/records/transform/entry_types.py b/cds_migrator_kit/rdm/records/transform/entry_types.py new file mode 100644 index 00000000..ce321252 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/entry_types.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Typed shapes for the migration ETL entry. + +``TypedDict`` values are plain ``dict``s at runtime (no behavior change, +nothing enforced), so these exist purely to make the envelope shapes +produced by ``transform.py`` and consumed by ``load.py``/ +``ep_approval_entry.py`` visible at the definition site, instead of having +to be reconstructed by grepping across those files. + +Deliberately NOT modeled here: the RDM record body itself +(``RecordEntry["json"]["metadata"|"pids"|"files"|"custom_fields"]``) - that +shape is governed by invenio_rdm_records' own record schema, not by this +package, and is already comparatively well documented by the field mappers +in ``mappers/``. +""" +from pathlib import Path +from typing import Any, Dict, List, Optional, TypedDict, Union + +from arrow import Arrow + + +class RecordJsonOutputRequired(TypedDict): + """Required keys of ``RecordEntry["json"]``.""" + + files: dict + pids: dict + metadata: dict + access_grants: List[dict] + + +class RecordJsonOutput(RecordJsonOutputRequired, total=False): + """The RDM record body: ``RecordEntry["json"]``. + + ``current_rdm_records_service.create()``'s ``data`` argument - built by + ``CDSToRDMRecordEntry.transform()``. Deliberately excludes ``access``: + that's set per-version, after creation, via ``VersionEntry["access"]`` + (see ``load.py::_load_record_access``) rather than at record-create time. + """ + + custom_fields: dict + internal_notes: Any + + +class RecordEntry(TypedDict): + """A single record's content - ``MigrationEntry["record"]``. + + Built by ``CDSToRDMRecordEntry.transform()``. Everything here is + record-scoped (as opposed to ``MigrationEntry``'s other top-level keys, + which are ETL-envelope-scoped - see that type's docstring). + """ + + created: str + updated: str + version_id: int + index: int + recid: str + communities: List[str] + json: RecordJsonOutput + # None when RecordFlaggedCuration was raised and caught - see + # CDSToRDMRecordEntry._access()/.transform(). + access_status: Optional[str] + owned_by: Union[str, int] + # Community-inclusion request payload for this record, if any. + _request_data: Optional[dict] + # EP approval workflow entries for this record, if any (possibly []). + ep_approval: List[dict] + + +class ParentAccess(TypedDict): + """``ParentEntry["json"]["access"]``.""" + + owned_by: Dict[str, Union[str, int]] + + +class ParentCommunities(TypedDict, total=False): + """``ParentEntry["json"]["communities"]``.""" + + ids: List[str] + default: str + + +class ParentJson(TypedDict): + """``ParentEntry["json"]``.""" + + id: str + access: ParentAccess + communities: ParentCommunities + + +class ParentEntry(TypedDict): + """The parent record - ``MigrationEntry["parent"]``. Built by ``CDSToRDMRecordTransform._parent()``.""" + + created: str + updated: str + version_id: int + json: ParentJson + + +class VersionAccessObj(TypedDict): + """``VersionAccess["access_obj"]`` - mirrors the RDM record access schema.""" + + record: Optional[str] + files: Optional[str] + + +class VersionAccess(TypedDict, total=False): + """A version's access - ``VersionEntry["access"]``. + + Set directly on the record post-create via + ``load.py::_load_record_access`` (``record.access = access_dict["access_obj"]``). + """ + + access_obj: VersionAccessObj + # Raw legacy file-restriction status string, present only when an + # individual file carried its own restriction - see + # CDSToRDMRecordTransform._versions()::compute_access(). + meta: str + + +class VersionFileMetadata(TypedDict): + """``VersionFileEntry["metadata"]``.""" + + description: Optional[str] + name: str + status: str + original_path: str + comment: Optional[str] + + +class VersionFileEntry(TypedDict): + """One file within ``VersionEntry["files"]``, keyed by its ``full_name``.""" + + eos_tmp_path: Path + id_bibdoc: int + key: str + metadata: VersionFileMetadata + mimetype: str + checksum: str + version: int + access: str + type: str + creation_date: str + + +class VersionEntry(TypedDict): + """One record version - a value in ``MigrationEntry["versions"]``. + + Built by ``CDSToRDMRecordTransform._versions()``, keyed there by legacy + file version number (int), starting at 1. + """ + + files: Dict[str, VersionFileEntry] + # Arrow instance when derived from a file's creation date; a plain ISO + # date string in the no-files fallback branch (copied straight from + # RecordEntry["json"]["metadata"]["publication_date"]) - see + # CDSToRDMRecordTransform._versions(). + publication_date: Union[Arrow, str] + access: VersionAccess + + +class MigrationEntry(TypedDict): + """The full ETL entry yielded by ``CDSToRDMRecordTransform.run()``. + + Consumed by ``CDSRecordServiceLoad``/``ep_approval_entry.py``. Keys + outside ``"record"`` are ETL-envelope-scoped (about this migration run, + not about the record's own content): ``versions``/``parent`` are + computed by ``CDSToRDMRecordTransform`` itself, while + ``_original_dump``/``_clc_sync`` are carried alongside it - see + ``CDSToRDMRecordTransform._transform()``. + """ + + record: RecordEntry + versions: Dict[int, VersionEntry] + parent: ParentEntry + _original_dump: dict + _clc_sync: Any diff --git a/cds_migrator_kit/rdm/records/transform/mappers/__init__.py b/cds_migrator_kit/rdm/records/transform/mappers/__init__.py new file mode 100644 index 00000000..eb15ac05 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM record field mappers. + +Each mapper owns the derivation of a single ``metadata`` or +``custom_fields`` value from the legacy record entry, composed together by +``CDSToRDMRecordEntry`` in ``transform.py``. +""" diff --git a/cds_migrator_kit/rdm/records/transform/mappers/base.py b/cds_migrator_kit/rdm/records/transform/mappers/base.py new file mode 100644 index 00000000..eacf1f14 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/base.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Base classes shared by all CDS-RDM record field mappers.""" +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + + +@dataclass +class RecordTransformContext: + """Shared, mutable context passed to field mappers building one record. + + ``metadata`` and ``custom_fields`` are the in-progress output dicts: + mappers write their own value into them (metadata mappers return a + value that the caller assigns; custom_fields list-accumulator mappers + mutate ``custom_fields`` directly) and later mappers in the same phase + may read earlier ones (e.g. the title mapper reads the already-resolved + ``metadata["resource_type"]``). + """ + + json_entry: dict + entry: dict + migration_logger: object = None + affiliations_mapping: object = None + access_grants_view: object = None + json_output: dict = None + metadata: dict = field(default_factory=dict) + custom_fields: dict = field(default_factory=dict) + + def flag_curation(self, exc): + """Log a caught ``RecordFlaggedCuration`` for curation follow-up.""" + self.migration_logger.add_information( + self.json_entry["recid"], + {"message": exc.message, "value": exc.value}, + ) + + +class FieldMapper(ABC): + """Derives a single output value, identified by ``id``, from the entry. + + Used both for ``metadata.`` fields and for other top-level + ``record_json_output`` fields (e.g. ``access_grants``) - the caller + decides where the returned value is assigned. + """ + + id: str + + @abstractmethod + def map_value(self, ctx: RecordTransformContext): + """Return the value for this field, or a falsy value to omit it.""" + raise NotImplementedError + + +class PassthroughMapper(FieldMapper): + """Copies a key from the source entry through unchanged.""" + + def __init__(self, id): + """Constructor.""" + self.id = id + + def map_value(self, ctx): + """Return the raw value of ``self.id`` from the source entry.""" + return ctx.json_entry.get(self.id) + + +class CustomFieldMapper(ABC): + """Writes one or more ``custom_fields`` keys into ``ctx.custom_fields``. + + Every mapper owns its own soft-fail (curation) handling internally, via + ``ctx.flag_curation()`` - mirroring how affiliation matching handles its + own curation flags in ``mappers/contributors.py``. Only a genuinely hard + failure (e.g. ``UnexpectedValue``) is left to propagate and abort the + whole record. This lets the caller run every mapper the same way, in a + single uninterrupted loop, with no branching per mapper. + """ + + @abstractmethod + def apply(self, ctx: RecordTransformContext): + """Apply the mapper, writing into ``ctx.custom_fields``.""" + raise NotImplementedError + + +class PassthroughCustomFieldMapper(CustomFieldMapper): + """Copies a custom_fields key from the source entry through unchanged.""" + + def __init__(self, id, default=None): + """Constructor.""" + self.id = id + self.default = default + + def apply(self, ctx): + """Copy ``self.id`` from the source custom_fields, or use the default.""" + source = ctx.json_entry.get("custom_fields", {}) + ctx.custom_fields[self.id] = source.get(self.id, self.default) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/contributors.py b/cds_migrator_kit/rdm/records/transform/mappers/contributors.py new file mode 100644 index 00000000..0ad6ff12 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/contributors.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Creators/contributors field mapping: affiliations and person-id lookup.""" +from copy import deepcopy + +from idutils import normalize_ror +from idutils.validators import is_ror +from invenio_accounts.models import UserIdentity +from invenio_db import db +from invenio_vocabularies.contrib.affiliations.models import AffiliationsMetadata +from invenio_vocabularies.contrib.names.models import NamesMetadata + +from cds_migrator_kit.errors import ManualImportRequired, RecordFlaggedCuration +from cds_migrator_kit.rdm.migration_config import VOCABULARIES_NAMES_SCHEMES +from cds_migrator_kit.rdm.records.transform.mappers.base import FieldMapper + + +def match_affiliation(affiliation_name, ctx): + """Match an affiliation against `CDSMigrationAffiliationMapping` db table.""" + json_entry = ctx.json_entry + if is_ror(affiliation_name): + ror = normalize_ror(affiliation_name) + name = AffiliationsMetadata.query.filter_by(pid=ror).one_or_none() + if name is None: + raise ManualImportRequired( + message="Affiliation {ror} does not exist in the AffiliationMetadata table".format( + ror=ror + ), + field="validation", + stage="transform", + description="Add this affiliation", + recid=json_entry["recid"], + priority="critical", + value=None, + subfield=None, + ) + return {"id": normalize_ror(affiliation_name)} + # Step 1: search in the affiliation mapping (ROR organizations) + match = ctx.affiliations_mapping.query.filter_by( + legacy_affiliation_input=affiliation_name + ).one_or_none() + if match: + # Step 1: check if there is a curated input + if match.curated_affiliation: + return match.curated_affiliation + # Step 2: check if there is an exact match + if match.ror_exact_match: + return {"id": normalize_ror(match.ror_exact_match)} + # Step 3: check if there is not exact match + if match.ror_not_exact_match: + _affiliation_ror_id = normalize_ror(match.ror_not_exact_match) + raise RecordFlaggedCuration( + subfield="u", + value={"id": _affiliation_ror_id}, + field="author", + message=f"Affiliation {_affiliation_ror_id} not found as an exact match, ROR id should be checked.", + stage="vocabulary match", + ) + # Step 4: set the originally inserted value from legacy (no match, or match + # found but has no ROR id of any kind) + raise RecordFlaggedCuration( + subfield="u", + value={"name": affiliation_name}, + field="author", + message=f"Affiliation {affiliation_name} not found as an exact match, custom value should be checked.", + stage="vocabulary match", + ) + + +def _creator_affiliations(creator, ctx): + affiliations = creator.get("affiliations", []) + transformed_aff = [] + + for affiliation_name in affiliations: + try: + affiliation = match_affiliation(affiliation_name, ctx) + if affiliation not in transformed_aff: + transformed_aff.append(affiliation) + except RecordFlaggedCuration as exc: + # Save not exact match affiliation and reraise to flag the record + ctx.flag_curation(exc) + aff = {"name": affiliation_name} + if aff not in transformed_aff: + transformed_aff.append({"name": affiliation_name}) + creator["affiliations"] = transformed_aff + + +def _creator_identifiers(creator): + processed_identifiers = [] + inner_dict = creator.get("person_or_org", {}) + identifiers = inner_dict.get("identifiers", []) + for identifier in identifiers: + # we check for unknown schemes + if identifier["scheme"] in VOCABULARIES_NAMES_SCHEMES.keys(): + processed_identifiers.append(identifier) + if processed_identifiers: + inner_dict["identifiers"] = processed_identifiers + else: + inner_dict.pop("identifiers", None) + + +def _lookup_person_id(creator): + migrated_identifiers = deepcopy( + creator.get("person_or_org", {}).get("identifiers", []) + ) + name = None + # lookup person_id + person_id = next( + ( + identifier + for identifier in migrated_identifiers + if identifier["scheme"] == "cern" + ), + {}, + ).get("identifier") + if person_id: + ui = UserIdentity.query.filter_by(id=person_id).one_or_none() + if ui: + user_id = ui.user.id + names = NamesMetadata.query.filter_by(internal_id=str(user_id)).all() + name = next( + ( + name + for name in names + if "unlisted" not in name.json.get("tags", []) + ), + None, + ) + # filter out cern person_id + creator["person_or_org"]["identifiers"] = [ + identifier + for identifier in migrated_identifiers + if identifier["scheme"] != "cern" + ] + if name: + # update identifiers of the authors to the latest known + ids = creator["person_or_org"]["identifiers"] + # check ids supplied by the names vocabulary and add missing + for identifier in name.json.get("identifiers", []): + if identifier not in ids and identifier.get("scheme") != "cern": + ids.append(identifier) + + # copy names identifiers and json to assign explicitly json object + # due to how postgres assignment of json is handled + json_copy = deepcopy(name.json) + existing_ids = deepcopy(name.json.get("identifiers", [])) + # update the names vocab to contain other ids found during migration + for identifier in ids: + if identifier not in existing_ids: + existing_ids.append(identifier) + + if existing_ids: + # assign json explicitly to names entry + json_copy["identifiers"] = existing_ids + name.json = json_copy + + db.session.add(name) + # db.session.commit() + + +def creators_for(ctx, key="creators"): + """Build the creators/contributors list for ``key``.""" + _creators = deepcopy(ctx.json_entry.get(key, [])) + _creators = list(filter(lambda x: x is not None, _creators)) + for creator in _creators: + _creator_affiliations(creator, ctx) + _lookup_person_id(creator) + _creator_identifiers(creator) + return _creators + + +class CreatorsMapper(FieldMapper): + """Maps the creators list, resolving affiliations and person ids.""" + + id = "creators" + + def map_value(self, ctx): + """Build the creators list.""" + return creators_for(ctx, key="creators") + + +class ContributorsMapper(FieldMapper): + """Maps the contributors list, resolving affiliations and person ids.""" + + id = "contributors" + + def map_value(self, ctx): + """Build the contributors list.""" + return creators_for(ctx, key="contributors") diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py new file mode 100644 index 00000000..3cf5a649 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""``custom_fields`` mappers for CDS to RDM record transformation.""" +from cds_migrator_kit.errors import RecordFlaggedCuration, UnexpectedValue +from cds_migrator_kit.rdm.records.transform.config import EXPERIMENT_ALIASES +from cds_migrator_kit.rdm.records.transform.mappers.base import CustomFieldMapper +from cds_migrator_kit.rdm.records.transform.mappers.vocabulary import search_vocabulary + + +class ExperimentsMapper(CustomFieldMapper): + """Sets cern:experiments. + + Raises ``UnexpectedValue`` (a hard, whole-record failure) for the first + unmatched experiment name, having first added it as a subject fallback + so it's visible if the raw dump is inspected. + """ + + def apply(self, ctx): + """Set ctx.custom_fields["cern:experiments"].""" + experiments_out = ctx.custom_fields["cern:experiments"] = [] + experiments = ctx.json_entry.get("custom_fields", {}).get( + "cern:experiments", [] + ) + for experiment in experiments: + if experiment.lower().strip() in ["not applicable", "xx"]: + continue + experiment = EXPERIMENT_ALIASES.get( + experiment.lower().strip(), experiment + ) + result = search_vocabulary(experiment, "experiments") + if result and result not in experiments_out: + experiments_out.append(result) + elif not result: + subj = ctx.json_output["metadata"].get("subjects", []) + subj.append({"subject": experiment}) + ctx.json_output["metadata"]["subjects"] = subj + raise UnexpectedValue( + subfield="e", + value=experiment, + field="693", + message=f"Experiment {experiment} not found", + stage="vocabulary match", + ) + + +class DepartmentsMapper(CustomFieldMapper): + """Sets cern:departments. + + For the first unmatched department, adds it as the administrative unit + and as a subject fallback, and flags the record for curation (soft + fail - handled locally, doesn't abort the record). + """ + + def apply(self, ctx): + """Set ctx.custom_fields["cern:departments"].""" + departments_out = ctx.custom_fields["cern:departments"] = [] + departments = ctx.json_entry.get("custom_fields", {}).get( + "cern:departments", [] + ) + for department in departments: + if "-" in department: + dep = department.split("-")[0] + else: + dep = department + result = search_vocabulary(dep, "departments") + if result and result not in departments_out: + departments_out.append(result) + elif not result: + if department.lower() == "cern?": + continue + subj = ctx.json_output["metadata"].get("subjects", []) + subj.append({"subject": department}) + ctx.json_output["metadata"]["subjects"] = subj + ctx.custom_fields["cern:administrative_unit"] = department + ctx.flag_curation( + RecordFlaggedCuration( + subfield="a", + value=department, + field="department", + message=f"Department {department} not found. " + f"Added as unit and subject", + stage="vocabulary match", + ) + ) + # first unmatched department halts department processing for + # this record (matching the original single-raise behavior); + # other custom_fields mappers still run. + return + + +class AcceleratorsMapper(CustomFieldMapper): + """Sets cern:accelerators. + + Raises ``UnexpectedValue`` (hard failure) for an unmatched accelerator. + """ + + def apply(self, ctx): + """Set ctx.custom_fields["cern:accelerators"].""" + accelerators_out = ctx.custom_fields["cern:accelerators"] = [] + accelerators = ctx.json_entry.get("custom_fields", {}).get( + "cern:accelerators", [] + ) + for accelerator in accelerators: + if accelerator.lower().strip() in ["not applicable", "xx", "fermi"]: + continue + result = search_vocabulary(accelerator, "accelerators") + if result and result not in accelerators_out: + accelerators_out.append(result) + elif not result: + raise UnexpectedValue( + subfield="a", + value=accelerator, + field="accelerators", + message=f"Accelerator {accelerator} not found.", + stage="vocabulary match", + ) + + +class BeamsMapper(CustomFieldMapper): + """Sets cern:beams. + + Raises ``UnexpectedValue`` (hard failure) for an unmatched beam. + """ + + def apply(self, ctx): + """Set ctx.custom_fields["cern:beams"].""" + beams_out = ctx.custom_fields["cern:beams"] = [] + beams = ctx.json_entry.get("custom_fields", {}).get("cern:beams", []) + for beam in beams: + if beam.lower().strip() == "not applicable": + continue + result = search_vocabulary(beam, "beams") + if result and result not in beams_out: + beams_out.append(result) + elif not result: + raise UnexpectedValue( + subfield="a", + value=beam, + field="beams", + message=f"Beam {beam} not found.", + stage="vocabulary match", + ) + + +class ProgrammesMapper(CustomFieldMapper): + """Sets cern:programmes, defaulting theses without one to {"id": "None"}. + + Left unset (rather than set to None) when not applicable, so it's + dropped from the final record the same way an absent key would be. + """ + + def apply(self, ctx): + """Set ctx.custom_fields["cern:programmes"], or leave it unset.""" + record_json = ctx.json_entry + programme = record_json.get("custom_fields", {}).get("cern:programmes") + if programme: + result = search_vocabulary(programme, "programmes") + if not result: + raise UnexpectedValue( + value=programme, + field="programme", + message=f"programme {programme} not found", + stage="vocabulary match", + ) + ctx.custom_fields["cern:programmes"] = result + elif record_json["resource_type"] == "publication-thesis": + ctx.custom_fields["cern:programmes"] = {"id": "None"} + + +class JournalMapper(CustomFieldMapper): + """Sets journal:journal. + + Flags a partial (titleless) journal field for curation (soft fail) and + drops it, rather than aborting the record. + """ + + def apply(self, ctx): + """Set ctx.custom_fields["journal:journal"].""" + journal = ctx.json_entry.get("custom_fields", {}).get("journal:journal", {}) + if journal and not journal.get("title"): + ctx.flag_curation( + RecordFlaggedCuration( + message="found partial journal field, to be checked", + stage="transform", + field="773", + ) + ) + journal = {} + ctx.custom_fields["journal:journal"] = journal diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py new file mode 100644 index 00000000..8852ef6f --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""``metadata`` field mappers for CDS to RDM record transformation.""" +from dateutil.parser import parse + +from cds_migrator_kit.errors import MissingRequiredField, UnexpectedValue +from cds_migrator_kit.rdm.migration_config import RDM_RECORDS_IDENTIFIERS_SCHEMES +from cds_migrator_kit.rdm.records.transform.config import ( + IDENTIFIERS_SCHEMES_TO_DROP, + IDENTIFIERS_VALUES_TO_DROP, +) +from cds_migrator_kit.rdm.records.transform.mappers.base import FieldMapper + + +class ResourceTypeMapper(FieldMapper): + """Maps resource_type, requiring it to have been resolved upstream.""" + + id = "resource_type" + + def map_value(self, ctx): + """Return resource_type, dropping the upstream ranking scratch key.""" + json_entry = ctx.json_entry + # `_resource_type_rank` is bookkeeping for the 980__/697C_ + # resource_type rule and research_committee.py's report-number + # detection (see research.py:resource_type) - drop it before it + # reaches the final record. + json_entry.pop("_resource_type_rank", None) + try: + return json_entry["resource_type"] + except KeyError: + raise MissingRequiredField(message="resource_type", field="980") + + +class TitleMapper(FieldMapper): + """Maps title, falling back to the meeting title for conference proceedings.""" + + id = "title" + + def map_value(self, ctx): + """Return title, or the 111__a meeting title as a fallback.""" + json_entry = ctx.json_entry + title = json_entry.get("title") + if title: + return title + # 245 (title) is sometimes absent on conference proceedings + # records; fall back to the conference name (111__a) stored on + # the first meeting entry. + resource_type = ctx.metadata.get("resource_type") or {} + if resource_type.get("id") == "publication-conferenceproceeding": + meetings = json_entry.get("custom_fields", {}).get("meeting:meeting", []) + for meeting_entry in meetings: + meeting_title = meeting_entry.get("title") + if meeting_title: + return meeting_title + return title + + +class PublicationDateMapper(FieldMapper): + """Maps publication_date, falling back to status week or file creation date.""" + + id = "publication_date" + + def map_value(self, ctx): + """Return publication_date, requiring at least one date source.""" + json_entry = ctx.json_entry + pub_date = json_entry.get("publication_date") + created = json_entry.get("status_week_date") + files = ctx.entry["files"] + if not (pub_date or created or files): + raise MissingRequiredField( + message="missing creation or publication date", field="916" + ) + if not pub_date: + if created: + pub_date = json_entry["status_week_date"] + elif not created and files: + pub_date = parse(files[0]["creation_date"]).date().isoformat() + return pub_date + + +class SubjectsMapper(FieldMapper): + """Maps subjects, dropping placeholder "xx"/"talk" entries.""" + + id = "subjects" + + def map_value(self, ctx): + """Return the subjects list with placeholder entries removed.""" + subjects = ctx.json_entry.get("subjects") + if subjects: + for subject in reversed(subjects): + if subject.get("subject", "").lower() in ["xx", "talk"]: + subjects.remove(subject) + elif subject.get("id", "").lower() in ["xx", "talk"]: + subjects.remove(subject) + return subjects + + +class TableOfContentsMapper(FieldMapper): + """Folds table_of_content into additional_descriptions.""" + + id = "additional_descriptions" + + def map_value(self, ctx): + """Move table_of_content into additional_descriptions and return it.""" + json_entry = ctx.json_entry + toc = json_entry.get("table_of_content", []) + additional_desc = json_entry.get("additional_descriptions", []) + if toc: + additional_desc.append( + {"description": toc, "type": {"id": "table-of-contents"}} + ) + json_entry["additional_descriptions"] = additional_desc + json_entry.pop("table_of_content") + return json_entry.get("additional_descriptions") + + +class IdentifiersMapper(FieldMapper): + """Maps identifiers, dropping unwanted schemes and validating the rest.""" + + id = "identifiers" + + def map_value(self, ctx): + """Return identifiers filtered/validated against known schemes.""" + identifiers = ctx.json_entry.get("identifiers", []) + for item in reversed(identifiers): + # drop unwanted schemes + if item is None or "scheme" not in item: + raise UnexpectedValue( + field="identifiers", + value=item, + subfield="9", + message="IDENTIFIER SCHEME MISSING", + priority="warning", + stage="transform", + ) + if ( + item["scheme"].upper() in IDENTIFIERS_SCHEMES_TO_DROP + or IDENTIFIERS_VALUES_TO_DROP in item["identifier"] + ): + identifiers.remove(item) + continue + if item["scheme"] not in RDM_RECORDS_IDENTIFIERS_SCHEMES.keys(): + raise UnexpectedValue( + field="identifiers", + subfield="9", + message="IDENTIFIER SCHEME INVALID", + priority="warning", + stage="transform", + value=item, + ) + return identifiers + + +# Fields that pass through unchanged from json_entry - kept explicit in the +# composed list (mappers/config equivalent) rather than open-ended, so the +# "forgotten metadata key" completeness check in CDSToRDMRecordEntry._metadata +# still catches any newly introduced json_entry key nobody has mapped yet. +PASSTHROUGH_METADATA_FIELDS = ( + "description", + "publisher", + "additional_titles", + "languages", + "dates", + "funding", + "related_identifiers", + "rights", + "copyright", +) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/record.py b/cds_migrator_kit/rdm/records/transform/mappers/record.py new file mode 100644 index 00000000..11d4aa0d --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/record.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Top-level ``record_json_output`` field mappers.""" +from cds_migrator_kit.rdm.records.transform.mappers.base import FieldMapper + + +class AccessGrantsMapper(FieldMapper): + """Maps access_grants, appending any configured collection-wide view grants.""" + + id = "access_grants" + + def map_value(self, ctx): + """Return access_grants extended with configured view grants.""" + access_grants = ctx.json_entry.get("access_grants", []) + if ctx.access_grants_view: + for grant in ctx.access_grants_view: + access_grants.append({str(grant): "view"}) + return access_grants diff --git a/cds_migrator_kit/rdm/records/transform/mappers/registry.py b/cds_migrator_kit/rdm/records/transform/mappers/registry.py new file mode 100644 index 00000000..3b08bf01 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/registry.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Composed lists of field mappers used by CDSToRDMRecordEntry.""" +from cds_migrator_kit.rdm.records.transform.mappers.base import ( + PassthroughCustomFieldMapper, + PassthroughMapper, +) +from cds_migrator_kit.rdm.records.transform.mappers.contributors import ( + ContributorsMapper, + CreatorsMapper, +) +from cds_migrator_kit.rdm.records.transform.mappers.custom_fields import ( + AcceleratorsMapper, + BeamsMapper, + DepartmentsMapper, + ExperimentsMapper, + JournalMapper, + ProgrammesMapper, +) +from cds_migrator_kit.rdm.records.transform.mappers.metadata import ( + PASSTHROUGH_METADATA_FIELDS, + IdentifiersMapper, + PublicationDateMapper, + ResourceTypeMapper, + SubjectsMapper, + TableOfContentsMapper, + TitleMapper, +) + +# Order matters: TableOfContentsMapper must run before "additional_descriptions" +# is read elsewhere, and ResourceTypeMapper must run before TitleMapper (which +# reads the already-resolved metadata["resource_type"]). +METADATA_MAPPERS = ( + TableOfContentsMapper(), + ResourceTypeMapper(), + TitleMapper(), + CreatorsMapper(), + ContributorsMapper(), + PublicationDateMapper(), + SubjectsMapper(), + IdentifiersMapper(), + *(PassthroughMapper(field_name) for field_name in PASSTHROUGH_METADATA_FIELDS), +) + +# custom_fields keys copied straight from the legacy custom_fields dict, +# each with its own default when absent. +CUSTOM_FIELDS_PASSTHROUGH_DEFAULTS = { + "cern:administrative_unit": [], + "cern:projects": [], + "cern:facilities": [], + "cern:studies": [], + "cern:committees": None, + "cern:oa_funding_model": None, + "thesis:thesis": {}, + "imprint:imprint": {}, + "meeting:meeting": {}, +} + +# Every custom_fields mapper writes its own key(s) directly into +# ctx.custom_fields and handles its own curation flagging internally, so +# this list is run as a single uninterrupted loop - see +# CustomFieldMapper in mappers/base.py. Order matters only for +# "cern:administrative_unit": DepartmentsMapper overrides the passthrough +# default when a department can't be matched, so it must run after it. +CUSTOM_FIELD_MAPPERS = ( + *( + PassthroughCustomFieldMapper(field_name, default) + for field_name, default in CUSTOM_FIELDS_PASSTHROUGH_DEFAULTS.items() + ), + JournalMapper(), + ProgrammesMapper(), + ExperimentsMapper(), + DepartmentsMapper(), + AcceleratorsMapper(), + BeamsMapper(), +) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py b/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py new file mode 100644 index 00000000..ac22daed --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""YAML-backed vocabulary lookup used by custom_fields mappers.""" +from pathlib import Path + +import yaml +from flask import current_app + +_VOCAB_FILENAMES = { + "experiments": "experiments.yaml", + "departments": "departments.yaml", + "programmes": "programmes.yaml", + "accelerators": "accelerators.yaml", + "beams": "beams.yaml", +} + + +class VocabularyCache: + """Vocabulary lookup cache loaded once from YAML files at startup.""" + + def __init__(self, default_dir, override_dir=None): + """Load all vocabularies into memory. + + For each vocabulary file, ``override_dir`` (e.g. a test-local + directory overriding just a subset of files) is preferred when it + contains that file, falling back to ``default_dir`` otherwise. + """ + self._cache = {} + default_dir = Path(default_dir) + override_dir = Path(override_dir) if override_dir else None + for vocab_type, filename in _VOCAB_FILENAMES.items(): + filepath = default_dir / filename + if override_dir and (override_dir / filename).exists(): + filepath = override_dir / filename + self._cache[vocab_type] = self._load(filepath) + + @staticmethod + def _load(filepath): + """Build a case-insensitive term→id lookup from a vocabulary YAML.""" + with open(filepath) as f: + entries = yaml.safe_load(f) + lookup = {} + for entry in entries: + entry_id = entry["id"] + lookup[entry_id.lower()] = entry_id + title = entry.get("title", {}).get("en", "") + if title and title.lower() != entry_id.lower(): + lookup[title.lower()] = entry_id + return lookup + + def get(self, term, vocab_type): + """Return {"id": vocab_id} if term matches, else None.""" + entry_id = self._cache[vocab_type].get(term.strip().lower()) + return {"id": entry_id} if entry_id else None + + +_vocabulary_cache = None + + +def _get_vocabulary_cache(): + global _vocabulary_cache + if _vocabulary_cache is None: + import cds_rdm + + default_dir = Path(cds_rdm.__file__).parent / "app_data" / "vocabularies" + override_dir = current_app.config.get("CDS_MIGRATOR_KIT_VOCABULARIES_DIR") + _vocabulary_cache = VocabularyCache(default_dir, override_dir) + return _vocabulary_cache + + +def search_vocabulary(term, vocab_type): + """Look up a vocabulary term using the pre-loaded YAML cache. + + Returns {"id": vocab_id} if found, else None. + """ + return _get_vocabulary_cache().get(term, vocab_type) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index 44d86083..d5da74f1 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -12,27 +12,20 @@ from collections import OrderedDict from copy import deepcopy from pathlib import Path +from typing import Any, Dict, NamedTuple, Optional import arrow -import yaml from cds_dojson.marc21.utils import create_record from cds_rdm.legacy.models import CDSMigrationAffiliationMapping from cds_rdm.legacy.resolver import get_pid_by_legacy_recid -from dateutil.parser import ParserError, parse +from dateutil.parser import ParserError from flask import current_app -from idutils import normalize_ror -from idutils.validators import is_doi, is_ror +from idutils.validators import is_doi from invenio_access.permissions import system_identity -from invenio_accounts.models import User, UserIdentity -from invenio_db import db +from invenio_accounts.models import User from invenio_pidstore.models import PersistentIdentifier, PIDStatus -from invenio_rdm_migrator.streams.records.transform import ( - RDMRecordEntry, - RDMRecordTransform, -) +from invenio_rdm_migrator.logging import Logger from invenio_rdm_records.proxies import current_rdm_records_service, current_record_communities_service -from invenio_vocabularies.contrib.affiliations.models import AffiliationsMetadata -from invenio_vocabularies.contrib.names.models import NamesMetadata from sqlalchemy.exc import NoResultFound from cds_migrator_kit.errors import ( @@ -43,95 +36,40 @@ RestrictedFileDetected, UnexpectedValue, ) -from cds_migrator_kit.rdm.migration_config import ( - RDM_RECORDS_IDENTIFIERS_SCHEMES, - VOCABULARIES_NAMES_SCHEMES, -) from cds_migrator_kit.rdm.records.transform.config import ( - EXPERIMENT_ALIASES, FILE_SUBFORMATS_TO_DROP, - IDENTIFIERS_SCHEMES_TO_DROP, - IDENTIFIERS_VALUES_TO_DROP, PIDS_SCHEMES_ALLOWED, PIDS_SCHEMES_TO_DROP, ) +from cds_migrator_kit.rdm.records.transform.entry_types import ( + MigrationEntry, + ParentAccess, + ParentEntry, + ParentJson, + RecordEntry, + VersionEntry, +) +from cds_migrator_kit.rdm.records.transform.mappers.base import RecordTransformContext +from cds_migrator_kit.rdm.records.transform.mappers.record import AccessGrantsMapper +from cds_migrator_kit.rdm.records.transform.mappers.registry import ( + CUSTOM_FIELD_MAPPERS, + METADATA_MAPPERS, +) from cds_migrator_kit.transform.dumper import CDSRecordDump from cds_migrator_kit.transform.errors import LossyConversion cli_logger = logging.getLogger("migrator") -_VOCAB_FILENAMES = { - "experiments": "experiments.yaml", - "departments": "departments.yaml", - "programmes": "programmes.yaml", - "accelerators": "accelerators.yaml", - "beams": "beams.yaml", -} - - -class VocabularyCache: - """Vocabulary lookup cache loaded once from YAML files at startup.""" - - def __init__(self, default_dir, override_dir=None): - """Load all vocabularies into memory. - - For each vocabulary file, ``override_dir`` (e.g. a test-local - directory overriding just a subset of files) is preferred when it - contains that file, falling back to ``default_dir`` otherwise. - """ - self._cache = {} - default_dir = Path(default_dir) - override_dir = Path(override_dir) if override_dir else None - for vocab_type, filename in _VOCAB_FILENAMES.items(): - filepath = default_dir / filename - if override_dir and (override_dir / filename).exists(): - filepath = override_dir / filename - self._cache[vocab_type] = self._load(filepath) - - @staticmethod - def _load(filepath): - """Build a case-insensitive term→id lookup from a vocabulary YAML.""" - with open(filepath) as f: - entries = yaml.safe_load(f) - lookup = {} - for entry in entries: - entry_id = entry["id"] - lookup[entry_id.lower()] = entry_id - title = entry.get("title", {}).get("en", "") - if title and title.lower() != entry_id.lower(): - lookup[title.lower()] = entry_id - return lookup - - def get(self, term, vocab_type): - """Return {"id": vocab_id} if term matches, else None.""" - entry_id = self._cache[vocab_type].get(term.strip().lower()) - return {"id": entry_id} if entry_id else None - - -_vocabulary_cache = None - -def _get_vocabulary_cache(): - global _vocabulary_cache - if _vocabulary_cache is None: - import cds_rdm +class CDSToRDMRecordEntry: + """Transform CDS record to RDM record. - default_dir = Path(cds_rdm.__file__).parent / "app_data" / "vocabularies" - override_dir = current_app.config.get("CDS_MIGRATOR_KIT_VOCABULARIES_DIR") - _vocabulary_cache = VocabularyCache(default_dir, override_dir) - return _vocabulary_cache - - -def search_vocabulary(term, vocab_type): - """Look up a vocabulary term using the pre-loaded YAML cache. - - Returns {"id": vocab_id} if found, else None. + Builds the ``record`` content dict consumed by + ``CDSToRDMRecordTransform`` - not the invenio_rdm_migrator "generic RDM + record" envelope (this class deliberately does not use that framework's + ``RDMRecordEntry.transform()``/``_load_partial`` orchestration, since the + CDS legacy shape and the CDS loader's needs don't match it). """ - return _get_vocabulary_cache().get(term, vocab_type) - - -class CDSToRDMRecordEntry(RDMRecordEntry): - """Transform CDS record to RDM record.""" def __init__( self, @@ -148,6 +86,7 @@ def __init__( preferred_model=None, ): """Constructor.""" + self.partial = partial self.missing_users_dir = missing_users_dir self.missing_users_filename = missing_users_filename self.affiliations_mapping = affiliations_mapping @@ -157,6 +96,12 @@ def __init__( self.access_grants_view = access_grants_view self.migration_logger = migration_logger self.record_state_logger = record_state_logger + # populated by transform(); an ETL-envelope concern (does the + # parent need a CLC sync after load), not record content, so it + # isn't part of the dict transform() returns - the caller + # (CDSToRDMRecordTransform._record()) reads it off this instance + # instead. See CDSToRDMRecordTransform._transform()'s docstring. + self.clc_sync = None self.preferred_model = preferred_model self.ep_approval_request = None super().__init__(partial) @@ -193,18 +138,6 @@ def _recid(self, record_dump): """Returns the recid of the record.""" return str(record_dump.data["recid"]) - def _bucket_id(self, json_entry): - return - - def _id(self, entry): - return - - def _media_bucket_id(self, entry): - return - - def _media_files(self, entry): - return {} - def _pids(self, json_entry): DATACITE_PREFIX = current_app.config["DATACITE_PREFIX"] @@ -256,288 +189,21 @@ def _communities(self, json_entry): def _owner(self, json_entry): email = json_entry.get("submitter") - if not email: - return "system" - try: - user = User.query.filter_by(email=email).one() - return user.id - except NoResultFound: - raise UnexpectedValue( - message=f"{email} not found - did you run user migration?", - stage="transform", - recid=json_entry["legacy_recid"], - value=email, - priority="critical", - ) + return email - def _match_affiliation(self, affiliation_name, json_entry): - """Match an affiliation against `CDSMigrationAffiliationMapping` db table.""" - if is_ror(affiliation_name): - ror = normalize_ror(affiliation_name) - name = AffiliationsMetadata.query.filter_by(pid=ror).one_or_none() - if name is None: - raise ManualImportRequired( - message="Affiliation {ror} does not exist in the AffiliationMetadata table".format( - ror=ror - ), - field="validation", - stage="transform", - description="Add this affiliation", - recid=json_entry["recid"], - priority="critical", - value=None, - subfield=None, - ) - return {"id": normalize_ror(affiliation_name)} - # Step 1: search in the affiliation mapping (ROR organizations) - match = self.affiliations_mapping.query.filter_by( - legacy_affiliation_input=affiliation_name - ).one_or_none() - if match: - # Step 1: check if there is a curated input - if match.curated_affiliation: - return match.curated_affiliation - # Step 2: check if there is an exact match - if match.ror_exact_match: - return {"id": normalize_ror(match.ror_exact_match)} - # Step 3: check if there is not exact match - if match.ror_not_exact_match: - _affiliation_ror_id = normalize_ror(match.ror_not_exact_match) - raise RecordFlaggedCuration( - subfield="u", - value={"id": _affiliation_ror_id}, - field="author", - message=f"Affiliation {_affiliation_ror_id} not found as an exact match, ROR id should be checked.", - stage="vocabulary match", - ) - # Step 4: set the originally inserted value from legacy (no match, or match - # found but has no ROR id of any kind) - raise RecordFlaggedCuration( - subfield="u", - value={"name": affiliation_name}, - field="author", - message=f"Affiliation {affiliation_name} not found as an exact match, custom value should be checked.", - stage="vocabulary match", + def _metadata(self, json_entry, entry): + """Build the metadata dict by running the composed field mappers.""" + ctx = RecordTransformContext( + json_entry=json_entry, + entry=entry, + migration_logger=self.migration_logger, + affiliations_mapping=self.affiliations_mapping, ) - - def _metadata(self, json_entry, record_dump): - - def creator_affiliations(creator): - affiliations = creator.get("affiliations", []) - transformed_aff = [] - - for affiliation_name in affiliations: - try: - affiliation = self._match_affiliation(affiliation_name, json_entry) - if affiliation not in transformed_aff: - transformed_aff.append(affiliation) - except RecordFlaggedCuration as exc: - # Save not exact match affiliation and reraise to flag the record - self.migration_logger.add_information( - json_entry["recid"], - {"message": exc.message, "value": exc.value}, - ) - aff = {"name": affiliation_name} - if aff not in transformed_aff: - transformed_aff.append({"name": affiliation_name}) - creator["affiliations"] = transformed_aff - - def creator_identifiers(creator): - processed_identifiers = [] - inner_dict = creator.get("person_or_org", {}) - identifiers = inner_dict.get("identifiers", []) - for identifier in identifiers: - # we check for unknown schemes - if identifier["scheme"] in VOCABULARIES_NAMES_SCHEMES.keys(): - processed_identifiers.append(identifier) - if processed_identifiers: - inner_dict["identifiers"] = processed_identifiers - else: - inner_dict.pop("identifiers", None) - - def lookup_person_id(creator): - migrated_identifiers = deepcopy( - creator.get("person_or_org", {}).get("identifiers", []) - ) - name = None - # lookup person_id - person_id = next( - ( - identifier - for identifier in migrated_identifiers - if identifier["scheme"] == "cern" - ), - {}, - ).get("identifier") - if person_id: - ui = UserIdentity.query.filter_by(id=person_id).one_or_none() - if ui: - user_id = ui.user.id - names = NamesMetadata.query.filter_by( - internal_id=str(user_id) - ).all() - name = next( - ( - name - for name in names - if "unlisted" not in name.json.get("tags", []) - ), - None, - ) - # filter out cern person_id - creator["person_or_org"]["identifiers"] = [ - identifier - for identifier in migrated_identifiers - if identifier["scheme"] != "cern" - ] - if name: - # update identifiers of the authors to the latest known - ids = creator["person_or_org"]["identifiers"] - # check ids supplied by the names vocabulary and add missing - for identifier in name.json.get("identifiers", []): - if identifier not in ids and identifier.get("scheme") != "cern": - ids.append(identifier) - - # copy names identifiers and json to assign explicitly json object - # due to how postgres assignment of json is handled - json_copy = deepcopy(name.json) - existing_ids = deepcopy(name.json.get("identifiers", [])) - # update the names vocab to contain other ids found during migration - for identifier in ids: - if identifier not in existing_ids: - existing_ids.append(identifier) - - if existing_ids: - # assign json explicitly to names entry - json_copy["identifiers"] = existing_ids - name.json = json_copy - - db.session.add(name) - # db.session.commit() - - def creators(json, key="creators"): - _creators = deepcopy(json.get(key, [])) - _creators = list(filter(lambda x: x is not None, _creators)) - for creator in _creators: - creator_affiliations(creator) - lookup_person_id(creator) - creator_identifiers(creator) - return _creators - - def _resource_type(entry): - # `_resource_type_rank` is bookkeeping for the 980__/697C_ - # resource_type rule and research_committee.py's report-number - # detection (see research.py:resource_type) - drop it before it - # reaches the final record. - entry.pop("_resource_type_rank", None) - try: - return entry["resource_type"] - except KeyError: - raise MissingRequiredField(message="resource_type", field="980") - - def _title(entry, resource_type): - title = entry.get("title") - if title: - return title - # 245 (title) is sometimes absent on conference proceedings - # records; fall back to the conference name (111__a) stored on - # the first meeting entry. - if resource_type.get("id") == "publication-conferenceproceeding": - meetings = entry.get("custom_fields", {}).get("meeting:meeting", []) - for meeting_entry in meetings: - meeting_title = meeting_entry.get("title") - if meeting_title: - return meeting_title - return title - - def _publication_date(entry, dump_record): - pub_date = entry.get("publication_date") - created = entry.get("status_week_date") - files = dump_record["files"] - if not (pub_date or created or files): - raise MissingRequiredField( - message="missing creation or publication date", field="916" - ) - if not pub_date: - if created: - pub_date = entry["status_week_date"] - elif not created and files: - pub_date = parse(files[0]["creation_date"]).date().isoformat() - return pub_date - - def _identifiers(json_entry): - identifiers = json_entry.get("identifiers", []) - for item in reversed(identifiers): - # drop unwanted schemes - if item is None or "scheme" not in item: - raise UnexpectedValue( - field="identifiers", - value=item, - subfield="9", - message="IDENTIFIER SCHEME MISSING", - priority="warning", - stage="transform", - ) - if ( - item["scheme"].upper() in IDENTIFIERS_SCHEMES_TO_DROP - or IDENTIFIERS_VALUES_TO_DROP in item["identifier"] - ): - identifiers.remove(item) - continue - if item["scheme"] not in RDM_RECORDS_IDENTIFIERS_SCHEMES.keys(): - raise UnexpectedValue( - field="identifiers", - subfield="9", - message="IDENTIFIER SCHEME INVALID", - priority="warning", - stage="transform", - value=item, - ) - return identifiers - - def table_of_contents(json_entry): - toc = json_entry.get("table_of_content", []) - additional_desc = json_entry.get("additional_descriptions", []) - if toc: - additional_desc.append( - {"description": toc, "type": {"id": "table-of-contents"}} - ) - json_entry["additional_descriptions"] = additional_desc - json_entry.pop("table_of_content") - - def subjects(json_entry): - _subjects = json_entry.get("subjects") - if _subjects: - for subject in reversed(_subjects): - if subject.get("subject", "").lower() in ["xx", "talk"]: - _subjects.remove(subject) - elif subject.get("id", "").lower() in ["xx", "talk"]: - _subjects.remove(subject) - return _subjects - - _subjects = subjects(json_entry) - table_of_contents(json_entry) - - _resource_type_value = _resource_type(json_entry) - metadata = { - "creators": creators(json_entry), - "title": _title(json_entry, _resource_type_value), - "resource_type": _resource_type_value, - "description": json_entry.get("description"), - "publication_date": _publication_date(json_entry, record_dump), - "contributors": creators(json_entry, key="contributors"), - "subjects": _subjects, - "publisher": json_entry.get("publisher"), - "additional_descriptions": json_entry.get("additional_descriptions"), - "additional_titles": json_entry.get("additional_titles"), - "identifiers": _identifiers(json_entry), - "languages": json_entry.get("languages"), - "dates": json_entry.get("dates"), - "funding": json_entry.get("funding"), - "related_identifiers": json_entry.get("related_identifiers"), - "rights": json_entry.get("rights"), - "copyright": json_entry.get("copyright"), - } + metadata = ctx.metadata + # Order matters: ResourceTypeMapper must run before TitleMapper reads + # metadata["resource_type"]; see mappers/registry.py. + for mapper in METADATA_MAPPERS: + metadata[mapper.id] = mapper.map_value(ctx) # filter empty keys helper_keys = [ @@ -553,8 +219,6 @@ def subjects(json_entry): "internal_notes", "ep_approval", ] - self.ep_approval_request = json_entry.get("ep_approval", []) - keys = deepcopy(list(json_entry.keys())) for item in helper_keys: if item in keys: @@ -566,169 +230,16 @@ def subjects(json_entry): return {k: v for k, v in metadata.items() if v} def _custom_fields(self, json_entry, json_output): - - def field_experiments(record_json, custom_fields_dict): - experiments = record_json.get("custom_fields", {}).get( - "cern:experiments", [] - ) - for experiment in experiments: - if experiment.lower().strip() in ["not applicable", "xx"]: - continue - experiment = EXPERIMENT_ALIASES.get( - experiment.lower().strip(), experiment - ) - result = search_vocabulary(experiment, "experiments") - if result and result not in custom_fields_dict["cern:experiments"]: - custom_fields_dict["cern:experiments"].append(result) - elif not result: - subj = json_output["metadata"].get("subjects", []) - subj.append({"subject": experiment}) - json_output["metadata"]["subjects"] = subj - raise UnexpectedValue( - subfield="e", - value=experiment, - field="693", - message=f"Experiment {experiment} not found", - stage="vocabulary match", - ) - - def field_programmes(record_json): - programme = record_json.get("custom_fields", {}).get("cern:programmes") - if programme: - result = search_vocabulary(programme, "programmes") - if result: - return result - else: - raise UnexpectedValue( - value=programme, - field="programme", - message=f"programme {programme} not found", - stage="vocabulary match", - ) - else: - if record_json["resource_type"] == "publication-thesis": - return {"id": "None"} - else: - return - - def field_departments(record_json, custom_fields_dict): - departments = record_json.get("custom_fields", {}).get( - "cern:departments", [] - ) - for department in departments: - if "-" in department: - units = department.split("-") - dep = units[0] - else: - dep = department - result = search_vocabulary(dep, "departments") - if result and result not in custom_fields_dict["cern:departments"]: - custom_fields_dict["cern:departments"].append(result) - elif not result: - if department.lower() == "cern?": - continue - subj = json_output["metadata"].get("subjects", []) - subj.append({"subject": department}) - json_output["metadata"]["subjects"] = subj - custom_fields_dict["cern:administrative_unit"] = department - raise RecordFlaggedCuration( - subfield="a", - value=department, - field="department", - message=f"Department {department} not found. " - f"Added as unit and subject", - stage="vocabulary match", - ) - - def field_accelerators(record_json, custom_fields_dict): - accelerators = record_json.get("custom_fields", {}).get( - "cern:accelerators", [] - ) - for accelerator in accelerators: - if accelerator.lower().strip() in ["not applicable", "xx", "fermi"]: - continue - result = search_vocabulary(accelerator, "accelerators") - if result and result not in custom_fields_dict["cern:accelerators"]: - custom_fields_dict["cern:accelerators"].append(result) - elif not result: - raise UnexpectedValue( - subfield="a", - value=accelerator, - field="accelerators", - message=f"Accelerator {accelerator} not found.", - stage="vocabulary match", - ) - - def field_beams(record_json, custom_fields_dict): - beams = record_json.get("custom_fields", {}).get("cern:beams", []) - for beam in beams: - if beam.lower().strip() == "not applicable": - continue - result = search_vocabulary(beam, "beams") - if result and result not in custom_fields_dict["cern:beams"]: - custom_fields_dict["cern:beams"].append(result) - elif not result: - raise UnexpectedValue( - subfield="a", - value=beam, - field="beams", - message=f"Beam {beam} not found.", - stage="vocabulary match", - ) - - def field_journal(record_json): - """Raise if title is missing in journal field""" - journal = record_json.get("custom_fields", {}).get("journal:journal", {}) - if journal: - if not journal.get("title"): - raise RecordFlaggedCuration( - message="found partial journal field, to be checked", - stage="transform", - field="773", - ) - return journal - return {} - - _cf = json_entry.get("custom_fields", {}) - try: - journal = field_journal(json_entry) - except RecordFlaggedCuration as e: - self.migration_logger.add_information( - json_entry["recid"], - {"message": e.message, "value": e.value}, - ) - journal = {} - custom_fields = { - "cern:experiments": [], - "cern:departments": [], - "cern:accelerators": [], - "cern:administrative_unit": _cf.get("cern:administrative_unit", []), - "cern:projects": _cf.get("cern:projects", []), - "cern:facilities": _cf.get("cern:facilities", []), - "cern:studies": _cf.get("cern:studies", []), - "cern:beams": [], - "cern:programmes": field_programmes(json_entry), - "cern:committees": _cf.get("cern:committees"), - "cern:oa_funding_model": _cf.get("cern:oa_funding_model"), - "thesis:thesis": _cf.get("thesis:thesis", {}), - "journal:journal": journal, - "imprint:imprint": _cf.get("imprint:imprint", {}), - "meeting:meeting": _cf.get("meeting:meeting", {}), - } - try: - field_experiments(json_entry, custom_fields) - field_departments(json_entry, custom_fields) - - except RecordFlaggedCuration as exc: - self.migration_logger.add_information( - json_entry["recid"], - {"message": exc.message, "value": exc.value}, - ) - field_accelerators(json_entry, custom_fields) - field_beams(json_entry, custom_fields) - - if custom_fields["cern:programmes"] is None: - del custom_fields["cern:programmes"] + """Build the custom_fields dict by running the composed field mappers.""" + ctx = RecordTransformContext( + json_entry=json_entry, + entry=json_entry, + migration_logger=self.migration_logger, + json_output=json_output, + ) + for mapper in CUSTOM_FIELD_MAPPERS: + mapper.apply(ctx) + custom_fields = ctx.custom_fields forgotten_keys = [ key @@ -762,15 +273,7 @@ def _verify_publication_date(self, entry, json_data): subfield=None, ) - def _access_grants(self, json_data, record_json_output): - access_grants = json_data.get("access_grants", []) - if self.access_grants_view: - for grant in self.access_grants_view: - access_grants.append({str(grant): "view"}) - if access_grants: - record_json_output.update({"access_grants": access_grants}) - - def transform(self, entry): + def transform(self, entry) -> RecordEntry: """Transform a record single entry.""" record_dump = CDSRecordDump( entry, @@ -800,7 +303,7 @@ def transform(self, entry): self.record_state_logger.add_record(json_data) - clc_sync = deepcopy(json_data.get("_clc_sync", False)) + self.clc_sync = deepcopy(json_data.get("_clc_sync", False)) if "_clc_sync" in json_data: del json_data["_clc_sync"] @@ -813,13 +316,18 @@ def transform(self, entry): error, ) + record_ctx = RecordTransformContext( + json_entry=json_data, + entry=entry, + access_grants_view=self.access_grants_view, + ) record_json_output = { "files": self._files(record_dump), "pids": self._pids(json_data), "metadata": self._metadata(json_data, entry), + "access_grants": AccessGrantsMapper().map_value(record_ctx), } - self._access_grants(json_data, record_json_output) custom_fields = self._custom_fields(json_data, record_json_output) internal_notes = json_data.get("internal_notes") @@ -846,18 +354,34 @@ def transform(self, entry): "recid": self._recid(record_dump), "communities": self._communities(json_data), "json": record_json_output, - "access": access, + "access_status": access, "owned_by": self._owner(json_data), - # keep the original extracted entry for storing it - "_original_dump": entry, + # record-scoped extras, read by load.py nested under "record" "_request_data": request_data, - "_clc_sync": clc_sync, - "ep_approval": self.ep_approval_request, + "ep_approval": entry.get("ep_approval", []), } -class CDSToRDMRecordTransform(RDMRecordTransform): - """CDSToRDMRecordTransform.""" +class RecordBuildResult(NamedTuple): + """Private return type of ``CDSToRDMRecordTransform._record()``. + + Pairs the record content with the one ETL-envelope extra + (``clc_sync``) that can only be computed as a side effect of building + it - see ``CDSToRDMRecordEntry.clc_sync``. + """ + + record: RecordEntry + clc_sync: Any + + +class CDSToRDMRecordTransform: + """Assembles the ETL entry consumed by ``CDSRecordServiceLoad``. + + Wraps the ``record`` content built by ``CDSToRDMRecordEntry`` together + with ``versions``/``parent`` (computed here) and the ETL-envelope extras + that aren't record content (currently just ``_original_dump`` and + ``_clc_sync`` - see ``_transform()``). + """ def __init__( self, @@ -876,6 +400,9 @@ def __init__( preferred_model=None, ): """Constructor.""" + self._workers = workers + self._throw = throw + self._logger = None self.files_dump_dir = Path(files_dump_dir).absolute().as_posix() self.missing_users_dir = Path(missing_users).absolute().as_posix() self.communities_ids = communities_ids @@ -888,7 +415,13 @@ def __init__( self.record_state_logger = record_state_logger self.preferred_model = preferred_model self.db_state = {"affiliations": CDSMigrationAffiliationMapping} - super().__init__(workers, throw) + + @property + def logger(self): + """Return the base logger.""" + if self._logger is None: + self._logger = Logger.get_logger() + return self._logger def _communities_ids(self, entry, record): communities = record.get("communities", []) @@ -897,47 +430,52 @@ def _communities_ids(self, entry, record): return {"ids": communities, "default": self.communities_ids[0]} return {} - def _parent(self, entry, record): - if record["owned_by"] == "system": + def _parent(self, entry, record: RecordEntry) -> ParentEntry: + + email = record["owned_by"] + if not email: owner = "system" else: try: - owner = int(record["owned_by"]) - except (ValueError, TypeError): - owner = "system" - parent = { - "created": record["created"], # same as the record - "updated": record["updated"], # same as the record - "version_id": record["version_id"], - "json": { + user = User.query.filter_by(email=email).one() + owner = user.id + except NoResultFound: + raise UnexpectedValue( + message=f"{email} not found - did you run user migration?", + stage="transform", + recid=entry["legacy_recid"], + value=email, + priority="critical", + ) + + return ParentEntry( + created=record["created"], # same as the record + updated=record["updated"], # same as the record + version_id=record["version_id"], + json=ParentJson( # loader is responsible for creating/updating if the PID exists. # this part will be simply omitted - "id": f'{record["recid"]}-parent', - "access": { - "owned_by": {"user": owner}, - }, - "communities": self._communities_ids(entry, record), - }, - } - - return parent + id=f'{record["recid"]}-parent', + access=ParentAccess(owned_by={"user": owner}), + communities=self._communities_ids(entry, record), + ), + ) - def _transform(self, entry): + def _transform(self, entry) -> Optional[MigrationEntry]: """Transform a single entry.""" # creates the output structure for load step migration_logger = self.migration_logger try: - record = self._record(entry) - original_dump = record.pop("_original_dump", {}) - clc_sync = record.pop("_clc_sync", {}) + built = self._record(entry) + record = built.record if record: return { "record": record, "versions": self._versions(entry, record), "parent": self._parent(entry, record), - "_original_dump": original_dump, - "_clc_sync": clc_sync, + "_original_dump": entry, + "_clc_sync": built.clc_sync, } except ( LossyConversion, @@ -949,10 +487,9 @@ def _transform(self, entry): ) as e: migration_logger.add_log(e, record=entry) - def _record(self, entry): + def _record(self, entry) -> RecordBuildResult: # could be in draft as well, depends on how we decide to publish - - return CDSToRDMRecordEntry( + entry_builder = CDSToRDMRecordEntry( missing_users_dir=self.missing_users_dir, affiliations_mapping=self.db_state["affiliations"], dry_run=self.dry_run, @@ -962,7 +499,9 @@ def _record(self, entry): migration_logger=self.migration_logger, record_state_logger=self.record_state_logger, preferred_model=self.preferred_model, - ).transform(entry) + ) + record = entry_builder.transform(entry) + return RecordBuildResult(record=record, clc_sync=entry_builder.clc_sync) def _draft(self, entry): return None @@ -970,7 +509,7 @@ def _draft(self, entry): def _parse_file_status(self, file_status): pass - def _versions(self, entry, record): + def _versions(self, entry, record: RecordEntry) -> Dict[int, VersionEntry]: def compute_access(file, record_access): @@ -1079,7 +618,7 @@ def compute_files(file_dump, versions_dict): # we start versions from files (because this is the only way of # mapping version of files to version of records from legacy) _files = entry["files"] - record_access = record["access"] + record_access = record["access_status"] for file in _files: if should_skip_file(file): continue diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index e40379d8..a2890613 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -304,6 +304,8 @@ def report_number(self, key, value): scheme = "handle" if scheme == "arXiv:reportnumber": scheme = "cdsrn" + if scheme.lower() == "submitter": + scheme = "cdsrn" if ( scheme.upper() in PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS @@ -316,7 +318,7 @@ def report_number(self, key, value): raise UnexpectedValue(field=key, value=value, subfield="n") scheme = "handle" if (key == "037__" and not scheme) or (identifier and key == "088__"): - # if there is no scheme, it meaens report number + # if there is no scheme, it means report number scheme = "cdsrn" # if there is no identifier it means something else was stored in __9 From 422a5271dfd1e8d707383bf323d66225c23d16f5 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Fri, 21 Aug 2026 11:05:08 +0200 Subject: [PATCH 2/9] chore(transform): create entities to handle record split - encapsulate data keys handling within specific entity - isolate responsibilities of transforming and validation - remove data layer access from rules transformation --- .../rdm/records/load/ep_approval_entry.py | 32 +- .../rdm/records/load/ep_approval_load.py | 6 +- cds_migrator_kit/rdm/records/load/load.py | 141 +--- .../records/transform/entities/__init__.py | 0 .../records/transform/entities/migration.py | 45 ++ .../rdm/records/transform/entities/parent.py | 211 ++++++ .../rdm/records/transform/entities/record.py | 338 +++++++++ .../rdm/records/transform/entities/request.py | 150 ++++ .../rdm/records/transform/entities/version.py | 230 ++++++ .../rdm/records/transform/entry_types.py | 182 ----- .../rdm/records/transform/mappers/__init__.py | 2 +- .../rdm/records/transform/mappers/base.py | 1 - .../transform/mappers/custom_fields.py | 26 +- .../rdm/records/transform/mappers/metadata.py | 2 +- .../rdm/records/transform/mappers/registry.py | 2 +- .../rdm/records/transform/transform.py | 665 +++--------------- .../records/transform/transform_versions.py | 122 ++++ .../xml_processing/quality/reviewers.py | 68 -- .../xml_processing/rules/research.py | 21 +- cds_migrator_kit/users/load.py | 3 +- tests/cds-rdm/test_ep_approval_entry.py | 80 ++- tests/cds-rdm/test_load_reviewers.py | 52 +- tests/cds-rdm/test_publications_rules.py | 10 +- .../cds-rdm/test_transform_metadata_title.py | 4 +- tests/cds-rdm/test_transform_versions.py | 44 +- 25 files changed, 1369 insertions(+), 1068 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/entities/__init__.py create mode 100644 cds_migrator_kit/rdm/records/transform/entities/migration.py create mode 100644 cds_migrator_kit/rdm/records/transform/entities/parent.py create mode 100644 cds_migrator_kit/rdm/records/transform/entities/record.py create mode 100644 cds_migrator_kit/rdm/records/transform/entities/request.py create mode 100644 cds_migrator_kit/rdm/records/transform/entities/version.py delete mode 100644 cds_migrator_kit/rdm/records/transform/entry_types.py create mode 100644 cds_migrator_kit/rdm/records/transform/transform_versions.py delete mode 100644 cds_migrator_kit/rdm/records/transform/xml_processing/quality/reviewers.py diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py b/cds_migrator_kit/rdm/records/load/ep_approval_entry.py index ddca2423..5390051f 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py +++ b/cds_migrator_kit/rdm/records/load/ep_approval_entry.py @@ -14,10 +14,8 @@ from flask import current_app from cds_migrator_kit.errors import UnexpectedValue -from cds_migrator_kit.rdm.records.transform.entry_types import ( - MigrationEntry, - VersionEntry, -) +from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry +from cds_migrator_kit.rdm.records.transform.entities.version import VersionEntry EPPHAPP_FILE_TYPE = "EPPHAPP_FILE" EP_APPROVAL_REPORT_NUMBER_PREFIX = "CERN-EP" @@ -52,14 +50,14 @@ def identifiers(self, identifiers): def build(self) -> MigrationEntry: """Return a load entry with split files and modified metadata.""" split = deepcopy(self.entry) - split["record"].pop("ep_approval", None) + split.pop("ep_approval", None) split["versions"] = self._build_versions(split) self._apply_metadata(split) self._apply_entry_modifications(split) return split def _apply_metadata(self, split): - metadata = split["record"]["json"]["metadata"] + metadata = split["record"]["body"]["metadata"] metadata["identifiers"] = self.identifiers(metadata.get("identifiers", [])) self._remove_doi_pid(split) @@ -195,21 +193,20 @@ def identifiers(self, identifiers): return kept def _apply_entry_modifications(self, split): - split["record"].pop("_request_data", None) + split.pop("_request_data", None) split["record"]["owned_by"] = "system" - split["parent"]["json"]["access"]["owned_by"] = {"user": "system"} + split["parent"].body["access"]["owned_by"] = {"user": "system"} self._add_cern_scientific_community(split) def _add_cern_scientific_community(self, entry): community_id = _cern_scientific_community_id() - communities = entry.get("parent", {}).get("json", {}).get("communities", {}) + # mutating in place: entry["parent"].communities is the same dict + # object, no need to set it back. + communities = entry["parent"].communities ids = list(communities.get("ids", [])) if community_id not in ids: ids.append(community_id) communities["ids"] = ids - entry.setdefault("parent", {}).setdefault("json", {})[ - "communities" - ] = communities class RestrictedEntry(MetadataEntry): @@ -226,16 +223,15 @@ def _remove_cern_scientific_community(self, entry): it (see _add_cern_scientific_community). """ community_id = _cern_scientific_community_id() - communities = entry.get("parent", {}).get("json", {}).get("communities", {}) + # mutating in place: entry["parent"].communities is the same dict + # object, no need to set it back. + communities = entry["parent"].communities ids = [ cid for cid in communities.get("ids", []) if cid != community_id ] communities["ids"] = ids if communities.get("default") == community_id: communities["default"] = ids[0] if ids else None - entry.setdefault("parent", {}).setdefault("json", {})[ - "communities" - ] = communities def _has_restricted_files(self, split): return any( @@ -341,8 +337,8 @@ def identifiers(self, identifiers): def _remove_doi_pid(self, split): """Remove DOI PID from restricted record.""" recid = split.get("record", {}).get("recid") - record_json = split.get("record", {}).get("json", {}) - pids = record_json.get("pids") + record_body = split.get("record", {}).get("body", {}) + pids = record_body.get("pids") if not pids or "doi" not in pids: return diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_load.py b/cds_migrator_kit/rdm/records/load/ep_approval_load.py index 1636f9a6..8f2cf0c2 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_load.py +++ b/cds_migrator_kit/rdm/records/load/ep_approval_load.py @@ -90,7 +90,7 @@ def _load(self, entry): self.migration_logger.finalise_record(recid) return - ep_approval = entry.get("record", {}).get("ep_approval") + ep_approval = entry.get("ep_approval") if not ep_approval: raise UnexpectedValue( message="EP approval request not found", @@ -98,8 +98,8 @@ def _load(self, entry): recid=recid, priority="critical", ) - record_json = entry.get("record", {}).get("json", {}) - metadata = record_json.get("metadata", {}) + record_body = entry.get("record", {}).get("body", {}) + metadata = record_body.get("metadata", {}) self.approval_request = ApprovalRequest( ep_approval=ep_approval, diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 0a9829b2..b7f49ab7 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -10,7 +10,6 @@ import datetime import json import os -import re from copy import deepcopy from typing import Dict @@ -50,8 +49,8 @@ RecordFlaggedCuration, UnexpectedValue, ) -from cds_migrator_kit.rdm.records.transform.entry_types import ( - MigrationEntry, +from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry +from cds_migrator_kit.rdm.records.transform.entities.version import ( VersionAccess, VersionFileEntry, ) @@ -197,11 +196,12 @@ def _load_files( def _load_parent_access_and_communities(self, draft, entry: MigrationEntry): """Load access rights and communities in a single parent commit.""" parent = draft._record.parent - parent.access = entry["parent"]["json"]["access"] - communities = entry["parent"]["json"]["communities"]["ids"] + record_parent = entry["parent"] + parent.access = record_parent.body["access"] + communities = record_parent.communities["ids"] for community in communities: parent.communities.add(community) - parent.communities.default = entry["parent"]["json"]["communities"]["default"] + parent.communities.default = record_parent.communities["default"] parent.commit() def _load_record_access(self, draft, access_dict: VersionAccess): @@ -229,7 +229,7 @@ def _after_publish_update_dois(self, identity, record, entry, uow): """Update migrated DOIs post publish.""" if not self._is_final_record: return - migrated_pids = entry["record"]["json"]["pids"] + migrated_pids = entry["record"]["body"]["pids"] for pid_type, identifier in migrated_pids.items(): if pid_type == "doi": # If a DOI was already minted from legacy then on publish the datacite @@ -248,107 +248,19 @@ def _after_publish_load_parent_access_grants( self, draft, version, entry: MigrationEntry ): """Load access grants from metadata and record grants efficiently.""" - - def _normalize_group_name(subject): - if subject.endswith(" [CERN]"): - subject = subject.replace(" [CERN]", "") - return subject.strip() - access_dict = entry["versions"][version]["access"] parent = draft._record.parent identity = system_identity - record_grants = entry["record"]["json"].get("access_grants", []) + record_parent = entry["parent"] specific_file_restrictions = access_dict.get("meta", "") - if not specific_file_restrictions and not record_grants: + if not specific_file_restrictions and not record_parent.access_grants: return default_permission = "view" - groups = set() - emails = set() - grants_with_perms = {} - email_pattern = re.compile(r"[^@]+@[^@]+\.[^@]+") - - # ----Parse file status metadata----# - if specific_file_restrictions: - - group_mappings = current_app.config.get("CDS_ACCESS_GROUP_MAPPINGS", {}) - - if specific_file_restrictions in group_mappings: - try: - groups.update(group_mappings[specific_file_restrictions]) - except KeyError as e: - raise ManualImportRequired( - message="Missing permission mapping", - field="access", - subfield="subject.id", - stage="load", - recid=entry["record"]["recid"], - priority="critical", - value=specific_file_restrictions, - ) - elif specific_file_restrictions == "restricted": - # https://cds.cern.ch/admin/webaccess/webaccessadmin.py/showroledetails?id_role=69 - groups.add("cern-personnel") - elif specific_file_restrictions.strip().endswith("[CERN]") and not any( - kw in specific_file_restrictions for kw in ("firerole:", "allow ") - ): - # bare CERN e-group name, e.g. - # "cds-ph-ep-publications-referee-non-lhc [CERN]" - groups.add(_normalize_group_name(specific_file_restrictions)) - else: - if not any( - kw in specific_file_restrictions - for kw in ("firerole: allow group", "allow email") - ): - raise ManualImportRequired( - message="Unexpected permission format.", - field="access", - subfield="subject.id", - stage="load", - recid=entry["record"]["recid"], - priority="critical", - value=specific_file_restrictions, - ) - - meta_str = specific_file_restrictions.replace("\r\n", "\n") - - # Parse groups - group_matches = re.search( - r'allow group\s+((?:"[^"]+",?\s*)+)', meta_str - ) - if group_matches: - group_values = re.findall(r'"([^"]+)"', group_matches.group(1)) - for g in group_values: - groups.add(_normalize_group_name(g)) - - # Parse emails - email_matches = re.search( - r'allow email\s+((?:"[^"]+",?\s*)+)', meta_str - ) - if email_matches: - email_values = re.findall(r'"([^"]+)"', email_matches.group(1)) - emails.update(email_values) - - # ----Parse record access grants----# - - for grant_info in record_grants: - if not isinstance(grant_info, dict) or not grant_info: - continue - - subject, permission = next(iter(grant_info.items())) - permission = permission or default_permission - grants_with_perms[subject] = permission - - # attention! - # this is important - if there was no specific restrictions on the file, - # then the record grands takes over - but if file had specific status, - # then we take the least possible access - if not specific_file_restrictions: - if email_pattern.match(subject): - emails.add(subject) - else: - groups.add(_normalize_group_name(subject)) + groups, emails, grants_with_perms = record_parent.resolve_grants( + specific_file_restrictions + ) def _create_grant(subject_type, subject_id, permission): grant_data = { @@ -608,21 +520,14 @@ def _after_publish( self._after_publish_update_files_created(published_record, entry, version) self._after_publish_load_parent_access_grants(published_record, version, entry) self._after_publish_set_committee_approval(published_record, entry["record"], uow) - request_data = entry["record"].get("_request_data", {}) + record_request = entry.get("_request_data") - if request_data and not self.create_inclusion_request: - raise ManualImportRequired( - message="Detected request data, enable the requests", - field="validation", - stage="load", - recid=entry["record"]["recid"], - priority="warning", - subfield=None, - ) - if self.create_inclusion_request and request_data: - self._after_publish_add_submission_request( - request_data, published_record, entry, uow - ) + if record_request: + record_request.ensure_enabled(self.create_inclusion_request) + if self.create_inclusion_request: + self._after_publish_add_submission_request( + record_request.data, published_record, entry, uow + ) # db.session.commit() def _assign_rep_numbers(self, draft): @@ -669,7 +574,7 @@ def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): # we decided to skip it and act normal try: draft = current_rdm_records_service.create( - identity, data=entry["record"]["json"], uow=uow + identity, data=entry["record"]["body"], uow=uow ) self._assign_rep_numbers(draft) except (UniqueViolation, IntegrityError) as e: @@ -694,7 +599,7 @@ def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): draft_dict = draft.to_dict() if not self.update_new_version_publication_date: publication_date = arrow.get( - entry["record"]["json"]["metadata"]["publication_date"] + entry["record"]["body"]["metadata"]["publication_date"] ) else: publication_date = versions[version]["publication_date"] @@ -750,7 +655,7 @@ def _load_versions(self, entry: MigrationEntry, uow): def _dry_load(self, entry: MigrationEntry): current_rdm_records_service.schema.load( - entry["record"]["json"], + entry["record"]["body"], context=dict( identity=system_identity, ), @@ -893,7 +798,7 @@ def _load(self, entry: MigrationEntry, uow=None): del entry["_clc_sync"] try: - ep_approval = entry.get("record", {}).get("ep_approval") + ep_approval = entry.get("ep_approval") if ep_approval: raise UnexpectedValue( message="EP approval records must be loaded with the '--ep-approval' flag", diff --git a/cds_migrator_kit/rdm/records/transform/entities/__init__.py b/cds_migrator_kit/rdm/records/transform/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cds_migrator_kit/rdm/records/transform/entities/migration.py b/cds_migrator_kit/rdm/records/transform/entities/migration.py new file mode 100644 index 00000000..129dcfa2 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/entities/migration.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""The full ETL entry yielded by ``CDSToRDMRecordTransform.run()``.""" +from typing import Any, Dict, List, TypedDict + +from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntryData +from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest +from cds_migrator_kit.rdm.records.transform.entities.version import VersionEntry + + +class MigrationEntry(TypedDict): + """The full ETL entry yielded by ``CDSToRDMRecordTransform.run()``. + + Consumed by ``CDSRecordServiceLoad``/``ep_approval_entry.py``. Keys + outside ``"record"`` are ETL-envelope-scoped (about this migration run, + not about the record's own content): ``versions``/``parent`` are + computed by ``CDSToRDMRecordTransform`` itself, while + ``_original_dump``/``_clc_sync``/``_request_data``/``ep_approval`` are + carried alongside it - see ``CDSToRDMRecordTransform._transform()``. + None of the four need ``RecordEntry`` to build: ``_original_dump`` + and ``ep_approval`` are read straight off the raw harvested entry, + which the transform already has; ``_clc_sync`` and ``_request_data`` + are popped off ``raw_json_entry`` in ``_transform()``, before + ``RecordEntry.transform()`` is even called. + + ``parent`` is a real ``RecordParent`` object, not a dict - see + ``entities/parent.py``. + """ + + record: RecordEntryData + versions: Dict[int, VersionEntry] + parent: RecordParent + # Community-inclusion request for this record - a RecordRequest object, + # not a plain dict; see entities/request.py. + _request_data: RecordRequest + # EP approval workflow entries for this record, if any (possibly []). + ep_approval: List[dict] + _original_dump: dict + _clc_sync: Any diff --git a/cds_migrator_kit/rdm/records/transform/entities/parent.py b/cds_migrator_kit/rdm/records/transform/entities/parent.py new file mode 100644 index 00000000..15723d83 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/entities/parent.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""The RDM parent record for one migrated CDS record.""" +import re + +from flask import current_app +from invenio_accounts.models import User +from sqlalchemy.exc import NoResultFound + +from cds_migrator_kit.errors import ManualImportRequired, UnexpectedValue +from cds_migrator_kit.rdm.records.transform.mappers.base import RecordTransformContext +from cds_migrator_kit.rdm.records.transform.mappers.record import AccessGrantsMapper + +EMAIL_PATTERN = re.compile(r"[^@]+@[^@]+\.[^@]+") + + +class RecordParent: + """The parent record for one migrated CDS record. + + Owns everything needed to build the RDM parent's access, community + membership, and access grants - built by ``build()``, called from + ``CDSToRDMRecordTransform._transform()`` (the one place that has both + the already-built record content and the DOJSON-processed + ``json_entry`` that access-grant resolution needs). + """ + + def __init__(self, record, entry, json_entry, communities_ids, access_grants_view): + """Constructor. + + :param record: the already-built ``RecordEntryData`` dict (needs + ``owned_by``/``recid``/``communities``). + :param entry: the original harvested legacy entry (needs + ``legacy_recid``, for error reporting). + :param json_entry: the DOJSON-processed record data - required to + resolve access grants (see ``_build_access_grants()``). + :param communities_ids: configured target community ids for this + migration run (``CDSToRDMRecordTransform.communities_ids``). + :param access_grants_view: configured collection-wide view grants + (``CDSToRDMRecordTransform.access_grants_view``). + """ + self.record = record + self.entry = entry + self.json_entry = json_entry + self.communities_ids = communities_ids + self.access_grants_view = access_grants_view + self.body = None + self.communities = None + self.access_grants = None + + def build(self): + """Populate ``body``/``communities``/``access_grants``; returns self.""" + access = self._build_access() + self.communities = self._build_communities() + self.access_grants = self._build_access_grants_from_record_marc() + self.body = { + # loader is responsible for creating/updating if the PID exists, + # this part will be simply omitted. + "id": f'{self.record["recid"]}-parent', + "access": access, + "communities": self.communities, + } + return self + + def _build_access(self): + """Resolve the owner and return the parent's access dict.""" + email = self.record["owned_by"] + if not email: + owner = "system" + else: + try: + user = User.query.filter_by(email=email).one() + owner = user.id + except NoResultFound: + raise UnexpectedValue( + message=f"{email} not found - did you run user migration?", + stage="transform", + recid=self.entry["legacy_recid"], + value=email, + priority="critical", + ) + return {"owned_by": {"user": owner}} + + def _build_communities(self): + """Combine the configured target communities with the record's own.""" + communities = self.record.get("communities", []) + communities = self.communities_ids + [slug for slug in communities] + if communities: + return {"ids": communities, "default": self.communities_ids[0]} + return {} + + def _build_access_grants_from_record_marc(self): + """Compute the access grants to create on this parent after publish.""" + ctx = RecordTransformContext( + json_entry=self.json_entry, + entry=self.entry, + access_grants_view=self.access_grants_view, + ) + return AccessGrantsMapper().map_value(ctx) + + def resolve_grants(self, specific_file_restrictions=""): + """Resolve which groups/emails/permissions get access grants for a version. + + Combines this parent's own ``access_grants`` (record-level, from + legacy access-grant metadata) with ``specific_file_restrictions`` (a + version-specific file-restriction status string, e.g. "firerole: + allow group ..."). Pure computation - the caller (``load.py``) is + responsible for actually creating the grants against the RDM parent. + + :param specific_file_restrictions: the ``meta`` value from a + version's access dict (``VersionEntry["access"]["meta"]``), or + "" if that version has no individual file restriction. + :return: ``(groups, emails, grants_with_perms)`` - see + ``load.py::_after_publish_load_parent_access_grants()`` for how + these are consumed to create the actual grants. + """ + default_permission = "view" + groups = set() + emails = set() + grants_with_perms = {} + + # ----Parse file status metadata----# + if specific_file_restrictions: + group_mappings = current_app.config.get("CDS_ACCESS_GROUP_MAPPINGS", {}) + + if specific_file_restrictions in group_mappings: + try: + groups.update(group_mappings[specific_file_restrictions]) + except KeyError: + raise ManualImportRequired( + message="Missing permission mapping", + field="access", + subfield="subject.id", + stage="load", + recid=self.record["recid"], + priority="critical", + value=specific_file_restrictions, + ) + elif specific_file_restrictions == "restricted": + # https://cds.cern.ch/admin/webaccess/webaccessadmin.py/showroledetails?id_role=69 + groups.add("cern-personnel") + elif specific_file_restrictions.strip().endswith("[CERN]") and not any( + kw in specific_file_restrictions for kw in ("firerole:", "allow ") + ): + # bare CERN e-group name, e.g. + # "cds-ph-ep-publications-referee-non-lhc [CERN]" + groups.add(self._normalize_group_name(specific_file_restrictions)) + else: + if not any( + kw in specific_file_restrictions + for kw in ("firerole: allow group", "allow email") + ): + raise ManualImportRequired( + message="Unexpected permission format.", + field="access", + subfield="subject.id", + stage="load", + recid=self.record["recid"], + priority="critical", + value=specific_file_restrictions, + ) + + meta_str = specific_file_restrictions.replace("\r\n", "\n") + + # Parse groups + group_matches = re.search( + r'allow group\s+((?:"[^"]+",?\s*)+)', meta_str + ) + if group_matches: + group_values = re.findall(r'"([^"]+)"', group_matches.group(1)) + for g in group_values: + groups.add(self._normalize_group_name(g)) + + # Parse emails + email_matches = re.search( + r'allow email\s+((?:"[^"]+",?\s*)+)', meta_str + ) + if email_matches: + email_values = re.findall(r'"([^"]+)"', email_matches.group(1)) + emails.update(email_values) + + # ----Parse record access grants----# + for grant_info in self.access_grants: + if not isinstance(grant_info, dict) or not grant_info: + continue + + subject, permission = next(iter(grant_info.items())) + permission = permission or default_permission + grants_with_perms[subject] = permission + + # attention! + # this is important - if there was no specific restrictions on the file, + # then the record grands takes over - but if file had specific status, + # then we take the least possible access + if not specific_file_restrictions: + if EMAIL_PATTERN.match(subject): + emails.add(subject) + else: + groups.add(self._normalize_group_name(subject)) + + return groups, emails, grants_with_perms + + @staticmethod + def _normalize_group_name(subject): + if subject.endswith(" [CERN]"): + subject = subject.replace(" [CERN]", "") + return subject.strip() diff --git a/cds_migrator_kit/rdm/records/transform/entities/record.py b/cds_migrator_kit/rdm/records/transform/entities/record.py new file mode 100644 index 00000000..8276ff58 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/entities/record.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""The RDM record's own content - ``MigrationEntry["record"]``.""" +from copy import deepcopy +from typing import Any, List, Optional, TypedDict, Union + +from flask import current_app +from idutils.validators import is_doi + +from cds_migrator_kit.errors import ( + ManualImportRequired, + RecordFlaggedCuration, + UnexpectedValue, +) +from cds_migrator_kit.rdm.records.transform.config import ( + PIDS_SCHEMES_ALLOWED, + PIDS_SCHEMES_TO_DROP, +) +from cds_migrator_kit.rdm.records.transform.mappers.base import RecordTransformContext +from cds_migrator_kit.rdm.records.transform.mappers.registry import ( + CUSTOM_FIELD_MAPPERS, + METADATA_MAPPERS, +) + + +class RecordBodyRequired(TypedDict): + """Required keys of ``RecordEntryData["body"]``.""" + + files: dict + pids: dict + metadata: dict + + +class RecordBody(RecordBodyRequired, total=False): + """The RDM record body: ``RecordEntryData["body"]``. + + ``current_rdm_records_service.create()``'s ``data`` argument - built by + ``RecordEntry.transform()``. Deliberately excludes: + + - ``access``: set per-version, after creation, via + ``VersionEntry["access"]`` (see ``load.py::_load_record_access``) + rather than at record-create time. + - ``access_grants``: RDM parent-level, not record-level - see + ``RecordParent.access_grants`` in ``entities/parent.py``. + """ + + custom_fields: dict + internal_notes: Any + + +class RecordEntryData(TypedDict): + """A single record's content - ``MigrationEntry["record"]``. + + Built by ``RecordEntry.transform()``. Everything here is + record-scoped (as opposed to ``MigrationEntry``'s other top-level keys, + which are ETL-envelope-scoped - see that type's docstring). + """ + + created: str + updated: str + version_id: int + index: int + recid: str + communities: List[str] + # The record's actual content, handed wholesale to + # current_rdm_records_service.create()/schema.load() - see RecordBody. + body: RecordBody + # None when RecordFlaggedCuration was raised and caught - see + # RecordEntry._access()/.transform(). + access_status: Optional[str] + owned_by: Union[str, int] + + +class RecordEntry: + """Transform CDS record to RDM record. + + Builds the ``record`` content dict consumed by + ``CDSToRDMRecordTransform`` - not the invenio_rdm_migrator "generic RDM + record" envelope (this class deliberately does not use that framework's + ``RDMRecordEntry.transform()``/``_load_partial`` orchestration, since the + CDS legacy shape and the CDS loader's needs don't match it). + """ + + def __init__( + self, + partial=False, + missing_users_dir=None, + missing_users_filename="people.csv", + affiliations_mapping=None, + dry_run=False, + collection=None, + restricted=False, + migration_logger=None, + record_state_logger=None, + ): + """Constructor.""" + self.partial = partial + self.missing_users_dir = missing_users_dir + self.missing_users_filename = missing_users_filename + self.affiliations_mapping = affiliations_mapping + self.dry_run = dry_run + self.collection = collection + self.restricted = restricted + self.migration_logger = migration_logger + self.record_state_logger = record_state_logger + + def _created(self, entry): + return entry["created"] + + def _updated(self, record_dump): + """Returns the creation date of the record.""" + return record_dump.data["record"][0]["modification_datetime"] + + def _version_id(self, entry): + """Returns the version id of the record.""" + return 1 + + def _access(self, entry, record_dump): + record_restriction = ( + r[0] if isinstance(r := entry.get("record_restriction"), list) else r + ) + restrictions = "restricted" if self.restricted else record_restriction + if not restrictions: + raise RecordFlaggedCuration( + message="record restriction not found make sure the record should be public", + stage="transform", + field="record_restriction", + ) + return restrictions + + def _index(self, record_dump): + """Returns the version index of the record.""" + return 1 # in legacy we start at 0 + + def _recid(self, record_dump): + """Returns the recid of the record.""" + return str(record_dump.data["recid"]) + + def _pids(self, json_entry): + DATACITE_PREFIX = current_app.config["DATACITE_PREFIX"] + + pids = json_entry.get("_pids", {}) + output_pids = deepcopy(pids) + for key, identifier in pids.items(): + # ignoring some pids + if key.upper() in PIDS_SCHEMES_TO_DROP: + del output_pids[key] + + elif key and key.upper() not in PIDS_SCHEMES_ALLOWED: + raise UnexpectedValue( + field=key, + subfield="2", + message="Unexpected PID scheme (should be DOI)", + priority="warning", + stage="transform", + value=identifier, + ) + elif not key and is_doi(identifier): + # assume it is DOI + key = "DOI" + if key.upper() == "DOI": + doi_identifier = deepcopy(identifier) + doi = identifier["identifier"] + + if doi.startswith(DATACITE_PREFIX): + doi_identifier["provider"] = "datacite" + else: + doi_identifier["provider"] = "external" + + if doi.startswith(DATACITE_PREFIX) or doi.startswith("10.5170"): + if not json_entry.get("publisher"): + json_entry["publisher"] = "CERN" + output_pids["doi"] = doi_identifier + if output_pids: + return output_pids + else: + return {} + + def _files(self, record_dump): + """Transform the files of a record.""" + record_dump.prepare_files() + files = record_dump.files + return {"enabled": bool(files)} + + def _communities(self, json_entry): + return json_entry.get("communities", []) + + def _owner(self, json_entry): + email = json_entry.get("submitter") + return email + + def _metadata(self, json_entry, entry): + """Build the metadata dict by running the composed field mappers.""" + ctx = RecordTransformContext( + json_entry=json_entry, + entry=entry, + migration_logger=self.migration_logger, + affiliations_mapping=self.affiliations_mapping, + ) + metadata = ctx.metadata + # Order matters: ResourceTypeMapper must run before TitleMapper reads + # metadata["resource_type"]; see mappers/registry.py. + for mapper in METADATA_MAPPERS: + metadata[mapper.id] = mapper.map_value(ctx) + + # filter empty keys + helper_keys = [ + "recid", + "legacy_recid", + "agency_code", + "submitter", + "status_week_date", + "record_restriction", + "access_grants", + "custom_fields", + "_pids", + "internal_notes", + "ep_approval", + ] + keys = deepcopy(list(json_entry.keys())) + for item in helper_keys: + if item in keys: + keys.remove(item) + + forgotten_keys = [key for key in keys if key not in list(metadata.keys())] + if forgotten_keys: + raise ManualImportRequired("Unassigned metadata key", value=forgotten_keys) + return {k: v for k, v in metadata.items() if v} + + def _custom_fields(self, json_entry): + """Build the custom_fields dict by running the composed field mappers. + + Must run before ``_metadata()``: a couple of these mappers add a + fallback ``json_entry["subjects"]`` entry when a vocabulary lookup + fails, which metadata's own SubjectsMapper then picks up like any + other subject - see DepartmentsMapper. + """ + ctx = RecordTransformContext( + json_entry=json_entry, + entry=json_entry, + migration_logger=self.migration_logger, + ) + for mapper in CUSTOM_FIELD_MAPPERS: + mapper.apply(ctx) + custom_fields = ctx.custom_fields + + forgotten_keys = [ + key + for key in json_entry["custom_fields"].keys() + if key not in custom_fields.keys() + ] + if forgotten_keys: + raise ManualImportRequired( + "Unassigned custom field key", value=forgotten_keys + ) + # filter out null values + return {k: v for k, v in custom_fields.items() if v} + + def _verify_publication_date(self, entry, json_data): + """Verify creation date. + + If the record has no files (file creation date will be used as record + creation date) and no creation date, raise an exception. + """ + if not entry.get("files") and not ( + json_data.get("status_week_date") or json_data.get("publication_date") + ): + raise ManualImportRequired( + message="Record missing publication date", + field="validation", + stage="transform", + description="Record has no files and no publication date", + recid=entry["recid"], + priority="warning", + value=None, + subfield=None, + ) + + def transform(self, entry, record_dump, json_data) -> RecordEntryData: + """Transform a record single entry. + + :param entry: the original harvested legacy entry. + :param record_dump: the ``CDSRecordDump`` for ``entry`` - produced + by ``CDSToRDMRecordTransform.transform_xml_to_json()``. + :param json_data: the DOJSON-processed record data + (``record_dump.latest_revision``'s content) - also produced by + ``transform_xml_to_json()``, passed in separately since this + method mutates it in place as it builds the record. + """ + self._verify_publication_date(entry, json_data) + + # custom_fields runs before metadata: see _custom_fields()'s docstring. + custom_fields = self._custom_fields(json_data) + + record_json_output = { + "files": self._files(record_dump), + "pids": self._pids(json_data), + "metadata": self._metadata(json_data, entry), + "internal_notes": json_data.get("internal_notes"), + "custom_fields": custom_fields, + } + # drop empty optional keys rather than sending them to the RDM + # record schema as null/{} + for key in ("internal_notes", "custom_fields"): + if not record_json_output[key]: + del record_json_output[key] + + access = None + try: + access = self._access(json_data, record_dump) + except RecordFlaggedCuration as exc: + self.migration_logger.add_information( + entry["recid"], + {"message": exc.message, "value": exc.value}, + ) + return { + "created": record_dump.first_created, + "updated": self._updated(record_dump), + "version_id": self._version_id(record_dump), + "index": self._index(record_dump), + "recid": self._recid(record_dump), + "communities": self._communities(json_data), + "body": record_json_output, + "access_status": access, + "owned_by": self._owner(json_data), + # _request_data/ep_approval are no longer record content here - + # CDSToRDMRecordTransform._transform() assigns them directly on + # MigrationEntry, since neither needs anything only this method + # has: ep_approval reads the raw entry (already available to + # the caller), and _request_data's pop off json_data happens + # in CDSToRDMRecordTransform._transform(), before this method + # even runs. + } diff --git a/cds_migrator_kit/rdm/records/transform/entities/request.py b/cds_migrator_kit/rdm/records/transform/entities/request.py new file mode 100644 index 00000000..a1fe983f --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/entities/request.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""A community-inclusion request for one migrated CDS record.""" +from invenio_accounts.models import User +from invenio_db import db + +from cds_migrator_kit.errors import ManualImportRequired, RecordFlaggedCuration + + +class RecordRequest: + """A community-inclusion request for one migrated CDS record. + + Built by ``CDSToRDMRecordTransform._transform()`` - pops the raw + request data off ``json_entry`` (before any field mapper can see it, + since it isn't part of the RDM record schema), resolves raw reviewer + name/email strings to actual user accounts (a DB lookup, which is why + this lives here and not in a dojson rule - rules must stay DB-free), + and logs any resolution failures. Consumed by ``load.py`` after + publish, which owns actually creating the RDM request. + """ + + def __init__(self, json_entry, recid, migration_logger): + """Constructor. + + :param json_entry: the DOJSON-processed record data - request_data + is popped off it. + :param recid: this record's legacy recid, for error reporting. + :param migration_logger: for reviewer-error/validation logging. + """ + self.json_entry = json_entry + self.recid = recid + self.migration_logger = migration_logger + self.data = None + + def build(self): + """Pop request_data off ``json_entry``, resolve reviewers; return self.""" + request_data = self.json_entry.pop("request_data", None) + if request_data: + reviewer_names = request_data.pop("reviewer_names", []) + # merge into whatever's already there rather than overwriting - + # some rules (e.g. faser_publication.py's status rule) add + # already-resolved reviewer entries directly (no DB lookup + # needed, e.g. a group reviewer), and those must be kept. + reviewers = request_data.setdefault("reviewers", []) + for reviewer_entry in self._resolve_reviewers(reviewer_names): + if reviewer_entry not in reviewers: + reviewers.append(reviewer_entry) + self.data = request_data + return self + + def __bool__(self): + """True when there's request data to act on.""" + return bool(self.data) + + def ensure_enabled(self, create_inclusion_request): + """Raise if this record has request data but requests aren't enabled. + + :param create_inclusion_request: whether the current load run is + configured to create community-inclusion requests - see + ``CDSRecordServiceLoad.create_inclusion_request``. + """ + if self.data and not create_inclusion_request: + raise ManualImportRequired( + message="Detected request data, enable the requests", + field="validation", + stage="load", + recid=self.recid, + priority="warning", + subfield=None, + ) + + def _resolve_reviewers(self, reviewer_names): + """Resolve raw reviewer name/email strings to RDM reviewer entries. + + A reviewer that can't be matched to a user account is flagged for + curation (logged) and represented by a "-1" placeholder user id, + rather than aborting the whole record. + """ + resolved = [] + for reviewer_name in reviewer_names: + try: + user = self._find_reviewer(reviewer_name) + reviewer_entry = {"user": str(user.id)} + except RecordFlaggedCuration as exc: + self.migration_logger.add_information( + self.recid, {"message": exc.message, "value": exc.value} + ) + reviewer_entry = {"user": "-1"} + if reviewer_entry not in resolved: + resolved.append(reviewer_entry) + return resolved + + @staticmethod + def _find_reviewer(reviewer): + """Resolve a reviewer string (email or name) to a User. + + :param reviewer: email address, or a "Family, Given"/"Given Family" name. + :raises RecordFlaggedCuration: if no matching user is found, so the + record is flagged for manual curation instead of failing outright. + """ + reviewer = reviewer.strip() + if RecordRequest._is_email(reviewer): + user = User.query.filter_by(email=reviewer).one_or_none() + else: + family_name, given_name = RecordRequest._parse_reviewer_name(reviewer) + query = User.query.filter( + db.func.lower(User._user_profile["family_name"].as_string()) + == family_name.lower() + ) + if given_name: + query = query.filter( + db.func.lower(User._user_profile["given_name"].as_string()) + == given_name.lower() + ) + user = query.one_or_none() + + if user is None: + raise RecordFlaggedCuration( + message=f"Reviewer '{reviewer}' could not be matched to an account.", + field="request_reviewers", + stage="transform", + value=reviewer, + ) + return user + + @staticmethod + def _is_email(value): + """Return True if the reviewer value looks like an email address.""" + return "@" in value + + @staticmethod + def _parse_reviewer_name(name): + """Split a 'Family, Given' or 'Given Family' string into (family, given). + + ``request_reviewers`` (906__p) stores names as "Given Family" (comma + already resolved), but legacy data can also arrive as "Family, Given". + """ + name = name.strip() + if "," in name: + family, _, given = name.partition(",") + return family.strip(), given.strip() + parts = name.split() + if len(parts) > 1: + return parts[-1], " ".join(parts[:-1]) + return name, "" diff --git a/cds_migrator_kit/rdm/records/transform/entities/version.py b/cds_migrator_kit/rdm/records/transform/entities/version.py new file mode 100644 index 00000000..67d29646 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/entities/version.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""One record version - a value in ``MigrationEntry["versions"]``.""" +from pathlib import Path +from typing import Dict, Optional, TypedDict, Union + +import arrow +from arrow import Arrow + +LEGACY_FILES_PATH_ROOT = Path("/opt/cdsweb/var/data/files/") + + +class VersionAccessObj(TypedDict): + """``VersionAccess["access_obj"]`` - mirrors the RDM record access schema.""" + + record: Optional[str] + files: Optional[str] + + +class VersionAccess(TypedDict, total=False): + """A version's access - ``VersionEntry["access"]``. + + Set directly on the record post-create via + ``load.py::_load_record_access`` (``record.access = access_dict["access_obj"]``). + """ + + access_obj: VersionAccessObj + # Raw legacy file-restriction status string, present only when an + # individual file carried its own restriction - see + # RecordVersion.compute_access(). + meta: str + + +class VersionFileMetadata(TypedDict): + """``VersionFileEntry["metadata"]``.""" + + description: Optional[str] + name: str + status: str + original_path: str + comment: Optional[str] + + +class VersionFileEntry(TypedDict): + """One file within ``VersionEntry["files"]``, keyed by its ``full_name``.""" + + eos_tmp_path: Path + id_bibdoc: int + key: str + metadata: VersionFileMetadata + mimetype: str + checksum: str + version: int + access: str + type: str + creation_date: str + + +class VersionEntry(TypedDict): + """One record version - a value in ``MigrationEntry["versions"]``. + + Built by ``RecordVersionsTransform``, keyed there by legacy file + version number (int), starting at 1. + """ + + files: Dict[str, VersionFileEntry] + # Arrow instance when derived from a file's creation date; a plain ISO + # date string in the no-files fallback branch (copied straight from + # RecordEntryData["body"]["metadata"]["publication_date"]) - see + # RecordVersionsTransform.build(). + publication_date: Union[Arrow, str] + access: VersionAccess + + +class RecordVersion: + """One record version - a value in ``MigrationEntry["versions"]``. + + Built by ``RecordVersionsTransform``, which resolves the cross-version + file carry-forward (a version includes every earlier version's files + too - see that class) after each ``RecordVersion`` computes its own + files/access from its own raw legacy file dumps. + """ + + def __init__( + self, + record_access, + files_dump_dir, + migration_logger, + representative_file=None, + own_file_dumps=None, + publication_date=None, + ): + """Constructor. + + :param record_access: the record's overall access status + (``RecordEntryData["access_status"]``). + :param files_dump_dir: local EOS mirror root for file content. + :param migration_logger: for individual-file-restriction logging. + :param representative_file: the raw legacy file dump this version's + access is derived from (the first raw file dump encountered + for this version - see ``RecordVersionsTransform``), or + ``None`` for the metadata-only fallback version (no files at + all for the record). + :param own_file_dumps: this version's own raw legacy file dumps - + NOT including files carried forward from earlier versions, + that's ``RecordVersionsTransform``'s job. + :param publication_date: used as-is when ``representative_file`` + is ``None`` (the metadata-only fallback - a plain ISO date + string copied from the record's own publication date, rather + than an Arrow instance derived from a file's creation date). + """ + self.record_access = record_access + self.files_dump_dir = files_dump_dir + self.migration_logger = migration_logger + self.representative_file = representative_file + self.own_file_dumps = own_file_dumps or [] + self.publication_date = publication_date + self.files = None + self.access = None + + def build(self): + """Populate ``files``/``access``/``publication_date``; return this version's dict.""" + self.files = self.compute_files() + self.access = self.compute_access() + if self.representative_file is not None: + self.publication_date = arrow.get( + self.representative_file["creation_date"] + ).replace(tzinfo=None) + return { + "files": self.files, + "publication_date": self.publication_date, + "access": self.access, + } + + def compute_access(self): + """Return this version's access dict, from its representative file.""" + file = self.representative_file + record_access = self.record_access + if file is None or not file["status"]: + return { + "access_obj": { + "record": record_access, + "files": record_access, + } + } + # if we have anything in the status string, it means the file is + # restricted; we pass this information to parse later in load step + self.migration_logger.add_information( + str(file["recid"]), + { + "message": "Record has individual file restrictions", + "value": file["status"], + }, + ) + return { + "access_obj": {"record": record_access, "files": "restricted"}, + "meta": file["status"], + } + + def compute_files(self): + """Transform this version's own raw file dumps into RDM file entries.""" + files = {} + for file_dump in self.own_file_dumps: + files[file_dump["full_name"]] = self._compute_file(file_dump) + return files + + def _compute_file(self, file_dump): + tmp_eos_root = Path(self.files_dump_dir) + full_path = Path(file_dump["full_path"]) + return { + "eos_tmp_path": tmp_eos_root + / full_path.relative_to(LEGACY_FILES_PATH_ROOT), + "id_bibdoc": file_dump["bibdocid"], + "key": file_dump["full_name"], + "metadata": { + "description": file_dump["description"], + "name": file_dump["name"], + "status": file_dump["status"], + "original_path": file_dump["path"], + "comment": file_dump["comment"], + }, + "mimetype": file_dump["mime"], + "checksum": file_dump["checksum"], + "version": file_dump["version"], + "access": file_dump["status"], + "type": file_dump["type"], + "creation_date": arrow.get(file_dump["creation_date"]) + .replace(tzinfo=None) + .date() + .isoformat(), + } + + +# ATTENTION -leave this comment as it describes an example file dump +# +# "files": [ +# { +# "comment": null, +# "status": "firerole: allow group \"council-full [CERN]\"\ndeny until \"1996-02-01\"\nallow all", +# "version": 1, +# "encoding": null, +# "creation_date": "2009-11-03T12:29:06+00:00", +# "bibdocid": 502379, +# "mime": "application/pdf", +# "full_name": "CM-P00080632-e.pdf", +# "superformat": ".pdf", +# "recids_doctype": [[32097, "Main", "CM-P00080632-e.pdf"]], +# "path": "/opt/cdsweb/var/data/files/g50/502379/CM-P00080632-e.pdf;1", +# "size": 5033532, +# "license": {}, +# "modification_date": "2009-11-03T12:29:06+00:00", +# "copyright": {}, +# "url": "http://cds.cern.ch/record/32097/files/CM-P00080632-e.pdf", +# "checksum": "ed797ce5d024dcff0040db79c3396da9", +# "description": "English", +# "format": ".pdf", +# "name": "CM-P00080632-e", +# "subformat": "", +# "etag": "\"502379.pdf1\"", +# "recid": 32097, +# "flags": [], +# "hidden": false, +# "type": "Main", +# "full_path": "/opt/cdsweb/var/data/files/g50/502379/CM-P00080632-e.pdf;1" +# },] diff --git a/cds_migrator_kit/rdm/records/transform/entry_types.py b/cds_migrator_kit/rdm/records/transform/entry_types.py deleted file mode 100644 index ce321252..00000000 --- a/cds_migrator_kit/rdm/records/transform/entry_types.py +++ /dev/null @@ -1,182 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2022-2026 CERN. -# -# CDS-RDM is free software; you can redistribute it and/or modify it under -# the terms of the MIT License; see LICENSE file for more details. - -"""Typed shapes for the migration ETL entry. - -``TypedDict`` values are plain ``dict``s at runtime (no behavior change, -nothing enforced), so these exist purely to make the envelope shapes -produced by ``transform.py`` and consumed by ``load.py``/ -``ep_approval_entry.py`` visible at the definition site, instead of having -to be reconstructed by grepping across those files. - -Deliberately NOT modeled here: the RDM record body itself -(``RecordEntry["json"]["metadata"|"pids"|"files"|"custom_fields"]``) - that -shape is governed by invenio_rdm_records' own record schema, not by this -package, and is already comparatively well documented by the field mappers -in ``mappers/``. -""" -from pathlib import Path -from typing import Any, Dict, List, Optional, TypedDict, Union - -from arrow import Arrow - - -class RecordJsonOutputRequired(TypedDict): - """Required keys of ``RecordEntry["json"]``.""" - - files: dict - pids: dict - metadata: dict - access_grants: List[dict] - - -class RecordJsonOutput(RecordJsonOutputRequired, total=False): - """The RDM record body: ``RecordEntry["json"]``. - - ``current_rdm_records_service.create()``'s ``data`` argument - built by - ``CDSToRDMRecordEntry.transform()``. Deliberately excludes ``access``: - that's set per-version, after creation, via ``VersionEntry["access"]`` - (see ``load.py::_load_record_access``) rather than at record-create time. - """ - - custom_fields: dict - internal_notes: Any - - -class RecordEntry(TypedDict): - """A single record's content - ``MigrationEntry["record"]``. - - Built by ``CDSToRDMRecordEntry.transform()``. Everything here is - record-scoped (as opposed to ``MigrationEntry``'s other top-level keys, - which are ETL-envelope-scoped - see that type's docstring). - """ - - created: str - updated: str - version_id: int - index: int - recid: str - communities: List[str] - json: RecordJsonOutput - # None when RecordFlaggedCuration was raised and caught - see - # CDSToRDMRecordEntry._access()/.transform(). - access_status: Optional[str] - owned_by: Union[str, int] - # Community-inclusion request payload for this record, if any. - _request_data: Optional[dict] - # EP approval workflow entries for this record, if any (possibly []). - ep_approval: List[dict] - - -class ParentAccess(TypedDict): - """``ParentEntry["json"]["access"]``.""" - - owned_by: Dict[str, Union[str, int]] - - -class ParentCommunities(TypedDict, total=False): - """``ParentEntry["json"]["communities"]``.""" - - ids: List[str] - default: str - - -class ParentJson(TypedDict): - """``ParentEntry["json"]``.""" - - id: str - access: ParentAccess - communities: ParentCommunities - - -class ParentEntry(TypedDict): - """The parent record - ``MigrationEntry["parent"]``. Built by ``CDSToRDMRecordTransform._parent()``.""" - - created: str - updated: str - version_id: int - json: ParentJson - - -class VersionAccessObj(TypedDict): - """``VersionAccess["access_obj"]`` - mirrors the RDM record access schema.""" - - record: Optional[str] - files: Optional[str] - - -class VersionAccess(TypedDict, total=False): - """A version's access - ``VersionEntry["access"]``. - - Set directly on the record post-create via - ``load.py::_load_record_access`` (``record.access = access_dict["access_obj"]``). - """ - - access_obj: VersionAccessObj - # Raw legacy file-restriction status string, present only when an - # individual file carried its own restriction - see - # CDSToRDMRecordTransform._versions()::compute_access(). - meta: str - - -class VersionFileMetadata(TypedDict): - """``VersionFileEntry["metadata"]``.""" - - description: Optional[str] - name: str - status: str - original_path: str - comment: Optional[str] - - -class VersionFileEntry(TypedDict): - """One file within ``VersionEntry["files"]``, keyed by its ``full_name``.""" - - eos_tmp_path: Path - id_bibdoc: int - key: str - metadata: VersionFileMetadata - mimetype: str - checksum: str - version: int - access: str - type: str - creation_date: str - - -class VersionEntry(TypedDict): - """One record version - a value in ``MigrationEntry["versions"]``. - - Built by ``CDSToRDMRecordTransform._versions()``, keyed there by legacy - file version number (int), starting at 1. - """ - - files: Dict[str, VersionFileEntry] - # Arrow instance when derived from a file's creation date; a plain ISO - # date string in the no-files fallback branch (copied straight from - # RecordEntry["json"]["metadata"]["publication_date"]) - see - # CDSToRDMRecordTransform._versions(). - publication_date: Union[Arrow, str] - access: VersionAccess - - -class MigrationEntry(TypedDict): - """The full ETL entry yielded by ``CDSToRDMRecordTransform.run()``. - - Consumed by ``CDSRecordServiceLoad``/``ep_approval_entry.py``. Keys - outside ``"record"`` are ETL-envelope-scoped (about this migration run, - not about the record's own content): ``versions``/``parent`` are - computed by ``CDSToRDMRecordTransform`` itself, while - ``_original_dump``/``_clc_sync`` are carried alongside it - see - ``CDSToRDMRecordTransform._transform()``. - """ - - record: RecordEntry - versions: Dict[int, VersionEntry] - parent: ParentEntry - _original_dump: dict - _clc_sync: Any diff --git a/cds_migrator_kit/rdm/records/transform/mappers/__init__.py b/cds_migrator_kit/rdm/records/transform/mappers/__init__.py index eb15ac05..33b9b31d 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/__init__.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/__init__.py @@ -9,5 +9,5 @@ Each mapper owns the derivation of a single ``metadata`` or ``custom_fields`` value from the legacy record entry, composed together by -``CDSToRDMRecordEntry`` in ``transform.py``. +``RecordEntry`` in ``entities/record.py``. """ diff --git a/cds_migrator_kit/rdm/records/transform/mappers/base.py b/cds_migrator_kit/rdm/records/transform/mappers/base.py index eacf1f14..6cbafb2d 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/base.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/base.py @@ -27,7 +27,6 @@ class RecordTransformContext: migration_logger: object = None affiliations_mapping: object = None access_grants_view: object = None - json_output: dict = None metadata: dict = field(default_factory=dict) custom_fields: dict = field(default_factory=dict) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index 3cf5a649..9228e9e9 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -36,9 +36,6 @@ def apply(self, ctx): if result and result not in experiments_out: experiments_out.append(result) elif not result: - subj = ctx.json_output["metadata"].get("subjects", []) - subj.append({"subject": experiment}) - ctx.json_output["metadata"]["subjects"] = subj raise UnexpectedValue( subfield="e", value=experiment, @@ -73,9 +70,24 @@ def apply(self, ctx): elif not result: if department.lower() == "cern?": continue - subj = ctx.json_output["metadata"].get("subjects", []) - subj.append({"subject": department}) - ctx.json_output["metadata"]["subjects"] = subj + # Written into the shared source entry (not the already-built + # metadata output) so metadata's own SubjectsMapper picks it + # up naturally - see RecordEntry.transform(), which + # runs custom_fields mappers before metadata mappers for + # exactly this reason. + ctx.json_entry.setdefault("subjects", []).append( + {"subject": department} + ) + + if ctx.custom_fields.get("cern:administrative_unit"): + raise UnexpectedValue( + subfield="5", + value=department, + field="710", + message=f"conflict on administrative unit " + f"{ctx.custom_fields["cern:administrative_unit"]} VS {department}", + stage="vocabulary match", + ) ctx.custom_fields["cern:administrative_unit"] = department ctx.flag_curation( RecordFlaggedCuration( @@ -83,7 +95,7 @@ def apply(self, ctx): value=department, field="department", message=f"Department {department} not found. " - f"Added as unit and subject", + f"Added as unit and subject", stage="vocabulary match", ) ) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index 8852ef6f..ef2499be 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -158,7 +158,7 @@ def map_value(self, ctx): # Fields that pass through unchanged from json_entry - kept explicit in the # composed list (mappers/config equivalent) rather than open-ended, so the -# "forgotten metadata key" completeness check in CDSToRDMRecordEntry._metadata +# "forgotten metadata key" completeness check in RecordEntry._metadata # still catches any newly introduced json_entry key nobody has mapped yet. PASSTHROUGH_METADATA_FIELDS = ( "description", diff --git a/cds_migrator_kit/rdm/records/transform/mappers/registry.py b/cds_migrator_kit/rdm/records/transform/mappers/registry.py index 3b08bf01..a629f2ff 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/registry.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/registry.py @@ -5,7 +5,7 @@ # CDS-RDM is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -"""Composed lists of field mappers used by CDSToRDMRecordEntry.""" +"""Composed lists of field mappers used by RecordEntry.""" from cds_migrator_kit.rdm.records.transform.mappers.base import ( PassthroughCustomFieldMapper, PassthroughMapper, diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index d5da74f1..a100d5aa 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -6,381 +6,52 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform step module.""" -import datetime import logging import re from collections import OrderedDict from copy import deepcopy from pathlib import Path -from typing import Any, Dict, NamedTuple, Optional +from typing import Optional -import arrow -from cds_dojson.marc21.utils import create_record from cds_rdm.legacy.models import CDSMigrationAffiliationMapping from cds_rdm.legacy.resolver import get_pid_by_legacy_recid -from dateutil.parser import ParserError -from flask import current_app -from idutils.validators import is_doi from invenio_access.permissions import system_identity -from invenio_accounts.models import User from invenio_pidstore.models import PersistentIdentifier, PIDStatus from invenio_rdm_migrator.logging import Logger -from invenio_rdm_records.proxies import current_rdm_records_service, current_record_communities_service +from invenio_rdm_records.proxies import current_rdm_records_service, \ + current_record_communities_service from sqlalchemy.exc import NoResultFound from cds_migrator_kit.errors import ( ManualImportRequired, MissingRequiredField, MultipleModelsMatched, - RecordFlaggedCuration, RestrictedFileDetected, UnexpectedValue, ) -from cds_migrator_kit.rdm.records.transform.config import ( - FILE_SUBFORMATS_TO_DROP, - PIDS_SCHEMES_ALLOWED, - PIDS_SCHEMES_TO_DROP, -) -from cds_migrator_kit.rdm.records.transform.entry_types import ( - MigrationEntry, - ParentAccess, - ParentEntry, - ParentJson, +from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry +from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent +from cds_migrator_kit.rdm.records.transform.entities.record import ( RecordEntry, - VersionEntry, -) -from cds_migrator_kit.rdm.records.transform.mappers.base import RecordTransformContext -from cds_migrator_kit.rdm.records.transform.mappers.record import AccessGrantsMapper -from cds_migrator_kit.rdm.records.transform.mappers.registry import ( - CUSTOM_FIELD_MAPPERS, - METADATA_MAPPERS, + RecordEntryData, ) +from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest +from cds_migrator_kit.rdm.records.transform.transform_versions import \ + RecordVersionsTransform from cds_migrator_kit.transform.dumper import CDSRecordDump from cds_migrator_kit.transform.errors import LossyConversion cli_logger = logging.getLogger("migrator") -class CDSToRDMRecordEntry: - """Transform CDS record to RDM record. - - Builds the ``record`` content dict consumed by - ``CDSToRDMRecordTransform`` - not the invenio_rdm_migrator "generic RDM - record" envelope (this class deliberately does not use that framework's - ``RDMRecordEntry.transform()``/``_load_partial`` orchestration, since the - CDS legacy shape and the CDS loader's needs don't match it). - """ - - def __init__( - self, - partial=False, - missing_users_dir=None, - missing_users_filename="people.csv", - affiliations_mapping=None, - dry_run=False, - collection=None, - restricted=False, - migration_logger=None, - record_state_logger=None, - access_grants_view=None, - preferred_model=None, - ): - """Constructor.""" - self.partial = partial - self.missing_users_dir = missing_users_dir - self.missing_users_filename = missing_users_filename - self.affiliations_mapping = affiliations_mapping - self.dry_run = dry_run - self.collection = collection - self.restricted = restricted - self.access_grants_view = access_grants_view - self.migration_logger = migration_logger - self.record_state_logger = record_state_logger - # populated by transform(); an ETL-envelope concern (does the - # parent need a CLC sync after load), not record content, so it - # isn't part of the dict transform() returns - the caller - # (CDSToRDMRecordTransform._record()) reads it off this instance - # instead. See CDSToRDMRecordTransform._transform()'s docstring. - self.clc_sync = None - self.preferred_model = preferred_model - self.ep_approval_request = None - super().__init__(partial) - - def _created(self, entry): - return entry["created"] - - def _updated(self, record_dump): - """Returns the creation date of the record.""" - return record_dump.data["record"][0]["modification_datetime"] - - def _version_id(self, entry): - """Returns the version id of the record.""" - return 1 - - def _access(self, entry, record_dump): - record_restriction = ( - r[0] if isinstance(r := entry.get("record_restriction"), list) else r - ) - restrictions = "restricted" if self.restricted else record_restriction - if not restrictions: - raise RecordFlaggedCuration( - message="record restriction not found make sure the record should be public", - stage="transform", - field="record_restriction", - ) - return restrictions - - def _index(self, record_dump): - """Returns the version index of the record.""" - return 1 # in legacy we start at 0 - - def _recid(self, record_dump): - """Returns the recid of the record.""" - return str(record_dump.data["recid"]) - - def _pids(self, json_entry): - DATACITE_PREFIX = current_app.config["DATACITE_PREFIX"] - - pids = json_entry.get("_pids", {}) - output_pids = deepcopy(pids) - for key, identifier in pids.items(): - # ignoring some pids - if key.upper() in PIDS_SCHEMES_TO_DROP: - del output_pids[key] - - elif key and key.upper() not in PIDS_SCHEMES_ALLOWED: - raise UnexpectedValue( - field=key, - subfield="2", - message="Unexpected PID scheme (should be DOI)", - priority="warning", - stage="transform", - value=identifier, - ) - elif not key and is_doi(identifier): - # assume it is DOI - key = "DOI" - if key.upper() == "DOI": - doi_identifier = deepcopy(identifier) - doi = identifier["identifier"] - - if doi.startswith(DATACITE_PREFIX): - doi_identifier["provider"] = "datacite" - else: - doi_identifier["provider"] = "external" - - if doi.startswith(DATACITE_PREFIX) or doi.startswith("10.5170"): - if not json_entry.get("publisher"): - json_entry["publisher"] = "CERN" - output_pids["doi"] = doi_identifier - if output_pids: - return output_pids - else: - return {} - - def _files(self, record_dump): - """Transform the files of a record.""" - record_dump.prepare_files() - files = record_dump.files - return {"enabled": bool(files)} - - def _communities(self, json_entry): - return json_entry.get("communities", []) - - def _owner(self, json_entry): - email = json_entry.get("submitter") - return email - - def _metadata(self, json_entry, entry): - """Build the metadata dict by running the composed field mappers.""" - ctx = RecordTransformContext( - json_entry=json_entry, - entry=entry, - migration_logger=self.migration_logger, - affiliations_mapping=self.affiliations_mapping, - ) - metadata = ctx.metadata - # Order matters: ResourceTypeMapper must run before TitleMapper reads - # metadata["resource_type"]; see mappers/registry.py. - for mapper in METADATA_MAPPERS: - metadata[mapper.id] = mapper.map_value(ctx) - - # filter empty keys - helper_keys = [ - "recid", - "legacy_recid", - "agency_code", - "submitter", - "status_week_date", - "record_restriction", - "access_grants", - "custom_fields", - "_pids", - "internal_notes", - "ep_approval", - ] - keys = deepcopy(list(json_entry.keys())) - for item in helper_keys: - if item in keys: - keys.remove(item) - - forgotten_keys = [key for key in keys if key not in list(metadata.keys())] - if forgotten_keys: - raise ManualImportRequired("Unassigned metadata key", value=forgotten_keys) - return {k: v for k, v in metadata.items() if v} - - def _custom_fields(self, json_entry, json_output): - """Build the custom_fields dict by running the composed field mappers.""" - ctx = RecordTransformContext( - json_entry=json_entry, - entry=json_entry, - migration_logger=self.migration_logger, - json_output=json_output, - ) - for mapper in CUSTOM_FIELD_MAPPERS: - mapper.apply(ctx) - custom_fields = ctx.custom_fields - - forgotten_keys = [ - key - for key in json_entry["custom_fields"].keys() - if key not in custom_fields.keys() - ] - if forgotten_keys: - raise ManualImportRequired( - "Unassigned custom field key", value=forgotten_keys - ) - # filter out null values - return {k: v for k, v in custom_fields.items() if v} - - def _verify_publication_date(self, entry, json_data): - """Verify creation date. - - If the record has no files (file creation date will be used as record - creation date) and no creation date, raise an exception. - """ - if not entry.get("files") and not ( - json_data.get("status_week_date") or json_data.get("publication_date") - ): - raise ManualImportRequired( - message="Record missing publication date", - field="validation", - stage="transform", - description="Record has no files and no publication date", - recid=entry["recid"], - priority="warning", - value=None, - subfield=None, - ) - - def transform(self, entry) -> RecordEntry: - """Transform a record single entry.""" - record_dump = CDSRecordDump( - entry, - preferred_model=self.preferred_model, - ) - - record_dump.prepare_revisions() - - if record_dump.multiple_models_warning: - w = record_dump.multiple_models_warning - recid = entry.get("recid") or entry.get("record", {}).get("recid") - matched = re.findall(r"\['(\w+)',", w.message or "") - self.migration_logger.add_information( - str(recid), - { - "type": w.type, - "error": w.description, - "message": w.message, - "value": ", ".join(matched), - "priority": "warning", - }, - ) - - timestamp, json_data = record_dump.latest_revision - - self._verify_publication_date(entry, json_data) - - self.record_state_logger.add_record(json_data) - - self.clc_sync = deepcopy(json_data.get("_clc_sync", False)) - if "_clc_sync" in json_data: - del json_data["_clc_sync"] - - request_data = json_data.pop("request_data", None) - if request_data: - reviewer_errors = request_data.pop("_reviewer_errors", []) - for error in reviewer_errors: - self.migration_logger.add_information( - entry["recid"], - error, - ) - - record_ctx = RecordTransformContext( - json_entry=json_data, - entry=entry, - access_grants_view=self.access_grants_view, - ) - record_json_output = { - "files": self._files(record_dump), - "pids": self._pids(json_data), - "metadata": self._metadata(json_data, entry), - "access_grants": AccessGrantsMapper().map_value(record_ctx), - } - - custom_fields = self._custom_fields(json_data, record_json_output) - internal_notes = json_data.get("internal_notes") - - if custom_fields: - record_json_output.update({"custom_fields": custom_fields}) - if internal_notes: - record_json_output.update( - {"internal_notes": json_data.get("internal_notes")} - ) - - access = None - try: - access = self._access(json_data, record_dump) - except RecordFlaggedCuration as exc: - self.migration_logger.add_information( - entry["recid"], - {"message": exc.message, "value": exc.value}, - ) - return { - "created": record_dump.first_created, - "updated": self._updated(record_dump), - "version_id": self._version_id(record_dump), - "index": self._index(record_dump), - "recid": self._recid(record_dump), - "communities": self._communities(json_data), - "json": record_json_output, - "access_status": access, - "owned_by": self._owner(json_data), - # record-scoped extras, read by load.py nested under "record" - "_request_data": request_data, - "ep_approval": entry.get("ep_approval", []), - } - - -class RecordBuildResult(NamedTuple): - """Private return type of ``CDSToRDMRecordTransform._record()``. - - Pairs the record content with the one ETL-envelope extra - (``clc_sync``) that can only be computed as a side effect of building - it - see ``CDSToRDMRecordEntry.clc_sync``. - """ - - record: RecordEntry - clc_sync: Any - - class CDSToRDMRecordTransform: """Assembles the ETL entry consumed by ``CDSRecordServiceLoad``. - Wraps the ``record`` content built by ``CDSToRDMRecordEntry`` together - with ``versions``/``parent`` (computed here) and the ETL-envelope extras - that aren't record content (currently just ``_original_dump`` and - ``_clc_sync`` - see ``_transform()``). + Wraps the ``record`` content built by ``RecordEntry`` together + with ``versions``/``parent`` (computed here - ``parent`` is a + ``RecordParent``, built directly in ``_transform()``) and the + ETL-envelope extras that aren't record content (currently just + ``_original_dump`` and ``_clc_sync`` - see ``_transform()``). """ def __init__( @@ -415,6 +86,10 @@ def __init__( self.record_state_logger = record_state_logger self.preferred_model = preferred_model self.db_state = {"affiliations": CDSMigrationAffiliationMapping} + # the DOJSON-processed record data for the entry currently being + # transformed - populated by transform_xml_to_json(), read by + # _transform() to build this record's RecordParent (access grants). + self.raw_json_entry = None @property def logger(self): @@ -423,60 +98,67 @@ def logger(self): self._logger = Logger.get_logger() return self._logger - def _communities_ids(self, entry, record): - communities = record.get("communities", []) - communities = self.communities_ids + [slug for slug in communities] - if communities: - return {"ids": communities, "default": self.communities_ids[0]} - return {} - - def _parent(self, entry, record: RecordEntry) -> ParentEntry: + def _transform_xml_to_json(self, entry): + """Parse the legacy dump into the DOJSON-processed record. - email = record["owned_by"] - if not email: - owner = "system" - else: - try: - user = User.query.filter_by(email=email).one() - owner = user.id - except NoResultFound: - raise UnexpectedValue( - message=f"{email} not found - did you run user migration?", - stage="transform", - recid=entry["legacy_recid"], - value=email, - priority="critical", - ) + The one place per record that runs ``CDSRecordDump`` - populates + ``self.raw_json_entry`` with the DOJSON-mapped record content and + returns the ``CDSRecordDump`` instance itself, since record-level + facts derived from the dump (created/updated/recid/files) live on + that object, not as keys in ``raw_json_entry``. + """ + record_dump = CDSRecordDump(entry, preferred_model=self.preferred_model) + record_dump.prepare_revisions() + timestamp, json_data = record_dump.latest_revision + self.raw_json_entry = json_data + self.record_state_logger.add_record(json_data) + return record_dump - return ParentEntry( - created=record["created"], # same as the record - updated=record["updated"], # same as the record - version_id=record["version_id"], - json=ParentJson( - # loader is responsible for creating/updating if the PID exists. - # this part will be simply omitted - id=f'{record["recid"]}-parent', - access=ParentAccess(owned_by={"user": owner}), - communities=self._communities_ids(entry, record), - ), - ) + def _parent(self, entry, record): + return RecordParent( + record=record, + entry=entry, + json_entry=self.raw_json_entry, + communities_ids=self.communities_ids, + access_grants_view=self.access_grants_view, + ).build() def _transform(self, entry) -> Optional[MigrationEntry]: """Transform a single entry.""" # creates the output structure for load step migration_logger = self.migration_logger try: - built = self._record(entry) - record = built.record + # could be in draft as well, depends on how we decide to publish + record_dump = self._transform_xml_to_json(entry) + + # ETL-envelope concern, stripped before the record body is built - + # _metadata()'s forgotten-key check doesn't know this key. + clc_sync = deepcopy(self.raw_json_entry.get("_clc_sync", False)) + if "_clc_sync" in self.raw_json_entry: + del self.raw_json_entry["_clc_sync"] + + # same reason: "request_data" must be off raw_json_entry before + # RecordEntry.transform() runs _metadata()'s forgotten-key + # check, which doesn't know this key either. + record_request = RecordRequest( + json_entry=self.raw_json_entry, + recid=entry["recid"], + migration_logger=self.migration_logger, + ).build() + + record = self._record(entry, record_dump) if record: - return { - "record": record, - "versions": self._versions(entry, record), - "parent": self._parent(entry, record), - "_original_dump": entry, - "_clc_sync": built.clc_sync, - } + return MigrationEntry( + _original_dump=entry, + record=record, + versions=self._versions(entry, record), + parent=self._parent(entry, record), + _clc_sync=clc_sync, + _request_data=record_request, + ep_approval=entry.get("ep_approval", []), + ) + except ( LossyConversion, RestrictedFileDetected, @@ -487,178 +169,26 @@ def _transform(self, entry) -> Optional[MigrationEntry]: ) as e: migration_logger.add_log(e, record=entry) - def _record(self, entry) -> RecordBuildResult: - # could be in draft as well, depends on how we decide to publish - entry_builder = CDSToRDMRecordEntry( + def _record(self, entry, record_dump) -> RecordEntryData: + entry_builder = RecordEntry( missing_users_dir=self.missing_users_dir, affiliations_mapping=self.db_state["affiliations"], dry_run=self.dry_run, collection=self.collection, restricted=self.restricted, - access_grants_view=self.access_grants_view, migration_logger=self.migration_logger, record_state_logger=self.record_state_logger, - preferred_model=self.preferred_model, ) - record = entry_builder.transform(entry) - return RecordBuildResult(record=record, clc_sync=entry_builder.clc_sync) - - def _draft(self, entry): - return None - - def _parse_file_status(self, file_status): - pass - - def _versions(self, entry, record: RecordEntry) -> Dict[int, VersionEntry]: + return entry_builder.transform(entry, record_dump, self.raw_json_entry) - def compute_access(file, record_access): - - if file is None: - return { - "access_obj": { - "record": record_access, - "files": record_access, - } - } - - if not file["status"]: - return { - "access_obj": { - "record": record_access, - "files": record_access, - } - } - - if file["status"]: - # if we have anything in the status string, - # it means the file is restricted - # we pass this information to parse later in load step - self.migration_logger.add_information( - str(file["recid"]), - { - "message": "Record has individual file restrictions", - "value": file["status"], - }, - ) - return { - "access_obj": {"record": record_access, "files": "restricted"}, - "meta": file["status"], - } - - def should_skip_file(file_dump): - if file_dump["subformat"] in FILE_SUBFORMATS_TO_DROP: - self.migration_logger.add_information( - str(file_dump["recid"]), - { - "message": f"File subformat {file_dump['subformat']} dropped.", - "value": file_dump["full_name"], - }, - ) - return True - - if not self.plots and file_dump["type"] == "Plot": - # skip figures if configuration says so - self.migration_logger.add_information( - str(file_dump["recid"]), - { - "message": f"Plot file dropped.", - "value": file_dump["full_name"], - }, - ) - return True - if file_dump["hidden"]: - # skip hidden files - self.migration_logger.add_information( - str(file_dump["recid"]), - { - "message": f"Hidden file dropped.", - "value": file_dump["full_name"], - }, - ) - return True - return False - - def compute_files(file_dump, versions_dict): - legacy_path_root = Path("/opt/cdsweb/var/data/files/") - tmp_eos_root = Path(self.files_dump_dir) - full_path = Path(file_dump["full_path"]) - - versions_dict[file_dump["version"]]["files"].update( - { - file_dump["full_name"]: { - "eos_tmp_path": tmp_eos_root - / full_path.relative_to(legacy_path_root), - "id_bibdoc": file_dump["bibdocid"], - "key": file_dump["full_name"], - "metadata": { - "description": file_dump["description"], - "name": file_dump["name"], - "status": file_dump["status"], - "original_path": file_dump["path"], - "comment": file_dump["comment"], - }, - "mimetype": file_dump["mime"], - "checksum": file_dump["checksum"], - "version": file_dump["version"], - "access": file_dump["status"], - "type": file_dump["type"], - "creation_date": arrow.get(file_dump["creation_date"]) - .replace(tzinfo=None) - .date() - .isoformat(), - } - } - ) - - # grouping draft attributes by version - # we build temporary representation of each version - # {"1": {"access": {...}, "files": [], "publication_date": None} - # {"2": {"access": {...}, "files": [], "publication_date: "2021-04-21"} - versions = OrderedDict() - # we start versions from files (because this is the only way of - # mapping version of files to version of records from legacy) - _files = entry["files"] - record_access = record["access_status"] - for file in _files: - if should_skip_file(file): - continue - if file["version"] not in versions: - versions[file["version"]] = { - "files": {}, - "publication_date": arrow.get(file["creation_date"]).replace( - tzinfo=None - ), - "access": compute_access(file, record_access), - } - - compute_files(file, versions) - - versioned_files = {} - # creates a collection of files per each version - # lets say record has 2 files: A & B - # if for file A new version was uploaded (version 2), - # we need to preserve the file B for version 2 of the record - for version in versions.keys(): - versioned_files |= versions.get(version, {}).get("files") - versions[version]["files"] = deepcopy(versioned_files) - publication_date = record["json"]["metadata"]["publication_date"] - - if not versioned_files: - # Record has no files. Add metadata-only record as single version - versions[1] = { - "files": {}, - "publication_date": publication_date, - "access": compute_access( - None, record_access - ), # public metadata and files - } - - return versions - - def _record_files(self, entry, record): - """Record files entries transform.""" - # TO implement if we decide not to go via draft publish - return [] + def _versions(self, entry, record: RecordEntryData): + return RecordVersionsTransform( + entry=entry, + record=record, + files_dump_dir=self.files_dump_dir, + plots=self.plots, + migration_logger=self.migration_logger, + ).build() def _load_migrated_recids(self): """Load all already-migrated legacy record IDs into a set once.""" @@ -756,36 +286,3 @@ def run(self, entries): if self._throw: raise continue - - # - # - # "files": [ - # { - # "comment": null, - # "status": "firerole: allow group \"council-full [CERN]\"\ndeny until \"1996-02-01\"\nallow all", - # "version": 1, - # "encoding": null, - # "creation_date": "2009-11-03T12:29:06+00:00", - # "bibdocid": 502379, - # "mime": "application/pdf", - # "full_name": "CM-P00080632-e.pdf", - # "superformat": ".pdf", - # "recids_doctype": [[32097, "Main", "CM-P00080632-e.pdf"]], - # "path": "/opt/cdsweb/var/data/files/g50/502379/CM-P00080632-e.pdf;1", - # "size": 5033532, - # "license": {}, - # "modification_date": "2009-11-03T12:29:06+00:00", - # "copyright": {}, - # "url": "http://cds.cern.ch/record/32097/files/CM-P00080632-e.pdf", - # "checksum": "ed797ce5d024dcff0040db79c3396da9", - # "description": "English", - # "format": ".pdf", - # "name": "CM-P00080632-e", - # "subformat": "", - # "etag": "\"502379.pdf1\"", - # "recid": 32097, - # "flags": [], - # "hidden": false, - # "type": "Main", - # "full_path": "/opt/cdsweb/var/data/files/g50/502379/CM-P00080632-e.pdf;1" - # },] diff --git a/cds_migrator_kit/rdm/records/transform/transform_versions.py b/cds_migrator_kit/rdm/records/transform/transform_versions.py new file mode 100644 index 00000000..785acafc --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/transform_versions.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022-2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Builds all of a record's versions - ``MigrationEntry["versions"]``.""" +from collections import OrderedDict +from copy import deepcopy + +from cds_migrator_kit.rdm.records.transform.config import FILE_SUBFORMATS_TO_DROP +from cds_migrator_kit.rdm.records.transform.entities.version import RecordVersion + + +class RecordVersionsTransform: + """Builds all of a record's versions - ``MigrationEntry["versions"]``. + + Groups the legacy file dumps by version, builds each version's own + files/access via ``RecordVersion``, then carries files forward across + versions: lets say a record has 2 files, A & B - if a new version of + file A gets uploaded, the later record version still needs to include + file B too, so each version's file list is a cumulative snapshot, not + just its own delta. + """ + + def __init__(self, entry, record, files_dump_dir, plots, migration_logger): + """Constructor. + + :param entry: the original harvested legacy entry (needs "files"). + :param record: the already-built ``RecordEntry`` dict (needs + ``access_status`` and ``body["metadata"]["publication_date"]``). + :param files_dump_dir: local EOS mirror root for file content. + :param plots: whether to keep Plot-type files. + :param migration_logger: for skip/restriction logging. + """ + self.entry = entry + self.record = record + self.files_dump_dir = files_dump_dir + self.plots = plots + self.migration_logger = migration_logger + + def build(self): + """Group legacy files by version, build + carry files forward, return.""" + record_access = self.record["access_status"] + + # group non-skipped raw file dumps by legacy version number, in + # first-seen order - own_file_dumps[v] is version v's own files + # (not yet carrying anything forward from earlier versions). + own_file_dumps = OrderedDict() + representative_file = {} + for file_dump in self.entry["files"]: + if self._should_skip_file(file_dump): + continue + version_number = file_dump["version"] + own_file_dumps.setdefault(version_number, []).append(file_dump) + representative_file.setdefault(version_number, file_dump) + + versions = OrderedDict( + ( + version_number, + RecordVersion( + record_access=record_access, + files_dump_dir=self.files_dump_dir, + migration_logger=self.migration_logger, + representative_file=representative_file[version_number], + own_file_dumps=own_file_dumps[version_number], + ).build(), + ) + for version_number in own_file_dumps + ) + + # carry files forward across versions (see class docstring) + versioned_files = {} + for version_number in versions: + versioned_files |= versions[version_number]["files"] + versions[version_number]["files"] = deepcopy(versioned_files) + + if not versioned_files: + # Record has no files. Add metadata-only record as single version + versions[1] = RecordVersion( + record_access=record_access, + files_dump_dir=self.files_dump_dir, + migration_logger=self.migration_logger, + publication_date=self.record["body"]["metadata"]["publication_date"], + ).build() + + return versions + + def _should_skip_file(self, file_dump): + if file_dump["subformat"] in FILE_SUBFORMATS_TO_DROP: + self.migration_logger.add_information( + str(file_dump["recid"]), + { + "message": f"File subformat {file_dump['subformat']} dropped.", + "value": file_dump["full_name"], + }, + ) + return True + + if not self.plots and file_dump["type"] == "Plot": + # skip figures if configuration says so + self.migration_logger.add_information( + str(file_dump["recid"]), + { + "message": "Plot file dropped.", + "value": file_dump["full_name"], + }, + ) + return True + if file_dump["hidden"]: + # skip hidden files + self.migration_logger.add_information( + str(file_dump["recid"]), + { + "message": "Hidden file dropped.", + "value": file_dump["full_name"], + }, + ) + return True + return False + diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/quality/reviewers.py b/cds_migrator_kit/rdm/records/transform/xml_processing/quality/reviewers.py deleted file mode 100644 index c05447b6..00000000 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/quality/reviewers.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2022 CERN. -# -# CDS-RDM is free software; you can redistribute it and/or modify it under -# the terms of the MIT License; see LICENSE file for more details. - -"""Reviewer resolution utilities.""" - -from invenio_accounts.models import User -from invenio_db import db - -from cds_migrator_kit.errors import RecordFlaggedCuration - - -def _is_email(value): - """Return True if the reviewer value looks like an email address.""" - return "@" in value - - -def _parse_reviewer_name(name): - """Split a 'Family, Given' or 'Given Family' string into (family, given). - - ``request_reviewers`` (906__p) stores names as "Given Family" (comma - already resolved), but legacy data can also arrive as "Family, Given". - """ - name = name.strip() - if "," in name: - family, _, given = name.partition(",") - return family.strip(), given.strip() - parts = name.split() - if len(parts) > 1: - return parts[-1], " ".join(parts[:-1]) - return name, "" - - -def find_reviewer(reviewer): - """Resolve a reviewer string (email or name) to a User. - - :param reviewer: email address, or a "Family, Given"/"Given Family" name. - :raises RecordFlaggedCuration: if no matching user is found, so the - record is flagged for manual curation instead of failing outright. - """ - reviewer = reviewer.strip() - if _is_email(reviewer): - user = User.query.filter_by(email=reviewer).one_or_none() - else: - family_name, given_name = _parse_reviewer_name(reviewer) - query = User.query.filter( - db.func.lower(User._user_profile["family_name"].as_string()) - == family_name.lower() - ) - if given_name: - query = query.filter( - db.func.lower(User._user_profile["given_name"].as_string()) - == given_name.lower() - ) - user = query.one_or_none() - - if user is None: - raise RecordFlaggedCuration( - message=f"Reviewer '{reviewer}' could not be matched to an account.", - field="request_reviewers", - stage="transform", - value=reviewer, - ) - - return user diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index cb1d4805..4bc21a09 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -8,7 +8,7 @@ from idutils.normalizers import normalize_isbn, normalize_issn from isbnlib import NotValidISBNError -from cds_migrator_kit.errors import ManualImportRequired, RecordFlaggedCuration, UnexpectedValue +from cds_migrator_kit.errors import ManualImportRequired, UnexpectedValue from cds_migrator_kit.transform.xml_processing.quality.decorators import ( filter_list_values, for_each_value, @@ -22,7 +22,6 @@ ) from ...models.base_publication_record import rdm_base_publication_model as model from .base import normalize -from ..quality.reviewers import find_reviewer @model.over("isbns", "^020__", override_tag=True) @@ -424,19 +423,13 @@ def request_reviewers(self, key, value): reviewer = " ".join(part for part in (first, last) if part) if reviewer: + # resolving a name/email to an actual user account requires a DB + # lookup, which dojson rules must not do - RecordRequest + # (transform/entities/request.py) resolves these into + # request_data["reviewers"] later, once transform() runs. request_data = self.setdefault("request_data", {}) - reviewers = request_data.setdefault("reviewers", []) - - try: - user = find_reviewer(reviewer) - reviewer_entry = {"user": str(user.id)} - except RecordFlaggedCuration as exc: - reviewer_errors = request_data.setdefault("_reviewer_errors", []) - reviewer_errors.append({"message": exc.message, "value": exc.value}) - reviewer_entry = {"user": "-1"} - - if reviewer_entry not in reviewers: - reviewers.append(reviewer_entry) + reviewer_names = request_data.setdefault("reviewer_names", []) + reviewer_names.append(reviewer) raise IgnoreKey("request_reviewers") diff --git a/cds_migrator_kit/users/load.py b/cds_migrator_kit/users/load.py index c8826b89..fc7edf50 100644 --- a/cds_migrator_kit/users/load.py +++ b/cds_migrator_kit/users/load.py @@ -173,7 +173,8 @@ def _ensure_reviewer_profile_name(self, user_id, family_name, given_name): """Make sure family_name/given_name are set on the profile. MigrationUserAPI.create_user() only ever sets `full_name`, but - find_reviewer() (cds_migrator_kit/rdm/records/load/load.py) matches + RecordRequest._find_reviewer() + (cds_migrator_kit/rdm/records/transform/entities/request.py) matches reviewers by `family_name`/`given_name` - without this, a just-created reviewer account would still be unmatchable by name later on. Only fills in missing values, never overwrites an diff --git a/tests/cds-rdm/test_ep_approval_entry.py b/tests/cds-rdm/test_ep_approval_entry.py index 33bd9d3b..55c8f59a 100644 --- a/tests/cds-rdm/test_ep_approval_entry.py +++ b/tests/cds-rdm/test_ep_approval_entry.py @@ -20,6 +20,7 @@ PublicEntry, RestrictedEntry, ) +from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent RECID = "12345" APPROVED_REPORT_NUMBER = "CERN-EP-2020-001" @@ -63,6 +64,24 @@ def _epphapp_file(key=DRAFT_FILE_KEY, checksum="bbb", version=1, id_bibdoc=200): } +def _make_parent(owner="uploader", communities=None, access_grants=None): + """Build a RecordParent-shaped test double. + + Bypasses RecordParent.build()'s DB/mapper-dependent logic (owner + lookup, access grant resolution) - this module tests PublicEntry/ + RestrictedEntry's splitting logic, not RecordParent's own construction. + """ + parent = RecordParent.__new__(RecordParent) + parent.communities = communities if communities is not None else {} + parent.access_grants = access_grants if access_grants is not None else [] + parent.body = { + "id": f"{RECID}-parent", + "access": {"owned_by": {"user": owner}}, + "communities": parent.communities, + } + return parent + + def _make_entry( versions, recid=RECID, @@ -108,27 +127,22 @@ def _make_entry( return { "record": { "recid": recid, - "json": record_json, - "ep_approval": [ - { - "status": "waiting", - "ep_report_number": report_number, - }, - { - "status": "approved", - "ep_report_number": report_number, - }, - ], + "body": record_json, "owned_by": "uploader", - "_request_data": {"placeholder": True}, - }, - "parent": { - "json": { - "access": {"owned_by": {"user": "uploader"}}, - "communities": {"ids": ["example-community"]}, - } }, + "parent": _make_parent(communities={"ids": ["example-community"]}), "versions": versions, + "ep_approval": [ + { + "status": "waiting", + "ep_report_number": report_number, + }, + { + "status": "approved", + "ep_report_number": report_number, + }, + ], + "_request_data": {"placeholder": True}, } @@ -420,7 +434,7 @@ def test_public_removes_cern_ep_report_numbers(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - identifiers = result["record"]["json"]["metadata"]["identifiers"] + identifiers = result["record"]["body"]["metadata"]["identifiers"] cdsrn_values = {i["identifier"] for i in identifiers if i["scheme"] == "cdsrn"} assert APPROVED_REPORT_NUMBER not in cdsrn_values @@ -442,7 +456,7 @@ def test_public_keeps_non_ep_cdsrn(self, app): cdsrn_ids = [ i - for i in result["record"]["json"]["metadata"]["identifiers"] + for i in result["record"]["body"]["metadata"]["identifiers"] if i["scheme"] == "cdsrn" ] assert len(cdsrn_ids) == 1 @@ -460,7 +474,7 @@ def test_restricted_removes_matching_cern_ep_rn(self, app): cdsrn_values = { i["identifier"] - for i in result["record"]["json"]["metadata"]["identifiers"] + for i in result["record"]["body"]["metadata"]["identifiers"] if i["scheme"] == "cdsrn" } @@ -474,7 +488,7 @@ def test_restricted_keeps_draft_report_number(self, app): cdsrn_values = { i["identifier"] - for i in result["record"]["json"]["metadata"]["identifiers"] + for i in result["record"]["body"]["metadata"]["identifiers"] if i["scheme"] == "cdsrn" } @@ -497,7 +511,7 @@ def test_restricted_removes_doi_pid(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - assert "doi" not in result["record"]["json"].get("pids", {}) + assert "doi" not in result["record"]["body"].get("pids", {}) class TestPublicEntryModifications: @@ -509,7 +523,7 @@ def test_public_removes_request_data(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - assert "_request_data" not in result["record"] + assert "_request_data" not in result def test_public_sets_owned_by_system(self, app): entry = _make_entry(_versions_with_epphapp()) @@ -518,7 +532,7 @@ def test_public_sets_owned_by_system(self, app): ).build() assert result["record"]["owned_by"] == "system" - assert result["parent"]["json"]["access"]["owned_by"] == {"user": "system"} + assert result["parent"].body["access"]["owned_by"] == {"user": "system"} def test_public_adds_cern_scientific_community(self, app): entry = _make_entry(_versions_with_epphapp()) @@ -526,13 +540,11 @@ def test_public_adds_cern_scientific_community(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - assert CDS_CERN_SCIENTIFIC_COMMUNITY_ID in ( - result["parent"]["json"]["communities"]["ids"] - ) + assert CDS_CERN_SCIENTIFIC_COMMUNITY_ID in result["parent"].communities["ids"] def test_public_does_not_duplicate_community(self, app): entry = _make_entry(_versions_with_epphapp()) - entry["parent"]["json"]["communities"]["ids"] = [ + entry["parent"].communities["ids"] = [ "example-community", CDS_CERN_SCIENTIFIC_COMMUNITY_ID, ] @@ -540,7 +552,7 @@ def test_public_does_not_duplicate_community(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - community_ids = result["parent"]["json"]["communities"]["ids"] + community_ids = result["parent"].communities["ids"] assert community_ids.count(CDS_CERN_SCIENTIFIC_COMMUNITY_ID) == 1 @@ -553,8 +565,8 @@ def test_public_build_does_not_mutate_original(self, app): PublicEntry(entry, _make_approval_request(), _make_migration_logger()).build() assert ( - entry["record"]["json"]["metadata"]["identifiers"] - == original["record"]["json"]["metadata"]["identifiers"] + entry["record"]["body"]["metadata"]["identifiers"] + == original["record"]["body"]["metadata"]["identifiers"] ) def test_restricted_build_does_not_mutate_original(self, app): @@ -565,6 +577,6 @@ def test_restricted_build_does_not_mutate_original(self, app): ).build() assert ( - entry["record"]["json"]["metadata"]["identifiers"] - == original["record"]["json"]["metadata"]["identifiers"] + entry["record"]["body"]["metadata"]["identifiers"] + == original["record"]["body"]["metadata"]["identifiers"] ) diff --git a/tests/cds-rdm/test_load_reviewers.py b/tests/cds-rdm/test_load_reviewers.py index 2195b060..f46b9361 100644 --- a/tests/cds-rdm/test_load_reviewers.py +++ b/tests/cds-rdm/test_load_reviewers.py @@ -5,65 +5,67 @@ # CDS-RDM is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -"""Tests for reviewer resolution in load.py::_after_publish_add_inclusion_request.""" +"""Tests for reviewer resolution encapsulated in RecordRequest (entities/request.py).""" import pytest from invenio_accounts.testutils import create_test_user from cds_migrator_kit.errors import RecordFlaggedCuration -from cds_migrator_kit.rdm.records.transform.xml_processing.quality.reviewers import ( - _is_email, - _parse_reviewer_name, - find_reviewer, -) +from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest class TestIsEmail: - """Test the _is_email helper.""" + """Test the RecordRequest._is_email helper.""" def test_email_detected(self): """Test that a string with '@' is detected as an email.""" - assert _is_email("john.doe@cern.ch") is True + assert RecordRequest._is_email("john.doe@cern.ch") is True def test_name_not_detected_as_email(self): """Test that a plain name is not detected as an email.""" - assert _is_email("Doe, John") is False - assert _is_email("John Doe") is False + assert RecordRequest._is_email("Doe, John") is False + assert RecordRequest._is_email("John Doe") is False class TestParseReviewerName: - """Test the _parse_reviewer_name helper.""" + """Test the RecordRequest._parse_reviewer_name helper.""" def test_family_comma_given(self): """Test 'Family, Given' format.""" - assert _parse_reviewer_name("Doe, John") == ("Doe", "John") + assert RecordRequest._parse_reviewer_name("Doe, John") == ("Doe", "John") def test_given_family_no_comma(self): """Test 'Given Family' format (no comma).""" - assert _parse_reviewer_name("John Doe") == ("Doe", "John") + assert RecordRequest._parse_reviewer_name("John Doe") == ("Doe", "John") def test_multi_word_given_name(self): """Test a multi-word given name without a comma.""" - assert _parse_reviewer_name("John Michael Doe") == ("Doe", "John Michael") + assert RecordRequest._parse_reviewer_name("John Michael Doe") == ( + "Doe", + "John Michael", + ) def test_family_name_only(self): """Test a single-word name with no given name available.""" - assert _parse_reviewer_name("Doe") == ("Doe", "") + assert RecordRequest._parse_reviewer_name("Doe") == ("Doe", "") def test_strips_whitespace(self): """Test that surrounding and inner whitespace is stripped.""" - assert _parse_reviewer_name(" Doe , John ") == ("Doe", "John") + assert RecordRequest._parse_reviewer_name(" Doe , John ") == ( + "Doe", + "John", + ) class TestFindReviewer: - """Test find_reviewer() DB resolution (email or profile name match).""" + """Test RecordRequest._find_reviewer() DB resolution (email or profile name match).""" def test_find_reviewer_by_email(self, app, db): """Test that a reviewer given as an email is resolved by email.""" user = create_test_user(email="jane.smith@cern.ch") db.session.commit() - found = find_reviewer("jane.smith@cern.ch") + found = RecordRequest._find_reviewer("jane.smith@cern.ch") assert found.id == user.id def test_find_reviewer_by_profile_name_family_given(self, app, db): @@ -74,7 +76,7 @@ def test_find_reviewer_by_profile_name_family_given(self, app, db): ) db.session.commit() - found = find_reviewer("Doe, John") + found = RecordRequest._find_reviewer("Doe, John") assert found.id == user.id def test_find_reviewer_by_profile_name_given_family(self, app, db): @@ -85,7 +87,7 @@ def test_find_reviewer_by_profile_name_given_family(self, app, db): ) db.session.commit() - found = find_reviewer("Mary Jones") + found = RecordRequest._find_reviewer("Mary Jones") assert found.id == user.id def test_find_reviewer_name_match_is_case_insensitive(self, app, db): @@ -96,7 +98,7 @@ def test_find_reviewer_name_match_is_case_insensitive(self, app, db): ) db.session.commit() - found = find_reviewer("lee, ANNA") + found = RecordRequest._find_reviewer("lee, ANNA") assert found.id == user.id def test_find_reviewer_family_name_only(self, app, db): @@ -107,18 +109,18 @@ def test_find_reviewer_family_name_only(self, app, db): ) db.session.commit() - found = find_reviewer("Solo") + found = RecordRequest._find_reviewer("Solo") assert found.id == user.id def test_find_reviewer_by_email_not_found_raises(self, app, db): """Test that an unmatched email raises RecordFlaggedCuration.""" with pytest.raises(RecordFlaggedCuration): - find_reviewer("nobody@cern.ch") + RecordRequest._find_reviewer("nobody@cern.ch") def test_find_reviewer_by_name_not_found_raises(self, app, db): """Test that an unmatched name raises RecordFlaggedCuration.""" with pytest.raises(RecordFlaggedCuration): - find_reviewer("Nobody, Here") + RecordRequest._find_reviewer("Nobody, Here") def test_find_reviewer_name_wrong_given_name_raises(self, app, db): """Test that a family-name match with a mismatched given name raises.""" @@ -129,4 +131,4 @@ def test_find_reviewer_name_wrong_given_name_raises(self, app, db): db.session.commit() with pytest.raises(RecordFlaggedCuration): - find_reviewer("Doe, Someone Else") + RecordRequest._find_reviewer("Doe, Someone Else") diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index 8b259714..09181125 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -571,8 +571,8 @@ def test_962_book_with_different_artid_is_not_duplicate(self): """A mismatched book leaves titleless journal data flagged for curation.""" from unittest.mock import MagicMock - from cds_migrator_kit.rdm.records.transform.transform import ( - CDSToRDMRecordEntry, + from cds_migrator_kit.rdm.records.transform.entities.record import ( + RecordEntry, ) from cds_migrator_kit.rdm.records.transform.xml_processing.rules.research import ( related_identifiers, @@ -596,9 +596,9 @@ def test_962_book_with_different_artid_is_not_duplicate(self): record["resource_type"] = "publication-other" migration_logger = MagicMock() - custom_fields = CDSToRDMRecordEntry( - migration_logger=migration_logger - )._custom_fields(record, {"metadata": {}}) + custom_fields = RecordEntry(migration_logger=migration_logger)._custom_fields( + record + ) # titleless journal data is dropped (falsy values are filtered out), # not raised as a hard error diff --git a/tests/cds-rdm/test_transform_metadata_title.py b/tests/cds-rdm/test_transform_metadata_title.py index c8672f50..ff0266da 100644 --- a/tests/cds-rdm/test_transform_metadata_title.py +++ b/tests/cds-rdm/test_transform_metadata_title.py @@ -9,13 +9,13 @@ import pytest -from cds_migrator_kit.rdm.records.transform.transform import CDSToRDMRecordEntry +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry @pytest.fixture def entry(): """Transform entry instance (no DB/app context required).""" - return CDSToRDMRecordEntry() + return RecordEntry() def _dump(): diff --git a/tests/cds-rdm/test_transform_versions.py b/tests/cds-rdm/test_transform_versions.py index e172b5a0..97970948 100644 --- a/tests/cds-rdm/test_transform_versions.py +++ b/tests/cds-rdm/test_transform_versions.py @@ -25,12 +25,13 @@ def _file_dump( subformat="", recid=123, bibdocid=1, + status="", ): """Build a minimal legacy file dump entry.""" checksum = checksum or f"checksum-v{file_version}" return { "comment": None, - "status": "", + "status": status, "version": file_version, "encoding": None, "creation_date": creation_date, @@ -68,8 +69,8 @@ def _file_dump( def _record(): """Build a minimal record.""" return { - "access": "public", - "json": {"metadata": {"publication_date": "2020-01-01"}}, + "access_status": "public", + "body": {"metadata": {"publication_date": "2020-01-01"}}, } @@ -163,3 +164,40 @@ def test_versions_with_skipped_files(transform): assert versions[2]["files"]["main.pdf"]["version"] == 2 assert "plot.png" not in versions[1]["files"] assert "plot.png" not in versions[2]["files"] + + +def test_versions_no_files_falls_back_to_metadata_only_version(transform): + """A record with no files gets a single, empty, public version.""" + entry = {"recid": 123, "files": []} + + versions = transform._versions(entry, _record()) + + assert list(versions.keys()) == [1] + assert versions[1]["files"] == {} + assert versions[1]["publication_date"] == "2020-01-01" + assert versions[1]["access"] == { + "access_obj": {"record": "public", "files": "public"} + } + + +def test_versions_individual_file_restriction_sets_access_meta(transform): + """A file with its own restriction status flags that version as restricted.""" + status = ( + 'firerole: allow group "some-group [CERN]"\ndeny until "1996-02-01"\nallow all' + ) + entry = { + "recid": 123, + "files": [_file_dump(status=status)], + } + + versions = transform._versions(entry, _record()) + + assert versions[1]["access"] == { + "access_obj": {"record": "public", "files": "restricted"}, + "meta": status, + } + transform.migration_logger.add_information.assert_called_once() + recid, info = transform.migration_logger.add_information.call_args[0] + assert recid == "123" + assert info["message"] == "Record has individual file restrictions" + assert info["value"] == status From fe2a070e963bcb0dbf8bdbad5a7c359a075d4b5b Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Fri, 21 Aug 2026 16:00:36 +0200 Subject: [PATCH 3/9] chore(transform): rename data structure to avoid confusion --- .../rdm/records/load/ep_approval_entry.py | 15 +- .../rdm/records/load/ep_approval_load.py | 4 +- cds_migrator_kit/rdm/records/load/load.py | 30 +-- .../records/transform/entities/migration.py | 19 +- .../rdm/records/transform/entities/parent.py | 41 +-- .../rdm/records/transform/entities/record.py | 252 +++++++----------- .../rdm/records/transform/entities/request.py | 12 +- .../rdm/records/transform/mappers/base.py | 10 +- .../records/transform/mappers/contributors.py | 6 +- .../transform/mappers/custom_fields.py | 14 +- .../rdm/records/transform/mappers/metadata.py | 45 ++-- .../rdm/records/transform/mappers/record.py | 2 +- .../rdm/records/transform/transform.py | 165 +++++++----- .../records/transform/transform_versions.py | 15 +- cds_migrator_kit/reports/log.py | 7 +- tests/cds-rdm/test_ep_approval_entry.py | 42 +-- tests/cds-rdm/test_publications_rules.py | 8 +- .../cds-rdm/test_transform_metadata_title.py | 39 +-- tests/cds-rdm/test_transform_versions.py | 27 +- 19 files changed, 381 insertions(+), 372 deletions(-) diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py b/cds_migrator_kit/rdm/records/load/ep_approval_entry.py index 5390051f..c965006d 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py +++ b/cds_migrator_kit/rdm/records/load/ep_approval_entry.py @@ -57,7 +57,7 @@ def build(self) -> MigrationEntry: return split def _apply_metadata(self, split): - metadata = split["record"]["body"]["metadata"] + metadata = split["record"].body["metadata"] metadata["identifiers"] = self.identifiers(metadata.get("identifiers", [])) self._remove_doi_pid(split) @@ -79,7 +79,7 @@ def _is_restricted_file(file_data): ) def _log_removed_identifiers(self, removed, split_type): - recid = self.entry.get("record", {}).get("recid") + recid = self.entry["record"].recid self.migration_logger.add_information( recid, { @@ -161,7 +161,7 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: raise UnexpectedValue( message="No public files found to load for EP approval public split", stage="load", - recid=split["record"]["recid"], + recid=split["record"].recid, priority="critical", ) @@ -194,7 +194,6 @@ def identifiers(self, identifiers): def _apply_entry_modifications(self, split): split.pop("_request_data", None) - split["record"]["owned_by"] = "system" split["parent"].body["access"]["owned_by"] = {"user": "system"} self._add_cern_scientific_community(split) @@ -248,7 +247,7 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: if not has_restricted_files: self.migration_logger.add_information( - split["record"]["recid"], + split["record"].recid, { "message": ( "No restricted files found; public files used for the " @@ -298,7 +297,7 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: raise UnexpectedValue( message=("No files found to load for EP approval restricted split"), stage="load", - recid=split["record"]["recid"], + recid=split["record"].recid, priority="critical", ) @@ -336,8 +335,8 @@ def identifiers(self, identifiers): def _remove_doi_pid(self, split): """Remove DOI PID from restricted record.""" - recid = split.get("record", {}).get("recid") - record_body = split.get("record", {}).get("body", {}) + recid = split["record"].recid + record_body = split["record"].body pids = record_body.get("pids") if not pids or "doi" not in pids: diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_load.py b/cds_migrator_kit/rdm/records/load/ep_approval_load.py index 8f2cf0c2..4178222d 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_load.py +++ b/cds_migrator_kit/rdm/records/load/ep_approval_load.py @@ -75,7 +75,7 @@ def _load(self, entry): if not entry: return try: - recid = entry.get("record", {}).get("recid") + recid = entry["record"].recid # The same legacy recid can be cross-listed under multiple EP # collections (e.g. a joint ALEPH/DELPHI/L3/OPAL paper appears in @@ -98,7 +98,7 @@ def _load(self, entry): recid=recid, priority="critical", ) - record_body = entry.get("record", {}).get("body", {}) + record_body = entry["record"].body metadata = record_body.get("metadata", {}) self.approval_request = ApprovalRequest( diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index b7f49ab7..1721d20a 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -113,7 +113,7 @@ def _load_files( uow=None, ): """Load files to draft.""" - recid = entry.get("record", {}).get("recid", {}) + recid = entry["record"].recid identity = system_identity # Should we create an identity for the migration? for filename, file_data in version_files.items(): @@ -229,7 +229,7 @@ def _after_publish_update_dois(self, identity, record, entry, uow): """Update migrated DOIs post publish.""" if not self._is_final_record: return - migrated_pids = entry["record"]["body"]["pids"] + migrated_pids = entry["record"].body["pids"] for pid_type, identifier in migrated_pids.items(): if pid_type == "doi": # If a DOI was already minted from legacy then on publish the datacite @@ -293,7 +293,7 @@ def _create_grant(subject_type, subject_id, permission): field="access", subfield="subject.id", stage="load", - recid=entry["record"]["recid"], + recid=entry["record"].recid, priority="warning", value=subject_id, ) @@ -317,7 +317,7 @@ def _create_grant(subject_type, subject_id, permission): raise GrantCreationError( message=f"Users not found for emails: {', '.join(missing_emails)}", stage="load", - recid=entry["record"]["recid"], + recid=entry["record"].recid, value=list(missing_emails), priority="warning", ) @@ -340,7 +340,7 @@ def _after_publish_update_created(self, record, entry: MigrationEntry, version): 2. The record's creation date if there are no files. 3. Today's date if the original value and file creation date is missing. """ - creation_date = arrow.get(entry["record"]["created"]).datetime.replace( + creation_date = arrow.get(entry["record"].created).datetime.replace( tzinfo=None ) @@ -361,7 +361,7 @@ def _after_publish_mint_recid(self, record, entry: MigrationEntry, version): """Mint legacy ids for redirections assigned to the parent.""" if not self._is_final_record: return - legacy_recid = entry["record"]["recid"] + legacy_recid = entry["record"].recid if record._record.versions.index == 1: # it seems more intuitive if we mint the lrecid for parent # but then we get a double redirection @@ -369,7 +369,7 @@ def _after_publish_mint_recid(self, record, entry: MigrationEntry, version): def _after_publish_add_submission_request(self, request_data, record, entry, uow): """Create community inclusion request after publish.""" - legacy_recid = entry["record"]["recid"] + legacy_recid = entry["record"].recid request_number = f"lrecid:{legacy_recid}" # Defensive/idempotency guard: skip if a request for this record was @@ -574,7 +574,7 @@ def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): # we decided to skip it and act normal try: draft = current_rdm_records_service.create( - identity, data=entry["record"]["body"], uow=uow + identity, data=entry["record"].body, uow=uow ) self._assign_rep_numbers(draft) except (UniqueViolation, IntegrityError) as e: @@ -583,10 +583,10 @@ def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): raise ManualImportRequired(message=str(e)) if draft.errors: raise ManualImportRequired( - message=f"{str(draft.errors)}: {str(entry['record']['json'])}", + message=f"{str(draft.errors)}: {str(entry['record'].body)}", field="validation", stage="load", - recid=entry["record"]["recid"], + recid=entry["record"].recid, priority="warning", value=draft._record.pid.pid_value, subfield=None, @@ -599,7 +599,7 @@ def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): draft_dict = draft.to_dict() if not self.update_new_version_publication_date: publication_date = arrow.get( - entry["record"]["body"]["metadata"]["publication_date"] + entry["record"].body["metadata"]["publication_date"] ) else: publication_date = versions[version]["publication_date"] @@ -625,7 +625,7 @@ def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): def _load_versions(self, entry: MigrationEntry, uow): """Load other versions of the record.""" versions = entry["versions"] - legacy_recid = entry["record"]["recid"] + legacy_recid = entry["record"].recid identity = system_identity @@ -655,7 +655,7 @@ def _load_versions(self, entry: MigrationEntry, uow): def _dry_load(self, entry: MigrationEntry): current_rdm_records_service.schema.load( - entry["record"]["body"], + entry["record"].body, context=dict( identity=system_identity, ), @@ -747,7 +747,7 @@ def _save_original_dumped_record(self, entry: MigrationEntry, recid_state): json=_original_dump, parent_object_uuid=recid_state["parent_object_uuid"], migrated_record_object_uuid=recid_state["latest_version_object_uuid"], - legacy_recid=entry["record"]["recid"], + legacy_recid=entry["record"].recid, ) db.session.add(_original_dump_model) @@ -785,7 +785,7 @@ def _load(self, entry: MigrationEntry, uow=None): operations (e.g. the EP approval record split). """ if entry: - recid = entry.get("record", {}).get("recid", {}) + recid = entry["record"].recid if self._should_skip_recid(recid): self.migration_logger.add_information( recid, state={"message": "Record already migrated", "value": recid} diff --git a/cds_migrator_kit/rdm/records/transform/entities/migration.py b/cds_migrator_kit/rdm/records/transform/entities/migration.py index 129dcfa2..582ca77e 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/migration.py +++ b/cds_migrator_kit/rdm/records/transform/entities/migration.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, TypedDict from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent -from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntryData +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest from cds_migrator_kit.rdm.records.transform.entities.version import VersionEntry @@ -23,17 +23,18 @@ class MigrationEntry(TypedDict): computed by ``CDSToRDMRecordTransform`` itself, while ``_original_dump``/``_clc_sync``/``_request_data``/``ep_approval`` are carried alongside it - see ``CDSToRDMRecordTransform._transform()``. - None of the four need ``RecordEntry`` to build: ``_original_dump`` - and ``ep_approval`` are read straight off the raw harvested entry, - which the transform already has; ``_clc_sync`` and ``_request_data`` - are popped off ``raw_json_entry`` in ``_transform()``, before - ``RecordEntry.transform()`` is even called. + None of the four need ``RecordEntry`` to build: ``_original_dump`` is + read straight off the raw harvested entry, which the transform already + has; ``_clc_sync``/``ep_approval``/``_request_data`` are popped off + ``dojson_entry`` in ``_transform()``, before ``RecordEntry`` is even + constructed. - ``parent`` is a real ``RecordParent`` object, not a dict - see - ``entities/parent.py``. + ``record`` and ``parent`` are real objects, not dicts - ``record.body`` + is the RDM record content, ``created``/``recid``/``access_status`` on + it are supporting data - see ``entities/record.py``/``entities/parent.py``. """ - record: RecordEntryData + record: RecordEntry versions: Dict[int, VersionEntry] parent: RecordParent # Community-inclusion request for this record - a RecordRequest object, diff --git a/cds_migrator_kit/rdm/records/transform/entities/parent.py b/cds_migrator_kit/rdm/records/transform/entities/parent.py index 15723d83..00ae6f82 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/parent.py +++ b/cds_migrator_kit/rdm/records/transform/entities/parent.py @@ -26,26 +26,31 @@ class RecordParent: membership, and access grants - built by ``build()``, called from ``CDSToRDMRecordTransform._transform()`` (the one place that has both the already-built record content and the DOJSON-processed - ``json_entry`` that access-grant resolution needs). + ``dojson_entry`` that access-grant resolution needs). """ - def __init__(self, record, entry, json_entry, communities_ids, access_grants_view): + def __init__( + self, record, raw_dump_entry, dojson_entry, communities_ids, access_grants_view + ): """Constructor. - :param record: the already-built ``RecordEntryData`` dict (needs - ``owned_by``/``recid``/``communities``). - :param entry: the original harvested legacy entry (needs - ``legacy_recid``, for error reporting). - :param json_entry: the DOJSON-processed record data - required to - resolve access grants (see ``_build_access_grants()``). + :param record: the already-built ``RecordEntry`` (needs ``recid``). + :param raw_dump_entry: the original harvested legacy entry (needs + ``recid``, for error reporting). + :param dojson_entry: the DOJSON-processed record data - owner/ + communities/access grants are popped straight off it + (``submitter``/``communities``/``access_grants``) - this is the + last entity to touch it, so it's also where those keys stop + existing for the forgotten-keys check in + ``CDSToRDMRecordTransform._check_forgotten_keys()``. :param communities_ids: configured target community ids for this migration run (``CDSToRDMRecordTransform.communities_ids``). :param access_grants_view: configured collection-wide view grants (``CDSToRDMRecordTransform.access_grants_view``). """ self.record = record - self.entry = entry - self.json_entry = json_entry + self.raw_dump_entry = raw_dump_entry + self.dojson_entry = dojson_entry self.communities_ids = communities_ids self.access_grants_view = access_grants_view self.body = None @@ -60,7 +65,7 @@ def build(self): self.body = { # loader is responsible for creating/updating if the PID exists, # this part will be simply omitted. - "id": f'{self.record["recid"]}-parent', + "id": f"{self.record.recid}-parent", "access": access, "communities": self.communities, } @@ -68,7 +73,7 @@ def build(self): def _build_access(self): """Resolve the owner and return the parent's access dict.""" - email = self.record["owned_by"] + email = self.dojson_entry.pop("submitter", None) if not email: owner = "system" else: @@ -79,7 +84,7 @@ def _build_access(self): raise UnexpectedValue( message=f"{email} not found - did you run user migration?", stage="transform", - recid=self.entry["legacy_recid"], + recid=self.raw_dump_entry["recid"], value=email, priority="critical", ) @@ -87,7 +92,7 @@ def _build_access(self): def _build_communities(self): """Combine the configured target communities with the record's own.""" - communities = self.record.get("communities", []) + communities = self.dojson_entry.pop("communities", []) communities = self.communities_ids + [slug for slug in communities] if communities: return {"ids": communities, "default": self.communities_ids[0]} @@ -96,8 +101,8 @@ def _build_communities(self): def _build_access_grants_from_record_marc(self): """Compute the access grants to create on this parent after publish.""" ctx = RecordTransformContext( - json_entry=self.json_entry, - entry=self.entry, + dojson_entry=self.dojson_entry, + raw_dump_entry=self.raw_dump_entry, access_grants_view=self.access_grants_view, ) return AccessGrantsMapper().map_value(ctx) @@ -136,7 +141,7 @@ def resolve_grants(self, specific_file_restrictions=""): field="access", subfield="subject.id", stage="load", - recid=self.record["recid"], + recid=self.record.recid, priority="critical", value=specific_file_restrictions, ) @@ -159,7 +164,7 @@ def resolve_grants(self, specific_file_restrictions=""): field="access", subfield="subject.id", stage="load", - recid=self.record["recid"], + recid=self.record.recid, priority="critical", value=specific_file_restrictions, ) diff --git a/cds_migrator_kit/rdm/records/transform/entities/record.py b/cds_migrator_kit/rdm/records/transform/entities/record.py index 8276ff58..9fc2d9af 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/record.py +++ b/cds_migrator_kit/rdm/records/transform/entities/record.py @@ -7,7 +7,7 @@ """The RDM record's own content - ``MigrationEntry["record"]``.""" from copy import deepcopy -from typing import Any, List, Optional, TypedDict, Union +from typing import Any, TypedDict from flask import current_app from idutils.validators import is_doi @@ -29,7 +29,7 @@ class RecordBodyRequired(TypedDict): - """Required keys of ``RecordEntryData["body"]``.""" + """Required keys of ``RecordEntry.body``.""" files: dict pids: dict @@ -37,10 +37,10 @@ class RecordBodyRequired(TypedDict): class RecordBody(RecordBodyRequired, total=False): - """The RDM record body: ``RecordEntryData["body"]``. + """The RDM record body: ``RecordEntry.body``, returned by ``build()``. - ``current_rdm_records_service.create()``'s ``data`` argument - built by - ``RecordEntry.transform()``. Deliberately excludes: + ``current_rdm_records_service.create()``'s ``data`` argument. Deliberately + excludes: - ``access``: set per-version, after creation, via ``VersionEntry["access"]`` (see ``load.py::_load_record_access``) @@ -53,77 +53,61 @@ class RecordBody(RecordBodyRequired, total=False): internal_notes: Any -class RecordEntryData(TypedDict): - """A single record's content - ``MigrationEntry["record"]``. - - Built by ``RecordEntry.transform()``. Everything here is - record-scoped (as opposed to ``MigrationEntry``'s other top-level keys, - which are ETL-envelope-scoped - see that type's docstring). - """ - - created: str - updated: str - version_id: int - index: int - recid: str - communities: List[str] - # The record's actual content, handed wholesale to - # current_rdm_records_service.create()/schema.load() - see RecordBody. - body: RecordBody - # None when RecordFlaggedCuration was raised and caught - see - # RecordEntry._access()/.transform(). - access_status: Optional[str] - owned_by: Union[str, int] - - class RecordEntry: - """Transform CDS record to RDM record. + """A single record's content - ``MigrationEntry["record"]``. - Builds the ``record`` content dict consumed by - ``CDSToRDMRecordTransform`` - not the invenio_rdm_migrator "generic RDM - record" envelope (this class deliberately does not use that framework's - ``RDMRecordEntry.transform()``/``_load_partial`` orchestration, since the - CDS legacy shape and the CDS loader's needs don't match it). + Builds the RDM record's own content - not the invenio_rdm_migrator + "generic RDM record" envelope (this class deliberately does not use + that framework's ``RDMRecordEntry.transform()``/``_load_partial`` + orchestration, since the CDS legacy shape and the CDS loader's needs + don't match it). The constructor eagerly computes the cheap, + single-value supporting data (``created``/``recid``/``access_status``); + ``build()`` does the more involved work of assembling ``body`` - the + dict handed wholesale to + ``current_rdm_records_service.create()``/``schema.load()``. """ def __init__( self, - partial=False, - missing_users_dir=None, - missing_users_filename="people.csv", + dump, + dojson_entry, affiliations_mapping=None, - dry_run=False, - collection=None, restricted=False, migration_logger=None, - record_state_logger=None, ): - """Constructor.""" - self.partial = partial - self.missing_users_dir = missing_users_dir - self.missing_users_filename = missing_users_filename + """Constructor. + + :param dump: the ``CDSRecordDump`` for this entry - produced by + ``CDSToRDMRecordTransform._transform_xml_to_json()``. Its + ``.data`` is the raw harvested entry (``raw_dump_entry`` below). + :param dojson_entry: the DOJSON-processed record data + (``dump.latest_revision``'s content) - passed in separately + since ``build()`` mutates it in place as it builds the record. + """ + self.dump = dump + self.raw_dump_entry = dump.data + self.dojson_entry = dojson_entry self.affiliations_mapping = affiliations_mapping - self.dry_run = dry_run - self.collection = collection self.restricted = restricted self.migration_logger = migration_logger - self.record_state_logger = record_state_logger + self.body = None - def _created(self, entry): - return entry["created"] - - def _updated(self, record_dump): - """Returns the creation date of the record.""" - return record_dump.data["record"][0]["modification_datetime"] - - def _version_id(self, entry): - """Returns the version id of the record.""" - return 1 + self.created = dump.first_created + self.recid = self._recid(dump) + # None when RecordFlaggedCuration was raised and caught. + self.access_status = None + try: + self.access_status = self._access(dojson_entry) + except RecordFlaggedCuration as exc: + self.migration_logger.add_information( + self.raw_dump_entry["recid"], + {"message": exc.message, "value": exc.value}, + ) - def _access(self, entry, record_dump): - record_restriction = ( - r[0] if isinstance(r := entry.get("record_restriction"), list) else r - ) + def _access(self, dojson_entry): + record_restriction = dojson_entry.pop("record_restriction", None) + if isinstance(record_restriction, list): + record_restriction = record_restriction[0] restrictions = "restricted" if self.restricted else record_restriction if not restrictions: raise RecordFlaggedCuration( @@ -133,18 +117,14 @@ def _access(self, entry, record_dump): ) return restrictions - def _index(self, record_dump): - """Returns the version index of the record.""" - return 1 # in legacy we start at 0 - - def _recid(self, record_dump): + def _recid(self, dump): """Returns the recid of the record.""" - return str(record_dump.data["recid"]) + return str(dump.data["recid"]) - def _pids(self, json_entry): + def _pids(self, dojson_entry): DATACITE_PREFIX = current_app.config["DATACITE_PREFIX"] - pids = json_entry.get("_pids", {}) + pids = dojson_entry.pop("_pids", {}) output_pids = deepcopy(pids) for key, identifier in pids.items(): # ignoring some pids @@ -173,32 +153,33 @@ def _pids(self, json_entry): doi_identifier["provider"] = "external" if doi.startswith(DATACITE_PREFIX) or doi.startswith("10.5170"): - if not json_entry.get("publisher"): - json_entry["publisher"] = "CERN" + if not dojson_entry.get("publisher"): + dojson_entry["publisher"] = "CERN" output_pids["doi"] = doi_identifier if output_pids: return output_pids else: return {} - def _files(self, record_dump): + def _files(self, dump): """Transform the files of a record.""" - record_dump.prepare_files() - files = record_dump.files + dump.prepare_files() + files = dump.files return {"enabled": bool(files)} - def _communities(self, json_entry): - return json_entry.get("communities", []) - - def _owner(self, json_entry): - email = json_entry.get("submitter") - return email + def _metadata(self, dojson_entry, raw_dump_entry): + """Build the metadata dict by running the composed field mappers. - def _metadata(self, json_entry, entry): - """Build the metadata dict by running the composed field mappers.""" + Whether every ``dojson_entry`` key ended up consumed *somewhere* in + the pipeline is no longer this method's concern - see + ``CDSToRDMRecordTransform._check_forgotten_keys()``, which runs + once all entities (this one, ``RecordParent``, ...) have built and + popped their own keys, for a check across the whole entry rather + than just this record's metadata. + """ ctx = RecordTransformContext( - json_entry=json_entry, - entry=entry, + dojson_entry=dojson_entry, + raw_dump_entry=raw_dump_entry, migration_logger=self.migration_logger, affiliations_mapping=self.affiliations_mapping, ) @@ -207,42 +188,19 @@ def _metadata(self, json_entry, entry): # metadata["resource_type"]; see mappers/registry.py. for mapper in METADATA_MAPPERS: metadata[mapper.id] = mapper.map_value(ctx) - - # filter empty keys - helper_keys = [ - "recid", - "legacy_recid", - "agency_code", - "submitter", - "status_week_date", - "record_restriction", - "access_grants", - "custom_fields", - "_pids", - "internal_notes", - "ep_approval", - ] - keys = deepcopy(list(json_entry.keys())) - for item in helper_keys: - if item in keys: - keys.remove(item) - - forgotten_keys = [key for key in keys if key not in list(metadata.keys())] - if forgotten_keys: - raise ManualImportRequired("Unassigned metadata key", value=forgotten_keys) return {k: v for k, v in metadata.items() if v} - def _custom_fields(self, json_entry): + def _custom_fields(self, dojson_entry, raw_dump_entry): """Build the custom_fields dict by running the composed field mappers. Must run before ``_metadata()``: a couple of these mappers add a - fallback ``json_entry["subjects"]`` entry when a vocabulary lookup + fallback ``dojson_entry["subjects"]`` entry when a vocabulary lookup fails, which metadata's own SubjectsMapper then picks up like any other subject - see DepartmentsMapper. """ ctx = RecordTransformContext( - json_entry=json_entry, - entry=json_entry, + dojson_entry=dojson_entry, + raw_dump_entry=raw_dump_entry, migration_logger=self.migration_logger, ) for mapper in CUSTOM_FIELD_MAPPERS: @@ -251,7 +209,7 @@ def _custom_fields(self, json_entry): forgotten_keys = [ key - for key in json_entry["custom_fields"].keys() + for key in dojson_entry["custom_fields"].keys() if key not in custom_fields.keys() ] if forgotten_keys: @@ -261,47 +219,47 @@ def _custom_fields(self, json_entry): # filter out null values return {k: v for k, v in custom_fields.items() if v} - def _verify_publication_date(self, entry, json_data): + def _verify_publication_date(self, raw_dump_entry, dojson_entry): """Verify creation date. If the record has no files (file creation date will be used as record creation date) and no creation date, raise an exception. """ - if not entry.get("files") and not ( - json_data.get("status_week_date") or json_data.get("publication_date") + if not raw_dump_entry.get("files") and not ( + dojson_entry.get("status_week_date") + or dojson_entry.get("publication_date") ): raise ManualImportRequired( message="Record missing publication date", field="validation", stage="transform", description="Record has no files and no publication date", - recid=entry["recid"], + recid=raw_dump_entry["recid"], priority="warning", value=None, subfield=None, ) - def transform(self, entry, record_dump, json_data) -> RecordEntryData: - """Transform a record single entry. - - :param entry: the original harvested legacy entry. - :param record_dump: the ``CDSRecordDump`` for ``entry`` - produced - by ``CDSToRDMRecordTransform.transform_xml_to_json()``. - :param json_data: the DOJSON-processed record data - (``record_dump.latest_revision``'s content) - also produced by - ``transform_xml_to_json()``, passed in separately since this - method mutates it in place as it builds the record. - """ - self._verify_publication_date(entry, json_data) + def build(self) -> RecordBody: + """Build and return this record's content - the RDM record body.""" + raw_dump_entry, dump, dojson_entry = ( + self.raw_dump_entry, + self.dump, + self.dojson_entry, + ) + self._verify_publication_date(raw_dump_entry, dojson_entry) # custom_fields runs before metadata: see _custom_fields()'s docstring. - custom_fields = self._custom_fields(json_data) + custom_fields = self._custom_fields(dojson_entry, raw_dump_entry) + # popped ahead of CDSToRDMRecordTransform._check_forgotten_keys(), + # same reason as _pids()/_access()'s record_restriction pop. + internal_notes = dojson_entry.pop("internal_notes", None) record_json_output = { - "files": self._files(record_dump), - "pids": self._pids(json_data), - "metadata": self._metadata(json_data, entry), - "internal_notes": json_data.get("internal_notes"), + "files": self._files(dump), + "pids": self._pids(dojson_entry), + "metadata": self._metadata(dojson_entry, raw_dump_entry), + "internal_notes": internal_notes, "custom_fields": custom_fields, } # drop empty optional keys rather than sending them to the RDM @@ -310,29 +268,5 @@ def transform(self, entry, record_dump, json_data) -> RecordEntryData: if not record_json_output[key]: del record_json_output[key] - access = None - try: - access = self._access(json_data, record_dump) - except RecordFlaggedCuration as exc: - self.migration_logger.add_information( - entry["recid"], - {"message": exc.message, "value": exc.value}, - ) - return { - "created": record_dump.first_created, - "updated": self._updated(record_dump), - "version_id": self._version_id(record_dump), - "index": self._index(record_dump), - "recid": self._recid(record_dump), - "communities": self._communities(json_data), - "body": record_json_output, - "access_status": access, - "owned_by": self._owner(json_data), - # _request_data/ep_approval are no longer record content here - - # CDSToRDMRecordTransform._transform() assigns them directly on - # MigrationEntry, since neither needs anything only this method - # has: ep_approval reads the raw entry (already available to - # the caller), and _request_data's pop off json_data happens - # in CDSToRDMRecordTransform._transform(), before this method - # even runs. - } + self.body = record_json_output + return self.body diff --git a/cds_migrator_kit/rdm/records/transform/entities/request.py b/cds_migrator_kit/rdm/records/transform/entities/request.py index a1fe983f..6ce76d6e 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/request.py +++ b/cds_migrator_kit/rdm/records/transform/entities/request.py @@ -16,7 +16,7 @@ class RecordRequest: """A community-inclusion request for one migrated CDS record. Built by ``CDSToRDMRecordTransform._transform()`` - pops the raw - request data off ``json_entry`` (before any field mapper can see it, + request data off ``dojson_entry`` (before any field mapper can see it, since it isn't part of the RDM record schema), resolves raw reviewer name/email strings to actual user accounts (a DB lookup, which is why this lives here and not in a dojson rule - rules must stay DB-free), @@ -24,22 +24,22 @@ class RecordRequest: publish, which owns actually creating the RDM request. """ - def __init__(self, json_entry, recid, migration_logger): + def __init__(self, dojson_entry, recid, migration_logger): """Constructor. - :param json_entry: the DOJSON-processed record data - request_data + :param dojson_entry: the DOJSON-processed record data - request_data is popped off it. :param recid: this record's legacy recid, for error reporting. :param migration_logger: for reviewer-error/validation logging. """ - self.json_entry = json_entry + self.dojson_entry = dojson_entry self.recid = recid self.migration_logger = migration_logger self.data = None def build(self): - """Pop request_data off ``json_entry``, resolve reviewers; return self.""" - request_data = self.json_entry.pop("request_data", None) + """Pop request_data off ``dojson_entry``, resolve reviewers; return self.""" + request_data = self.dojson_entry.pop("request_data", None) if request_data: reviewer_names = request_data.pop("reviewer_names", []) # merge into whatever's already there rather than overwriting - diff --git a/cds_migrator_kit/rdm/records/transform/mappers/base.py b/cds_migrator_kit/rdm/records/transform/mappers/base.py index 6cbafb2d..73130fa5 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/base.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/base.py @@ -22,8 +22,8 @@ class RecordTransformContext: ``metadata["resource_type"]``). """ - json_entry: dict - entry: dict + dojson_entry: dict + raw_dump_entry: dict migration_logger: object = None affiliations_mapping: object = None access_grants_view: object = None @@ -33,7 +33,7 @@ class RecordTransformContext: def flag_curation(self, exc): """Log a caught ``RecordFlaggedCuration`` for curation follow-up.""" self.migration_logger.add_information( - self.json_entry["recid"], + self.dojson_entry["recid"], {"message": exc.message, "value": exc.value}, ) @@ -63,7 +63,7 @@ def __init__(self, id): def map_value(self, ctx): """Return the raw value of ``self.id`` from the source entry.""" - return ctx.json_entry.get(self.id) + return ctx.dojson_entry.get(self.id) class CustomFieldMapper(ABC): @@ -93,5 +93,5 @@ def __init__(self, id, default=None): def apply(self, ctx): """Copy ``self.id`` from the source custom_fields, or use the default.""" - source = ctx.json_entry.get("custom_fields", {}) + source = ctx.dojson_entry.get("custom_fields", {}) ctx.custom_fields[self.id] = source.get(self.id, self.default) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/contributors.py b/cds_migrator_kit/rdm/records/transform/mappers/contributors.py index 0ad6ff12..63764949 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/contributors.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/contributors.py @@ -22,7 +22,7 @@ def match_affiliation(affiliation_name, ctx): """Match an affiliation against `CDSMigrationAffiliationMapping` db table.""" - json_entry = ctx.json_entry + dojson_entry = ctx.dojson_entry if is_ror(affiliation_name): ror = normalize_ror(affiliation_name) name = AffiliationsMetadata.query.filter_by(pid=ror).one_or_none() @@ -34,7 +34,7 @@ def match_affiliation(affiliation_name, ctx): field="validation", stage="transform", description="Add this affiliation", - recid=json_entry["recid"], + recid=dojson_entry["recid"], priority="critical", value=None, subfield=None, @@ -165,7 +165,7 @@ def _lookup_person_id(creator): def creators_for(ctx, key="creators"): """Build the creators/contributors list for ``key``.""" - _creators = deepcopy(ctx.json_entry.get(key, [])) + _creators = deepcopy(ctx.dojson_entry.get(key, [])) _creators = list(filter(lambda x: x is not None, _creators)) for creator in _creators: _creator_affiliations(creator, ctx) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index 9228e9e9..8fbef9b1 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -23,7 +23,7 @@ class ExperimentsMapper(CustomFieldMapper): def apply(self, ctx): """Set ctx.custom_fields["cern:experiments"].""" experiments_out = ctx.custom_fields["cern:experiments"] = [] - experiments = ctx.json_entry.get("custom_fields", {}).get( + experiments = ctx.dojson_entry.get("custom_fields", {}).get( "cern:experiments", [] ) for experiment in experiments: @@ -56,7 +56,7 @@ class DepartmentsMapper(CustomFieldMapper): def apply(self, ctx): """Set ctx.custom_fields["cern:departments"].""" departments_out = ctx.custom_fields["cern:departments"] = [] - departments = ctx.json_entry.get("custom_fields", {}).get( + departments = ctx.dojson_entry.get("custom_fields", {}).get( "cern:departments", [] ) for department in departments: @@ -75,7 +75,7 @@ def apply(self, ctx): # up naturally - see RecordEntry.transform(), which # runs custom_fields mappers before metadata mappers for # exactly this reason. - ctx.json_entry.setdefault("subjects", []).append( + ctx.dojson_entry.setdefault("subjects", []).append( {"subject": department} ) @@ -114,7 +114,7 @@ class AcceleratorsMapper(CustomFieldMapper): def apply(self, ctx): """Set ctx.custom_fields["cern:accelerators"].""" accelerators_out = ctx.custom_fields["cern:accelerators"] = [] - accelerators = ctx.json_entry.get("custom_fields", {}).get( + accelerators = ctx.dojson_entry.get("custom_fields", {}).get( "cern:accelerators", [] ) for accelerator in accelerators: @@ -142,7 +142,7 @@ class BeamsMapper(CustomFieldMapper): def apply(self, ctx): """Set ctx.custom_fields["cern:beams"].""" beams_out = ctx.custom_fields["cern:beams"] = [] - beams = ctx.json_entry.get("custom_fields", {}).get("cern:beams", []) + beams = ctx.dojson_entry.get("custom_fields", {}).get("cern:beams", []) for beam in beams: if beam.lower().strip() == "not applicable": continue @@ -168,7 +168,7 @@ class ProgrammesMapper(CustomFieldMapper): def apply(self, ctx): """Set ctx.custom_fields["cern:programmes"], or leave it unset.""" - record_json = ctx.json_entry + record_json = ctx.dojson_entry programme = record_json.get("custom_fields", {}).get("cern:programmes") if programme: result = search_vocabulary(programme, "programmes") @@ -193,7 +193,7 @@ class JournalMapper(CustomFieldMapper): def apply(self, ctx): """Set ctx.custom_fields["journal:journal"].""" - journal = ctx.json_entry.get("custom_fields", {}).get("journal:journal", {}) + journal = ctx.dojson_entry.get("custom_fields", {}).get("journal:journal", {}) if journal and not journal.get("title"): ctx.flag_curation( RecordFlaggedCuration( diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index ef2499be..a9f13362 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -24,14 +24,14 @@ class ResourceTypeMapper(FieldMapper): def map_value(self, ctx): """Return resource_type, dropping the upstream ranking scratch key.""" - json_entry = ctx.json_entry + dojson_entry = ctx.dojson_entry # `_resource_type_rank` is bookkeeping for the 980__/697C_ # resource_type rule and research_committee.py's report-number # detection (see research.py:resource_type) - drop it before it # reaches the final record. - json_entry.pop("_resource_type_rank", None) + dojson_entry.pop("_resource_type_rank", None) try: - return json_entry["resource_type"] + return dojson_entry["resource_type"] except KeyError: raise MissingRequiredField(message="resource_type", field="980") @@ -43,8 +43,8 @@ class TitleMapper(FieldMapper): def map_value(self, ctx): """Return title, or the 111__a meeting title as a fallback.""" - json_entry = ctx.json_entry - title = json_entry.get("title") + dojson_entry = ctx.dojson_entry + title = dojson_entry.get("title") if title: return title # 245 (title) is sometimes absent on conference proceedings @@ -52,7 +52,7 @@ def map_value(self, ctx): # the first meeting entry. resource_type = ctx.metadata.get("resource_type") or {} if resource_type.get("id") == "publication-conferenceproceeding": - meetings = json_entry.get("custom_fields", {}).get("meeting:meeting", []) + meetings = dojson_entry.get("custom_fields", {}).get("meeting:meeting", []) for meeting_entry in meetings: meeting_title = meeting_entry.get("title") if meeting_title: @@ -67,17 +67,17 @@ class PublicationDateMapper(FieldMapper): def map_value(self, ctx): """Return publication_date, requiring at least one date source.""" - json_entry = ctx.json_entry - pub_date = json_entry.get("publication_date") - created = json_entry.get("status_week_date") - files = ctx.entry["files"] + dojson_entry = ctx.dojson_entry + pub_date = dojson_entry.get("publication_date") + created = dojson_entry.get("status_week_date") + files = ctx.raw_dump_entry["files"] if not (pub_date or created or files): raise MissingRequiredField( message="missing creation or publication date", field="916" ) if not pub_date: if created: - pub_date = json_entry["status_week_date"] + pub_date = dojson_entry["status_week_date"] elif not created and files: pub_date = parse(files[0]["creation_date"]).date().isoformat() return pub_date @@ -90,7 +90,7 @@ class SubjectsMapper(FieldMapper): def map_value(self, ctx): """Return the subjects list with placeholder entries removed.""" - subjects = ctx.json_entry.get("subjects") + subjects = ctx.dojson_entry.get("subjects") if subjects: for subject in reversed(subjects): if subject.get("subject", "").lower() in ["xx", "talk"]: @@ -107,16 +107,16 @@ class TableOfContentsMapper(FieldMapper): def map_value(self, ctx): """Move table_of_content into additional_descriptions and return it.""" - json_entry = ctx.json_entry - toc = json_entry.get("table_of_content", []) - additional_desc = json_entry.get("additional_descriptions", []) + dojson_entry = ctx.dojson_entry + toc = dojson_entry.get("table_of_content", []) + additional_desc = dojson_entry.get("additional_descriptions", []) if toc: additional_desc.append( {"description": toc, "type": {"id": "table-of-contents"}} ) - json_entry["additional_descriptions"] = additional_desc - json_entry.pop("table_of_content") - return json_entry.get("additional_descriptions") + dojson_entry["additional_descriptions"] = additional_desc + dojson_entry.pop("table_of_content") + return dojson_entry.get("additional_descriptions") class IdentifiersMapper(FieldMapper): @@ -126,7 +126,7 @@ class IdentifiersMapper(FieldMapper): def map_value(self, ctx): """Return identifiers filtered/validated against known schemes.""" - identifiers = ctx.json_entry.get("identifiers", []) + identifiers = ctx.dojson_entry.get("identifiers", []) for item in reversed(identifiers): # drop unwanted schemes if item is None or "scheme" not in item: @@ -156,10 +156,11 @@ def map_value(self, ctx): return identifiers -# Fields that pass through unchanged from json_entry - kept explicit in the +# Fields that pass through unchanged from dojson_entry - kept explicit in the # composed list (mappers/config equivalent) rather than open-ended, so the -# "forgotten metadata key" completeness check in RecordEntry._metadata -# still catches any newly introduced json_entry key nobody has mapped yet. +# "forgotten metadata key" completeness check in +# CDSToRDMRecordTransform._check_forgotten_keys() still catches any newly +# introduced dojson_entry key nobody has mapped yet. PASSTHROUGH_METADATA_FIELDS = ( "description", "publisher", diff --git a/cds_migrator_kit/rdm/records/transform/mappers/record.py b/cds_migrator_kit/rdm/records/transform/mappers/record.py index 11d4aa0d..f8a0f3f6 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/record.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/record.py @@ -16,7 +16,7 @@ class AccessGrantsMapper(FieldMapper): def map_value(self, ctx): """Return access_grants extended with configured view grants.""" - access_grants = ctx.json_entry.get("access_grants", []) + access_grants = ctx.dojson_entry.pop("access_grants", []) if ctx.access_grants_view: for grant in ctx.access_grants_view: access_grants.append({str(grant): "view"}) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index a100d5aa..edece5b0 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -31,10 +31,7 @@ ) from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent -from cds_migrator_kit.rdm.records.transform.entities.record import ( - RecordEntry, - RecordEntryData, -) +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest from cds_migrator_kit.rdm.records.transform.transform_versions import \ RecordVersionsTransform @@ -50,8 +47,11 @@ class CDSToRDMRecordTransform: Wraps the ``record`` content built by ``RecordEntry`` together with ``versions``/``parent`` (computed here - ``parent`` is a ``RecordParent``, built directly in ``_transform()``) and the - ETL-envelope extras that aren't record content (currently just - ``_original_dump`` and ``_clc_sync`` - see ``_transform()``). + ETL-envelope extras that aren't record content (currently + ``_original_dump``/``_clc_sync``/``ep_approval`` - see + ``_transform()``). Also + owns the "did every DOJSON-produced key get consumed by something" + check across the whole entry - see ``_check_forgotten_keys()``. """ def __init__( @@ -70,15 +70,18 @@ def __init__( access_grants_view=None, preferred_model=None, ): - """Constructor.""" - self._workers = workers + """Constructor. + + ``workers``/``missing_users``/``dry_run``/``collection`` are + accepted (not stored) purely because + ``cds_migrator_kit.runner.runner.Runner`` always passes them when + constructing this class from ``streams.yaml`` config - they aren't + used by this class or ``RecordEntry``. + """ self._throw = throw self._logger = None self.files_dump_dir = Path(files_dump_dir).absolute().as_posix() - self.missing_users_dir = Path(missing_users).absolute().as_posix() self.communities_ids = communities_ids - self.dry_run = dry_run - self.collection = collection self.restricted = restricted self.access_grants_view = access_grants_view self.plots = plots @@ -87,9 +90,9 @@ def __init__( self.preferred_model = preferred_model self.db_state = {"affiliations": CDSMigrationAffiliationMapping} # the DOJSON-processed record data for the entry currently being - # transformed - populated by transform_xml_to_json(), read by + # transformed - populated by _transform_xml_to_json(), read by # _transform() to build this record's RecordParent (access grants). - self.raw_json_entry = None + self.dojson_entry = None @property def logger(self): @@ -98,65 +101,84 @@ def logger(self): self._logger = Logger.get_logger() return self._logger - def _transform_xml_to_json(self, entry): + def _transform_xml_to_json(self, raw_dump_entry): """Parse the legacy dump into the DOJSON-processed record. The one place per record that runs ``CDSRecordDump`` - populates - ``self.raw_json_entry`` with the DOJSON-mapped record content and + ``self.dojson_entry`` with the DOJSON-mapped record content and returns the ``CDSRecordDump`` instance itself, since record-level facts derived from the dump (created/updated/recid/files) live on - that object, not as keys in ``raw_json_entry``. + that object, not as keys in ``dojson_entry``. """ - record_dump = CDSRecordDump(entry, preferred_model=self.preferred_model) - record_dump.prepare_revisions() - timestamp, json_data = record_dump.latest_revision - self.raw_json_entry = json_data - self.record_state_logger.add_record(json_data) - return record_dump + dump = CDSRecordDump(raw_dump_entry, preferred_model=self.preferred_model) + dump.prepare_revisions() + timestamp, dojson_entry = dump.latest_revision + self.dojson_entry = dojson_entry + self.record_state_logger.add_record(dojson_entry) + return dump - def _parent(self, entry, record): + def _parent(self, raw_dump_entry, record): return RecordParent( record=record, - entry=entry, - json_entry=self.raw_json_entry, + raw_dump_entry=raw_dump_entry, + dojson_entry=self.dojson_entry, communities_ids=self.communities_ids, access_grants_view=self.access_grants_view, ).build() - def _transform(self, entry) -> Optional[MigrationEntry]: + def _request(self, raw_dump_entry): + """Build the community-inclusion request for this entry. + + "request_data" must be off ``dojson_entry`` before + ``_check_forgotten_keys()`` runs - ``RecordRequest.build()`` pops + it right here, so that check never sees it either. + """ + return RecordRequest( + dojson_entry=self.dojson_entry, + recid=raw_dump_entry["recid"], + migration_logger=self.migration_logger, + ).build() + + def _transform(self, raw_dump_entry) -> Optional[MigrationEntry]: """Transform a single entry.""" # creates the output structure for load step migration_logger = self.migration_logger try: # could be in draft as well, depends on how we decide to publish - record_dump = self._transform_xml_to_json(entry) + dump = self._transform_xml_to_json(raw_dump_entry) # ETL-envelope concern, stripped before the record body is built - - # _metadata()'s forgotten-key check doesn't know this key. - clc_sync = deepcopy(self.raw_json_entry.get("_clc_sync", False)) - if "_clc_sync" in self.raw_json_entry: - del self.raw_json_entry["_clc_sync"] - - # same reason: "request_data" must be off raw_json_entry before - # RecordEntry.transform() runs _metadata()'s forgotten-key - # check, which doesn't know this key either. - record_request = RecordRequest( - json_entry=self.raw_json_entry, - recid=entry["recid"], - migration_logger=self.migration_logger, - ).build() + # _check_forgotten_keys() doesn't know this key. + clc_sync = deepcopy(self.dojson_entry.pop("_clc_sync", False)) - record = self._record(entry, record_dump) + # legacy_recid is produced by a dojson rule but never consumed + # anywhere - it always carries the same value as recid (just + # int vs str). Pop it so it doesn't trip _check_forgotten_keys(). + self.dojson_entry.pop("legacy_recid", None) + # ep_approval (the record's EP-approval workflow history, from + # the ^9031_ MARC tag) is an ETL-envelope concern like + # _clc_sync above - popped here rather than left as record + # content, and consumed directly below via MigrationEntry. + ep_approval = self.dojson_entry.pop("ep_approval", []) + record_request = self._request(raw_dump_entry) + record = self._record(raw_dump_entry, dump) if record: + versions = self._versions(raw_dump_entry, record) + # RecordParent is the last entity to touch dojson_entry - + # it pops submitter/communities/access_grants - so the + # forgotten-keys check must run after it. + parent = self._parent(raw_dump_entry, record) + self._check_forgotten_keys(record) + return MigrationEntry( - _original_dump=entry, + _original_dump=raw_dump_entry, record=record, - versions=self._versions(entry, record), - parent=self._parent(entry, record), + versions=versions, + parent=parent, _clc_sync=clc_sync, _request_data=record_request, - ep_approval=entry.get("ep_approval", []), + ep_approval=ep_approval, ) except ( @@ -167,23 +189,42 @@ def _transform(self, entry) -> Optional[MigrationEntry]: MissingRequiredField, MultipleModelsMatched, ) as e: - migration_logger.add_log(e, record=entry) + migration_logger.add_log(e, record=raw_dump_entry) + + def _check_forgotten_keys(self, record: RecordEntry): + """Ensure every key DOJSON produced for this entry got consumed. + + A "global" check across the whole ETL entry, run once all entities + (``RecordEntry``, ``RecordParent``, ...) have built - and popped + their own keys off ``self.dojson_entry`` - rather than + ``RecordEntry`` checking only its own record body/metadata. + """ + # recid/status_week_date are read live by field mappers (recid for + # error reporting, status_week_date by PublicationDateMapper to + # derive metadata["publication_date"]) but never popped, since + # they're needed for the duration of that mapping; custom_fields + # is read the same way by TitleMapper's meeting-title fallback. + helper_keys = ["recid", "status_week_date", "custom_fields", "agency_code"] + keys = [key for key in self.dojson_entry if key not in helper_keys] + metadata_keys = record.body["metadata"].keys() + forgotten_keys = [key for key in keys if key not in metadata_keys] + if forgotten_keys: + raise ManualImportRequired("Unassigned metadata key", value=forgotten_keys) - def _record(self, entry, record_dump) -> RecordEntryData: + def _record(self, raw_dump_entry, dump) -> RecordEntry: entry_builder = RecordEntry( - missing_users_dir=self.missing_users_dir, + dump=dump, + dojson_entry=self.dojson_entry, affiliations_mapping=self.db_state["affiliations"], - dry_run=self.dry_run, - collection=self.collection, restricted=self.restricted, migration_logger=self.migration_logger, - record_state_logger=self.record_state_logger, ) - return entry_builder.transform(entry, record_dump, self.raw_json_entry) + entry_builder.build() + return entry_builder - def _versions(self, entry, record: RecordEntryData): + def _versions(self, raw_dump_entry, record: RecordEntry): return RecordVersionsTransform( - entry=entry, + raw_dump_entry=raw_dump_entry, record=record, files_dump_dir=self.files_dump_dir, plots=self.plots, @@ -200,8 +241,8 @@ def _load_migrated_recids(self): ).all() } - def should_skip(self, entry): - return str(entry["recid"]) in self._migrated_recids + def should_skip(self, raw_dump_entry): + return str(raw_dump_entry["recid"]) in self._migrated_recids def _existing_record_is_restricted(self, record_id): """Check the current access state of an already-migrated RDM record. @@ -219,9 +260,9 @@ def _existing_record_is_restricted(self, record_id): def run(self, entries): """Run transformation step.""" self._migrated_recids = self._load_migrated_recids() - for entry in entries: - if self.should_skip(entry): - recid = entry["recid"] + for raw_dump_entry in entries: + if self.should_skip(raw_dump_entry): + recid = raw_dump_entry["recid"] try: parent_pid = get_pid_by_legacy_recid(str(recid)) # we don't check here if the record has 980:MIGRATED @@ -266,7 +307,7 @@ def run(self, entries): }, ) except ManualImportRequired as exc: - self.migration_logger.add_log(exc, record=entry) + self.migration_logger.add_log(exc, record=raw_dump_entry) self.migration_logger.add_information( recid, @@ -280,9 +321,9 @@ def run(self, entries): self.migration_logger.finalise_record(recid) continue try: - yield self._transform(entry) + yield self._transform(raw_dump_entry) except Exception: - self.logger.exception(entry, exc_info=True) + self.logger.exception(raw_dump_entry, exc_info=True) if self._throw: raise continue diff --git a/cds_migrator_kit/rdm/records/transform/transform_versions.py b/cds_migrator_kit/rdm/records/transform/transform_versions.py index 785acafc..9358a888 100644 --- a/cds_migrator_kit/rdm/records/transform/transform_versions.py +++ b/cds_migrator_kit/rdm/records/transform/transform_versions.py @@ -24,17 +24,18 @@ class RecordVersionsTransform: just its own delta. """ - def __init__(self, entry, record, files_dump_dir, plots, migration_logger): + def __init__(self, raw_dump_entry, record, files_dump_dir, plots, migration_logger): """Constructor. - :param entry: the original harvested legacy entry (needs "files"). - :param record: the already-built ``RecordEntry`` dict (needs + :param raw_dump_entry: the original harvested legacy entry (needs + "files"). + :param record: the already-built ``RecordEntry`` (needs ``access_status`` and ``body["metadata"]["publication_date"]``). :param files_dump_dir: local EOS mirror root for file content. :param plots: whether to keep Plot-type files. :param migration_logger: for skip/restriction logging. """ - self.entry = entry + self.raw_dump_entry = raw_dump_entry self.record = record self.files_dump_dir = files_dump_dir self.plots = plots @@ -42,14 +43,14 @@ def __init__(self, entry, record, files_dump_dir, plots, migration_logger): def build(self): """Group legacy files by version, build + carry files forward, return.""" - record_access = self.record["access_status"] + record_access = self.record.access_status # group non-skipped raw file dumps by legacy version number, in # first-seen order - own_file_dumps[v] is version v's own files # (not yet carrying anything forward from earlier versions). own_file_dumps = OrderedDict() representative_file = {} - for file_dump in self.entry["files"]: + for file_dump in self.raw_dump_entry["files"]: if self._should_skip_file(file_dump): continue version_number = file_dump["version"] @@ -82,7 +83,7 @@ def build(self): record_access=record_access, files_dump_dir=self.files_dump_dir, migration_logger=self.migration_logger, - publication_date=self.record["body"]["metadata"]["publication_date"], + publication_date=self.record.body["metadata"]["publication_date"], ).build() return versions diff --git a/cds_migrator_kit/reports/log.py b/cds_migrator_kit/reports/log.py index 9176e02a..698f6e04 100644 --- a/cds_migrator_kit/reports/log.py +++ b/cds_migrator_kit/reports/log.py @@ -124,7 +124,12 @@ def add_log(self, exc, record=None, key=None, value=None): recid = getattr(exc, "recid", None) if not recid and record: - recid = record.get("recid", None) or record.get("record", {}).get("recid") + # `record` is either the raw harvested entry (has "recid" + # directly) or a MigrationEntry (its "record" key is a real + # RecordEntry object, not a dict - see entities/record.py). + recid = record.get("recid", None) or getattr( + record.get("record"), "recid", None + ) subfield = f"subfield: {exc.subfield}" if getattr(exc, "subfield", None) else "" error_format = { diff --git a/tests/cds-rdm/test_ep_approval_entry.py b/tests/cds-rdm/test_ep_approval_entry.py index 55c8f59a..86ff7364 100644 --- a/tests/cds-rdm/test_ep_approval_entry.py +++ b/tests/cds-rdm/test_ep_approval_entry.py @@ -21,6 +21,7 @@ RestrictedEntry, ) from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry RECID = "12345" APPROVED_REPORT_NUMBER = "CERN-EP-2020-001" @@ -82,6 +83,22 @@ def _make_parent(owner="uploader", communities=None, access_grants=None): return parent +def _make_record(recid, body): + """Build a RecordEntry-shaped test double. + + Bypasses RecordEntry's constructor/build()'s mapper-dependent logic - + this module tests PublicEntry/RestrictedEntry's splitting logic, not + RecordEntry's own construction. The record's owner is a RecordParent + concern (see _make_parent's `owner`), not tracked on the record itself. + """ + record = RecordEntry.__new__(RecordEntry) + record.recid = recid + record.body = body + record.access_status = "public" + record.created = "2020-01-15T00:00:00+00:00" + return record + + def _make_entry( versions, recid=RECID, @@ -125,11 +142,7 @@ def _make_entry( } return { - "record": { - "recid": recid, - "body": record_json, - "owned_by": "uploader", - }, + "record": _make_record(recid, record_json), "parent": _make_parent(communities={"ids": ["example-community"]}), "versions": versions, "ep_approval": [ @@ -434,7 +447,7 @@ def test_public_removes_cern_ep_report_numbers(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - identifiers = result["record"]["body"]["metadata"]["identifiers"] + identifiers = result["record"].body["metadata"]["identifiers"] cdsrn_values = {i["identifier"] for i in identifiers if i["scheme"] == "cdsrn"} assert APPROVED_REPORT_NUMBER not in cdsrn_values @@ -456,7 +469,7 @@ def test_public_keeps_non_ep_cdsrn(self, app): cdsrn_ids = [ i - for i in result["record"]["body"]["metadata"]["identifiers"] + for i in result["record"].body["metadata"]["identifiers"] if i["scheme"] == "cdsrn" ] assert len(cdsrn_ids) == 1 @@ -474,7 +487,7 @@ def test_restricted_removes_matching_cern_ep_rn(self, app): cdsrn_values = { i["identifier"] - for i in result["record"]["body"]["metadata"]["identifiers"] + for i in result["record"].body["metadata"]["identifiers"] if i["scheme"] == "cdsrn" } @@ -488,7 +501,7 @@ def test_restricted_keeps_draft_report_number(self, app): cdsrn_values = { i["identifier"] - for i in result["record"]["body"]["metadata"]["identifiers"] + for i in result["record"].body["metadata"]["identifiers"] if i["scheme"] == "cdsrn" } @@ -511,7 +524,7 @@ def test_restricted_removes_doi_pid(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - assert "doi" not in result["record"]["body"].get("pids", {}) + assert "doi" not in result["record"].body.get("pids", {}) class TestPublicEntryModifications: @@ -531,7 +544,6 @@ def test_public_sets_owned_by_system(self, app): entry, _make_approval_request(), _make_migration_logger() ).build() - assert result["record"]["owned_by"] == "system" assert result["parent"].body["access"]["owned_by"] == {"user": "system"} def test_public_adds_cern_scientific_community(self, app): @@ -565,8 +577,8 @@ def test_public_build_does_not_mutate_original(self, app): PublicEntry(entry, _make_approval_request(), _make_migration_logger()).build() assert ( - entry["record"]["body"]["metadata"]["identifiers"] - == original["record"]["body"]["metadata"]["identifiers"] + entry["record"].body["metadata"]["identifiers"] + == original["record"].body["metadata"]["identifiers"] ) def test_restricted_build_does_not_mutate_original(self, app): @@ -577,6 +589,6 @@ def test_restricted_build_does_not_mutate_original(self, app): ).build() assert ( - entry["record"]["body"]["metadata"]["identifiers"] - == original["record"]["body"]["metadata"]["identifiers"] + entry["record"].body["metadata"]["identifiers"] + == original["record"].body["metadata"]["identifiers"] ) diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index 09181125..069ae098 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -596,9 +596,11 @@ def test_962_book_with_different_artid_is_not_duplicate(self): record["resource_type"] = "publication-other" migration_logger = MagicMock() - custom_fields = RecordEntry(migration_logger=migration_logger)._custom_fields( - record - ) + # RecordEntry double for calling _custom_fields() in isolation - + # bypasses the constructor, which needs a real dump/dojson_entry. + record_entry = RecordEntry.__new__(RecordEntry) + record_entry.migration_logger = migration_logger + custom_fields = record_entry._custom_fields(record, raw_dump_entry={}) # titleless journal data is dropped (falsy values are filtered out), # not raised as a hard error diff --git a/tests/cds-rdm/test_transform_metadata_title.py b/tests/cds-rdm/test_transform_metadata_title.py index ff0266da..7887ca55 100644 --- a/tests/cds-rdm/test_transform_metadata_title.py +++ b/tests/cds-rdm/test_transform_metadata_title.py @@ -14,17 +14,24 @@ @pytest.fixture def entry(): - """Transform entry instance (no DB/app context required).""" - return RecordEntry() + """RecordEntry double for calling _metadata() in isolation. + Bypasses the constructor (which needs a real dump/dojson_entry) - + this module only exercises _metadata() directly. + """ + record_entry = RecordEntry.__new__(RecordEntry) + record_entry.migration_logger = None + record_entry.affiliations_mapping = None + return record_entry -def _dump(): - """Minimal record dump for _publication_date().""" + +def _raw_dump_entry(): + """Minimal raw dump entry for _publication_date().""" return {"files": []} -def _json_entry(**overrides): - """Minimal json_entry with a resource_type and a creation date set.""" +def _dojson_entry(**overrides): + """Minimal dojson_entry with a resource_type and a creation date set.""" base = { "recid": 123, "resource_type": {"id": "publication-conferenceproceeding"}, @@ -40,42 +47,42 @@ class TestMetadataTitleFromMeeting: def test_title_falls_back_to_meeting_title_when_missing(self, entry): """Test that a missing title is filled from the meeting:meeting title when resource_type is publication-conferenceproceeding.""" - json_entry = _json_entry( + dojson_entry = _dojson_entry( custom_fields={"meeting:meeting": [{"title": "Some Conference"}]} ) - metadata = entry._metadata(json_entry, _dump()) + metadata = entry._metadata(dojson_entry, _raw_dump_entry()) assert metadata["title"] == "Some Conference" def test_title_not_overridden_when_already_present(self, entry): """Test that an existing title is not replaced by the meeting title.""" - json_entry = _json_entry( + dojson_entry = _dojson_entry( title="Real Title", custom_fields={"meeting:meeting": [{"title": "Some Conference"}]}, ) - metadata = entry._metadata(json_entry, _dump()) + metadata = entry._metadata(dojson_entry, _raw_dump_entry()) assert metadata["title"] == "Real Title" def test_title_not_filled_for_other_resource_types(self, entry): """Test that the meeting-title fallback only applies to publication-conferenceproceeding, not other resource types.""" - json_entry = _json_entry( + dojson_entry = _dojson_entry( resource_type={"id": "publication-article"}, custom_fields={"meeting:meeting": [{"title": "Some Conference"}]}, ) - metadata = entry._metadata(json_entry, _dump()) + metadata = entry._metadata(dojson_entry, _raw_dump_entry()) assert "title" not in metadata def test_title_missing_without_meeting_custom_field(self, entry): """Test that title stays unset when there is no meeting:meeting entry to fall back to, even for conference proceedings.""" - json_entry = _json_entry() - metadata = entry._metadata(json_entry, _dump()) + dojson_entry = _dojson_entry() + metadata = entry._metadata(dojson_entry, _raw_dump_entry()) assert "title" not in metadata def test_title_uses_first_meeting_entry_with_a_title(self, entry): """Test that the fallback skips meeting entries without a title and uses the first one that has one.""" - json_entry = _json_entry( + dojson_entry = _dojson_entry( custom_fields={ "meeting:meeting": [ {"place": "Geneva"}, @@ -83,5 +90,5 @@ def test_title_uses_first_meeting_entry_with_a_title(self, entry): ] } ) - metadata = entry._metadata(json_entry, _dump()) + metadata = entry._metadata(dojson_entry, _raw_dump_entry()) assert metadata["title"] == "Second Meeting Title" diff --git a/tests/cds-rdm/test_transform_versions.py b/tests/cds-rdm/test_transform_versions.py index 97970948..493e40b3 100644 --- a/tests/cds-rdm/test_transform_versions.py +++ b/tests/cds-rdm/test_transform_versions.py @@ -7,6 +7,7 @@ """Tests for record version file snapshot logic in transform._versions().""" +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -67,11 +68,11 @@ def _file_dump( def _record(): - """Build a minimal record.""" - return { - "access_status": "public", - "body": {"metadata": {"publication_date": "2020-01-01"}}, - } + """Build a minimal RecordEntry-shaped test double.""" + return SimpleNamespace( + access_status="public", + body={"metadata": {"publication_date": "2020-01-01"}}, + ) @pytest.fixture @@ -86,7 +87,7 @@ def transform(tmp_path): def test_versions_preserve_file_revision_per_record_version(transform): """Each record version keeps the file revision.""" - entry = { + raw_dump_entry = { "recid": 123, "files": [ _file_dump(file_version=1, checksum="checksum-v1"), @@ -95,7 +96,7 @@ def test_versions_preserve_file_revision_per_record_version(transform): ], } - versions = transform._versions(entry, _record()) + versions = transform._versions(raw_dump_entry, _record()) assert list(versions.keys()) == [1, 2] assert versions[1]["files"]["draft.pdf"]["version"] == 1 @@ -109,7 +110,7 @@ def test_versions_preserve_file_revision_per_record_version(transform): def test_versions_with_skipped_files(transform): """Versions with skipped files should not create extra record versions.""" - entry = { + raw_dump_entry = { "recid": 123, "files": [ _file_dump( @@ -155,7 +156,7 @@ def test_versions_with_skipped_files(transform): ], } - versions = transform._versions(entry, _record()) + versions = transform._versions(raw_dump_entry, _record()) assert list(versions.keys()) == [1, 2] assert set(versions[1]["files"]) == {"main.pdf"} @@ -168,9 +169,9 @@ def test_versions_with_skipped_files(transform): def test_versions_no_files_falls_back_to_metadata_only_version(transform): """A record with no files gets a single, empty, public version.""" - entry = {"recid": 123, "files": []} + raw_dump_entry = {"recid": 123, "files": []} - versions = transform._versions(entry, _record()) + versions = transform._versions(raw_dump_entry, _record()) assert list(versions.keys()) == [1] assert versions[1]["files"] == {} @@ -185,12 +186,12 @@ def test_versions_individual_file_restriction_sets_access_meta(transform): status = ( 'firerole: allow group "some-group [CERN]"\ndeny until "1996-02-01"\nallow all' ) - entry = { + raw_dump_entry = { "recid": 123, "files": [_file_dump(status=status)], } - versions = transform._versions(entry, _record()) + versions = transform._versions(raw_dump_entry, _record()) assert versions[1]["access"] == { "access_obj": {"record": "public", "files": "restricted"}, From 0e3c1bd55aa0994f34072fabaae31976b063d729 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Tue, 25 Aug 2026 13:27:23 +0200 Subject: [PATCH 4/9] change(load): merge EP and regular workflows * refactor code to specialise on responsibilities --- cds_migrator_kit/rdm/README.md | 14 +- cds_migrator_kit/rdm/cli.py | 13 +- cds_migrator_kit/rdm/migration_config.py | 1 - cds_migrator_kit/rdm/records/load/__init__.py | 5 +- .../rdm/records/load/approval_request.py | 399 --------- .../rdm/records/load/entities/__init__.py | 0 .../records/load/entities/approval_request.py | 209 +++++ .../load/entities/approval_request_load.py | 252 ++++++ .../ep_migration_entry_load.py} | 233 +++-- .../ep_split.py} | 160 ++-- .../rdm/records/load/entities/parent.py | 191 ++++ .../rdm/records/load/entities/record.py | 460 ++++++++++ .../rdm/records/load/entities/request.py | 145 +++ cds_migrator_kit/rdm/records/load/load.py | 847 ++---------------- cds_migrator_kit/rdm/records/streams.py | 20 +- .../records/transform/entities/migration.py | 2 +- .../rdm/records/transform/entities/request.py | 16 - .../rdm/records/transform/transform.py | 2 +- cds_migrator_kit/runner/runner.py | 1 - tests/cds-rdm/test_ep_approval_entry.py | 2 +- .../cds-rdm/test_research_committee_rules.py | 591 ++++++++++++ 21 files changed, 2140 insertions(+), 1423 deletions(-) delete mode 100644 cds_migrator_kit/rdm/records/load/approval_request.py create mode 100644 cds_migrator_kit/rdm/records/load/entities/__init__.py create mode 100644 cds_migrator_kit/rdm/records/load/entities/approval_request.py create mode 100644 cds_migrator_kit/rdm/records/load/entities/approval_request_load.py rename cds_migrator_kit/rdm/records/load/{ep_approval_load.py => entities/ep_migration_entry_load.py} (60%) rename cds_migrator_kit/rdm/records/load/{ep_approval_entry.py => entities/ep_split.py} (78%) create mode 100644 cds_migrator_kit/rdm/records/load/entities/parent.py create mode 100644 cds_migrator_kit/rdm/records/load/entities/record.py create mode 100644 cds_migrator_kit/rdm/records/load/entities/request.py create mode 100644 tests/cds-rdm/test_research_committee_rules.py diff --git a/cds_migrator_kit/rdm/README.md b/cds_migrator_kit/rdm/README.md index 66fea9bf..5c23c6bc 100644 --- a/cds_migrator_kit/rdm/README.md +++ b/cds_migrator_kit/rdm/README.md @@ -152,9 +152,9 @@ Run the below command to migrate records in the created community from before: invenio migration run ``` -#### EP approval records (`--ep-approval`) +#### EP approval records -EP approval records must be migrated in a **separate stream**. Do not mix them with regular records. +EP approval records are detected automatically per-record (via the `9031_:EPPHAPP` history popped during transform) and split into a public/restricted pair by the loader - no separate flag or stream is needed, they can be migrated in the same run as regular records. 1. **Dump** them filtered by `9031_:EPPHAPP`, for example (FASER): @@ -162,17 +162,15 @@ EP approval records must be migrated in a **separate stream**. Do not mix them w inveniomigrator dump records -q '980__a:ARTICLE or 980__a:PREPRINT and 693:"FASER" not 980:CONFERENCEPAPER not 591__b:"Draft" 9031_:EPPHAPP -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY' --file-prefix faser-papers-cern-ep --chunk-size=1000 ``` -2. Configure a dedicated collection entry in `streams.yaml` pointing at that dump. +2. Configure a collection entry in `streams.yaml` pointing at that dump. -3. **Migrate** with the `--ep-approval` flag (dry run first): +3. **Migrate** normally (dry run first): ```shell -invenio migration run --collection faser-ep --ep-approval --dry-run -invenio migration run --collection faser-ep --ep-approval +invenio migration run --collection faser-ep --dry-run +invenio migration run --collection faser-ep ``` -Without `--ep-approval`, the loader will reject EP approval records. - #### Comments migration Configure the collection under `comments` in `streams.yaml` (`dir_path`, `reviewers`). Paths and reviewers are loaded from there when you pass `--collection`. diff --git a/cds_migrator_kit/rdm/cli.py b/cds_migrator_kit/rdm/cli.py index 20f46b66..72488542 100644 --- a/cds_migrator_kit/rdm/cli.py +++ b/cds_migrator_kit/rdm/cli.py @@ -22,7 +22,6 @@ CommentsStreamDefinition, ) from cds_migrator_kit.rdm.records.streams import ( # UserStreamDefinition, - RecordEPApprovalStreamDefinition, RecordStreamDefinition, ) from cds_migrator_kit.rdm.stats.runner import RecordStatsRunner @@ -70,20 +69,12 @@ def migration(): "Can also be set per-collection in streams.yaml under transform.workers." ), ) -@click.option( - "--ep-approval", - is_flag=True, - help="Use the EP approval load stream (pre-EP draft snapshots without legacy minting).", -) @with_appcontext -def run(collection, dry_run=False, keep_logs=False, workers=None, ep_approval=False): +def run(collection, dry_run=False, keep_logs=False, workers=None): """Run.""" stream_config = current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"] - stream_definition = ( - RecordEPApprovalStreamDefinition if ep_approval else RecordStreamDefinition - ) runner = Runner( - stream_definitions=[stream_definition], + stream_definitions=[RecordStreamDefinition], # stream_definitions=[UserStreamDefinition], config_filepath=Path(stream_config).absolute(), dry_run=dry_run, diff --git a/cds_migrator_kit/rdm/migration_config.py b/cds_migrator_kit/rdm/migration_config.py index f9c86116..c4827504 100644 --- a/cds_migrator_kit/rdm/migration_config.py +++ b/cds_migrator_kit/rdm/migration_config.py @@ -87,7 +87,6 @@ def _(x): # needed to avoid start time failure with lazy strings # See https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://cds-rdm:cds-rdm@localhost/cds-rdm" -SQLALCHEMY_ENGINE_OPTIONS = {"connect_args": {"options": "-c timezone=UTC"}} # Invenio-App # =========== diff --git a/cds_migrator_kit/rdm/records/load/__init__.py b/cds_migrator_kit/rdm/records/load/__init__.py index 4ce3126c..a11d68fa 100644 --- a/cds_migrator_kit/rdm/records/load/__init__.py +++ b/cds_migrator_kit/rdm/records/load/__init__.py @@ -7,7 +7,6 @@ """CDS-RDM Migration load package.""" -from .ep_approval_load import CDSEPApprovalRecordServiceLoad -from .load import CDSRecordServiceLoad +from .load import CDSMigrationEntryLoad -__all__ = ("CDSEPApprovalRecordServiceLoad", "CDSRecordServiceLoad") +__all__ = ("CDSMigrationEntryLoad",) diff --git a/cds_migrator_kit/rdm/records/load/approval_request.py b/cds_migrator_kit/rdm/records/load/approval_request.py deleted file mode 100644 index 09a80b72..00000000 --- a/cds_migrator_kit/rdm/records/load/approval_request.py +++ /dev/null @@ -1,399 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2026 CERN. -# -# CDS-RDM is free software; you can redistribute it and/or modify it under -# the terms of the MIT License; see LICENSE file for more details. - -"""CDS-RDM EP approval request validation and creation.""" - -from datetime import datetime, timezone - -from cds_rdm.requests.committee_approval import APPRN_PID_TYPE, CommitteeApprovalRequest -from flask import current_app -from invenio_access.permissions import system_identity -from invenio_accounts.models import User -from invenio_db.uow import UnitOfWork -from invenio_pidstore.errors import PIDAlreadyExists -from invenio_pidstore.models import PersistentIdentifier, PIDStatus -from invenio_rdm_records.records.api import RDMParent -from invenio_records_resources.services.uow import RecordCommitOp -from invenio_requests.customizations.event_types import ( - LogEventType, - ReviewersUpdatedType, -) -from invenio_requests.proxies import current_events_service, current_requests_service -from invenio_requests.resolvers.registry import ResolverRegistry - -from cds_migrator_kit.errors import ManualImportRequired, UnexpectedValue - -EP_APPROVAL_WAITING_STATUS = "waiting" -EP_APPROVAL_APPROVED_STATUS = "approved" -EP_APPROVAL_REVIEWING_STATUS = "reviewing" - - -class ApprovalRequest: - """Validate and create a migrated EP committee approval request.""" - - def __init__( - self, - ep_approval, - legacy_recid, - title=None, - resource_type=None, - dry_run=False, - ): - self.ep_approval = ep_approval - self.legacy_recid = legacy_recid - self.title = title - self.resource_type = resource_type - self.dry_run = dry_run - self.waiting_entry = None - self.reviewing_entry = None - self.approved_entry = None - self.report_number = None - self.approved_at = None - - def validate(self): - """Validate EP approval data before creating any records.""" - waiting_entry, approved_entry, reviewing, report_number = self._parse_history() - - existing = self._existing_request() - if existing: - raise ManualImportRequired( - message=f"EP approval request {existing['id']} already exists", - stage="load", - priority="critical", - ) - if self._exists_apprn_pid(report_number): - raise ManualImportRequired( - message=f"APPRN PID {report_number} already exists", - stage="load", - priority="critical", - ) - - self.waiting_entry = waiting_entry - self.approved_entry = approved_entry - self.reviewing_entry = reviewing - self.report_number = report_number - - def create(self, restricted_record_state, uow=None): - """Create and approve EP approval request after restricted record exists. - - If ``uow`` is provided, the request is registered on it without - committing, so the caller can group this atomically with other - operations (e.g. the public record creation and linking). - """ - if self.dry_run: - return - - if not restricted_record_state: - raise UnexpectedValue( - message="Restricted record is required for EP approval.", - stage="load", - recid=self.legacy_recid, - priority="critical", - ) - - restricted_recid = restricted_record_state["latest_version"] - restricted_parent = RDMParent.get_record( - restricted_record_state["parent_object_uuid"] - ) - self._create_request( - restricted_recid, - restricted_parent, - uow=uow, - ) - self._mint_apprn_pid(restricted_record_state["latest_version_object_uuid"]) - - def _parse_history(self): - """Return waiting/approved history entries and the report number.""" - if len(self.ep_approval) > 3: - raise UnexpectedValue( - message="EP approval history has more/less than 3 entries", - stage="load", - priority="critical", - ) - history = self.ep_approval or [] - waiting = next( - ( - item - for item in history - if item.get("status") == EP_APPROVAL_WAITING_STATUS - ), - None, - ) - reviewing = next( - ( - item - for item in history - if item.get("status") == EP_APPROVAL_REVIEWING_STATUS - ), - None, - ) - approved = next( - ( - item - for item in history - if item.get("status") == EP_APPROVAL_APPROVED_STATUS - ), - None, - ) - if not waiting: - raise UnexpectedValue( - message="EP approval history has no waiting entry", - stage="load", - priority="critical", - ) - if not approved: - raise UnexpectedValue( - message="EP approval history has no approved entry", - stage="load", - priority="critical", - ) - - report_number = approved.get("ep_report_number") - if not report_number: - raise UnexpectedValue( - message="EP approval approved entry is missing ep_report_number", - stage="load", - priority="critical", - ) - if waiting.get("ep_report_number") != report_number: - raise UnexpectedValue( - message=( - "EP approval waiting entry has different ep_report_number " - "than approved entry" - ), - stage="load", - priority="critical", - ) - - self._resolve_user_by_email(waiting.get("submitted_by"), "submitter") - self._resolve_user_by_email(approved.get("submitted_by"), "approver") - - waiting_deadline = self.parse_legacy_datetime(waiting.get("deadline")) - approved_date = self.parse_legacy_datetime(approved.get("date")) - self.approved_at = approved_date - created_at = self.parse_legacy_datetime(waiting.get("date")) - if not created_at or not approved_date or not waiting_deadline: - raise UnexpectedValue( - message="EP approval history has missing timestamps", - stage="load", - priority="critical", - ) - - return waiting, approved, reviewing, report_number - - @staticmethod - def parse_legacy_datetime(value): - """Parse legacy EP approval timestamps into timezone-aware datetimes.""" - if not value: - return None - for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): - try: - return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc) - except ValueError: - continue - return None - - def _get_referee_group(self, restricted_parent): - """Get the EP approval referee group from the restricted record.""" - default_community_id = restricted_parent.get("communities", {}).get("default") - if not default_community_id: - raise UnexpectedValue( - message="Restricted record has no default community for EP approval", - stage="load", - priority="critical", - ) - ep_config = current_app.config.get( - "CDS_COMMITTEE_APPROVAL_COMMUNITIES", {} - ).get(default_community_id) - if not ep_config: - raise UnexpectedValue( - message=( - f"Community {default_community_id} is not enrolled in " - "CDS_COMMITTEE_APPROVAL_COMMUNITIES" - ), - stage="load", - priority="critical", - ) - return ep_config["referee_group"] - - @staticmethod - def _resolve_user_by_email(email, role): - """Resolve the user by email.""" - if not email: - raise UnexpectedValue( - message=f"EP approval {role} email is missing", - stage="load", - priority="critical", - ) - user = User.query.filter_by(email=email).one_or_none() - if not user: - raise UnexpectedValue( - message=f"EP approval {role} user not found: {email}", - stage="load", - priority="critical", - ) - return {"user": str(user.id)} - - def _existing_request(self): - """Check if the EP approval request already exists.""" - number = f"lrecid:{self.legacy_recid}:ep-approval" - results = current_requests_service.search( - system_identity, - params={"q": f'number:"{number}"', "size": 1}, - ) - hits = list(results.hits) - return hits[0] if hits else None - - @staticmethod - def _exists_apprn_pid(report_number): - """Check if the APPRN PID already exists.""" - existing = PersistentIdentifier.query.filter_by( - pid_type=APPRN_PID_TYPE, - pid_value=report_number, - ).one_or_none() - return bool(existing) - - def _mint_apprn_pid(self, restricted_version_uuid): - """Mint the APPRN PID.""" - try: - PersistentIdentifier.create( - pid_type=APPRN_PID_TYPE, - pid_value=self.report_number, - object_type="rec", - object_uuid=str(restricted_version_uuid), - status=PIDStatus.REGISTERED, - ) - except PIDAlreadyExists: - raise ManualImportRequired( - message=f"APPRN PID {self.report_number} already exists", - stage="load", - priority="critical", - ) - - def _create_accept_log_event(self, request, uow): - """Create the accept timeline event with the legacy approver as created_by.""" - approver_ref = self._resolve_user_by_email( - self.approved_entry.get("submitted_by"), - "approver", - ) - - event = current_events_service.record_cls.create( - {}, - request=request.model, - request_id=str(request.id), - type=LogEventType, - ) - event.update({"payload": {"event": "accepted"}}) - event.created_by = ResolverRegistry.resolve_entity_proxy( - approver_ref, raise_=True - ) - - approved_at = self.parse_legacy_datetime(self.approved_entry.get("date")) - if approved_at: - event.model.created = approved_at - - uow.register(RecordCommitOp(event, indexer=current_events_service.indexer)) - - - def _create_reviewing_log_event(self, request, uow): - """Create the reviewers-updated timeline event with the legacy reviewer as created_by.""" - if not self.reviewing_entry: - return - - reviewer_ref = self._resolve_user_by_email( - self.reviewing_entry.get("submitted_by"), - "reviewer", - ) - request.reviewers = [reviewer_ref] - - event = current_events_service.record_cls.create( - {}, - request=request.model, - request_id=str(request.id), - type=ReviewersUpdatedType, - ) - event.update( - { - "payload": { - "event": "reviewers_updated", - "content": self.reviewing_entry.get("description", ""), - "reviewers": [reviewer_ref], - } - } - ) - event.created_by = ResolverRegistry.resolve_entity_proxy( - reviewer_ref, raise_=True - ) - - reviewing_at = self.parse_legacy_datetime(self.reviewing_entry.get("date")) - if reviewing_at: - event.model.created = reviewing_at - - uow.register(RecordCommitOp(event, indexer=current_events_service.indexer)) - - def _apply_approved_entry(self, request, uow): - """Update an existing request to accepted using the legacy approved entry.""" - payload = dict(request.get("payload") or {}) - payload["approved_report_number"] = self.report_number - request["payload"] = payload - request.status = "accepted" - - approved_at = self.parse_legacy_datetime(self.approved_entry.get("date")) - if approved_at: - request.model.updated = approved_at - - self._create_accept_log_event(request, uow) - - def _create_request(self, restricted_recid, restricted_parent, uow=None): - """Create request from waiting entry, then update it with approved entry. - - If ``uow`` is provided, it is used as-is and left uncommitted for the - caller to commit; otherwise a unit of work is created and committed - here. - """ - if uow is not None: - self._build_request(restricted_recid, restricted_parent, uow) - return - - with UnitOfWork() as inner_uow: - self._build_request(restricted_recid, restricted_parent, inner_uow) - inner_uow.commit() - - def _build_request(self, restricted_recid, restricted_parent, uow): - """Register the request creation and its updates on the given uow.""" - expires_at = self.parse_legacy_datetime(self.waiting_entry.get("deadline")) - referee_group = self._get_referee_group(restricted_parent) - - request_item = current_requests_service.create( - system_identity, - data={ - "title": f'EP approval for "{self.title}"', - "payload": {}, - }, - request_type=CommitteeApprovalRequest, - receiver={"group": referee_group}, - creator=self._resolve_user_by_email( - self.waiting_entry.get("submitted_by"), "submitter" - ), - topic={"record": restricted_recid}, - expires_at=expires_at, - uow=uow, - ) - request = request_item._record - request.number = f"lrecid:{self.legacy_recid}:ep-approval" - request.status = "submitted" - - submitted_at = self.parse_legacy_datetime(self.waiting_entry.get("date")) - if submitted_at: - request.model.created = submitted_at - - self._create_reviewing_log_event(request, uow) - self._apply_approved_entry(request, uow) - - uow.register( - RecordCommitOp(request, indexer=current_requests_service.indexer) - ) diff --git a/cds_migrator_kit/rdm/records/load/entities/__init__.py b/cds_migrator_kit/rdm/records/load/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cds_migrator_kit/rdm/records/load/entities/approval_request.py b/cds_migrator_kit/rdm/records/load/entities/approval_request.py new file mode 100644 index 00000000..1ccad64c --- /dev/null +++ b/cds_migrator_kit/rdm/records/load/entities/approval_request.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Validates an EP committee approval request's legacy history.""" +from datetime import datetime, timezone + +from cds_rdm.requests.committee_approval import APPRN_PID_TYPE +from invenio_access.permissions import system_identity +from invenio_accounts.models import User +from invenio_pidstore.models import PersistentIdentifier +from invenio_requests.proxies import current_requests_service + +from cds_migrator_kit.errors import ManualImportRequired, UnexpectedValue + +EP_APPROVAL_WAITING_STATUS = "waiting" +EP_APPROVAL_APPROVED_STATUS = "approved" +EP_APPROVAL_REVIEWING_STATUS = "reviewing" + + +class ApprovalRequest: + """Validates an EP committee approval request's legacy history. + + The "build" counterpart of the EP-approval split - parses and validates + the ``ep_approval`` MARC history (waiting/reviewing/approved entries) + and checks for a pre-existing request/PID, computing everything + ``ApprovalRequestLoad`` needs before it creates anything. Mirrors + ``RecordParent``/``RecordRequest`` on the transform side: computation + and read-only DB lookups only, no writes - see ``ApprovalRequestLoad`` + for the persistence half. + """ + + def __init__( + self, + ep_approval, + legacy_recid, + title=None, + resource_type=None, + dry_run=False, + ): + self.ep_approval = ep_approval + self.legacy_recid = legacy_recid + self.title = title + self.resource_type = resource_type + self.dry_run = dry_run + self.waiting_entry = None + self.reviewing_entry = None + self.approved_entry = None + self.report_number = None + self.approved_at = None + + def validate(self): + """Validate EP approval data before creating any records.""" + waiting_entry, approved_entry, reviewing, report_number = self._parse_history() + + existing = self._existing_request() + if existing: + raise ManualImportRequired( + message=f"EP approval request {existing['id']} already exists", + stage="load", + priority="critical", + ) + if self._exists_apprn_pid(report_number): + raise ManualImportRequired( + message=f"APPRN PID {report_number} already exists", + stage="load", + priority="critical", + ) + + self.waiting_entry = waiting_entry + self.approved_entry = approved_entry + self.reviewing_entry = reviewing + self.report_number = report_number + + def _parse_history(self): + """Return waiting/approved history entries and the report number.""" + if len(self.ep_approval) > 3: + raise UnexpectedValue( + message="EP approval history has more/less than 3 entries", + stage="load", + priority="critical", + ) + history = self.ep_approval or [] + waiting = next( + ( + item + for item in history + if item.get("status") == EP_APPROVAL_WAITING_STATUS + ), + None, + ) + reviewing = next( + ( + item + for item in history + if item.get("status") == EP_APPROVAL_REVIEWING_STATUS + ), + None, + ) + approved = next( + ( + item + for item in history + if item.get("status") == EP_APPROVAL_APPROVED_STATUS + ), + None, + ) + if not waiting: + raise UnexpectedValue( + message="EP approval history has no waiting entry", + stage="load", + priority="critical", + ) + if not approved: + raise UnexpectedValue( + message="EP approval history has no approved entry", + stage="load", + priority="critical", + ) + + report_number = approved.get("ep_report_number") + if not report_number: + raise UnexpectedValue( + message="EP approval approved entry is missing ep_report_number", + stage="load", + priority="critical", + ) + if waiting.get("ep_report_number") != report_number: + raise UnexpectedValue( + message=( + "EP approval waiting entry has different ep_report_number " + "than approved entry" + ), + stage="load", + priority="critical", + ) + + self.resolve_user_by_email(waiting.get("submitted_by"), "submitter") + self.resolve_user_by_email(approved.get("submitted_by"), "approver") + + waiting_deadline = self.parse_legacy_datetime(waiting.get("deadline")) + approved_date = self.parse_legacy_datetime(approved.get("date")) + self.approved_at = approved_date + created_at = self.parse_legacy_datetime(waiting.get("date")) + if not created_at or not approved_date or not waiting_deadline: + raise UnexpectedValue( + message="EP approval history has missing timestamps", + stage="load", + priority="critical", + ) + + return waiting, approved, reviewing, report_number + + @staticmethod + def parse_legacy_datetime(value): + """Parse legacy EP approval timestamps into timezone-aware datetimes.""" + if not value: + return None + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): + try: + return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc) + except ValueError: + continue + return None + + @staticmethod + def resolve_user_by_email(email, role): + """Resolve the user by email. + + Public (not ``_``-prefixed) since ``ApprovalRequestLoad`` also calls + it, the way ``RecordParent.resolve_grants()`` is called from + ``ParentLoad``. + """ + if not email: + raise UnexpectedValue( + message=f"EP approval {role} email is missing", + stage="load", + priority="critical", + ) + user = User.query.filter_by(email=email).one_or_none() + if not user: + raise UnexpectedValue( + message=f"EP approval {role} user not found: {email}", + stage="load", + priority="critical", + ) + return {"user": str(user.id)} + + def _existing_request(self): + """Check if the EP approval request already exists.""" + number = f"lrecid:{self.legacy_recid}:ep-approval" + results = current_requests_service.search( + system_identity, + params={"q": f'number:"{number}"', "size": 1}, + ) + hits = list(results.hits) + return hits[0] if hits else None + + @staticmethod + def _exists_apprn_pid(report_number): + """Check if the APPRN PID already exists.""" + existing = PersistentIdentifier.query.filter_by( + pid_type=APPRN_PID_TYPE, + pid_value=report_number, + ).one_or_none() + return bool(existing) diff --git a/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py b/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py new file mode 100644 index 00000000..eec3e7d5 --- /dev/null +++ b/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Creates and approves a migrated EP committee approval request.""" +from cds_rdm.requests.committee_approval import APPRN_PID_TYPE, CommitteeApprovalRequest +from flask import current_app +from invenio_access.permissions import system_identity +from invenio_db.uow import UnitOfWork +from invenio_pidstore.errors import PIDAlreadyExists +from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from invenio_rdm_records.records.api import RDMParent +from invenio_records_resources.services.uow import RecordCommitOp +from invenio_requests.customizations.event_types import ( + LogEventType, + ReviewersUpdatedType, +) +from invenio_requests.proxies import current_events_service, current_requests_service +from invenio_requests.resolvers.registry import ResolverRegistry + +from cds_migrator_kit.errors import ManualImportRequired, UnexpectedValue + +from .approval_request import ApprovalRequest + + +class ApprovalRequestLoad: + """Creates and approves a migrated EP committee approval request. + + The "persist" counterpart of ``ApprovalRequest`` - takes the already- + validated entity and drives the request-service calls, committee- + approval PID minting, and timeline events that materialize it. Mirrors + ``RequestLoad``. + """ + + def __init__(self, approval_request: ApprovalRequest): + """Constructor. + + :param approval_request: the ``ApprovalRequest`` for this entry, + already ``.validate()``-d. + """ + self.approval_request = approval_request + + def create(self, restricted_record_state, uow=None): + """Create and approve EP approval request after restricted record exists. + + If ``uow`` is provided, the request is registered on it without + committing, so the caller can group this atomically with other + operations (e.g. the public record creation and linking). + """ + approval_request = self.approval_request + if approval_request.dry_run: + return + + if not restricted_record_state: + raise UnexpectedValue( + message="Restricted record is required for EP approval.", + stage="load", + recid=approval_request.legacy_recid, + priority="critical", + ) + + restricted_recid = restricted_record_state["latest_version"] + restricted_parent = RDMParent.get_record( + restricted_record_state["parent_object_uuid"] + ) + self._create_request( + restricted_recid, + restricted_parent, + uow=uow, + ) + self._mint_apprn_pid(restricted_record_state["latest_version_object_uuid"]) + + def _get_referee_group(self, restricted_parent): + """Get the EP approval referee group from the restricted record.""" + default_community_id = restricted_parent.get("communities", {}).get("default") + if not default_community_id: + raise UnexpectedValue( + message="Restricted record has no default community for EP approval", + stage="load", + priority="critical", + ) + ep_config = current_app.config.get( + "CDS_COMMITTEE_APPROVAL_COMMUNITIES", {} + ).get(default_community_id) + if not ep_config: + raise UnexpectedValue( + message=( + f"Community {default_community_id} is not enrolled in " + "CDS_COMMITTEE_APPROVAL_COMMUNITIES" + ), + stage="load", + priority="critical", + ) + return ep_config["referee_group"] + + def _mint_apprn_pid(self, restricted_version_uuid): + """Mint the APPRN PID.""" + report_number = self.approval_request.report_number + try: + PersistentIdentifier.create( + pid_type=APPRN_PID_TYPE, + pid_value=report_number, + object_type="rec", + object_uuid=str(restricted_version_uuid), + status=PIDStatus.REGISTERED, + ) + except PIDAlreadyExists: + raise ManualImportRequired( + message=f"APPRN PID {report_number} already exists", + stage="load", + priority="critical", + ) + + def _create_accept_log_event(self, request, uow): + """Create the accept timeline event with the legacy approver as created_by.""" + approval_request = self.approval_request + approver_ref = approval_request.resolve_user_by_email( + approval_request.approved_entry.get("submitted_by"), + "approver", + ) + + event = current_events_service.record_cls.create( + {}, + request=request.model, + request_id=str(request.id), + type=LogEventType, + ) + event.update({"payload": {"event": "accepted"}}) + event.created_by = ResolverRegistry.resolve_entity_proxy( + approver_ref, raise_=True + ) + + approved_at = approval_request.parse_legacy_datetime( + approval_request.approved_entry.get("date") + ) + if approved_at: + event.model.created = approved_at + + uow.register(RecordCommitOp(event, indexer=current_events_service.indexer)) + + def _create_reviewing_log_event(self, request, uow): + """Create the reviewers-updated timeline event with the legacy reviewer as created_by.""" + approval_request = self.approval_request + if not approval_request.reviewing_entry: + return + + reviewer_ref = approval_request.resolve_user_by_email( + approval_request.reviewing_entry.get("submitted_by"), + "reviewer", + ) + request.reviewers = [reviewer_ref] + + event = current_events_service.record_cls.create( + {}, + request=request.model, + request_id=str(request.id), + type=ReviewersUpdatedType, + ) + event.update( + { + "payload": { + "event": "reviewers_updated", + "content": approval_request.reviewing_entry.get("description", ""), + "reviewers": [reviewer_ref], + } + } + ) + event.created_by = ResolverRegistry.resolve_entity_proxy( + reviewer_ref, raise_=True + ) + + reviewing_at = approval_request.parse_legacy_datetime( + approval_request.reviewing_entry.get("date") + ) + if reviewing_at: + event.model.created = reviewing_at + + uow.register(RecordCommitOp(event, indexer=current_events_service.indexer)) + + def _apply_approved_entry(self, request, uow): + """Update an existing request to accepted using the legacy approved entry.""" + approval_request = self.approval_request + payload = dict(request.get("payload") or {}) + payload["approved_report_number"] = approval_request.report_number + request["payload"] = payload + request.status = "accepted" + + approved_at = approval_request.parse_legacy_datetime( + approval_request.approved_entry.get("date") + ) + if approved_at: + request.model.updated = approved_at + + self._create_accept_log_event(request, uow) + + def _create_request(self, restricted_recid, restricted_parent, uow=None): + """Create request from waiting entry, then update it with approved entry. + + If ``uow`` is provided, it is used as-is and left uncommitted for the + caller to commit; otherwise a unit of work is created and committed + here. + """ + if uow is not None: + self._build_request(restricted_recid, restricted_parent, uow) + return + + with UnitOfWork() as inner_uow: + self._build_request(restricted_recid, restricted_parent, inner_uow) + inner_uow.commit() + + def _build_request(self, restricted_recid, restricted_parent, uow): + """Register the request creation and its updates on the given uow.""" + approval_request = self.approval_request + expires_at = approval_request.parse_legacy_datetime( + approval_request.waiting_entry.get("deadline") + ) + referee_group = self._get_referee_group(restricted_parent) + + request_item = current_requests_service.create( + system_identity, + data={ + "title": f'EP approval for "{approval_request.title}"', + "payload": {}, + }, + request_type=CommitteeApprovalRequest, + receiver={"group": referee_group}, + creator=approval_request.resolve_user_by_email( + approval_request.waiting_entry.get("submitted_by"), "submitter" + ), + topic={"record": restricted_recid}, + expires_at=expires_at, + uow=uow, + ) + request = request_item._record + request.number = f"lrecid:{approval_request.legacy_recid}:ep-approval" + request.status = "submitted" + + submitted_at = approval_request.parse_legacy_datetime( + approval_request.waiting_entry.get("date") + ) + if submitted_at: + request.model.created = submitted_at + + self._create_reviewing_log_event(request, uow) + self._apply_approved_entry(request, uow) + + uow.register( + RecordCommitOp(request, indexer=current_requests_service.indexer) + ) diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_load.py b/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py similarity index 60% rename from cds_migrator_kit/rdm/records/load/ep_approval_load.py rename to cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py index 4178222d..fca8c0d8 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_load.py +++ b/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py @@ -5,99 +5,88 @@ # CDS-RDM is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -"""CDS-RDM migration load module for records with EP approval.""" -import json - -from cds_rdm.legacy.resolver import get_pid_by_legacy_recid -from cds_rdm.minters import legacy_recid_minter +"""Splits and loads an EP approval record as a public/restricted pair.""" from invenio_access.permissions import system_identity from invenio_db import db from invenio_db.uow import UnitOfWork -from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp -from invenio_pidstore.models import PersistentIdentifier -from invenio_rdm_migrator.load.base import Load from invenio_rdm_records.proxies import current_rdm_records_service from invenio_rdm_records.records.api import RDMParent from cds_migrator_kit.errors import ManualImportRequired, UnexpectedValue +from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry +from ..load import CDSMigrationEntryLoad from .approval_request import ApprovalRequest -from .ep_approval_entry import PublicEntry, RestrictedEntry -from .load import CDSRecordServiceLoad +from .approval_request_load import ApprovalRequestLoad +from .ep_split import PublicEntry, RestrictedEntry +from .parent import ParentLoad + + +class EPMigrationEntryLoad(CDSMigrationEntryLoad): + """Splits and loads an EP approval record as a public/restricted pair. + + Adds EP-approval splitting on top of the plain per-entry loader: builds + and validates the ``ApprovalRequest``, splits the entry into + ``PublicEntry``/``RestrictedEntry``, creates both records directly via + ``RecordLoad`` (not by delegating to ``super()._load()``, since the two + halves need different treatment - see ``_load_split()``), creates/ + approves the EP approval request, links the two records with + related_identifiers, and writes EP approval metadata onto both parents - + all inside a single unit of work so a partial failure rolls back + everything instead of leaving an orphaned restricted record and/or + approval request behind. + + Registered as the stream's ``load_cls`` in place of + ``CDSMigrationEntryLoad`` - every entry goes through this class, and + ``_load()`` decides per-entry whether to split or fall back to the + inherited plain behavior. + """ + def _load(self, entry: MigrationEntry): + """Route to the plain per-entry loader, or split, based on the entry. -class CDSEPApprovalRecordServiceLoad(Load): - """Load records with EP approval. + Overrides ``CDSMigrationEntryLoad._load()`` to add the ep_approval + check *before* delegating - the base class has no knowledge of + splitting at all. + """ + if not entry: + return - Splits a legacy record into two RDM records before load: - - a public record with non-EPPHAPP files - - a restricted record with restricted EPPHAPP files - """ + recid = entry["record"].recid + if self._should_skip_recid(recid): + return + + if not entry.get("ep_approval"): + return super()._load(entry) + + return self._load_split(entry, recid) + + def _load_split(self, entry: MigrationEntry, recid): + """Load a record with EP approval by splitting it into two records. - def __init__( - self, - db_uri, - data_dir, - entries=None, - dry_run=False, - legacy_pids_to_redirect=None, - collection=None, - update_new_version_publication_date=False, - create_inclusion_request=False, - migration_logger=None, - record_state_logger=None, - ): - self.dry_run = dry_run - self.legacy_pids_to_redirect = {} - self.clc_sync = False - self.collection = collection - self.update_new_version_publication_date = update_new_version_publication_date - self.create_inclusion_request = create_inclusion_request - self.migration_logger = migration_logger - self.record_state_logger = record_state_logger - self.approval_request = None - if legacy_pids_to_redirect is not None: - with open(legacy_pids_to_redirect, "r") as fp: - self.legacy_pids_to_redirect = json.load(fp) - - def _load(self, entry): - """ - Load the record with EP approval. Configure the 2 records by separating the files, then: 1. create the restricted record 2. create and approve the EP approval request 3. create the public record and link both with related_identifiers + 4. write EP approval metadata on both parents Steps 1-4 are grouped into a single unit of work so a failure at any point rolls back everything, instead of leaving an orphaned - restricted record and/or approval request committed behind. + restricted record and/or approval request committed behind. This + path has its own uow/error handling (rather than sharing the plain + single-record path's) since a partial failure here is more severe - + it can leave a public/restricted pair half-created - so unexpected + errors are logged as "critical" here, not "warning". + + Parent access grants and the community-inclusion request are + applied to the *restricted* half - it's the one carrying the actual + restricted files and (per ``RestrictedEntry``) any ``_request_data`` + (``PublicEntry`` strips it). Original-dump persistence and CLC sync + are applied to the *public* half - the discoverable/"final" record + for this legacy recid. """ - if not entry: - return try: - recid = entry["record"].recid - - # The same legacy recid can be cross-listed under multiple EP - # collections (e.g. a joint ALEPH/DELPHI/L3/OPAL paper appears in - # all four experiments' dumps). Once one pass has fully migrated - # it, later passes should skip cleanly instead of failing on the - # already-created approval request. - if CDSRecordServiceLoad._have_migrated_recid(recid): - self.migration_logger.add_information( - recid, - state={"message": "Record already migrated", "value": recid}, - ) - self.migration_logger.finalise_record(recid) - return - ep_approval = entry.get("ep_approval") - if not ep_approval: - raise UnexpectedValue( - message="EP approval request not found", - stage="load", - recid=recid, - priority="critical", - ) record_body = entry["record"].body metadata = record_body.get("metadata", {}) @@ -109,6 +98,7 @@ def _load(self, entry): dry_run=self.dry_run, ) self.approval_request.validate() + approval_request_load = ApprovalRequestLoad(self.approval_request) # Split the metadata and files public_entry = PublicEntry( @@ -122,48 +112,60 @@ def _load(self, entry): migration_logger=self.migration_logger, ).build() - restricted_record_service = CDSRecordServiceLoad( - dry_run=self.dry_run, - collection=self.collection, - create_inclusion_request=self.create_inclusion_request, - migration_logger=self.migration_logger, + restricted_record_load = self.record_load_cls( + restricted_entry["record"], + restricted_entry["parent"], + self.migration_logger, + is_final_record=False, + update_new_version_publication_date=self.update_new_version_publication_date, record_state_logger=self.record_state_logger, - legacy_pids_to_redirect=self.legacy_pids_to_redirect, - _is_final_record=False, ) - public_record_service = CDSRecordServiceLoad( - dry_run=self.dry_run, - collection=self.collection, - create_inclusion_request=self.create_inclusion_request, - migration_logger=self.migration_logger, + public_record_load = self.record_load_cls( + public_entry["record"], + public_entry["parent"], + self.migration_logger, + is_final_record=True, + update_new_version_publication_date=self.update_new_version_publication_date, record_state_logger=self.record_state_logger, - legacy_pids_to_redirect=self.legacy_pids_to_redirect, - _is_final_record=True, ) if self.dry_run: # 1. Create restricted record - restricted_record_state = restricted_record_service._load( - restricted_entry + restricted_records = restricted_record_load.load(restricted_entry) + restricted_record_state = restricted_record_load.build_record_state( + recid, restricted_records ) # 2. Create and approve EP approval request - self.approval_request.create(restricted_record_state) + approval_request_load.create(restricted_record_state) # 3. Create public record - public_record_service._load(public_entry) + public_record_load.load(public_entry) return with UnitOfWork(db.session) as uow: # 1. Create restricted record - restricted_record_state = restricted_record_service._load( + restricted_records = restricted_record_load.load( restricted_entry, uow=uow ) + restricted_record_state = restricted_record_load.build_record_state( + recid, restricted_records + ) # 2. Create and approve EP approval request - self.approval_request.create(restricted_record_state, uow=uow) + approval_request_load.create(restricted_record_state, uow=uow) + + # Parent access grants + community-inclusion request for + # the restricted half. + self.parent_load_cls( + restricted_entry, self.migration_logger, restricted_record_state + ).load(published_record=restricted_records[-1]) + self.request_load_cls(restricted_entry).load( + restricted_records, self.create_inclusion_request, uow + ) # 3. Create public record - public_record_state = public_record_service._load( - public_entry, uow=uow + public_records = public_record_load.load(public_entry, uow=uow) + public_record_state = public_record_load.build_record_state( + recid, public_records ) if not public_record_state: raise UnexpectedValue( @@ -173,6 +175,9 @@ def _load(self, entry): priority="critical", ) + # Original-dump persistence for the public half. + self._save_original_dumped_record(public_entry, public_record_state) + # Link the records with related_identifiers self._append_related_identifier( public_record_state["latest_version"], @@ -203,10 +208,18 @@ def _load(self, entry): uow.commit() + # Only log to disk once the unit of work has actually + # committed - logging any earlier risks recording a record + # that a later failure in the same uow rolls back. + public_record_load.log_record_state(public_record_state) + # The public record is the final one; finalise it only now # that the whole split has actually committed (see the - # matching `uow is None` guard in CDSRecordServiceLoad._load). + # matching `uow is None` guard in the plain path). self.migration_logger.finalise_record(recid) + + # CLC sync for the public half, run after commit. + self._apply_clc_sync(public_record_state, public_entry) except (UnexpectedValue, ManualImportRequired) as e: self.migration_logger.add_log(e, record=entry) except Exception as e: @@ -266,7 +279,7 @@ def _write_parent_ep_approvals( public_record_state["parent_object_uuid"] ) - self._write_parent_ep_approval( + ParentLoad.write_committee_approval( restricted_parent, { "reportnumber": report_number, @@ -277,7 +290,7 @@ def _write_parent_ep_approvals( }, uow, ) - self._write_parent_ep_approval( + ParentLoad.write_committee_approval( public_parent, { "reportnumber": report_number, @@ -286,18 +299,6 @@ def _write_parent_ep_approvals( uow, ) - def _write_parent_ep_approval( - self, - parent, - ep_approval, - uow, - ): - """Write the EP approval metadata to the parent record.""" - pf = parent.get("permission_flags") or {} - pf["committee_approval"] = ep_approval - parent["permission_flags"] = pf - uow.register(ParentRecordCommitOp(parent)) - def _append_related_identifier( self, record_id, target_id, relation_id, resource_type, uow=None ): @@ -327,21 +328,3 @@ def _append_related_identifier( ) current_rdm_records_service.publish(system_identity, id_=draft.id, uow=uow) return True - - def _cleanup(self, *args, **kwargs): - """Post migration process.""" - for legacy_src_pid, legacy_dest_pid in self.legacy_pids_to_redirect.items(): - if CDSRecordServiceLoad._have_migrated_recid(legacy_src_pid): - continue - try: - parent_dest_pid = get_pid_by_legacy_recid(str(legacy_dest_pid)) - assert str(parent_dest_pid.status) == "R" - legacy_recid_minter(legacy_src_pid, parent_dest_pid.object_uuid) - db.session.commit() - self.migration_logger.finalise_record(legacy_src_pid) - except Exception as exc: - db.session.rollback() - self.migration_logger.add_log( - f"Failed to redirect {legacy_src_pid} to {legacy_dest_pid}: {str(exc)}", - record={"recid": legacy_src_pid}, - ) diff --git a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py b/cds_migrator_kit/rdm/records/load/entities/ep_split.py similarity index 78% rename from cds_migrator_kit/rdm/records/load/ep_approval_entry.py rename to cds_migrator_kit/rdm/records/load/entities/ep_split.py index c965006d..f666fa66 100644 --- a/cds_migrator_kit/rdm/records/load/ep_approval_entry.py +++ b/cds_migrator_kit/rdm/records/load/entities/ep_split.py @@ -94,43 +94,41 @@ def _remove_doi_pid(self, split): """Remove DOI PID from record.""" pass - def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: - """Return versioned files for this split; override in subclasses.""" + # Set by subclasses - written onto ``access_obj["record"]``/``["files"]`` + # for every version, and reused as the ``split_type`` label passed to + # ``_log_removed_identifiers()``. + _access_status = None + # Whether to drop the version's file-restriction ``meta`` string - kept + # for the restricted split (``ParentLoad.load_access_grants()`` reads it + # to resolve access grants), stripped for the public one (never + # restricted, so there's nothing to resolve grants from). + _strip_access_meta = False + + def _include_file(self, file_data, context): + """Return whether a file belongs to this split; override in subclasses.""" raise NotImplementedError - @staticmethod - def _version_signature(versioned_files): - return tuple( - sorted( - ( - key, - file_data.get("checksum"), - file_data.get("id_bibdoc"), - file_data.get("version"), - file_data.get("type"), - file_data.get("access"), - ) - for key, file_data in versioned_files.items() - ) - ) - + def _version_build_context(self, split): + """Hook for subclass precomputation before filtering files; default no-op.""" + return None -class PublicEntry(MetadataEntry): - """Build the public EP approval split entry.""" + def _no_versions_error_message(self): + """Error message when this split ends up with no versions; override in subclasses.""" + raise NotImplementedError def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: + """Return versioned files for this split, filtered/tagged per subclass.""" new_versions = OrderedDict() versioned_files = OrderedDict() previous_signature = None + context = self._version_build_context(split) for _, version_data in split.get("versions", {}).items(): - current_version_files = OrderedDict() - - for key, file_data in version_data.get("files", {}).items(): - if self._is_restricted_file(file_data): - continue - - current_version_files[key] = deepcopy(file_data) + current_version_files = OrderedDict( + (key, deepcopy(file_data)) + for key, file_data in version_data.get("files", {}).items() + if self._include_file(file_data, context) + ) if not current_version_files: continue @@ -146,9 +144,10 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: version_access = deepcopy(version_data.get("access", {})) access_obj = deepcopy(version_access.get("access_obj", {})) - access_obj["record"] = "public" - access_obj["files"] = "public" - version_access.pop("meta", None) + access_obj["record"] = self._access_status + access_obj["files"] = self._access_status + if self._strip_access_meta: + version_access.pop("meta", None) version_access["access_obj"] = access_obj new_version_data = deepcopy(version_data) @@ -159,7 +158,7 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: if not new_versions: raise UnexpectedValue( - message="No public files found to load for EP approval public split", + message=self._no_versions_error_message(), stage="load", recid=split["record"].recid, priority="critical", @@ -167,6 +166,35 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: return new_versions + @staticmethod + def _version_signature(versioned_files): + return tuple( + sorted( + ( + key, + file_data.get("checksum"), + file_data.get("id_bibdoc"), + file_data.get("version"), + file_data.get("type"), + file_data.get("access"), + ) + for key, file_data in versioned_files.items() + ) + ) + + +class PublicEntry(MetadataEntry): + """Build the public EP approval split entry.""" + + _access_status = "public" + _strip_access_meta = True + + def _include_file(self, file_data, context): + return not self._is_restricted_file(file_data) + + def _no_versions_error_message(self): + return "No public files found to load for EP approval public split" + def identifiers(self, identifiers): kept = [] removed = [] @@ -188,7 +216,7 @@ def identifiers(self, identifiers): ) if removed: - self._log_removed_identifiers(removed, "public") + self._log_removed_identifiers(removed, self._access_status) return kept @@ -211,6 +239,8 @@ def _add_cern_scientific_community(self, entry): class RestrictedEntry(MetadataEntry): """Build the restricted EP approval split entry.""" + _access_status = "restricted" + def _apply_entry_modifications(self, split): self._remove_cern_scientific_community(split) @@ -239,12 +269,14 @@ def _has_restricted_files(self, split): for file_data in version_data.get("files", {}).values() ) - def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: - new_versions = OrderedDict() - versioned_files = OrderedDict() - previous_signature = None - has_restricted_files = self._has_restricted_files(split) + def _version_build_context(self, split): + """Return whether this record has any restricted files. + Logged as a fallback notice when it doesn't, since the restricted + split then has to fall back to using all (public) files - see + ``_include_file()``. + """ + has_restricted_files = self._has_restricted_files(split) if not has_restricted_files: self.migration_logger.add_information( split["record"].recid, @@ -256,52 +288,16 @@ def _build_versions(self, split: MigrationEntry) -> Dict[int, VersionEntry]: "value": "public files", }, ) + return has_restricted_files - for _, version_data in split.get("versions", {}).items(): - current_version_files = OrderedDict() - - for key, file_data in version_data.get("files", {}).items(): - is_restricted = self._is_restricted_file(file_data) - - # If restricted files exist, use only those; otherwise fall - # back to using all (public) files for the restricted record. - if not is_restricted and has_restricted_files: - continue - - current_version_files[key] = deepcopy(file_data) - - if not current_version_files: - continue + def _include_file(self, file_data, context): + has_restricted_files = context + # If restricted files exist, use only those; otherwise fall back to + # using all (public) files for the restricted record. + return self._is_restricted_file(file_data) or not has_restricted_files - versioned_files.update(current_version_files) - - signature = self._version_signature(versioned_files) - if signature == previous_signature: - continue - - previous_signature = signature - - version_access = deepcopy(version_data.get("access", {})) - access_obj = deepcopy(version_access.get("access_obj", {})) - access_obj["record"] = "restricted" - access_obj["files"] = "restricted" - version_access["access_obj"] = access_obj - - new_version_data = deepcopy(version_data) - new_version_data["files"] = deepcopy(versioned_files) - new_version_data["access"] = version_access - - new_versions[len(new_versions) + 1] = new_version_data - - if not new_versions: - raise UnexpectedValue( - message=("No files found to load for EP approval restricted split"), - stage="load", - recid=split["record"].recid, - priority="critical", - ) - - return new_versions + def _no_versions_error_message(self): + return "No files found to load for EP approval restricted split" def identifiers(self, identifiers): kept = [] @@ -329,7 +325,7 @@ def identifiers(self, identifiers): kept.append(id_entry) if removed: - self._log_removed_identifiers(removed, "restricted") + self._log_removed_identifiers(removed, self._access_status) return kept diff --git a/cds_migrator_kit/rdm/records/load/entities/parent.py b/cds_migrator_kit/rdm/records/load/entities/parent.py new file mode 100644 index 00000000..33a887ab --- /dev/null +++ b/cds_migrator_kit/rdm/records/load/entities/parent.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Materializes a ``RecordParent`` against a real RDM parent record.""" +from flask import current_app +from invenio_access.permissions import system_identity +from invenio_accounts.models import User +from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp +from invenio_rdm_records.proxies import current_rdm_records_service + +from cds_migrator_kit.errors import GrantCreationError, ManualImportRequired +from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry +from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent + + +class ParentLoad: + """Loads a ``RecordParent``'s access/communities/grants onto a real RDM parent. + + Takes the already-built transform-side ``RecordParent`` (access/ + communities computed, access grants resolved from legacy metadata) and + applies it to the parent of a draft/published record - the load-side + counterpart of ``RecordParent``, the way ``load.py``'s record-loading + methods are the counterpart of ``RecordEntry``. + """ + + def __init__(self, entry: MigrationEntry, migration_logger, record_state): + """Constructor. + + :param record_parent: the built ``RecordParent`` for this entry + (``MigrationEntry["parent"]``). + :param migration_logger: kept for parity with the other load-side + entity collaborators; not used directly by this class today. + """ + self.record_parent = entry["parent"] + self.migration_logger = migration_logger + self.record_state = record_state + self.migration_entry = entry + self.versions = entry["versions"] + + def load(self, published_record): + """Load access/communities, then access grants for every version. + + Called once from ``CDSMigrationEntryLoad._load()`` after the whole + entry has published successfully, with the latest published + record. Access/communities are parent-level (shared across all + versions), so applying them via the latest record still sets them + correctly for the parent as a whole. + + Order matters: ``load_access_and_communities`` replaces the whole + ``parent.access`` field wholesale from ``record_parent.body``, so it + must run *before* ``load_access_grants`` - otherwise it would + overwrite the grants just created (each committed on its own + freshly-read parent instance, which ``load_access_and_communities``'s + stale in-memory parent doesn't see). + """ + self.load_access_and_communities(published_record) + self.load_access_grants(published_record) + + def load_access_and_communities(self, draft): + """Load access rights and communities in a single parent commit.""" + parent = draft._record.parent + parent.access = self.record_parent.body["access"] + for community in self.record_parent.communities["ids"]: + parent.communities.add(community) + parent.communities.default = self.record_parent.communities["default"] + parent.commit() + + def load_access_grants(self, published_record): + """Load access grants from metadata and record grants efficiently. + + :param draft: the draft/published record whose parent grants are set. + :param recid: this record's legacy recid, for error reporting. + """ + recid = self.migration_entry["record"].recid + # `record_state["versions"]` is keyed by invenio's own sequential + # `record.versions.index` (always 1, 2, 3, ...), while `self.versions` + # (`entry["versions"]`) is keyed by the *legacy* version number, which + # can start above 1 when the legacy version 1 was hard-deleted (see + # ``RecordLoad.pre_publish``). Both lists are built by iterating the + # same records in the same order (``RecordLoad._load_versions``), so + # match them up by position rather than by (possibly mismatched) key. + legacy_versions = list(self.versions.keys()) + for legacy_version, version_state in zip( + legacy_versions, self.record_state["versions"] + ): + access_dict = self.versions[legacy_version]["access"] + published_record = current_rdm_records_service.read( + system_identity, version_state["new_recid"] + ) + + parent = published_record._record.parent + identity = system_identity + + specific_file_restrictions = access_dict.get("meta", "") + if not specific_file_restrictions and not self.record_parent.access_grants: + continue + default_permission = "view" + + groups, emails, grants_with_perms = self.record_parent.resolve_grants( + specific_file_restrictions + ) + + def _create_grant(subject_type, subject_id, permission): + grant_data = { + "grants": [ + { + "subject": {"type": subject_type, "id": str(subject_id)}, + "permission": permission, + } + ] + } + current_rdm_records_service.access.schema_grants.load( + grant_data, + context={"identity": identity}, + raise_errors=True, + ) + + grant = parent.access.grants.create( + subject_type=subject_type, + subject_id=subject_id, + permission=permission, + origin="migrated", + ) + is_local_dev = current_app.config.get("CDS_MIGRATOR_KIT_ENV") == "local" + is_valid = current_rdm_records_service.access._validate_grant_subject( + identity, grant + ) + if not is_local_dev and not is_valid: + raise ManualImportRequired( + message="Verification of access subject failed (likely not existing entry)", + field="access", + subfield="subject.id", + stage="load", + recid=recid, + priority="warning", + value=subject_id, + ) + + # Create grants for groups + for group in groups: + _create_grant( + subject_type="role", + subject_id=group.lower(), + permission=grants_with_perms.get(group, default_permission), + ) + + # Fetch existing users + existing_users = { + user.email: user.id + for user in User.query.filter(User.email.in_(emails)).all() + } + # raise error for missing user + missing_emails = emails - existing_users.keys() + if missing_emails: + raise GrantCreationError( + message=f"Users not found for emails: {', '.join(missing_emails)}", + stage="load", + recid=recid, + value=list(missing_emails), + priority="warning", + ) + + # Create grants for users + for email, user_id in existing_users.items(): + _create_grant( + subject_type="user", + subject_id=user_id, + permission=grants_with_perms.get(email, default_permission), + ) + + parent.commit() + + @staticmethod + def write_committee_approval(parent, ep_approval, uow): + """Write EP approval metadata onto an already-published parent. + + :param parent: an ``RDMParent`` fetched by uuid (post-publish - not + the parent of a draft in this uow; see the EP approval split's + restricted/public parents in ``load.py``). + :param ep_approval: the ``permission_flags.committee_approval`` dict + to write. + :param uow: registers the commit op on it without committing. + """ + pf = parent.get("permission_flags") or {} + pf["committee_approval"] = ep_approval + parent["permission_flags"] = pf + uow.register(ParentRecordCommitOp(parent)) diff --git a/cds_migrator_kit/rdm/records/load/entities/record.py b/cds_migrator_kit/rdm/records/load/entities/record.py new file mode 100644 index 00000000..fe8cb3cd --- /dev/null +++ b/cds_migrator_kit/rdm/records/load/entities/record.py @@ -0,0 +1,460 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Creates, versions, and publishes a single RDM record from a ``RecordEntry``.""" +import os +from typing import Dict + +import arrow +from cds_rdm.minters import legacy_recid_minter +from flask import current_app +from invenio_access.permissions import system_identity +from invenio_db import db +from invenio_pidstore.errors import PIDAlreadyExists +from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from invenio_rdm_records.proxies import current_rdm_records_service +from psycopg2.errors import UniqueViolation +from sqlalchemy.exc import IntegrityError + +from cds_migrator_kit.errors import ManualImportRequired +from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry +from cds_migrator_kit.rdm.records.transform.entities.version import ( + VersionAccess, + VersionFileEntry, +) + +from .parent import ParentLoad + + +def import_legacy_files(filepath): + """Download file from legacy.""" + if current_app.config["CDS_MIGRATOR_KIT_ENV"] == "local": + import cds_migrator_kit + + base_path = os.path.dirname(os.path.realpath(cds_migrator_kit.__file__)) + filepath = os.path.join(base_path, "rdm/data/files/dummy.pdf") + filestream = open(filepath, "rb") + return filestream + + +class RecordLoad: + """Creates, versions, and publishes a single RDM record from a ``RecordEntry``. + + The load-side counterpart of ``RecordEntry`` - takes the already-built + transform-side entity (plus its sibling ``RecordParent``, needed when + creating the very first version) and drives the record-service calls + that materialize/version/publish it - mirrors how ``ParentLoad`` is the + counterpart of ``RecordParent``. + """ + + def __init__( + self, + record_entry: RecordEntry, + record_parent: RecordParent, + migration_logger, + is_final_record=True, + update_new_version_publication_date=False, + record_state_logger=None, + ): + """Constructor. + + :param record_entry: the built ``RecordEntry`` for this entry + (``MigrationEntry["record"]``). + :param record_parent: the built ``RecordParent`` for this entry - + only used on the first version, to set up the new draft's + parent access/communities (see ``pre_publish()``). + :param is_final_record: whether this is the "real"/final record for + its legacy recid - False for the restricted half of an EP + approval split, which exists only as an internal source record + and must skip legacy-recid-facing side effects (rep-number PID + assignment, DOI updates, recid minting) - see + ``CDSMigrationEntryLoad._is_final_record``. + :param record_state_logger: used by ``build_record_state()`` to + dump the computed record state, for later stats migration. + """ + self.record_entry = record_entry + self.record_parent = record_parent + self.migration_logger = migration_logger + self.is_final_record = is_final_record + self.update_new_version_publication_date = update_new_version_publication_date + self.record_state_logger = record_state_logger + + def load_files( + self, + draft, + version_files: Dict[str, VersionFileEntry], + uow=None, + ): + """Load files to draft.""" + recid = self.record_entry.recid + identity = system_identity # Should we create an identity for the migration? + + for filename, file_data in version_files.items(): + + file_data = version_files[filename] + + try: + current_rdm_records_service.draft_files.init_files( + identity, + draft.id, + data=[ + { + "key": file_data["key"], + "metadata": { + **file_data["metadata"], + "legacy_file_id": file_data["id_bibdoc"], + "legacy_recid": recid, + }, + "access": {"hidden": False}, + } + ], + uow=uow, + ) + current_rdm_records_service.draft_files.set_file_content( + identity, + draft.id, + file_data["key"], + import_legacy_files(file_data["eos_tmp_path"]), + uow=uow, + ) + result = current_rdm_records_service.draft_files.commit_file( + identity, draft.id, file_data["key"], uow=uow + ) + legacy_checksum = f"md5:{file_data['checksum']}" + new_checksum = result.to_dict()["checksum"] + if current_app.config["CDS_MIGRATOR_KIT_ENV"] != "local": + try: + assert legacy_checksum == new_checksum + except AssertionError: + raise ManualImportRequired( + message=f"Files checksum failed legacy:{legacy_checksum} calculated new: {new_checksum}", + field="checksum", + stage="load", + recid=recid, + priority="critical", + value=file_data["key"], + subfield=None, + ) + + except Exception as e: + exc = ManualImportRequired( + recid=recid, + message=str(e), + field="filename", + value=file_data["key"], + stage="file load", + priority="critical", + ) + self.migration_logger.add_log(exc, record={"record": self.record_entry}) + raise e + + def load_access(self, draft, access_dict: VersionAccess): + """Set this version's access on the published record.""" + record = draft._record + record.access = access_dict["access_obj"] + record.commit() + + def assign_rep_numbers(self, draft): + """Mint ``cdsrn`` PIDs for this draft's report-number identifiers.""" + if not self.is_final_record: + return + draft_report_nums = {} + for index, id in enumerate(draft.data["metadata"].get("identifiers", [])): + if id["scheme"] == "cdsrn": + draft_report_nums[id["identifier"]] = index + + if not draft_report_nums: + # If no mintable identifiers, return early + return + + for report_number, index in draft_report_nums.items(): + try: + PersistentIdentifier.create( + pid_type="cdsrn", + pid_value=report_number, + object_type="rec", + object_uuid=draft._record.parent.id, + status=PIDStatus.REGISTERED, + ) + except PIDAlreadyExists as e: + pid = PersistentIdentifier.get( + pid_type="cdsrn", pid_value=report_number + ) + if pid.object_uuid != draft._record.parent.id: + # raise only if different parent uuid found, meaning they are 2 + # different records and the repnum is duplicated + raise ManualImportRequired( + f"Report number {report_number} already exists." + ) + + def pre_publish(self, identity, versions, version, draft, uow): + """Create (or version) and process a draft before publish.""" + files = versions[version]["files"] + access = versions[version]["access"] + + if version == 1 or (version > 1 and draft is None): + # when draft is None, it means the initial version one was hard deleted + # and we don't have index 1 + # we decided to skip it and act normal + try: + draft = current_rdm_records_service.create( + identity, data=self.record_entry.body, uow=uow + ) + self.assign_rep_numbers(draft) + except (UniqueViolation, IntegrityError) as e: + raise ManualImportRequired(message=str(e)) + except Exception as e: + raise ManualImportRequired(message=str(e)) + if draft.errors: + raise ManualImportRequired( + message=f"{str(draft.errors)}: {str(self.record_entry.body)}", + field="validation", + stage="load", + recid=self.record_entry.recid, + priority="warning", + value=draft._record.pid.pid_value, + subfield=None, + ) + else: + draft = current_rdm_records_service.new_version( + identity, draft["id"], uow=uow + ) + draft_dict = draft.to_dict() + if not self.update_new_version_publication_date: + publication_date = arrow.get( + self.record_entry.body["metadata"]["publication_date"] + ) + else: + publication_date = versions[version]["publication_date"] + missing_data = { + **draft_dict, + "metadata": { + # copy over the previous draft metadata + **draft_dict["metadata"], + # add missing publication date based + # on the time of creation of the new file version + "publication_date": publication_date.date().isoformat(), + }, + } + draft = current_rdm_records_service.update_draft( + identity, draft["id"], data=missing_data, uow=uow + ) + + self.load_access(draft, access) + self.load_files(draft, files, uow=uow) + + return draft + + def after_publish_update_dois(self, identity, record, uow): + """Update migrated DOIs post publish.""" + if not self.is_final_record: + return + migrated_pids = self.record_entry.body["pids"] + for pid_type, identifier in migrated_pids.items(): + if pid_type == "doi": + # If a DOI was already minted from legacy then on publish the datacite + # will return a warning that "This DOI has already been taken" + # In that case, we edit and republish to force an update of the doi with + # the new published metadata as in the new system we have more information available + _draft = current_rdm_records_service.edit( + identity, record["id"], uow=uow + ) + record = current_rdm_records_service.publish( + identity, _draft["id"], uow=uow + ) + return record + + def after_publish_update_created(self, record, version_data, version): + """Update created timestamp post publish. + + Ensures that the `created` timestamp is correctly set, preferring: + 1. The original legacy system value for the version. + 2. The record's creation date if there are no files. + 3. Today's date if the original value and file creation date is missing. + """ + creation_date = arrow.get(self.record_entry.created).datetime.replace( + tzinfo=None + ) + + if version_data.get("files") and version != 1: + # Subsequent versions should use the file creation date, instead of the record creation date, + # which is stored as the publication date in the version data + creation_date = version_data["publication_date"].datetime.replace( + tzinfo=None + ) + + record._record.model.created = creation_date + db.session.add(record._record.model) + + def after_publish_mint_recid(self, record): + """Mint legacy ids for redirections assigned to the parent.""" + if not self.is_final_record: + return + legacy_recid = self.record_entry.recid + if record._record.versions.index == 1: + # it seems more intuitive if we mint the lrecid for parent + # but then we get a double redirection + legacy_recid_minter(legacy_recid, record._record.parent.model.id) + + def after_publish_update_files_created(self, record, version_data): + """Update the created date of the files post publish.""" + # Fix the `created` timestamp forcing the one from the legacy system + # Force the created date. This can be done after publish as the service + # overrides the `created` date otherwise. + files = version_data.get("files", {}) + for _, file_data in files.items(): + file = record._record.files.entries[file_data["key"]] + file.model.created = arrow.get(file_data["creation_date"]).datetime.replace( + tzinfo=None + ) + db.session.add(file.model) + + def _after_publish(self, identity, published_record, entry, version, uow): + """Run fixes after record publish.""" + record = self.after_publish_update_dois(identity, published_record, uow) + if record: + published_record = record + version_data = entry.get("versions", {}).get(version, {}) + self.after_publish_update_created(published_record, version_data, version) + self.after_publish_mint_recid(published_record) + self.after_publish_update_files_created(published_record, version_data) + + def _load_versions(self, entry, uow): + """Create, publish, and run after-publish fixes for every version.""" + identity = system_identity + records = [] + # initial value of draft. If different file versions identified then the first + # created draft is used to populate all newer versions + draft = None + for version in entry["versions"].keys(): + # Create and prepare draft + draft = self.pre_publish(identity, entry["versions"], version, draft, uow) + + # Publish draft + published_record = current_rdm_records_service.publish( + identity, draft["id"], uow=uow + ) + # Run after publish fixes + self._after_publish(identity, published_record, entry, version, uow) + records.append(published_record) + + return records + + def load(self, entry, uow=None): + """Load all versions of the record; return the list of published records.""" + return self._load_versions(entry, uow) + + def dry_load(self): + """Validate the record body via the service schema without persisting it.""" + current_rdm_records_service.schema.load( + self.record_entry.body, + context=dict( + identity=system_identity, + ), + raise_errors=True, + ) + + def build_record_state(self, legacy_recid, records): + """Compute the record state for newly published records. + + Deliberately has no side effects - the caller must not treat this as + "the load succeeded", since parent/request loading can still fail + and roll back the surrounding unit of work. Call ``log_record_state`` + only once the whole unit of work has actually committed, otherwise + the record-state log ends up with entries for records that were + rolled back (see ``log_record_state``). + """ + if not records: + return None + return self._load_record_state(legacy_recid, records) + + def log_record_state(self, record_state_context): + """Persist the computed record state, for later stats migration. + + Must only be called after the unit of work that produced + ``record_state_context`` has committed - this writes straight to + disk, so calling it before commit (e.g. right after + ``build_record_state``) risks logging a record that later gets + rolled back by a failure elsewhere in the same load (e.g. a grant + creation failure). + """ + if self.is_final_record: + self.record_state_logger.add_record_state(record_state_context) + + def _load_record_state(self, legacy_recid, records): + """Compute state for legacy recid. + + Returns + { + "legacy_recid": "2884810", + "parent_recid": "zts3q-6ef46", + "parent_object_uuid": "435be22f-3038-49e0-9f17-9518eaac783a", + "latest_version": "1mae4-skq89" + "latest_version_object_uuid": "895be22f-3038-49e0-9f17-9518eaac783a", + "versions": [ + { + "new_recid": "1mae4-skq89", + "version": 2, + "files": [ + { + "legacy_file_id": 1568736, + "bucket_id": "155be22f-3038-49e0-9f17-9518eaac783a", + "file_key": "Summer student program report.pdf", + "file_id": "06cdb9d2-635f-4dbe-89fe-4b27afddeaa2", + "size": "1690854" + } + ] + } + ] + } + """ + + def convert_file_format(file_entries, bucket_id): + """Convert the file metadata into the required format.""" + return [ + { + "legacy_file_id": entry["metadata"]["legacy_file_id"], + "bucket_id": bucket_id, + "file_key": entry["key"], + "file_id": entry["file_id"], + "size": str(entry["size"]), + } + for entry in file_entries.values() + ] + + def extract_record_version(record): + """Extract relevant details from a single record.""" + bucket_id = str(record.files.bucket_id) + files = record.__class__.files.dump( + record, record.files, include_entries=True + ).get("entries", {}) + return { + "new_recid": record.pid.pid_value, + "version": record.versions.index, + "files": convert_file_format(files, bucket_id), + } + + recid_state = {"legacy_recid": legacy_recid, "versions": []} + parent_recid = None + + for record in records: + if parent_recid is None: + parent_id = str(record._record.parent.id) + parent_recid = record._record.parent.pid.pid_value + recid_state["parent_recid"] = parent_recid + recid_state["parent_object_uuid"] = parent_id + + recid_version = extract_record_version(record._record) + # Save the record versions for legacy recid + recid_state["versions"].append(recid_version) + + if "latest_version" not in recid_state: + rec = record._record.get_latest_by_parent(record._record.parent) + recid_state["latest_version"] = rec["id"] + recid_state["latest_version_object_uuid"] = str(rec.id) + return recid_state diff --git a/cds_migrator_kit/rdm/records/load/entities/request.py b/cds_migrator_kit/rdm/records/load/entities/request.py new file mode 100644 index 00000000..824c6d76 --- /dev/null +++ b/cds_migrator_kit/rdm/records/load/entities/request.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Creates a community-inclusion request from a ``RecordRequest``.""" +import datetime + +from invenio_access.permissions import system_identity +from invenio_rdm_records.requests import CommunitySubmission +from invenio_records_resources.services.uow import RecordCommitOp +from invenio_requests.customizations.event_types import LogEventType +from invenio_requests.proxies import current_events_service, current_requests_service +from invenio_requests.records.models import RequestMetadata + +from cds_migrator_kit.errors import ManualImportRequired +from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry +from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest + + +class RequestLoad: + """Creates a community-inclusion request from a ``RecordRequest``. + + The load-side counterpart of ``RecordRequest`` - takes the already-built + transform-side entity (reviewers resolved, ``request_data`` popped off + ``dojson_entry``) and drives the request-service calls that materialize + it once the record has been published - mirrors ``RecordLoad``/ + ``ParentLoad``. + """ + + def __init__(self, entry:MigrationEntry): + """Constructor. + + :param record_request: the built ``RecordRequest`` for this entry + (``MigrationEntry["_request_data"]``). + :param legacy_recid: this record's legacy recid, used to build the + request's idempotency ``number``. + """ + self.record_request = entry.get("_request_data") + self.legacy_recid = entry["record"].recid + + def load(self, records, create_inclusion_request, uow): + """Create the community-inclusion request against the latest published record. + + :param records: this entry's published records in version order + (``RecordLoad.load()``'s return value - service-level wrapper + objects, as returned by ``current_rdm_records_service.publish()``); + only the last (latest) one is used. + :param create_inclusion_request: whether the current load run is + configured to create community-inclusion requests. + """ + if self.record_request and create_inclusion_request: + self.create_submission_request(records[-1], uow) + elif self.record_request.data and not create_inclusion_request: + raise ManualImportRequired( + message="Detected request data, enable the requests", + field="validation", + stage="load", + recid=self.legacy_recid, + priority="warning", + subfield=None, + ) + + def create_submission_request(self, record, uow): + """Create community inclusion request after publish. + + :param record: the published record - the service-level wrapper + (e.g. the last item of ``load()``'s ``records``, as returned by + ``current_rdm_records_service.publish()``), hence + ``record._record.parent``/``record.id`` (not a bare + ``record.parent``/``record.pid``). + """ + request_data = self.record_request.data + request_number = f"lrecid:{self.legacy_recid}" + + # Defensive/idempotency guard: skip if a request for this record was + # already committed by a previous (partial) load attempt, otherwise + # re-creating it would violate the unique constraint on `number`. + if RequestMetadata.query.filter_by(number=request_number).first() is not None: + return + + status = request_data.get("status", "accepted") + reviewers = request_data.get("reviewers", []) + + created_at = datetime.datetime.fromisoformat(record["created"]) + + parent = record._record.parent + owner_id = parent.access.owned_by.owner_id + community = parent.communities.default + + creator = {"user": str(owner_id)} + receiver = {"community": str(community.id)} + + def create_event(request_model, payload, event_type, user): + """Create and register a request event.""" + event = current_events_service.record_cls.create( + {}, + request=request_model, + request_id=str(request_model.id), + type=event_type, + ) + event.update(payload=payload) + event.model.created = created_at + event.created_by = {"user": str(user)} + + uow.register(RecordCommitOp(event, indexer=current_events_service.indexer)) + + request_item = current_requests_service.create( + system_identity, + data={"title": record["metadata"]["title"]}, + request_type=CommunitySubmission, + receiver=receiver, + creator=creator, + topic={"record": record.id}, + uow=uow, + ) + + request = request_item._record + request.status = "submitted" + request.number = request_number + request.model.created = created_at + + if reviewers: + request.reviewers = reviewers + + if status: + request.status = status + if status == "accepted": + parent_to_request_relation = ( + parent.communities._m2m_model_cls.query.filter_by( + record_id=parent.id, community_id=community.id + ).one() + ) + parent_to_request_relation.request_id = request.id + + create_event( + request.model, + {"event": status}, + event_type=LogEventType, + user="system", + ) + + uow.register(RecordCommitOp(request, indexer=current_requests_service.indexer)) diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 1721d20a..160e7bfc 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -7,40 +7,19 @@ """CDS-RDM migration load module.""" -import datetime import json -import os -from copy import deepcopy -from typing import Dict -import arrow from cds_rdm.clc_sync.models import CDSToCLCSyncModel -from cds_rdm.clc_sync.proxies import current_clc_sync_service from cds_rdm.legacy.models import CDSMigrationLegacyRecord from cds_rdm.legacy.resolver import get_pid_by_legacy_recid from cds_rdm.minters import legacy_recid_minter -from flask import current_app -from invenio_access.permissions import system_identity -from invenio_accounts.models import User from invenio_db import db from invenio_db.uow import UnitOfWork from invenio_i18n import _ -from invenio_pidstore.errors import PIDAlreadyExists -from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from invenio_pidstore.models import PersistentIdentifier from invenio_rdm_migrator.load.base import Load -from invenio_rdm_records.proxies import current_rdm_records_service -from invenio_rdm_records.requests import CommunitySubmission from invenio_records.systemfields.relations import InvalidRelationValue -from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp -from invenio_records_resources.services.uow import RecordCommitOp -from invenio_requests.customizations.event_types import ( - LogEventType, -) -from invenio_requests.proxies import current_events_service, current_requests_service -from invenio_requests.records.models import RequestMetadata from marshmallow import ValidationError -from psycopg2.errors import UniqueViolation -from sqlalchemy.exc import IntegrityError from cds_migrator_kit.errors import ( CDSMigrationException, @@ -50,25 +29,14 @@ UnexpectedValue, ) from cds_migrator_kit.rdm.records.transform.entities.migration import MigrationEntry -from cds_migrator_kit.rdm.records.transform.entities.version import ( - VersionAccess, - VersionFileEntry, -) - -def import_legacy_files(filepath): - """Download file from legacy.""" - if current_app.config["CDS_MIGRATOR_KIT_ENV"] == "local": - import cds_migrator_kit +from .entities.parent import ParentLoad +from .entities.record import RecordLoad +from .entities.request import RequestLoad - base_path = os.path.dirname(os.path.realpath(cds_migrator_kit.__file__)) - filepath = os.path.join(base_path, "rdm/data/files/dummy.pdf") - filestream = open(filepath, "rb") - return filestream - -class CDSRecordServiceLoad(Load): - """CDSRecordServiceLoad.""" +class CDSMigrationEntryLoad(Load): + """Loads a plain (non-EP-approval) ``MigrationEntry`` end to end.""" def __init__( self, @@ -77,671 +45,40 @@ def __init__( entries=None, dry_run=False, legacy_pids_to_redirect=None, - collection=None, update_new_version_publication_date=False, create_inclusion_request=False, migration_logger=None, record_state_logger=None, - _is_final_record=True, ): """Constructor.""" self.dry_run = dry_run self.legacy_pids_to_redirect = {} - self.clc_sync = False - self.collection = collection self.update_new_version_publication_date = update_new_version_publication_date self.create_inclusion_request = create_inclusion_request self.migration_logger = migration_logger self.record_state_logger = record_state_logger - self._is_final_record = _is_final_record - if legacy_pids_to_redirect is not None: - if isinstance(legacy_pids_to_redirect, dict): - self.legacy_pids_to_redirect = legacy_pids_to_redirect - else: - with open(legacy_pids_to_redirect, "r") as fp: - self.legacy_pids_to_redirect = json.load(fp) - - def _prepare(self, entry): - """Prepare the record.""" - pass - - def _load_files( - self, - draft, - entry: MigrationEntry, - version_files: Dict[str, VersionFileEntry], - uow=None, - ): - """Load files to draft.""" - recid = entry["record"].recid - identity = system_identity # Should we create an identity for the migration? - - for filename, file_data in version_files.items(): - - file_data = version_files[filename] - - try: - current_rdm_records_service.draft_files.init_files( - identity, - draft.id, - data=[ - { - "key": file_data["key"], - "metadata": { - **file_data["metadata"], - "legacy_file_id": file_data["id_bibdoc"], - "legacy_recid": recid, - }, - "access": {"hidden": False}, - } - ], - uow=uow, - ) - # TODO change to eos move or xrootd command instead of going through the app - # TODO leave the init part to pre-create the destination folder - # TODO update checksum, size, commit (to be checked on how these methods work) - # if current_app.config["XROOTD_ENABLED"]: - # storage = current_files_rest.storage_factory - # current_rdm_records_service.draft_files.set_file_content( - # identity, - # draft.id, - # file["key"], - # BytesIO(b"Placeholder file"), - # ) - # obj = None - # for object in draft._record.files.objects: - # if object.key == file["key"]: - # obj = object - # path = obj.file.uri - # else: - # for local development - current_rdm_records_service.draft_files.set_file_content( - identity, - draft.id, - file_data["key"], - import_legacy_files(file_data["eos_tmp_path"]), - uow=uow, - ) - result = current_rdm_records_service.draft_files.commit_file( - identity, draft.id, file_data["key"], uow=uow - ) - legacy_checksum = f"md5:{file_data['checksum']}" - new_checksum = result.to_dict()["checksum"] - if current_app.config["CDS_MIGRATOR_KIT_ENV"] != "local": - try: - assert legacy_checksum == new_checksum - except AssertionError: - raise ManualImportRequired( - message=f"Files checksum failed legacy:{legacy_checksum} calculated new: {new_checksum}", - field="checksum", - stage="load", - recid=recid, - priority="critical", - value=file_data["key"], - subfield=None, - ) - - except Exception as e: - exc = ManualImportRequired( - recid=recid, - message=str(e), - field="filename", - value=file_data["key"], - stage="file load", - priority="critical", - ) - self.migration_logger.add_log(exc, record=entry) - raise e - - def _load_parent_access_and_communities(self, draft, entry: MigrationEntry): - """Load access rights and communities in a single parent commit.""" - parent = draft._record.parent - record_parent = entry["parent"] - parent.access = record_parent.body["access"] - communities = record_parent.communities["ids"] - for community in communities: - parent.communities.add(community) - parent.communities.default = record_parent.communities["default"] - parent.commit() - - def _load_record_access(self, draft, access_dict: VersionAccess): - record = draft._record - record.access = access_dict["access_obj"] - record.commit() - - def _after_commit_run_clc_sync(self, record_state): - """Run the CLC sync after UOW commit.""" - if not self._is_final_record: - return - if self.clc_sync: - clc_sync_entry = current_clc_sync_service.read( - system_identity, record_state["parent_recid"] - ).to_dict() - clc_sync_entry["record"] = current_rdm_records_service.read( - system_identity, record_state["latest_version"] - ).to_dict() - clc_sync_entry["auto_sync"] = True - current_clc_sync_service.update( - system_identity, clc_sync_entry["id"], clc_sync_entry - ) - - def _after_publish_update_dois(self, identity, record, entry, uow): - """Update migrated DOIs post publish.""" - if not self._is_final_record: - return - migrated_pids = entry["record"].body["pids"] - for pid_type, identifier in migrated_pids.items(): - if pid_type == "doi": - # If a DOI was already minted from legacy then on publish the datacite - # will return a warning that "This DOI has already been taken" - # In that case, we edit and republish to force an update of the doi with - # the new published metadata as in the new system we have more information available - _draft = current_rdm_records_service.edit( - identity, record["id"], uow=uow - ) - record = current_rdm_records_service.publish( - identity, _draft["id"], uow=uow - ) - return record - - def _after_publish_load_parent_access_grants( - self, draft, version, entry: MigrationEntry - ): - """Load access grants from metadata and record grants efficiently.""" - access_dict = entry["versions"][version]["access"] - parent = draft._record.parent - identity = system_identity - - record_parent = entry["parent"] - specific_file_restrictions = access_dict.get("meta", "") - if not specific_file_restrictions and not record_parent.access_grants: - return - default_permission = "view" - - groups, emails, grants_with_perms = record_parent.resolve_grants( - specific_file_restrictions - ) - - def _create_grant(subject_type, subject_id, permission): - grant_data = { - "grants": [ - { - "subject": {"type": subject_type, "id": str(subject_id)}, - "permission": permission, - } - ] - } - current_rdm_records_service.access.schema_grants.load( - grant_data, - context={"identity": identity}, - raise_errors=True, - ) - - grant = parent.access.grants.create( - subject_type=subject_type, - subject_id=subject_id, - permission=permission, - origin="migrated", - ) - is_local_dev = current_app.config.get("CDS_MIGRATOR_KIT_ENV") == "local" - is_valid = current_rdm_records_service.access._validate_grant_subject( - identity, grant - ) - if not is_local_dev and not is_valid: - raise ManualImportRequired( - message="Verification of access subject failed (likely not existing entry)", - field="access", - subfield="subject.id", - stage="load", - recid=entry["record"].recid, - priority="warning", - value=subject_id, - ) - - # Create grants for groups - for group in groups: - _create_grant( - subject_type="role", - subject_id=group.lower(), - permission=grants_with_perms.get(group, default_permission), - ) - - # Fetch existing users - existing_users = { - user.email: user.id - for user in User.query.filter(User.email.in_(emails)).all() - } - # raise error for missing user - missing_emails = emails - existing_users.keys() - if missing_emails: - raise GrantCreationError( - message=f"Users not found for emails: {', '.join(missing_emails)}", - stage="load", - recid=entry["record"].recid, - value=list(missing_emails), - priority="warning", - ) - - # Create grants for users - for email, user_id in existing_users.items(): - _create_grant( - subject_type="user", - subject_id=user_id, - permission=grants_with_perms.get(email, default_permission), - ) - - parent.commit() - - def _after_publish_update_created(self, record, entry: MigrationEntry, version): - """Update created timestamp post publish. - - Ensures that the `created` timestamp is correctly set, preferring: - 1. The original legacy system value for the version. - 2. The record's creation date if there are no files. - 3. Today's date if the original value and file creation date is missing. - """ - creation_date = arrow.get(entry["record"].created).datetime.replace( - tzinfo=None - ) - - versions = entry.get("versions", {}) - version_data = versions.get(version, {}) - - if version_data.get("files") and version != 1: - # Subsequent versions should use the file creation date, instead of the record creation date, - # which is stored as the publication date in the version data - creation_date = version_data["publication_date"].datetime.replace( - tzinfo=None - ) - - record._record.model.created = creation_date - db.session.add(record._record.model) - - def _after_publish_mint_recid(self, record, entry: MigrationEntry, version): - """Mint legacy ids for redirections assigned to the parent.""" - if not self._is_final_record: - return - legacy_recid = entry["record"].recid - if record._record.versions.index == 1: - # it seems more intuitive if we mint the lrecid for parent - # but then we get a double redirection - legacy_recid_minter(legacy_recid, record._record.parent.model.id) - - def _after_publish_add_submission_request(self, request_data, record, entry, uow): - """Create community inclusion request after publish.""" - legacy_recid = entry["record"].recid - request_number = f"lrecid:{legacy_recid}" - - # Defensive/idempotency guard: skip if a request for this record was - # already committed by a previous (partial) load attempt, otherwise - # re-creating it would violate the unique constraint on `number`. - if RequestMetadata.query.filter_by(number=request_number).first() is not None: - return - - status = request_data.get("status", "accepted") - reviewers = request_data.get("reviewers", []) - - created_at = datetime.datetime.fromisoformat(record["created"]) - - parent = record._record.parent - owner_id = parent.access.owned_by.owner_id - community = parent.communities.default - - creator = {"user": str(owner_id)} - receiver = {"community": str(community.id)} - - def create_event(request_model, payload, event_type, user): - """Create and register a request event.""" - event = current_events_service.record_cls.create( - {}, - request=request_model, - request_id=str(request_model.id), - type=event_type, - ) - event.update(payload=payload) - event.model.created = created_at - event.created_by = {"user": str(user)} - - uow.register(RecordCommitOp(event, indexer=current_events_service.indexer)) - - request_item = current_requests_service.create( - system_identity, - data={"title": record["metadata"]["title"]}, - request_type=CommunitySubmission, - receiver=receiver, - creator=creator, - topic={"record": record.id}, - uow=uow, - ) - - request = request_item._record - request.status = "submitted" - request.number = request_number - request.model.created = created_at - - if reviewers: - request.reviewers = reviewers - - if status: - request.status = status - if status == "accepted": - parent_to_request_relation = ( - parent.communities._m2m_model_cls.query.filter_by( - record_id=parent.id, community_id=community.id - ).one() - ) - parent_to_request_relation.request_id = request.id - - create_event( - request.model, - {"event": status}, - event_type=LogEventType, - user="system", - ) - - uow.register(RecordCommitOp(request, indexer=current_requests_service.indexer)) - - def _after_publish_update_files_created( - self, record, entry: MigrationEntry, version - ): - """Update the created date of the files post publish.""" - # Fix the `created` timestamp forcing the one from the legacy system - # Force the created date. This can be done after publish as the service - # overrides the `created` date otherwise. - versions = entry.get("versions", {}) - version_data = versions.get(version, {}) - files = version_data.get("files", {}) - for _, file_data in files.items(): - file = record._record.files.entries[file_data["key"]] - file.model.created = arrow.get(file_data["creation_date"]).datetime.replace( - tzinfo=None - ) - db.session.add(file.model) - - def _after_publish_set_committee_approval(self, published_record, entry, uow): - """Write committee_approval to parent for records already EP-approved pre-migration. - - Only runs when: - 1. entry["ep_approval"] is empty — record did NOT go through the - 9031_/EPPHAPP path (those are handled by CDSEPApprovalRecordServiceLoad - which already writes committee_approval correctly for both records). - 2. The record carries at least one apprn identifier. - - For these records the migrated record IS the final public version (no - separate internal draft exists). We write the same "public side" - committee_approval block that ep_approval_load writes, pointing - source_internal_version at the record's own PID so that - get_committee_approval_state returns is_public_approved_record=True. - """ - if entry.get("ep_approval"): - return - - record = published_record._record - identifiers = record.get("metadata", {}).get("identifiers", []) - apprn_ids = [i["identifier"] for i in identifiers if i.get("scheme") == "apprn"] - if not apprn_ids: - return - - parent = record.parent - pf = parent.get("permission_flags") or {} - if pf.get("committee_approval", {}).get("source_internal_version"): - return # idempotency: already written - - pf["committee_approval"] = { - "source_internal_version": str(record.pid.pid_value), - "reportnumber": apprn_ids[0], - } - parent["permission_flags"] = pf - uow.register(ParentRecordCommitOp(parent)) - - # Mint apprn PIDs in pidstore — same logic as ApprovalRequest._mint_apprn_pid. - from cds_rdm.requests.committee_approval import APPRN_PID_TYPE - for apprn_value in apprn_ids: - try: - PersistentIdentifier.create( - pid_type=APPRN_PID_TYPE, - pid_value=apprn_value, - object_type="rec", - object_uuid=str(record.id), - status=PIDStatus.REGISTERED, - ) - except PIDAlreadyExists: - pass # already minted on a previous run — idempotent - - def _after_publish( - self, identity, published_record, entry: MigrationEntry, version, uow - ): - """Run fixes after record publish.""" - record = self._after_publish_update_dois(identity, published_record, entry, uow) - if record: - published_record = record - self._after_publish_update_created(published_record, entry, version) - self._after_publish_mint_recid(published_record, entry, version) - self._after_publish_update_files_created(published_record, entry, version) - self._after_publish_load_parent_access_grants(published_record, version, entry) - self._after_publish_set_committee_approval(published_record, entry["record"], uow) - record_request = entry.get("_request_data") - - if record_request: - record_request.ensure_enabled(self.create_inclusion_request) - if self.create_inclusion_request: - self._after_publish_add_submission_request( - record_request.data, published_record, entry, uow - ) - # db.session.commit() - - def _assign_rep_numbers(self, draft): - if not self._is_final_record: - return - draft_report_nums = {} - for index, id in enumerate(draft.data["metadata"].get("identifiers", [])): - if id["scheme"] == "cdsrn": - draft_report_nums[id["identifier"]] = index - - if not draft_report_nums: - # If no mintable identifiers, return early - return - - for report_number, index in draft_report_nums.items(): - try: - PersistentIdentifier.create( - pid_type="cdsrn", - pid_value=report_number, - object_type="rec", - object_uuid=draft._record.parent.id, - status=PIDStatus.REGISTERED, - ) - except PIDAlreadyExists as e: - pid = PersistentIdentifier.get( - pid_type="cdsrn", pid_value=report_number - ) - if pid.object_uuid != draft._record.parent.id: - # raise only if different parent uuid found, meaning they are 2 - # different records and the repnum is duplicated - raise ManualImportRequired( - f"Report number {report_number} already exists." - ) - - def _pre_publish(self, identity, entry: MigrationEntry, version, draft, uow): - """Create and process draft before publish.""" - versions = entry["versions"] - files = versions[version]["files"] - access = versions[version]["access"] - - if version == 1 or (version > 1 and draft is None): - # when draft is None, it means the initial version one was hard deleted - # and we don't have index 1 - # we decided to skip it and act normal - try: - draft = current_rdm_records_service.create( - identity, data=entry["record"].body, uow=uow - ) - self._assign_rep_numbers(draft) - except (UniqueViolation, IntegrityError) as e: - raise ManualImportRequired(message=str(e)) - except Exception as e: - raise ManualImportRequired(message=str(e)) - if draft.errors: - raise ManualImportRequired( - message=f"{str(draft.errors)}: {str(entry['record'].body)}", - field="validation", - stage="load", - recid=entry["record"].recid, - priority="warning", - value=draft._record.pid.pid_value, - subfield=None, - ) - self._load_parent_access_and_communities(draft, entry) - else: - draft = current_rdm_records_service.new_version( - identity, draft["id"], uow=uow - ) - draft_dict = draft.to_dict() - if not self.update_new_version_publication_date: - publication_date = arrow.get( - entry["record"].body["metadata"]["publication_date"] - ) - else: - publication_date = versions[version]["publication_date"] - missing_data = { - **draft_dict, - "metadata": { - # copy over the previous draft metadata - **draft_dict["metadata"], - # add missing publication date based - # on the time of creation of the new file version - "publication_date": publication_date.date().isoformat(), - }, - } - draft = current_rdm_records_service.update_draft( - identity, draft["id"], data=missing_data, uow=uow - ) - - self._load_record_access(draft, access) - self._load_files(draft, entry, files, uow=uow) - - return draft - - def _load_versions(self, entry: MigrationEntry, uow): - """Load other versions of the record.""" - versions = entry["versions"] - legacy_recid = entry["record"].recid - - identity = system_identity - - records = [] - # initial value of draft. If different file versions identified then the first - # created draft is used to populate all newer versions - draft = None - for version in versions.keys(): - # Create and prepare draft - draft = self._pre_publish(identity, entry, version, draft, uow) - - # Publish draft - published_record = current_rdm_records_service.publish( - identity, draft["id"], uow=uow + self.parent_load_cls = ParentLoad + self.record_load_cls = RecordLoad + self.request_load_cls = RequestLoad + with open(legacy_pids_to_redirect, "r") as fp: + self.legacy_pids_to_redirect = json.load(fp) + + def _apply_clc_sync(self, record_state, entry: MigrationEntry): + """Create the CLC sync entry after the load has committed.""" + if entry.get("_clc_sync", False): + sync = CDSToCLCSyncModel( + parent_record_pid=record_state["parent_recid"], + status="P", + auto_sync=True, ) - # Run after publish fixes - self._after_publish(identity, published_record, entry, version, uow) - records.append(published_record._record) - - if records: - record_state_context = self._load_record_state(legacy_recid, records) - # Dump the computed record state. This is useful to migrate then the record stats - if record_state_context: - if self._is_final_record: - self.record_state_logger.add_record_state(record_state_context) - return record_state_context - - def _dry_load(self, entry: MigrationEntry): - current_rdm_records_service.schema.load( - entry["record"].body, - context=dict( - identity=system_identity, - ), - raise_errors=True, - ) - - def _load_record_state(self, legacy_recid, records): - """Compute state for legacy recid. - - Returns - { - "legacy_recid": "2884810", - "parent_recid": "zts3q-6ef46", - "parent_object_uuid": "435be22f-3038-49e0-9f17-9518eaac783a", - "latest_version": "1mae4-skq89" - "latest_version_object_uuid": "895be22f-3038-49e0-9f17-9518eaac783a", - "versions": [ - { - "new_recid": "1mae4-skq89", - "version": 2, - "files": [ - { - "legacy_file_id": 1568736, - "bucket_id": "155be22f-3038-49e0-9f17-9518eaac783a", - "file_key": "Summer student program report.pdf", - "file_id": "06cdb9d2-635f-4dbe-89fe-4b27afddeaa2", - "size": "1690854" - } - ] - } - ] - } - """ - - def convert_file_format(file_entries, bucket_id): - """Convert the file metadata into the required format.""" - return [ - { - "legacy_file_id": entry["metadata"]["legacy_file_id"], - "bucket_id": bucket_id, - "file_key": entry["key"], - "file_id": entry["file_id"], - "size": str(entry["size"]), - } - for entry in file_entries.values() - ] - - def extract_record_version(record): - """Extract relevant details from a single record.""" - bucket_id = str(record.files.bucket_id) - files = record.__class__.files.dump( - record, record.files, include_entries=True - ).get("entries", {}) - return { - "new_recid": record.pid.pid_value, - "version": record.versions.index, - "files": convert_file_format(files, bucket_id), - } - - recid_state = {"legacy_recid": legacy_recid, "versions": []} - parent_recid = None - - for record in records: - if parent_recid is None: - parent_id = str(record.parent.id) - parent_recid = record.parent.pid.pid_value - recid_state["parent_recid"] = parent_recid - recid_state["parent_object_uuid"] = parent_id - - recid_version = extract_record_version(record) - # Save the record versions for legacy recid - recid_state["versions"].append(recid_version) - - if "latest_version" not in recid_state: - rec = record.get_latest_by_parent(record.parent) - recid_state["latest_version"] = rec["id"] - recid_state["latest_version_object_uuid"] = str(rec.id) - return recid_state + db.session.add(sync) + db.session.commit() def _save_original_dumped_record(self, entry: MigrationEntry, recid_state): """Save the original dumped record. This is the originally extracted record before any transformation. """ - if not self._is_final_record: - return _original_dump = entry["_original_dump"] _original_dump_model = CDSMigrationLegacyRecord( json=_original_dump, @@ -763,96 +100,78 @@ def _have_migrated_recid(recid): def _should_skip_recid(self, recid): """Check if recid should be skipped.""" if recid in self.legacy_pids_to_redirect or self._have_migrated_recid(recid): + self.migration_logger.add_information( + recid, state={"message": "Record already migrated", "value": recid} + ) + self.migration_logger.finalise_record(recid) return True return False - def _after_load_clc_sync(self, record_state): - if not self._is_final_record: + def _load(self, entry: MigrationEntry): + """Use the services to load the entry.""" + if not entry: return - if self.clc_sync: - sync = CDSToCLCSyncModel( - parent_record_pid=record_state["parent_recid"], - status="P", - auto_sync=False, - ) - db.session.add(sync) - - def _load(self, entry: MigrationEntry, uow=None): - """Use the services to load the entries. - - If ``uow`` is provided, operations are registered on it without - committing, so the caller can group this load atomically with other - operations (e.g. the EP approval record split). - """ - if entry: - recid = entry["record"].recid - if self._should_skip_recid(recid): - self.migration_logger.add_information( - recid, state={"message": "Record already migrated", "value": recid} - ) - self.migration_logger.finalise_record(recid) - return - self.clc_sync = deepcopy(entry.get("_clc_sync", False)) - if "_clc_sync" in entry: - del entry["_clc_sync"] + recid = entry["record"].recid + if self._should_skip_recid(recid): + return - try: - ep_approval = entry.get("ep_approval") - if ep_approval: - raise UnexpectedValue( - message="EP approval records must be loaded with the '--ep-approval' flag", - stage="load", - recid=recid, - priority="critical", + record_load = RecordLoad( + entry["record"], + entry["parent"], + self.migration_logger, + is_final_record=True, + update_new_version_publication_date=self.update_new_version_publication_date, + record_state_logger=self.record_state_logger, + ) + try: + if self.dry_run: + record_load.dry_load() + recid_state_after_load = None + else: + with UnitOfWork(db.session) as uow: + records = record_load.load(entry, uow=uow) + recid_state_after_load = record_load.build_record_state( + recid, records ) - if self.dry_run: - self._dry_load(entry) - recid_state_after_load = None - elif uow is not None: - recid_state_after_load = self._load_versions(entry, uow) if recid_state_after_load: self._save_original_dumped_record(entry, recid_state_after_load) - self._after_load_clc_sync(recid_state_after_load) - else: - with UnitOfWork(db.session) as inner_uow: - recid_state_after_load = self._load_versions(entry, inner_uow) - if recid_state_after_load: - self._save_original_dumped_record( - entry, recid_state_after_load - ) - self._after_load_clc_sync(recid_state_after_load) - inner_uow.commit() - if self._is_final_record and uow is None: - # When an external uow is provided, the caller owns the - # commit boundary and is responsible for finalising the - # record only after it actually commits. + self.parent_load_cls( + entry, self.migration_logger, recid_state_after_load + ).load(published_record=records[-1]) + self.request_load_cls(entry).load( + records, self.create_inclusion_request, uow + ) + uow.commit() + if recid_state_after_load: + # only log to disk once the unit of work has actually + # committed - logging any earlier risks recording a + # record that a later failure in the same uow rolls back + record_load.log_record_state(recid_state_after_load) self.migration_logger.finalise_record(recid) - # Run the CLC sync after UOW commit - self._after_commit_run_clc_sync(recid_state_after_load) - return recid_state_after_load - except (UnexpectedValue, ManualImportRequired) as e: - self.migration_logger.add_log(e, record=entry) - except GrantCreationError as e: - self.migration_logger.add_log(e, record=entry) - except (CDSMigrationException, ValidationError, InvalidRelationValue) as e: - exc = ManualImportRequired( - message=str(e), - field="validation", - stage="load", - recid=recid, - priority="warning", - ) - self.migration_logger.add_log(exc, record=entry) - except Exception as e: - exc = ManualImportRequired( - message=str(e), - field="validation", - stage="load", - recid=recid, - priority="warning", - ) - self.migration_logger.add_log(exc, record=entry) + # apply after record fully finished (does not sync at the spot, only enabled) + self._apply_clc_sync(recid_state_after_load, entry) + return recid_state_after_load + except (UnexpectedValue, ManualImportRequired, GrantCreationError) as e: + self.migration_logger.add_log(e, record=entry) + except (CDSMigrationException, ValidationError, InvalidRelationValue) as e: + exc = ManualImportRequired( + message=str(e), + field="validation", + stage="load", + recid=recid, + priority="warning", + ) + self.migration_logger.add_log(exc, record=entry) + except Exception as e: + exc = ManualImportRequired( + message=str(e), + field="validation", + stage="load", + recid=recid, + priority="warning", + ) + self.migration_logger.add_log(exc, record=entry) def _cleanup(self, *args, **kwargs): """Post migration process.""" diff --git a/cds_migrator_kit/rdm/records/streams.py b/cds_migrator_kit/rdm/records/streams.py index eccfe125..0e7689bd 100644 --- a/cds_migrator_kit/rdm/records/streams.py +++ b/cds_migrator_kit/rdm/records/streams.py @@ -11,20 +11,20 @@ from cds_migrator_kit.extract.extract import LegacyExtract from cds_migrator_kit.rdm.records.transform.transform import CDSToRDMRecordTransform -from .load import CDSEPApprovalRecordServiceLoad, CDSRecordServiceLoad +from .load.entities.ep_migration_entry_load import EPMigrationEntryLoad RecordStreamDefinition = StreamDefinition( name="records", extract_cls=LegacyExtract, transform_cls=CDSToRDMRecordTransform, - load_cls=CDSRecordServiceLoad, + load_cls=EPMigrationEntryLoad, ) -"""ETL stream for CDS to RDM records.""" +"""ETL stream for CDS to RDM records. -RecordEPApprovalStreamDefinition = StreamDefinition( - name="records", - extract_cls=LegacyExtract, - transform_cls=CDSToRDMRecordTransform, - load_cls=CDSEPApprovalRecordServiceLoad, -) -"""ETL stream for CDS to RDM records with EP approval.""" +Handles both plain records and EP approval records (split into a +public/restricted pair) - which path an entry takes is decided per-record by +``EPMigrationEntryLoad._load()``, based on whether the entry carries +``ep_approval`` data, not by a separate stream/CLI flag. ``EPMigrationEntryLoad`` +subclasses ``CDSMigrationEntryLoad`` (the plain per-entry loader) and falls +back to it for non-EP-approval entries. +""" diff --git a/cds_migrator_kit/rdm/records/transform/entities/migration.py b/cds_migrator_kit/rdm/records/transform/entities/migration.py index 582ca77e..3c3ff7f6 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/migration.py +++ b/cds_migrator_kit/rdm/records/transform/entities/migration.py @@ -17,7 +17,7 @@ class MigrationEntry(TypedDict): """The full ETL entry yielded by ``CDSToRDMRecordTransform.run()``. - Consumed by ``CDSRecordServiceLoad``/``ep_approval_entry.py``. Keys + Consumed by ``CDSMigrationEntryLoad``/``EPMigrationEntryLoad``. Keys outside ``"record"`` are ETL-envelope-scoped (about this migration run, not about the record's own content): ``versions``/``parent`` are computed by ``CDSToRDMRecordTransform`` itself, while diff --git a/cds_migrator_kit/rdm/records/transform/entities/request.py b/cds_migrator_kit/rdm/records/transform/entities/request.py index 6ce76d6e..7fd45a11 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/request.py +++ b/cds_migrator_kit/rdm/records/transform/entities/request.py @@ -57,22 +57,6 @@ def __bool__(self): """True when there's request data to act on.""" return bool(self.data) - def ensure_enabled(self, create_inclusion_request): - """Raise if this record has request data but requests aren't enabled. - - :param create_inclusion_request: whether the current load run is - configured to create community-inclusion requests - see - ``CDSRecordServiceLoad.create_inclusion_request``. - """ - if self.data and not create_inclusion_request: - raise ManualImportRequired( - message="Detected request data, enable the requests", - field="validation", - stage="load", - recid=self.recid, - priority="warning", - subfield=None, - ) def _resolve_reviewers(self, reviewer_names): """Resolve raw reviewer name/email strings to RDM reviewer entries. diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index edece5b0..36e52f96 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -42,7 +42,7 @@ class CDSToRDMRecordTransform: - """Assembles the ETL entry consumed by ``CDSRecordServiceLoad``. + """Assembles the ETL entry consumed by ``CDSMigrationEntryLoad``. Wraps the ``record`` content built by ``RecordEntry`` together with ``versions``/``parent`` (computed here - ``parent`` is a diff --git a/cds_migrator_kit/runner/runner.py b/cds_migrator_kit/runner/runner.py index 871aeb18..1ffe142c 100644 --- a/cds_migrator_kit/runner/runner.py +++ b/cds_migrator_kit/runner/runner.py @@ -109,7 +109,6 @@ def __init__( db_uri=self.db_uri, data_dir=data_dir, dry_run=dry_run, - collection=collection, update_new_version_publication_date=self.update_new_version_publication_date, create_inclusion_request=self.create_inclusion_request, migration_logger=self.migration_logger, diff --git a/tests/cds-rdm/test_ep_approval_entry.py b/tests/cds-rdm/test_ep_approval_entry.py index 86ff7364..483fc232 100644 --- a/tests/cds-rdm/test_ep_approval_entry.py +++ b/tests/cds-rdm/test_ep_approval_entry.py @@ -15,7 +15,7 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.migration_config import CDS_CERN_SCIENTIFIC_COMMUNITY_ID -from cds_migrator_kit.rdm.records.load.ep_approval_entry import ( +from cds_migrator_kit.rdm.records.load.entities.ep_split import ( EPPHAPP_FILE_TYPE, PublicEntry, RestrictedEntry, diff --git a/tests/cds-rdm/test_research_committee_rules.py b/tests/cds-rdm/test_research_committee_rules.py new file mode 100644 index 00000000..14fae731 --- /dev/null +++ b/tests/cds-rdm/test_research_committee_rules.py @@ -0,0 +1,591 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for research_committee.py migration rules.""" + +import pytest +from dojson.errors import IgnoreKey +from dojson.utils import GroupableOrderedDict + +from cds_migrator_kit.errors import MissingRequiredField +from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry +from cds_migrator_kit.rdm.records.transform.models.research_committee import ( + research_comm_model, +) +from cds_migrator_kit.rdm.records.transform.xml_processing.rules.research_committee import ( + _RANK_REPORT_NUMBER, + _RANK_SERIES, + _RANK_TITLE, + imprint, + report_number, + series_information, + title, +) + + +class TestCommitteeReportNumberType: + """Test the -- report number pattern detection + (e.g. "SPSC-I-170", see https://cds.cern.ch/record/493774 for a real + "SPSLC-P-282" example) and its resource_type/subject derivation. + """ + + def _call(self, record, key, value): + """Plain committee report numbers (no scheme/provenance subfield) + always resolve to the "cdsrn" scheme, which base.report_number + stores directly on `identifiers` and signals via IgnoreKey - see + TestCommitteeReportNumberBaseBehaviourPreserved. Swallow it here so + each test can focus on the resource_type/subject side effect. + """ + with pytest.raises(IgnoreKey): + report_number(record, key, value) + + def test_spsc_i_matches_letter(self): + """The example from the request: SPSC-I-170 -> publication-letter.""" + record = {} + self._call(record, "088__", {"a": "SPSC-I-170"}) + assert record["resource_type"] == {"id": "publication-letter"} + + def test_real_record_spslc_p_282_matches_proposal(self): + """https://cds.cern.ch/record/493774 has 088__a "SPSLC-P-282".""" + record = {} + self._call(record, "088__", {"a": "SPSLC-P-282"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_real_record_292374_drdc_p_matches_proposal(self): + """https://cds.cern.ch/record/292374 has 088__a "DRDC-P-2" (plus + the unrelated plain report number "CERN-DRDC-90-25").""" + record = {} + self._call(record, "088__", {"a": "DRDC-P-2"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_real_record_291072_status_report_matches_report(self): + """https://cds.cern.ch/record/291072 has 088__a + "DRDC-Status-report-RD-30": a two-token type ("Status-report") + followed by a number that itself contains a hyphen ("RD-30").""" + record = {} + self._call(record, "088__", {"a": "DRDC-Status-report-RD-30"}) + assert record["resource_type"] == {"id": "publication-report"} + + def test_spsc_m_is_memorandum(self): + """M is SPSC-specific: publication-memorandum, not meeting minutes.""" + record = {} + self._call(record, "088__", {"a": "SPSC-M-12"}) + assert record["resource_type"] == {"id": "publication-memorandum"} + + def test_non_spsc_m_is_meeting_minutes(self): + """M defaults to publication-meetingminutes for other committees.""" + record = {} + self._call(record, "088__", {"a": "TCC-M-12"}) + assert record["resource_type"] == {"id": "publication-meetingminutes"} + + def test_type_a_is_meeting_agenda(self): + record = {} + self._call(record, "088__", {"a": "TCC-A-3"}) + assert record["resource_type"] == {"id": "publication-meetingagenda"} + + def test_type_g_is_other(self): + record = {} + self._call(record, "088__", {"a": "TCC-G-3"}) + assert record["resource_type"] == {"id": "publication-other"} + + def test_type_p_is_proposal(self): + record = {} + self._call(record, "088__", {"a": "TCC-P-3"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_type_t_is_technical_note(self): + record = {} + self._call(record, "088__", {"a": "TCC-T-3"}) + assert record["resource_type"] == {"id": "publication-technicalnote"} + + def test_type_tdr_is_report(self): + record = {} + self._call(record, "088__", {"a": "TCC-TDR-3"}) + assert record["resource_type"] == {"id": "publication-report"} + + def test_type_sr_is_report(self): + record = {} + self._call(record, "088__", {"a": "TCC-SR-3"}) + assert record["resource_type"] == {"id": "publication-report"} + + def test_type_ug_is_report_with_upgrade_cost_group_subject(self): + record = {} + self._call(record, "088__", {"a": "TCC-UG-3"}) + assert record["resource_type"] == {"id": "publication-report"} + assert record["subjects"] == [{"subject": "collection:upgrade cost group"}] + + def test_spsc_r_is_report_with_recommendation_subject(self): + """R is only defined for SPSC: publication-report + subject.""" + record = {} + self._call(record, "088__", {"a": "SPSC-R-3"}) + assert record["resource_type"] == {"id": "publication-report"} + assert record["subjects"] == [{"subject": "recommendation"}] + + def test_non_spsc_r_is_not_mapped(self): + """R has no default mapping outside SPSC - left untouched.""" + record = {} + self._call(record, "088__", {"a": "TCC-R-3"}) + assert "resource_type" not in record + assert "subjects" not in record + + def test_unknown_committee_prefix_not_matched(self): + record = {} + self._call(record, "088__", {"a": "XYZ-M-5"}) + assert "resource_type" not in record + + def test_unknown_type_code_not_matched(self): + record = {} + self._call(record, "088__", {"a": "SPSC-Z-5"}) + assert "resource_type" not in record + + def test_generic_cern_report_number_not_matched(self): + """ "CERN-SPSLC-94-025" (from record 493774) is a plain CERN report + number, not a -- code - must not match.""" + record = {} + self._call(record, "088__", {"a": "CERN-SPSLC-94-025"}) + assert "resource_type" not in record + + def test_037_field_also_detected(self): + """The pattern is checked on both 037__ and 088__.""" + record = {} + self._call(record, "037__", {"a": "SPSC-I-170"}) + assert record["resource_type"] == {"id": "publication-letter"} + + def test_match_sets_rank_below_any_980_priority(self): + """The rank sentinel set alongside resource_type must outrank + anything the generic 980__ rule (research.py:resource_type) can + produce, so a committee-report-derived type is never clobbered by + it - see TestResearchCommitteeModelIntegration for the full + end-to-end behaviour.""" + record = {} + self._call(record, "088__", {"a": "SPSC-I-170"}) + assert record["_resource_type_rank"] == _RANK_REPORT_NUMBER + + +class TestGenericReportNumberType: + """Test detection of a type token anywhere in a report number that + doesn't follow the -- convention - CERN's + generic CERN--- department report numbering, e.g. + "CERN-NP-MEMO-7840".""" + + def _call(self, record, key, value): + with pytest.raises(IgnoreKey): + report_number(record, key, value) + + def test_real_example_cern_np_memo(self): + record = {} + self._call(record, "088__", {"a": "CERN-NP-MEMO-7840"}) + assert record["resource_type"] == {"id": "publication-memorandum"} + assert record["_resource_type_rank"] == _RANK_REPORT_NUMBER + + def test_tdr_token_matched(self): + record = {} + self._call(record, "088__", {"a": "CERN-LHC-TDR-2020"}) + assert record["resource_type"] == {"id": "publication-report"} + + def test_status_report_token_matched(self): + """The two-token type is matched here too, not just when anchored + to a known committee - see TestCommitteeReportNumberType's + DRDC-Status-report-RD-30 case.""" + record = {} + self._call(record, "088__", {"a": "CERN-EP-STATUS-REPORT-99"}) + assert record["resource_type"] == {"id": "publication-report"} + + def test_single_letter_codes_not_matched_unanchored(self): + """M, I, A, G, P, T (and N, S) are only trusted right after a known + committee prefix (see TestCommitteeReportNumberType) - unanchored, + they're too likely to be a coincidental match.""" + record = {} + self._call(record, "088__", {"a": "CERN-EP-T-123"}) + assert "resource_type" not in record + + def test_unrelated_report_number_not_matched(self): + record = {} + self._call(record, "088__", {"a": "CERN-EP-2020-042"}) + assert "resource_type" not in record + + def test_committee_anchored_match_still_takes_precedence(self): + """Both detectors run on every 037__/088__ occurrence, but they + agree here since MEMO is a token in both - just confirming the + two don't conflict for a genuine committee report number.""" + record = {} + self._call(record, "088__", {"a": "SPSC-M-12"}) + assert record["resource_type"] == {"id": "publication-memorandum"} + + def test_does_not_override_existing_series_match(self): + """490__ is processed after 037__/088__, but a report-number match + (committee-anchored or generic) shares the top priority tier - see + TestSeriesResourceType.test_does_not_override_existing_resource_type + for the reverse ordering check.""" + record = {} + self._call(record, "088__", {"a": "CERN-NP-MEMO-7840"}) + series_information(record, "490__", {"a": "Proposal"}) + assert record["resource_type"] == {"id": "publication-memorandum"} + + +class TestSeriesResourceType: + """Test resource_type derived from 490__$a free text (e.g. + "Memorandum", "Proposal", "Letter of Intent") for research committee + records that don't carry a -- report number.""" + + def test_memorandum(self): + record = {} + result = series_information(record, "490__", {"a": "Memorandum"}) + assert record["resource_type"] == {"id": "publication-memorandum"} + assert record["_resource_type_rank"] == _RANK_SERIES + assert result == [ + {"description": "Memorandum", "type": {"id": "series-information"}} + ] + + def test_proposal(self): + record = {} + series_information(record, "490__", {"a": "Proposal"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_letter_of_intent(self): + record = {} + series_information(record, "490__", {"a": "Letter of Intent"}) + assert record["resource_type"] == {"id": "publication-letter"} + + def test_case_insensitive(self): + record = {} + series_information(record, "490__", {"a": "PROPOSAL"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_unknown_phrase_not_matched(self): + record = {} + series_information(record, "490__", {"a": "Yellow Report Series"}) + assert "resource_type" not in record + + def test_base_series_information_behaviour_preserved(self): + """additional_descriptions is still populated exactly like the + generic 490__ rule (base.series_information).""" + record = {} + result = series_information(record, "490__", {"a": "Proposal", "v": "12"}) + assert result == [ + {"description": "Proposal (12)", "type": {"id": "series-information"}} + ] + + def test_does_not_override_existing_resource_type(self): + """037__/088__ is processed before 490__ (fields are visited in tag + order - see CdsOverdo.do), so a committee report number match must + win over conflicting 490__ free text.""" + record = {} + with pytest.raises(IgnoreKey): + report_number(record, "088__", {"a": "DRDC-P-2"}) + series_information(record, "490__", {"a": "Memorandum"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + +class TestEditionResourceType: + """Test resource_type derived from 250__$a free text (edition + statement) for research committee records.""" + + def _call(self, record, value): + """imprint always raises IgnoreKey("imprint_info") - see + research.imprint - it stores the edition on custom_fields itself + rather than returning it. Swallow it here so each test can focus + on the resource_type side effect.""" + with pytest.raises(IgnoreKey): + imprint(record, "250__", value) + + def test_real_record_1005022_addendum_matches_other(self): + """https://cds.cern.ch/record/1005022 has 250__a "Addendum".""" + record = {} + self._call(record, {"a": "Addendum"}) + assert record["resource_type"] == {"id": "publication-other"} + assert record["_resource_type_rank"] == _RANK_SERIES + + def test_numbered_addendum_matched(self): + """Unlike 490__ series, 250__ values are often followed by a + number, e.g. "Addendum 1", "addendum 2" - an exact match wouldn't + catch these.""" + for value in ("Addendum 1", "addendum 2"): + record = {} + self._call(record, {"a": value}) + assert record["resource_type"] == {"id": "publication-other"}, value + + def test_case_insensitive(self): + record = {} + self._call(record, {"a": "ADDENDUM"}) + assert record["resource_type"] == {"id": "publication-other"} + + def test_unknown_phrase_not_matched(self): + record = {} + self._call(record, {"a": "2nd ed."}) + assert "resource_type" not in record + + def test_base_imprint_behaviour_preserved(self): + """custom_fields.imprint:imprint.edition is still populated exactly + like the generic 250__ rule (research.imprint).""" + record = {} + self._call(record, {"a": "Addendum"}) + assert record["custom_fields"]["imprint:imprint"]["edition"] == "Addendum" + + def test_does_not_override_existing_report_number_match(self): + """037__/088__ is processed before 250__, so a committee report + number match must win over conflicting 250__ free text.""" + record = {} + with pytest.raises(IgnoreKey): + report_number(record, "088__", {"a": "DRDC-P-2"}) + self._call(record, {"a": "Addendum"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_wins_over_weaker_title_guess(self): + """245__ is processed before 250__, but a title-derived guess is + the weakest signal - a 250__ edition match must still override + it.""" + record = {} + title(record, "245__", {"a": "Proposal for a new experiment"}) + self._call(record, {"a": "Addendum"}) + assert record["resource_type"] == {"id": "publication-other"} + + +class TestTitleResourceType: + """Test resource_type derived from a document type mentioned anywhere + in the 245__ title (e.g. "Letter of Intent for ...", "Draft minutes of + ...") for research committee records that carry neither a report + number nor a 490__ series value.""" + + def test_letter_of_intent(self): + record = {} + title( + record, + "245__", + {"a": "Letter of Intent for a General Purpose Detector at LHC"}, + ) + assert record["resource_type"] == {"id": "publication-letter"} + assert record["_resource_type_rank"] == _RANK_TITLE + + def test_status_report(self): + record = {} + title(record, "245__", {"a": "Status report on the readout electronics"}) + assert record["resource_type"] == {"id": "publication-report"} + + def test_real_record_1015008_draft_minutes_matches_meetingminutes(self): + """https://cds.cern.ch/record/1015008 has 245__a "Draft minutes of + the third meeting of the EEC held on 21 June, 1961" - "minutes" + isn't the first word of the title.""" + record = {} + title( + record, + "245__", + { + "a": "Draft minutes of the third meeting of the EEC held on 21 June, 1961" + }, + ) + assert record["resource_type"] == {"id": "publication-meetingminutes"} + + def test_phrase_matches_anywhere_in_title(self): + """The phrase doesn't need to open the title - it's matched + wherever it appears.""" + record = {} + title( + record, + "245__", + {"a": "A study of memorandum handling in distributed systems"}, + ) + assert record["resource_type"] == {"id": "publication-memorandum"} + + def test_case_insensitive(self): + record = {} + title(record, "245__", {"a": "PROPOSAL for a new experiment"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_phrase_must_match_a_word_boundary(self): + """ "Reported" must not match "report".""" + record = {} + title(record, "245__", {"a": "Reported observations of the detector"}) + assert "resource_type" not in record + + def test_unrelated_title_not_matched(self): + record = {} + title(record, "245__", {"a": "A study of fluoride crystals for LHC"}) + assert "resource_type" not in record + + def test_subtitle_245__b_also_checked(self): + """A phrase in the subtitle (245__$b) must be detected the same + way as in the title itself (245__$a).""" + record = {} + title( + record, + "245__", + {"a": "On the readout electronics", "b": "Status report"}, + ) + assert record["resource_type"] == {"id": "publication-report"} + + def test_title_wins_over_conflicting_subtitle(self): + """$a is checked before $b, so a match in the title takes priority + over a conflicting match in the subtitle.""" + record = {} + title( + record, + "245__", + {"a": "Proposal for a new experiment", "b": "Draft minutes"}, + ) + assert record["resource_type"] == {"id": "publication-proposal"} + + def test_base_title_behaviour_preserved(self): + """title is still populated exactly like the generic 245__ rule + (base.title).""" + record = {} + result = title(record, "245__", {"a": "Proposal for a new experiment"}) + assert result == "Proposal for a new experiment" + + def test_series_still_overrides_weaker_title_guess(self): + """245__ is processed before 490__ (fields are visited in tag order + - see CdsOverdo.do), but a title-derived guess is the weakest of + the three signals - a later, more reliable 490__ series match must + still override it.""" + record = {} + title(record, "245__", {"a": "Proposal for a new experiment"}) + series_information(record, "490__", {"a": "Memorandum"}) + assert record["resource_type"] == {"id": "publication-memorandum"} + + def test_does_not_override_existing_report_number_match(self): + """037__/088__ is processed before 245__, so a committee report + number match must win over a conflicting title guess.""" + record = {} + with pytest.raises(IgnoreKey): + report_number(record, "088__", {"a": "DRDC-P-2"}) + title(record, "245__", {"a": "Memorandum on the status"}) + assert record["resource_type"] == {"id": "publication-proposal"} + + +class TestCommitteeReportNumberBaseBehaviourPreserved: + """The generic report_number (identifiers/related_identifiers) behaviour + from base.py must be fully preserved for committee records.""" + + def test_plain_report_number_goes_to_identifiers(self): + record = {} + with pytest.raises(IgnoreKey): + report_number(record, "088__", {"a": "SPSLC-P-282"}) + assert record["identifiers"] == [ + {"scheme": "cdsrn", "identifier": "SPSLC-P-282"} + ] + + def test_generic_cern_report_number_goes_to_identifiers(self): + record = {} + with pytest.raises(IgnoreKey): + report_number(record, "088__", {"a": "CERN-SPSLC-94-025"}) + assert record["identifiers"] == [ + {"scheme": "cdsrn", "identifier": "CERN-SPSLC-94-025"} + ] + + def test_arxiv_still_handled(self): + """arXiv identifiers still go through the arXiv branch of the base + rule and are excluded from `identifiers`.""" + record = {} + with pytest.raises(IgnoreKey): + report_number(record, "037__", {"a": "arXiv:1234.5678", "9": "arXiv"}) + assert record["related_identifiers"][0]["scheme"] == "arxiv" + + +class TestResearchCommitteeModelIntegration: + """End-to-end test through research_comm_model.do(), reproducing the + real record https://cds.cern.ch/record/493774 (SPSLC committee, 088__ + "CERN-SPSLC-94-025" and "SPSLC-P-282").""" + + def test_record_493774_resource_type_detected_alongside_committee(self): + blob = GroupableOrderedDict( + ( + ( + "088__", + ({"a": "CERN-SPSLC-94-025"}, {"a": "SPSLC-P-282"}), + ), + ("980__", {"a": "SCICOMMPUBLSPSLC"}), + ) + ) + out = research_comm_model.do(blob) + + assert out["resource_type"] == {"id": "publication-proposal"} + assert out["custom_fields"]["cern:committees"] == [{"id": "SPSLC"}] + assert {"scheme": "cdsrn", "identifier": "CERN-SPSLC-94-025"} in out[ + "identifiers" + ] + assert {"scheme": "cdsrn", "identifier": "SPSLC-P-282"} in out["identifiers"] + + def test_committee_report_type_not_clobbered_by_generic_980_type(self): + """037__/088__ is processed before 980__ (fields are visited in tag + order - see CdsOverdo.do). A generic 980__ document-type tag + arriving after a committee report number must not override the + more specific, report-number-derived resource_type.""" + blob = GroupableOrderedDict( + ( + ("088__", {"a": "SPSC-I-170"}), + ("980__", ({"a": "ARTICLE"}, {"a": "SCICOMMPUBLSPSC"})), + ) + ) + out = research_comm_model.do(blob) + + assert out["resource_type"] == {"id": "publication-letter"} + + +class TestResourceTypeFinalizer: + """`_resource_type` (inside RecordEntry._metadata) must strip + the `_resource_type_rank` bookkeeping key and raise when no rule ever + resolved a resource_type, rather than silently defaulting.""" + + @pytest.fixture + def entry(self): + """RecordEntry double for calling _metadata() in isolation. + + Bypasses the constructor (which needs a real dump/dojson_entry) - + this class only exercises _metadata() directly. + """ + record_entry = RecordEntry.__new__(RecordEntry) + record_entry.migration_logger = None + record_entry.affiliations_mapping = None + return record_entry + + def _raw_dump_entry(self): + return {"files": []} + + def test_resource_type_passed_through(self, entry): + dojson_entry = { + "recid": 1, + "resource_type": {"id": "publication-proposal"}, + "status_week_date": "1994-01-01", + "_resource_type_rank": float("-inf"), + } + metadata = entry._metadata(dojson_entry, self._raw_dump_entry()) + assert metadata["resource_type"] == {"id": "publication-proposal"} + + def test_rank_scratch_key_removed_from_entry(self, entry): + dojson_entry = { + "recid": 1, + "resource_type": {"id": "publication-proposal"}, + "status_week_date": "1994-01-01", + "_resource_type_rank": float("-inf"), + } + entry._metadata(dojson_entry, self._raw_dump_entry()) + assert "_resource_type_rank" not in dojson_entry + + def test_raises_when_no_resource_type_resolved_at_all(self, entry): + """A record for which no rule ever resolved a resource_type (and no + committee-report-number type either) must raise, not silently fall + back to publication-other.""" + dojson_entry = { + "recid": 1, + "status_week_date": "1994-01-01", + } + with pytest.raises(MissingRequiredField): + entry._metadata(dojson_entry, self._raw_dump_entry()) + + +class TestResearchCommitteeModelDoesNotDefaultResourceType: + """ResearchCommitteeModel must not seed resource_type with a default - + a record where no 980__/697C_ occurrence or committee report number + resolves a real type should end up with no resource_type key at all, + so it's caught as a missing required field downstream.""" + + def test_committee_only_record_has_no_resource_type(self): + """A record with only a committee tag (no resolvable document type) + must not end up with resource_type=publication-other.""" + blob = GroupableOrderedDict((("980__", {"a": "SCICOMMPUBLSPSLC"}),)) + out = research_comm_model.do(blob) + assert "resource_type" not in out From 66007e569a591e2745d05bc3222cfb78a22b7c85 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Tue, 25 Aug 2026 13:45:23 +0200 Subject: [PATCH 5/9] chore(global): fix 3.9 compat, fix cache config --- cds_migrator_kit/config.py | 4 +++- .../rdm/records/transform/mappers/custom_fields.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cds_migrator_kit/config.py b/cds_migrator_kit/config.py index e115566d..1b8b2603 100644 --- a/cds_migrator_kit/config.py +++ b/cds_migrator_kit/config.py @@ -86,7 +86,9 @@ def _(x): # Cache # ===== -CACHE_TYPE = "null" +# flask caching v2.5 +# https://github.com/inveniosoftware/invenio-cache/blob/master/invenio_cache/config.py#L19C1-L20C1 +CACHE_TYPE = "flask_caching.backends.RedisCache" # JSONSchemas # =========== diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index 8fbef9b1..ea220869 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -85,7 +85,7 @@ def apply(self, ctx): value=department, field="710", message=f"conflict on administrative unit " - f"{ctx.custom_fields["cern:administrative_unit"]} VS {department}", + f"{ctx.custom_fields['cern:administrative_unit']} VS {department}", stage="vocabulary match", ) ctx.custom_fields["cern:administrative_unit"] = department From 3f60b4ac8f749dc343b97d76237eb8de4414d6da Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 26 Aug 2026 13:54:10 +0200 Subject: [PATCH 6/9] chore(transform and load): adapt changes to committee approval --- .../rdm/records/load/entities/parent.py | 56 ++++++++++++++++++- cds_migrator_kit/rdm/records/load/load.py | 2 +- .../rdm/records/transform/transform.py | 15 +++++ .../transform/xml_processing/rules/base.py | 7 ++- 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/cds_migrator_kit/rdm/records/load/entities/parent.py b/cds_migrator_kit/rdm/records/load/entities/parent.py index 33a887ab..b09c21c1 100644 --- a/cds_migrator_kit/rdm/records/load/entities/parent.py +++ b/cds_migrator_kit/rdm/records/load/entities/parent.py @@ -10,6 +10,8 @@ from invenio_access.permissions import system_identity from invenio_accounts.models import User from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp +from invenio_pidstore.errors import PIDAlreadyExists +from invenio_pidstore.models import PersistentIdentifier, PIDStatus from invenio_rdm_records.proxies import current_rdm_records_service from cds_migrator_kit.errors import GrantCreationError, ManualImportRequired @@ -40,8 +42,9 @@ def __init__(self, entry: MigrationEntry, migration_logger, record_state): self.record_state = record_state self.migration_entry = entry self.versions = entry["versions"] + self.ep_approval = entry.get("ep_approval") - def load(self, published_record): + def load(self, published_record, uow): """Load access/communities, then access grants for every version. Called once from ``CDSMigrationEntryLoad._load()`` after the whole @@ -59,6 +62,7 @@ def load(self, published_record): """ self.load_access_and_communities(published_record) self.load_access_grants(published_record) + self._set_committee_approval(published_record, uow) def load_access_and_communities(self, draft): """Load access rights and communities in a single parent commit.""" @@ -189,3 +193,53 @@ def write_committee_approval(parent, ep_approval, uow): pf["committee_approval"] = ep_approval parent["permission_flags"] = pf uow.register(ParentRecordCommitOp(parent)) + + def _set_committee_approval(self, published_record, uow): + """Write committee_approval to parent for records already EP-approved pre-migration. + + Only runs when: + 1. entry["ep_approval"] is empty — record did NOT go through the + 9031_/EPPHAPP path (those are handled by CDSEPApprovalRecordServiceLoad + which already writes committee_approval correctly for both records). + 2. The record carries at least one apprn identifier. + + For these records the migrated record IS the final public version (no + separate internal draft exists). We write the same "public side" + committee_approval block that ep_approval_load writes, pointing + source_internal_version at the record's own PID so that + get_committee_approval_state returns is_public_approved_record=True. + """ + if self.ep_approval: + return + + record = published_record._record + identifiers = record.get("metadata", {}).get("identifiers", []) + apprn_ids = [i["identifier"] for i in identifiers if i.get("scheme") == "apprn"] + if not apprn_ids: + return + + parent = record.parent + pf = parent.get("permission_flags") or {} + if pf.get("committee_approval", {}).get("source_internal_version"): + return # idempotency: already written + + pf["committee_approval"] = { + "source_internal_version": str(record.pid.pid_value), + "reportnumber": apprn_ids[0], + } + parent["permission_flags"] = pf + uow.register(ParentRecordCommitOp(parent)) + + # Mint apprn PIDs in pidstore — same logic as ApprovalRequest._mint_apprn_pid. + from cds_rdm.requests.committee_approval import APPRN_PID_TYPE + for apprn_value in apprn_ids: + try: + PersistentIdentifier.create( + pid_type=APPRN_PID_TYPE, + pid_value=apprn_value, + object_type="rec", + object_uuid=str(record.id), + status=PIDStatus.REGISTERED, + ) + except PIDAlreadyExists: + pass # already minted on a previous run — idempotent diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 160e7bfc..33d9e462 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -138,7 +138,7 @@ def _load(self, entry: MigrationEntry): self._save_original_dumped_record(entry, recid_state_after_load) self.parent_load_cls( entry, self.migration_logger, recid_state_after_load - ).load(published_record=records[-1]) + ).load(published_record=records[-1], uow=uow) self.request_load_cls(entry).load( records, self.create_inclusion_request, uow ) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index 36e52f96..a965e4b8 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -112,6 +112,21 @@ def _transform_xml_to_json(self, raw_dump_entry): """ dump = CDSRecordDump(raw_dump_entry, preferred_model=self.preferred_model) dump.prepare_revisions() + if dump.multiple_models_warning: + w = dump.multiple_models_warning + recid = raw_dump_entry.get("recid") or raw_dump_entry.get("record", {}).get( + "recid") + matched = re.findall(r"\['(\w+)',", w.message or "") + self.migration_logger.add_information( + str(recid), + { + "type": w.type, + "error": w.description, + "message": w.message, + "value": ", ".join(matched), + "priority": "warning", + }, + ) timestamp, dojson_entry = dump.latest_revision self.dojson_entry = dojson_entry self.record_state_logger.add_record(dojson_entry) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index a2890613..614403c2 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -23,10 +23,9 @@ RDM_RECORDS_IDENTIFIERS_SCHEMES, RDM_RECORDS_RELATED_IDENTIFIERS_SCHEMES, ) -from cds_migrator_kit.rdm.records.load.ep_approval_entry import ( +from cds_migrator_kit.rdm.records.load.entities.ep_split import ( EP_APPROVAL_REPORT_NUMBER_RE, ) - from cds_migrator_kit.rdm.records.transform.config import ( CONTROLLED_SUBJECTS_SCHEMES, IDENTIFIERS_SCHEMES_TO_DROP, @@ -693,7 +692,9 @@ def licenses(self, key, value): if not license_id: # 540__f/b without a license id — side effects (funding model, copyright) already applied - raise UnexpectedValue("License title missing", field=key, subfield="a", value=value) + raise UnexpectedValue( + "License title missing", field=key, subfield="a", value=value + ) # 2897660, 2694245, 684383 if license_id in [ From c57be14ef8deffbd9f32d5b47ff0dcec29717310 Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Fri, 28 Aug 2026 16:00:14 +0200 Subject: [PATCH 7/9] fix(load): ensure consistent uow usage, rename some methods --- .../load/entities/ep_migration_entry_load.py | 15 ++++++------ .../rdm/records/load/entities/parent.py | 23 +++++++++++-------- .../rdm/records/load/entities/record.py | 23 +++++++++++-------- cds_migrator_kit/rdm/records/load/load.py | 10 ++++---- .../records/transform/entities/migration.py | 5 +++- .../rdm/records/transform/entities/version.py | 4 +++- 6 files changed, 47 insertions(+), 33 deletions(-) diff --git a/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py b/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py index fca8c0d8..c1a018b9 100644 --- a/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py +++ b/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Splits and loads an EP approval record as a public/restricted pair.""" + from invenio_access.permissions import system_identity from invenio_db import db from invenio_db.uow import UnitOfWork @@ -157,7 +158,7 @@ def _load_split(self, entry: MigrationEntry, recid): # the restricted half. self.parent_load_cls( restricted_entry, self.migration_logger, restricted_record_state - ).load(published_record=restricted_records[-1]) + ).load(published_record=restricted_records[-1], uow=uow) self.request_load_cls(restricted_entry).load( restricted_records, self.create_inclusion_request, uow ) @@ -176,7 +177,9 @@ def _load_split(self, entry: MigrationEntry, recid): ) # Original-dump persistence for the public half. - self._save_original_dumped_record(public_entry, public_record_state) + self._save_original_dumped_record( + public_entry, public_record_state, uow + ) # Link the records with related_identifiers self._append_related_identifier( @@ -275,11 +278,9 @@ def _write_parent_ep_approvals( restricted_parent = RDMParent.get_record( restricted_record_state["parent_object_uuid"] ) - public_parent = RDMParent.get_record( - public_record_state["parent_object_uuid"] - ) + public_parent = RDMParent.get_record(public_record_state["parent_object_uuid"]) - ParentLoad.write_committee_approval( + ParentLoad.write_committee_approval_obj( restricted_parent, { "reportnumber": report_number, @@ -290,7 +291,7 @@ def _write_parent_ep_approvals( }, uow, ) - ParentLoad.write_committee_approval( + ParentLoad.write_committee_approval_obj( public_parent, { "reportnumber": report_number, diff --git a/cds_migrator_kit/rdm/records/load/entities/parent.py b/cds_migrator_kit/rdm/records/load/entities/parent.py index b09c21c1..9abcfa7a 100644 --- a/cds_migrator_kit/rdm/records/load/entities/parent.py +++ b/cds_migrator_kit/rdm/records/load/entities/parent.py @@ -6,6 +6,8 @@ # the terms of the MIT License; see LICENSE file for more details. """Materializes a ``RecordParent`` against a real RDM parent record.""" + +from cds_rdm.requests.committee_approval import APPRN_PID_TYPE from flask import current_app from invenio_access.permissions import system_identity from invenio_accounts.models import User @@ -60,20 +62,20 @@ def load(self, published_record, uow): freshly-read parent instance, which ``load_access_and_communities``'s stale in-memory parent doesn't see). """ - self.load_access_and_communities(published_record) - self.load_access_grants(published_record) - self._set_committee_approval(published_record, uow) + self.load_access_and_communities(published_record, uow) + self.load_access_grants(published_record, uow) + self.set_stateless_committee_approval(published_record, uow) - def load_access_and_communities(self, draft): + def load_access_and_communities(self, draft, uow): """Load access rights and communities in a single parent commit.""" parent = draft._record.parent parent.access = self.record_parent.body["access"] for community in self.record_parent.communities["ids"]: parent.communities.add(community) parent.communities.default = self.record_parent.communities["default"] - parent.commit() + uow.register(ParentRecordCommitOp(parent)) - def load_access_grants(self, published_record): + def load_access_grants(self, published_record, uow): """Load access grants from metadata and record grants efficiently. :param draft: the draft/published record whose parent grants are set. @@ -117,6 +119,8 @@ def _create_grant(subject_type, subject_id, permission): } ] } + + # Verify the grant meets the schema current_rdm_records_service.access.schema_grants.load( grant_data, context={"identity": identity}, @@ -176,10 +180,10 @@ def _create_grant(subject_type, subject_id, permission): permission=grants_with_perms.get(email, default_permission), ) - parent.commit() + uow.register(ParentRecordCommitOp(parent)) @staticmethod - def write_committee_approval(parent, ep_approval, uow): + def write_committee_approval_obj(parent, ep_approval, uow): """Write EP approval metadata onto an already-published parent. :param parent: an ``RDMParent`` fetched by uuid (post-publish - not @@ -194,7 +198,7 @@ def write_committee_approval(parent, ep_approval, uow): parent["permission_flags"] = pf uow.register(ParentRecordCommitOp(parent)) - def _set_committee_approval(self, published_record, uow): + def set_stateless_committee_approval(self, published_record, uow): """Write committee_approval to parent for records already EP-approved pre-migration. Only runs when: @@ -231,7 +235,6 @@ def _set_committee_approval(self, published_record, uow): uow.register(ParentRecordCommitOp(parent)) # Mint apprn PIDs in pidstore — same logic as ApprovalRequest._mint_apprn_pid. - from cds_rdm.requests.committee_approval import APPRN_PID_TYPE for apprn_value in apprn_ids: try: PersistentIdentifier.create( diff --git a/cds_migrator_kit/rdm/records/load/entities/record.py b/cds_migrator_kit/rdm/records/load/entities/record.py index fe8cb3cd..e94fc617 100644 --- a/cds_migrator_kit/rdm/records/load/entities/record.py +++ b/cds_migrator_kit/rdm/records/load/entities/record.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Creates, versions, and publishes a single RDM record from a ``RecordEntry``.""" + import os from typing import Dict @@ -14,9 +15,11 @@ from flask import current_app from invenio_access.permissions import system_identity from invenio_db import db +from invenio_db.uow import ModelCommitOp from invenio_pidstore.errors import PIDAlreadyExists from invenio_pidstore.models import PersistentIdentifier, PIDStatus from invenio_rdm_records.proxies import current_rdm_records_service +from invenio_records_resources.services.uow import RecordCommitOp from psycopg2.errors import UniqueViolation from sqlalchemy.exc import IntegrityError @@ -153,11 +156,11 @@ def load_files( self.migration_logger.add_log(exc, record={"record": self.record_entry}) raise e - def load_access(self, draft, access_dict: VersionAccess): + def load_access(self, draft, access_dict: VersionAccess, uow): """Set this version's access on the published record.""" record = draft._record record.access = access_dict["access_obj"] - record.commit() + uow.register(RecordCommitOp(record)) def assign_rep_numbers(self, draft): """Mint ``cdsrn`` PIDs for this draft's report-number identifiers.""" @@ -245,8 +248,8 @@ def pre_publish(self, identity, versions, version, draft, uow): identity, draft["id"], data=missing_data, uow=uow ) - self.load_access(draft, access) - self.load_files(draft, files, uow=uow) + self.load_access(draft, access, uow) + self.load_files(draft, files, uow) return draft @@ -269,7 +272,7 @@ def after_publish_update_dois(self, identity, record, uow): ) return record - def after_publish_update_created(self, record, version_data, version): + def after_publish_update_created(self, record, version_data, version, uow): """Update created timestamp post publish. Ensures that the `created` timestamp is correctly set, preferring: @@ -289,7 +292,7 @@ def after_publish_update_created(self, record, version_data, version): ) record._record.model.created = creation_date - db.session.add(record._record.model) + uow.register(ModelCommitOp(record._record.model)) def after_publish_mint_recid(self, record): """Mint legacy ids for redirections assigned to the parent.""" @@ -301,7 +304,7 @@ def after_publish_mint_recid(self, record): # but then we get a double redirection legacy_recid_minter(legacy_recid, record._record.parent.model.id) - def after_publish_update_files_created(self, record, version_data): + def after_publish_update_files_created(self, record, version_data, uow): """Update the created date of the files post publish.""" # Fix the `created` timestamp forcing the one from the legacy system # Force the created date. This can be done after publish as the service @@ -312,7 +315,7 @@ def after_publish_update_files_created(self, record, version_data): file.model.created = arrow.get(file_data["creation_date"]).datetime.replace( tzinfo=None ) - db.session.add(file.model) + uow.register(ModelCommitOp(file.model)) def _after_publish(self, identity, published_record, entry, version, uow): """Run fixes after record publish.""" @@ -320,9 +323,9 @@ def _after_publish(self, identity, published_record, entry, version, uow): if record: published_record = record version_data = entry.get("versions", {}).get(version, {}) - self.after_publish_update_created(published_record, version_data, version) + self.after_publish_update_created(published_record, version_data, version, uow) self.after_publish_mint_recid(published_record) - self.after_publish_update_files_created(published_record, version_data) + self.after_publish_update_files_created(published_record, version_data, uow) def _load_versions(self, entry, uow): """Create, publish, and run after-publish fixes for every version.""" diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 33d9e462..5f81dd83 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -14,7 +14,7 @@ from cds_rdm.legacy.resolver import get_pid_by_legacy_recid from cds_rdm.minters import legacy_recid_minter from invenio_db import db -from invenio_db.uow import UnitOfWork +from invenio_db.uow import ModelCommitOp, UnitOfWork from invenio_i18n import _ from invenio_pidstore.models import PersistentIdentifier from invenio_rdm_migrator.load.base import Load @@ -74,7 +74,7 @@ def _apply_clc_sync(self, record_state, entry: MigrationEntry): db.session.add(sync) db.session.commit() - def _save_original_dumped_record(self, entry: MigrationEntry, recid_state): + def _save_original_dumped_record(self, entry: MigrationEntry, recid_state, uow): """Save the original dumped record. This is the originally extracted record before any transformation. @@ -86,7 +86,7 @@ def _save_original_dumped_record(self, entry: MigrationEntry, recid_state): migrated_record_object_uuid=recid_state["latest_version_object_uuid"], legacy_recid=entry["record"].recid, ) - db.session.add(_original_dump_model) + uow.register(ModelCommitOp(_original_dump_model)) @staticmethod def _have_migrated_recid(recid): @@ -135,7 +135,9 @@ def _load(self, entry: MigrationEntry): recid, records ) if recid_state_after_load: - self._save_original_dumped_record(entry, recid_state_after_load) + self._save_original_dumped_record( + entry, recid_state_after_load, uow + ) self.parent_load_cls( entry, self.migration_logger, recid_state_after_load ).load(published_record=records[-1], uow=uow) diff --git a/cds_migrator_kit/rdm/records/transform/entities/migration.py b/cds_migrator_kit/rdm/records/transform/entities/migration.py index 3c3ff7f6..304931c5 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/migration.py +++ b/cds_migrator_kit/rdm/records/transform/entities/migration.py @@ -6,8 +6,11 @@ # the terms of the MIT License; see LICENSE file for more details. """The full ETL entry yielded by ``CDSToRDMRecordTransform.run()``.""" + from typing import Any, Dict, List, TypedDict +from typing_extensions import NotRequired + from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest @@ -41,6 +44,6 @@ class MigrationEntry(TypedDict): # not a plain dict; see entities/request.py. _request_data: RecordRequest # EP approval workflow entries for this record, if any (possibly []). - ep_approval: List[dict] + ep_approval: NotRequired[List[dict]] _original_dump: dict _clc_sync: Any diff --git a/cds_migrator_kit/rdm/records/transform/entities/version.py b/cds_migrator_kit/rdm/records/transform/entities/version.py index 67d29646..0ea31b4f 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/version.py +++ b/cds_migrator_kit/rdm/records/transform/entities/version.py @@ -6,11 +6,13 @@ # the terms of the MIT License; see LICENSE file for more details. """One record version - a value in ``MigrationEntry["versions"]``.""" + from pathlib import Path from typing import Dict, Optional, TypedDict, Union import arrow from arrow import Arrow +from typing_extensions import Required LEGACY_FILES_PATH_ROOT = Path("/opt/cdsweb/var/data/files/") @@ -29,7 +31,7 @@ class VersionAccess(TypedDict, total=False): ``load.py::_load_record_access`` (``record.access = access_dict["access_obj"]``). """ - access_obj: VersionAccessObj + access_obj: Required[VersionAccessObj] # Raw legacy file-restriction status string, present only when an # individual file carried its own restriction - see # RecordVersion.compute_access(). From a5f7dac93f6178a5e45e76d427229ec30fac4bfb Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Mon, 31 Aug 2026 09:30:22 +0200 Subject: [PATCH 8/9] fix(load): finalise the log even on dry runs --- .../rdm/records/load/entities/ep_migration_entry_load.py | 3 +++ cds_migrator_kit/rdm/records/load/load.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py b/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py index c1a018b9..9ccd6889 100644 --- a/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py +++ b/cds_migrator_kit/rdm/records/load/entities/ep_migration_entry_load.py @@ -140,6 +140,9 @@ def _load_split(self, entry: MigrationEntry, recid): approval_request_load.create(restricted_record_state) # 3. Create public record public_record_load.load(public_entry) + # We need to finalise here, or the log would only list the + # records that failed. + self.migration_logger.finalise_record(recid) return with UnitOfWork(db.session) as uow: diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 5f81dd83..6fad6305 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -60,8 +60,12 @@ def __init__( self.parent_load_cls = ParentLoad self.record_load_cls = RecordLoad self.request_load_cls = RequestLoad - with open(legacy_pids_to_redirect, "r") as fp: - self.legacy_pids_to_redirect = json.load(fp) + + if legacy_pids_to_redirect is not None: + with open(legacy_pids_to_redirect, "r") as fp: + self.legacy_pids_to_redirect = json.load(fp) + else: + self.legacy_pids_to_redirect = {} def _apply_clc_sync(self, record_state, entry: MigrationEntry): """Create the CLC sync entry after the load has committed.""" @@ -128,6 +132,7 @@ def _load(self, entry: MigrationEntry): if self.dry_run: record_load.dry_load() recid_state_after_load = None + self.migration_logger.finalise_record(recid) else: with UnitOfWork(db.session) as uow: records = record_load.load(entry, uow=uow) From f2a08e61ee1aa6456e8e5c5e66fb9317c81a0d2c Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Mon, 31 Aug 2026 09:30:36 +0200 Subject: [PATCH 9/9] fix(transform): prevent errors in mappers --- .../transform/mappers/custom_fields.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index ea220869..9a28a298 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -6,7 +6,12 @@ # the terms of the MIT License; see LICENSE file for more details. """``custom_fields`` mappers for CDS to RDM record transformation.""" -from cds_migrator_kit.errors import RecordFlaggedCuration, UnexpectedValue + +from cds_migrator_kit.errors import ( + MissingRequiredField, + RecordFlaggedCuration, + UnexpectedValue, +) from cds_migrator_kit.rdm.records.transform.config import EXPERIMENT_ALIASES from cds_migrator_kit.rdm.records.transform.mappers.base import CustomFieldMapper from cds_migrator_kit.rdm.records.transform.mappers.vocabulary import search_vocabulary @@ -29,9 +34,7 @@ def apply(self, ctx): for experiment in experiments: if experiment.lower().strip() in ["not applicable", "xx"]: continue - experiment = EXPERIMENT_ALIASES.get( - experiment.lower().strip(), experiment - ) + experiment = EXPERIMENT_ALIASES.get(experiment.lower().strip(), experiment) result = search_vocabulary(experiment, "experiments") if result and result not in experiments_out: experiments_out.append(result) @@ -85,7 +88,7 @@ def apply(self, ctx): value=department, field="710", message=f"conflict on administrative unit " - f"{ctx.custom_fields['cern:administrative_unit']} VS {department}", + f"{ctx.custom_fields['cern:administrative_unit']} VS {department}", stage="vocabulary match", ) ctx.custom_fields["cern:administrative_unit"] = department @@ -95,7 +98,7 @@ def apply(self, ctx): value=department, field="department", message=f"Department {department} not found. " - f"Added as unit and subject", + f"Added as unit and subject", stage="vocabulary match", ) ) @@ -170,6 +173,10 @@ def apply(self, ctx): """Set ctx.custom_fields["cern:programmes"], or leave it unset.""" record_json = ctx.dojson_entry programme = record_json.get("custom_fields", {}).get("cern:programmes") + resource_type = record_json.get("resource_type") + if resource_type is None: + raise MissingRequiredField(message="resource_type", field="980") + if programme: result = search_vocabulary(programme, "programmes") if not result: @@ -180,7 +187,7 @@ def apply(self, ctx): stage="vocabulary match", ) ctx.custom_fields["cern:programmes"] = result - elif record_json["resource_type"] == "publication-thesis": + elif resource_type == "publication-thesis": ctx.custom_fields["cern:programmes"] = {"id": "None"}