From dd7fe751440b7fc268be6469ac2e756da671c27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=C3=BCbeyde=20Civelek?= Date: Fri, 21 Aug 2026 10:36:34 +0200 Subject: [PATCH 1/3] change(tests): include rdm and videos packages --- run-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run-tests.sh b/run-tests.sh index fa5ae42f..e8bf5213 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -40,10 +40,10 @@ for arg in $@; do keep_services=1 ;; rdm) - pytest_args+=( "tests/cds-rdm" "tests/test_cds_migrator_kit.py" ) + pytest_args+=( "tests/cds-rdm" "tests/test_cds_migrator_kit.py" "cds_migrator_kit/rdm" ) ;; videos) - pytest_args+=( "tests/cds-videos" "tests/test_cds_migrator_kit.py" ) + pytest_args+=( "tests/cds-videos" "tests/test_cds_migrator_kit.py" "cds_migrator_kit/videos/weblecture_migration" ) ;; *) pytest_args+=( ${arg} ) From 323731d81a08abb7c8c6bb934c4786485f119744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=C3=BCbeyde=20Civelek?= Date: Fri, 11 Sep 2026 15:12:45 +0200 Subject: [PATCH 2/3] fix(rdm): black, isort and pydocstyle --- cds_migrator_kit/rdm/affiliations/__init__.py | 1 + cds_migrator_kit/rdm/affiliations/load.py | 1 + cds_migrator_kit/rdm/affiliations/streams.py | 1 + .../rdm/affiliations/transform.py | 5 ++- cds_migrator_kit/rdm/cli.py | 1 + cds_migrator_kit/rdm/comments/load.py | 1 + cds_migrator_kit/rdm/comments/log.py | 4 +- cds_migrator_kit/rdm/comments/runner.py | 8 ++-- cds_migrator_kit/rdm/comments/streams.py | 1 + .../rdm/records/load/entities/__init__.py | 1 + .../records/load/entities/approval_request.py | 2 + .../load/entities/approval_request_load.py | 5 +-- .../rdm/records/load/entities/ep_split.py | 3 ++ .../rdm/records/load/entities/request.py | 3 +- cds_migrator_kit/rdm/records/streams.py | 1 + .../records/transform/entities/__init__.py | 1 + .../rdm/records/transform/entities/parent.py | 1 + .../rdm/records/transform/entities/record.py | 4 +- .../rdm/records/transform/entities/request.py | 2 +- .../rdm/records/transform/mappers/base.py | 1 + .../records/transform/mappers/contributors.py | 7 +--- .../rdm/records/transform/mappers/metadata.py | 1 + .../rdm/records/transform/mappers/record.py | 1 + .../rdm/records/transform/mappers/registry.py | 1 + .../records/transform/mappers/vocabulary.py | 1 + .../rdm/records/transform/models/ab_dep.py | 13 ++++--- .../records/transform/models/annual_report.py | 1 + .../rdm/records/transform/models/antares.py | 3 +- .../rdm/records/transform/models/at.py | 2 + .../models/base_publication_record.py | 2 + .../rdm/records/transform/models/beams.py | 1 + .../rdm/records/transform/models/books.py | 1 + .../transform/models/bulletin_issue.py | 1 + .../rdm/records/transform/models/courier.py | 1 + .../rdm/records/transform/models/en.py | 1 + .../rdm/records/transform/models/fap.py | 1 + .../rdm/records/transform/models/hr.py | 1 + .../rdm/records/transform/models/it.py | 1 + .../records/transform/models/it_meetings.py | 1 + .../rdm/records/transform/models/lcd.py | 3 +- .../rdm/records/transform/models/lep.py | 11 +++--- .../rdm/records/transform/models/lhcf.py | 1 + .../rdm/records/transform/models/mous.py | 1 + .../rdm/records/transform/models/research.py | 1 + .../transform/models/research_committee.py | 1 + .../transform/models/staff_association.py | 1 + .../transform/models/summer_student_report.py | 1 + .../rdm/records/transform/models/sy.py | 1 + .../rdm/records/transform/models/te.py | 1 + .../transform/models/technical_support.py | 15 +++++--- .../transform/models/yellow_reports.py | 1 + .../rdm/records/transform/transform.py | 37 +++++++++++-------- .../records/transform/transform_versions.py | 2 +- .../xml_processing/rules/annual_rep.py | 5 ++- .../transform/xml_processing/rules/at.py | 4 ++ .../transform/xml_processing/rules/base.py | 3 +- .../transform/xml_processing/rules/beams.py | 2 + .../transform/xml_processing/rules/books.py | 4 +- .../xml_processing/rules/bulletin_issue.py | 12 +++++- .../xml_processing/rules/cms_notes.py | 1 - .../transform/xml_processing/rules/courier.py | 6 ++- .../transform/xml_processing/rules/en.py | 2 + .../transform/xml_processing/rules/hr.py | 5 +++ .../transform/xml_processing/rules/it.py | 5 ++- .../xml_processing/rules/it_meetings.py | 2 + .../xml_processing/rules/research.py | 13 ++++++- .../rules/research_committee.py | 32 +++++++++++----- .../xml_processing/rules/small_experiments.py | 6 ++- .../xml_processing/rules/staff_association.py | 2 + .../rules/summer_student_report.py | 1 + .../transform/xml_processing/rules/sy.py | 2 + .../transform/xml_processing/rules/te.py | 2 + .../xml_processing/rules/technical_support.py | 12 ++++-- .../transform/xml_processing/rules/thesis.py | 5 +++ .../xml_processing/rules/yellow_report.py | 2 + cds_migrator_kit/rdm/stats/event_generator.py | 1 + cds_migrator_kit/rdm/stats/load.py | 5 ++- cds_migrator_kit/rdm/stats/search.py | 6 ++- cds_migrator_kit/rdm/stats/streams.py | 1 + cds_migrator_kit/rdm/users/api.py | 1 + cds_migrator_kit/rdm/users/streams.py | 1 + .../rdm/users/transform/__init__.py | 1 + cds_migrator_kit/rdm/users/transform/users.py | 2 + .../xml_processing/rules/reviewers.py | 2 +- .../xml_processing/quality/contributors.py | 8 +++- setup.cfg | 4 ++ 86 files changed, 240 insertions(+), 85 deletions(-) diff --git a/cds_migrator_kit/rdm/affiliations/__init__.py b/cds_migrator_kit/rdm/affiliations/__init__.py index 80489cc6..eae0d9ec 100644 --- a/cds_migrator_kit/rdm/affiliations/__init__.py +++ b/cds_migrator_kit/rdm/affiliations/__init__.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration stats module.""" + from cds_dojson.overdo import OverdoBase affiliations_migrator_marc21 = OverdoBase( diff --git a/cds_migrator_kit/rdm/affiliations/load.py b/cds_migrator_kit/rdm/affiliations/load.py index f87c13c4..7824e554 100644 --- a/cds_migrator_kit/rdm/affiliations/load.py +++ b/cds_migrator_kit/rdm/affiliations/load.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration load module.""" + import json import logging import os diff --git a/cds_migrator_kit/rdm/affiliations/streams.py b/cds_migrator_kit/rdm/affiliations/streams.py index 256039db..bb7bc9bb 100644 --- a/cds_migrator_kit/rdm/affiliations/streams.py +++ b/cds_migrator_kit/rdm/affiliations/streams.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration streams module.""" + from invenio_rdm_migrator.streams import StreamDefinition from invenio_rdm_migrator.transform import IdentityTransform diff --git a/cds_migrator_kit/rdm/affiliations/transform.py b/cds_migrator_kit/rdm/affiliations/transform.py index 5ffe592d..ae16b9a8 100644 --- a/cds_migrator_kit/rdm/affiliations/transform.py +++ b/cds_migrator_kit/rdm/affiliations/transform.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform step module.""" + import logging from copy import deepcopy @@ -44,7 +45,7 @@ def get_ror_affiliation(affiliation): except Exception as err: cli_logger.exception(err) - (chosen, affiliation) = get_ror_affiliation(affiliation_name) + chosen, affiliation = get_ror_affiliation(affiliation_name) return (chosen, affiliation) @@ -76,7 +77,7 @@ def _affiliations(self, json_entry, key): "original_input": affiliation_name, } - (chosen, match_or_suggestions) = affiliations_search(affiliation_name) + chosen, match_or_suggestions = affiliations_search(affiliation_name) if chosen: _affiliation.update( diff --git a/cds_migrator_kit/rdm/cli.py b/cds_migrator_kit/rdm/cli.py index 72488542..cea59734 100644 --- a/cds_migrator_kit/rdm/cli.py +++ b/cds_migrator_kit/rdm/cli.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM command line module.""" + import logging from datetime import datetime from pathlib import Path diff --git a/cds_migrator_kit/rdm/comments/load.py b/cds_migrator_kit/rdm/comments/load.py index 9bbe896d..08e56c67 100644 --- a/cds_migrator_kit/rdm/comments/load.py +++ b/cds_migrator_kit/rdm/comments/load.py @@ -69,6 +69,7 @@ def get_attached_files_for_comment(self, recid, comment_id): return [] def get_oldest_record(self, parent_pid_value): + """Return the oldest version of a record by parent PID.""" latest_record = current_rdm_records_service.read_latest( identity=system_identity, id_=parent_pid_value ) diff --git a/cds_migrator_kit/rdm/comments/log.py b/cds_migrator_kit/rdm/comments/log.py index ffab778c..f7bac34d 100644 --- a/cds_migrator_kit/rdm/comments/log.py +++ b/cds_migrator_kit/rdm/comments/log.py @@ -35,9 +35,7 @@ def __init__(self, log_dir, collection=None): self.log_dir = Path(log_dir) self.collection = collection # Per-collection file logs + CSV; shared dir when collection is unset - self.files_dir = ( - self.log_dir / collection if collection else self.log_dir - ) + self.files_dir = self.log_dir / collection if collection else self.log_dir os.makedirs(self.files_dir, exist_ok=True) # Initializes logging format and file handlers for logging module (not CSV report). diff --git a/cds_migrator_kit/rdm/comments/runner.py b/cds_migrator_kit/rdm/comments/runner.py index 8051e92f..17319927 100644 --- a/cds_migrator_kit/rdm/comments/runner.py +++ b/cds_migrator_kit/rdm/comments/runner.py @@ -36,8 +36,10 @@ def __init__( self.log_dir = Path(log_dir) self.logger = CommentsLogger(self.log_dir, collection) - collection_dirpath=collection_config["dir_path"] - comments_metadata_filepath = os.path.join(collection_dirpath, "comments_metadata.json") + collection_dirpath = collection_config["dir_path"] + comments_metadata_filepath = os.path.join( + collection_dirpath, "comments_metadata.json" + ) self.stream = Stream( stream_definition.name, @@ -71,7 +73,7 @@ def __init__( """Constructor.""" config = read_config(config_filepath) collection_config = config["comments"][collection] - dirpath=collection_config["dir_path"] + dirpath = collection_config["dir_path"] missing_users_dir = os.path.join(dirpath, "users") filename = "missing_commentors_from_ldap.json" diff --git a/cds_migrator_kit/rdm/comments/streams.py b/cds_migrator_kit/rdm/comments/streams.py index 046f7252..a12b3841 100644 --- a/cds_migrator_kit/rdm/comments/streams.py +++ b/cds_migrator_kit/rdm/comments/streams.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-Migrator-Kit comments streams module.""" + from invenio_rdm_migrator.streams import StreamDefinition from invenio_rdm_migrator.transform import IdentityTransform diff --git a/cds_migrator_kit/rdm/records/load/entities/__init__.py b/cds_migrator_kit/rdm/records/load/entities/__init__.py index e69de29b..5fed95ec 100644 --- a/cds_migrator_kit/rdm/records/load/entities/__init__.py +++ b/cds_migrator_kit/rdm/records/load/entities/__init__.py @@ -0,0 +1 @@ +"""CDS-RDM record load entities.""" diff --git a/cds_migrator_kit/rdm/records/load/entities/approval_request.py b/cds_migrator_kit/rdm/records/load/entities/approval_request.py index 1ccad64c..da726dad 100644 --- a/cds_migrator_kit/rdm/records/load/entities/approval_request.py +++ b/cds_migrator_kit/rdm/records/load/entities/approval_request.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Validates an EP committee approval request's legacy history.""" + from datetime import datetime, timezone from cds_rdm.requests.committee_approval import APPRN_PID_TYPE @@ -41,6 +42,7 @@ def __init__( resource_type=None, dry_run=False, ): + """Constructor.""" self.ep_approval = ep_approval self.legacy_recid = legacy_recid self.title = title diff --git a/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py b/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py index eec3e7d5..fc8ac616 100644 --- a/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py +++ b/cds_migrator_kit/rdm/records/load/entities/approval_request_load.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Creates and approves a migrated EP committee approval request.""" + from cds_rdm.requests.committee_approval import APPRN_PID_TYPE, CommitteeApprovalRequest from flask import current_app from invenio_access.permissions import system_identity @@ -247,6 +248,4 @@ def _build_request(self, restricted_recid, restricted_parent, uow): self._create_reviewing_log_event(request, uow) self._apply_approved_entry(request, uow) - uow.register( - RecordCommitOp(request, indexer=current_requests_service.indexer) - ) + uow.register(RecordCommitOp(request, indexer=current_requests_service.indexer)) diff --git a/cds_migrator_kit/rdm/records/load/entities/ep_split.py b/cds_migrator_kit/rdm/records/load/entities/ep_split.py index 9d2878f8..cf5b8895 100644 --- a/cds_migrator_kit/rdm/records/load/entities/ep_split.py +++ b/cds_migrator_kit/rdm/records/load/entities/ep_split.py @@ -58,6 +58,7 @@ class MetadataEntry: """Build a load entry for the public or restricted EP approval split.""" def __init__(self, entry: MigrationEntry, approval_request, migration_logger): + """Constructor.""" self.entry = entry self.approval_request = approval_request self.migration_logger = migration_logger @@ -215,6 +216,7 @@ def _no_versions_error_message(self): return "No public files found to load for EP approval public split" def identifiers(self, identifiers): + """Return identifiers for the public split, dropping EP approval RNs.""" kept = [] removed = [] for id_entry in identifiers: @@ -317,6 +319,7 @@ def _no_versions_error_message(self): return "No files found to load for EP approval restricted split" def identifiers(self, identifiers): + """Return identifiers for the restricted split, keeping draft RNs.""" kept = [] removed = [] for id_entry in identifiers: diff --git a/cds_migrator_kit/rdm/records/load/entities/request.py b/cds_migrator_kit/rdm/records/load/entities/request.py index 824c6d76..fabaf06b 100644 --- a/cds_migrator_kit/rdm/records/load/entities/request.py +++ b/cds_migrator_kit/rdm/records/load/entities/request.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Creates a community-inclusion request from a ``RecordRequest``.""" + import datetime from invenio_access.permissions import system_identity @@ -30,7 +31,7 @@ class RequestLoad: ``ParentLoad``. """ - def __init__(self, entry:MigrationEntry): + def __init__(self, entry: MigrationEntry): """Constructor. :param record_request: the built ``RecordRequest`` for this entry diff --git a/cds_migrator_kit/rdm/records/streams.py b/cds_migrator_kit/rdm/records/streams.py index 0e7689bd..8893fec7 100644 --- a/cds_migrator_kit/rdm/records/streams.py +++ b/cds_migrator_kit/rdm/records/streams.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration streams module.""" + from invenio_rdm_migrator.streams import StreamDefinition from cds_migrator_kit.extract.extract import LegacyExtract diff --git a/cds_migrator_kit/rdm/records/transform/entities/__init__.py b/cds_migrator_kit/rdm/records/transform/entities/__init__.py index e69de29b..7bd7b350 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/__init__.py +++ b/cds_migrator_kit/rdm/records/transform/entities/__init__.py @@ -0,0 +1 @@ +"""CDS-RDM record transform entities.""" diff --git a/cds_migrator_kit/rdm/records/transform/entities/parent.py b/cds_migrator_kit/rdm/records/transform/entities/parent.py index 00ae6f82..9f966be3 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/parent.py +++ b/cds_migrator_kit/rdm/records/transform/entities/parent.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """The RDM parent record for one migrated CDS record.""" + import re from flask import current_app diff --git a/cds_migrator_kit/rdm/records/transform/entities/record.py b/cds_migrator_kit/rdm/records/transform/entities/record.py index 9fc2d9af..5058ed30 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/record.py +++ b/cds_migrator_kit/rdm/records/transform/entities/record.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """The RDM record's own content - ``MigrationEntry["record"]``.""" + from copy import deepcopy from typing import Any, TypedDict @@ -226,8 +227,7 @@ def _verify_publication_date(self, raw_dump_entry, dojson_entry): creation date) and no creation date, raise an exception. """ if not raw_dump_entry.get("files") and not ( - dojson_entry.get("status_week_date") - or dojson_entry.get("publication_date") + dojson_entry.get("status_week_date") or dojson_entry.get("publication_date") ): raise ManualImportRequired( message="Record missing publication date", diff --git a/cds_migrator_kit/rdm/records/transform/entities/request.py b/cds_migrator_kit/rdm/records/transform/entities/request.py index 7fd45a11..9a57f759 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/request.py +++ b/cds_migrator_kit/rdm/records/transform/entities/request.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """A community-inclusion request for one migrated CDS record.""" + from invenio_accounts.models import User from invenio_db import db @@ -57,7 +58,6 @@ def __bool__(self): """True when there's request data to act on.""" return bool(self.data) - def _resolve_reviewers(self, reviewer_names): """Resolve raw reviewer name/email strings to RDM reviewer entries. diff --git a/cds_migrator_kit/rdm/records/transform/mappers/base.py b/cds_migrator_kit/rdm/records/transform/mappers/base.py index 73130fa5..ec811f5f 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/base.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/base.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Base classes shared by all CDS-RDM record field mappers.""" + from abc import ABC, abstractmethod from dataclasses import dataclass, field diff --git a/cds_migrator_kit/rdm/records/transform/mappers/contributors.py b/cds_migrator_kit/rdm/records/transform/mappers/contributors.py index 63764949..9cb52a71 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/contributors.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/contributors.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Creators/contributors field mapping: affiliations and person-id lookup.""" + from copy import deepcopy from idutils import normalize_ror @@ -124,11 +125,7 @@ def _lookup_person_id(creator): user_id = ui.user.id names = NamesMetadata.query.filter_by(internal_id=str(user_id)).all() name = next( - ( - name - for name in names - if "unlisted" not in name.json.get("tags", []) - ), + (name for name in names if "unlisted" not in name.json.get("tags", [])), None, ) # filter out cern person_id diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index a9f13362..81f8c383 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """``metadata`` field mappers for CDS to RDM record transformation.""" + from dateutil.parser import parse from cds_migrator_kit.errors import MissingRequiredField, UnexpectedValue diff --git a/cds_migrator_kit/rdm/records/transform/mappers/record.py b/cds_migrator_kit/rdm/records/transform/mappers/record.py index f8a0f3f6..1155691c 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/record.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/record.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Top-level ``record_json_output`` field mappers.""" + from cds_migrator_kit.rdm.records.transform.mappers.base import FieldMapper diff --git a/cds_migrator_kit/rdm/records/transform/mappers/registry.py b/cds_migrator_kit/rdm/records/transform/mappers/registry.py index a629f2ff..abe28f11 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/registry.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/registry.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Composed lists of field mappers used by RecordEntry.""" + from cds_migrator_kit.rdm.records.transform.mappers.base import ( PassthroughCustomFieldMapper, PassthroughMapper, diff --git a/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py b/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py index ac22daed..f207bcc3 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """YAML-backed vocabulary lookup used by custom_fields mappers.""" + from pathlib import Path import yaml diff --git a/cds_migrator_kit/rdm/records/transform/models/ab_dep.py b/cds_migrator_kit/rdm/records/transform/models/ab_dep.py index e8e2b547..e2e86514 100644 --- a/cds_migrator_kit/rdm/records/transform/models/ab_dep.py +++ b/cds_migrator_kit/rdm/records/transform/models/ab_dep.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) @@ -15,14 +16,14 @@ class LEPResearchModel(ResearchModel): """Translation model for research.""" - __query__ = '(710__.5:SI OR 710__.5:SC OR 710__.5:SL OR 710__.5:PS OR 710__.5:MPS OR 710__.5:ISR OR 710__.5:MSC OR 710__.5:AC OR 710__.5:SPS OR 710__.5:LEP OR 710__.5:AB OR 710__.5:AR) AND (980__:ARTICLE OR 980__:PREPRINT) OR 690C_.a:ISRRunning OR 690C_.a:ISRPerForm -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM -710__.5:TS -710__.5:ST -710__.5:MT -710__.5:EST -710__.5:SB -088:CERN-ALEPH-PUB-* -693__.e:ALEPH' + __query__ = "(710__.5:SI OR 710__.5:SC OR 710__.5:SL OR 710__.5:PS OR 710__.5:MPS OR 710__.5:ISR OR 710__.5:MSC OR 710__.5:AC OR 710__.5:SPS OR 710__.5:LEP OR 710__.5:AB OR 710__.5:AR) AND (980__:ARTICLE OR 980__:PREPRINT) OR 690C_.a:ISRRunning OR 690C_.a:ISRPerForm -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM -710__.5:TS -710__.5:ST -710__.5:MT -710__.5:EST -710__.5:SB -088:CERN-ALEPH-PUB-* -693__.e:ALEPH" __ignore_keys__ = { - "594__a", # can be ignored for this collection - "775__p", # can be ignored for this collection - title of another volume - "775__c", # year of volume - "596__a", # multivolume tag - "300__x", # drop the physical description + "594__a", # can be ignored for this collection + "775__p", # can be ignored for this collection - title of another volume + "775__c", # year of volume + "596__a", # multivolume tag + "300__x", # drop the physical description "8564_z", # file comment, migrated via file metadata "0248_a", "0248_p", diff --git a/cds_migrator_kit/rdm/records/transform/models/annual_report.py b/cds_migrator_kit/rdm/records/transform/models/annual_report.py index 1dd1fa28..21039237 100644 --- a/cds_migrator_kit/rdm/records/transform/models/annual_report.py +++ b/cds_migrator_kit/rdm/records/transform/models/annual_report.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/antares.py b/cds_migrator_kit/rdm/records/transform/models/antares.py index 849bc173..2d9cd17b 100644 --- a/cds_migrator_kit/rdm/records/transform/models/antares.py +++ b/cds_migrator_kit/rdm/records/transform/models/antares.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.research import ( research_model, ) @@ -18,7 +19,7 @@ class ANTARESResearchModel(CdsOverdo): __query__ = '980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' __ignore_keys__ = { - "923__r", # place of photo, present in posters but redundant + "923__r", # place of photo, present in posters but redundant "0248_a", "0248_p", "0248_q", diff --git a/cds_migrator_kit/rdm/records/transform/models/at.py b/cds_migrator_kit/rdm/records/transform/models/at.py index 52e02cb2..d3f7de8d 100644 --- a/cds_migrator_kit/rdm/records/transform/models/at.py +++ b/cds_migrator_kit/rdm/records/transform/models/at.py @@ -5,6 +5,8 @@ # CDS-RDM is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +"""CDS-RDM AT migration model.""" + from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, diff --git a/cds_migrator_kit/rdm/records/transform/models/base_publication_record.py b/cds_migrator_kit/rdm/records/transform/models/base_publication_record.py index 86fb8d81..ef79a20e 100644 --- a/cds_migrator_kit/rdm/records/transform/models/base_publication_record.py +++ b/cds_migrator_kit/rdm/records/transform/models/base_publication_record.py @@ -1,3 +1,5 @@ +"""CDS-RDM base publication migration model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/beams.py b/cds_migrator_kit/rdm/records/transform/models/beams.py index 0f3f31e2..b7e02687 100644 --- a/cds_migrator_kit/rdm/records/transform/models/beams.py +++ b/cds_migrator_kit/rdm/records/transform/models/beams.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM Beams model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/books.py b/cds_migrator_kit/rdm/records/transform/models/books.py index 9ace0328..5502ccdf 100644 --- a/cds_migrator_kit/rdm/records/transform/models/books.py +++ b/cds_migrator_kit/rdm/records/transform/models/books.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM Yellow report model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/bulletin_issue.py b/cds_migrator_kit/rdm/records/transform/models/bulletin_issue.py index 784e93d2..5e157849 100644 --- a/cds_migrator_kit/rdm/records/transform/models/bulletin_issue.py +++ b/cds_migrator_kit/rdm/records/transform/models/bulletin_issue.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/courier.py b/cds_migrator_kit/rdm/records/transform/models/courier.py index f7141582..c857458d 100644 --- a/cds_migrator_kit/rdm/records/transform/models/courier.py +++ b/cds_migrator_kit/rdm/records/transform/models/courier.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/en.py b/cds_migrator_kit/rdm/records/transform/models/en.py index edf56e1d..861aa901 100644 --- a/cds_migrator_kit/rdm/records/transform/models/en.py +++ b/cds_migrator_kit/rdm/records/transform/models/en.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/fap.py b/cds_migrator_kit/rdm/records/transform/models/fap.py index f6039ef1..b5554cc2 100644 --- a/cds_migrator_kit/rdm/records/transform/models/fap.py +++ b/cds_migrator_kit/rdm/records/transform/models/fap.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM FAP (Finance and Administrative Processes) model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/hr.py b/cds_migrator_kit/rdm/records/transform/models/hr.py index 90f3935f..fc9d8140 100644 --- a/cds_migrator_kit/rdm/records/transform/models/hr.py +++ b/cds_migrator_kit/rdm/records/transform/models/hr.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/it.py b/cds_migrator_kit/rdm/records/transform/models/it.py index 3dec8cd0..3ce490cb 100644 --- a/cds_migrator_kit/rdm/records/transform/models/it.py +++ b/cds_migrator_kit/rdm/records/transform/models/it.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM Yellow report model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/it_meetings.py b/cds_migrator_kit/rdm/records/transform/models/it_meetings.py index 02e13f7f..1f7b15a9 100644 --- a/cds_migrator_kit/rdm/records/transform/models/it_meetings.py +++ b/cds_migrator_kit/rdm/records/transform/models/it_meetings.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM IT Meetings report model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/lcd.py b/cds_migrator_kit/rdm/records/transform/models/lcd.py index 22564fec..c6517075 100644 --- a/cds_migrator_kit/rdm/records/transform/models/lcd.py +++ b/cds_migrator_kit/rdm/records/transform/models/lcd.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.research import ( research_model, ) @@ -15,7 +16,7 @@ class ANTARESResearchModel(CdsOverdo): """Translation model for research.""" - __query__ = '980__:LCD-Notes OR 980__:LCD-NOTES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + __query__ = "980__:LCD-Notes OR 980__:LCD-NOTES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM" __ignore_keys__ = { "710__b", diff --git a/cds_migrator_kit/rdm/records/transform/models/lep.py b/cds_migrator_kit/rdm/records/transform/models/lep.py index 4b5673e6..53fcadd4 100644 --- a/cds_migrator_kit/rdm/records/transform/models/lep.py +++ b/cds_migrator_kit/rdm/records/transform/models/lep.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) @@ -18,11 +19,11 @@ class LEPResearchModel(ResearchModel): __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:RE4 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 088__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* OR 088__:CERN-ALEPH-ARCH-DATA-* -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' __ignore_keys__ = { - "594__a", # can be ignored for this collection - "775__p", # can be ignored for this collection - title of another volume - "775__c", # year of volume - "596__a", # multivolume tag - "300__x", # drop the physical description + "594__a", # can be ignored for this collection + "775__p", # can be ignored for this collection - title of another volume + "775__c", # year of volume + "596__a", # multivolume tag + "300__x", # drop the physical description "8564_z", # file comment, migrated via file metadata "0248_a", "0248_p", diff --git a/cds_migrator_kit/rdm/records/transform/models/lhcf.py b/cds_migrator_kit/rdm/records/transform/models/lhcf.py index 3d04eff3..02346dfd 100644 --- a/cds_migrator_kit/rdm/records/transform/models/lhcf.py +++ b/cds_migrator_kit/rdm/records/transform/models/lhcf.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM LHCf model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/mous.py b/cds_migrator_kit/rdm/records/transform/models/mous.py index e538c787..6ed086b8 100644 --- a/cds_migrator_kit/rdm/records/transform/models/mous.py +++ b/cds_migrator_kit/rdm/records/transform/models/mous.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/research.py b/cds_migrator_kit/rdm/records/transform/models/research.py index 8cc21e8b..994bdc26 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research.py +++ b/cds_migrator_kit/rdm/records/transform/models/research.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/research_committee.py b/cds_migrator_kit/rdm/records/transform/models/research_committee.py index bb27821f..817e21f2 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/models/research_committee.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/staff_association.py b/cds_migrator_kit/rdm/records/transform/models/staff_association.py index a2505d37..77077303 100644 --- a/cds_migrator_kit/rdm/records/transform/models/staff_association.py +++ b/cds_migrator_kit/rdm/records/transform/models/staff_association.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM Staff Association model.""" + from cds_migrator_kit.rdm.records.transform.models.bulletin_issue import ( bull_issue_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/summer_student_report.py b/cds_migrator_kit/rdm/records/transform/models/summer_student_report.py index 213c13fa..c2580012 100644 --- a/cds_migrator_kit/rdm/records/transform/models/summer_student_report.py +++ b/cds_migrator_kit/rdm/records/transform/models/summer_student_report.py @@ -18,6 +18,7 @@ # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """CDS-RDM Summer student model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/sy.py b/cds_migrator_kit/rdm/records/transform/models/sy.py index b7c7b05c..fda1a2cd 100644 --- a/cds_migrator_kit/rdm/records/transform/models/sy.py +++ b/cds_migrator_kit/rdm/records/transform/models/sy.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/te.py b/cds_migrator_kit/rdm/records/transform/models/te.py index 02670c4f..f2a802c6 100644 --- a/cds_migrator_kit/rdm/records/transform/models/te.py +++ b/cds_migrator_kit/rdm/records/transform/models/te.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/models/technical_support.py b/cds_migrator_kit/rdm/records/transform/models/technical_support.py index 71166d83..2f39f9a3 100644 --- a/cds_migrator_kit/rdm/records/transform/models/technical_support.py +++ b/cds_migrator_kit/rdm/records/transform/models/technical_support.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM CMS note model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) @@ -25,15 +26,15 @@ class TechnicalSupportModel(CdsOverdo): __ignore_keys__ = { "030__a", "690c_a", - "595__i", # "INSPEC" id - "500__9", # source of notes - "035__z", + "595__i", # "INSPEC" id + "500__9", # source of notes + "035__z", "0248_a", "0248_p", "0248_q", "100__m", # email of contributor - "110__c", # Location of corporate author - "110__g", # Type of corporate author + "110__c", # Location of corporate author + "110__g", # Type of corporate author "340__a", # Physical medium "037__c", # arxiv subject "300__a", # number of pages @@ -62,7 +63,9 @@ class TechnicalSupportModel(CdsOverdo): "964__a", # number of physical copies } - _default_fields = {"custom_fields": {},} + _default_fields = { + "custom_fields": {}, + } technical_support_model = TechnicalSupportModel( diff --git a/cds_migrator_kit/rdm/records/transform/models/yellow_reports.py b/cds_migrator_kit/rdm/records/transform/models/yellow_reports.py index ef66c54f..25ba55e3 100644 --- a/cds_migrator_kit/rdm/records/transform/models/yellow_reports.py +++ b/cds_migrator_kit/rdm/records/transform/models/yellow_reports.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM Yellow report model.""" + from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index a965e4b8..bcb60b1b 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform step module.""" + import logging import re from collections import OrderedDict @@ -18,8 +19,10 @@ from invenio_access.permissions import system_identity from invenio_pidstore.models import PersistentIdentifier, PIDStatus from invenio_rdm_migrator.logging import Logger -from invenio_rdm_records.proxies import current_rdm_records_service, \ - current_record_communities_service +from invenio_rdm_records.proxies import ( + current_rdm_records_service, + current_record_communities_service, +) from sqlalchemy.exc import NoResultFound from cds_migrator_kit.errors import ( @@ -33,8 +36,9 @@ from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest -from cds_migrator_kit.rdm.records.transform.transform_versions import \ - RecordVersionsTransform +from cds_migrator_kit.rdm.records.transform.transform_versions import ( + RecordVersionsTransform, +) from cds_migrator_kit.transform.dumper import CDSRecordDump from cds_migrator_kit.transform.errors import LossyConversion @@ -115,7 +119,8 @@ def _transform_xml_to_json(self, raw_dump_entry): if dump.multiple_models_warning: w = dump.multiple_models_warning recid = raw_dump_entry.get("recid") or raw_dump_entry.get("record", {}).get( - "recid") + "recid" + ) matched = re.findall(r"\['(\w+)',", w.message or "") self.migration_logger.add_information( str(recid), @@ -197,12 +202,12 @@ def _transform(self, raw_dump_entry) -> Optional[MigrationEntry]: ) except ( - LossyConversion, - RestrictedFileDetected, - UnexpectedValue, - ManualImportRequired, - MissingRequiredField, - MultipleModelsMatched, + LossyConversion, + RestrictedFileDetected, + UnexpectedValue, + ManualImportRequired, + MissingRequiredField, + MultipleModelsMatched, ) as e: migration_logger.add_log(e, record=raw_dump_entry) @@ -257,6 +262,7 @@ def _load_migrated_recids(self): } def should_skip(self, raw_dump_entry): + """Return True if the legacy recid was already migrated.""" return str(raw_dump_entry["recid"]) in self._migrated_recids def _existing_record_is_restricted(self, record_id): @@ -266,9 +272,7 @@ def _existing_record_is_restricted(self, record_id): MARCXML, since access restrictions may have been changed in RDM after migration and the legacy record data would be stale. """ - record = current_rdm_records_service.read_latest( - system_identity, id_=record_id - ) + record = current_rdm_records_service.read_latest(system_identity, id_=record_id) access = record.data.get("access", {}) return access.get("record") != "public" or access.get("files") != "public" @@ -328,8 +332,9 @@ def run(self, entries): recid, { "message": "Record already migrated, skipping," - " added existing {} to communities {}".format( - recid, self.communities_ids), + " added existing {} to communities {}".format( + recid, self.communities_ids + ), "value": recid, }, ) diff --git a/cds_migrator_kit/rdm/records/transform/transform_versions.py b/cds_migrator_kit/rdm/records/transform/transform_versions.py index 9358a888..05a1f64d 100644 --- a/cds_migrator_kit/rdm/records/transform/transform_versions.py +++ b/cds_migrator_kit/rdm/records/transform/transform_versions.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Builds all of a record's versions - ``MigrationEntry["versions"]``.""" + from collections import OrderedDict from copy import deepcopy @@ -120,4 +121,3 @@ def _should_skip_file(self, file_dump): ) return True return False - diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/annual_rep.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/annual_rep.py index b8c7a1f8..2c5a28ad 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/annual_rep.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/annual_rep.py @@ -1,3 +1,5 @@ +"""CDS-RDM annual report rules.""" + from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey from idutils.normalizers import normalize_isbn @@ -22,6 +24,7 @@ def collection(self, key, value): @model.over("subjects", "^65017", override=True) def subjects(self, key, value): + """Ignore subjects for annual reports.""" raise IgnoreKey("subjects") @@ -29,7 +32,6 @@ def subjects(self, key, value): @for_each_value def related_identifiers_custom_fields(self, key, value): """Handles both custom fields and related identifiers from 962_.""" - # ------------------------------ # Related Identifiers # ------------------------------ @@ -73,6 +75,7 @@ def related_identifiers_custom_fields(self, key, value): @model.over("isbn", "(^020__)", override=True) def isbn(self, key, value): + """Translates ISBN identifiers.""" _isbn = StringValue(value.get("a", "")).parse() _isbn_material = StringValue(value.get("u", "")).parse() if _isbn: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py index b461d318..6a71d79e 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py @@ -5,6 +5,9 @@ # CDS-RDM is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. # + +"""CDS-RDM AT migration rules.""" + from dojson.errors import IgnoreKey from cds_migrator_kit.transform.xml_processing.quality.decorators import ( @@ -19,6 +22,7 @@ @model.over("contributors", "^541__") @for_each_value def contact_person(self, key, value): + """Translates contact person.""" contact_person = value.get("a", None) if contact_person is None: raise IgnoreKey("contributors") diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 5302a97e..fe10eb0a 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -265,7 +265,6 @@ def record_restriction(self, key, value): @for_each_value def report_number(self, key, value): """Translates report_number fields.""" - identifier = value.get("a", "") identifier = StringValue(identifier).parse() existing_ids = self.get("related_identifiers", []) @@ -1136,7 +1135,6 @@ def process(_note): @for_each_value def additional_titles(self, key, value): """Translates additional titles.""" - additional_desc_text = value.get("p") volume = value.get("n") if additional_desc_text: @@ -1207,6 +1205,7 @@ def validate_inspire_identifier(id_value, key): # Helper function def normalize(date_str): + """Normalize a date string to EDTF-compatible form.""" date_str = date_str.strip() if date_str.count("/") == 1: # Intervals diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/beams.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/beams.py index b5357545..66daefc9 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/beams.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/beams.py @@ -1,3 +1,5 @@ +"""CDS-RDM beams migration rules.""" + from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/books.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/books.py index 6b1d3aa0..a31adc54 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/books.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/books.py @@ -1,3 +1,5 @@ +"""CDS-RDM books migration rules.""" + import re from dateutil.parser import ParserError, parse @@ -20,7 +22,6 @@ @for_each_value def corpo_creator(self, key, value): """Translates creators field.""" - creator = value.get("a", "").strip() if not creator: @@ -47,6 +48,7 @@ def collection(self, key, value): @model.over("keywords", "^697C_") @for_each_value def keywords(self, key, value): + """Translates keywords into subjects.""" val = value.get("a") keyword_map = {"LEGSERLIB": "Legal service", "BOOKSHOP": "BOOKSHOP"} diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/bulletin_issue.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/bulletin_issue.py index b405f654..2ea12dd6 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/bulletin_issue.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/bulletin_issue.py @@ -1,3 +1,5 @@ +"""CDS-RDM bulletin issue rules.""" + import re from urllib.parse import ParseResult, urlparse @@ -39,7 +41,6 @@ def creators(self, key, value): @require(["a"]) def additional_titles_bulletin(self, key, value): """Translate additional titles.""" - # many records are missing main title, reuse the 246 field if missing title = value.get("a") if title and "title" not in self: @@ -53,7 +54,6 @@ def additional_titles_bulletin(self, key, value): @model.over("description", "^520__", override=True) def description(self, key, value): """Translates description.""" - description_text = value.get("a", "") description_text_b = value.get("b", "") description_text = description_text.replace("", "").strip() @@ -128,6 +128,7 @@ def imprint_info(self, key, value): @model.over("custom_fields", "(^773__)") def journal(self, key, value): + """Translates journal fields.""" _custom_fields = self.get("custom_fields", {}) journal_fields = _custom_fields.get("journal:journal", {}) @@ -149,6 +150,7 @@ def journal(self, key, value): @model.over("additional_descriptions", "(^500__)") @for_each_value def additional_descriptions(self, key, value): + """Translates additional descriptions.""" description = value.get("a", "").strip() curated_dm = value.get("9", "").strip() if "curated" in curated_dm: @@ -163,6 +165,7 @@ def additional_descriptions(self, key, value): @model.over("additional_descriptions", "(^590__)") @for_each_value def translated_description(self, key, value): + """Translates translated description.""" description_text = value.get("a", "") description_text_b = value.get("b", "") description_text = description_text.replace("", "").strip() @@ -185,6 +188,7 @@ def translated_description(self, key, value): @model.over("subjects", "(^650[12_][7_])|(^6531_)", override=True) @for_each_value def subjects_bulletin(self, key, value): + """Translates bulletin subjects.""" subject = value.get("a", "").strip() scheme = value.get("2", "").strip() if scheme in ["EuCARD2", "AIDA-2020"]: @@ -196,6 +200,7 @@ def subjects_bulletin(self, key, value): @model.over("url_identifiers", "^8564_", override=True) @for_each_value def urls_bulletin(self, key, value): + """Translates bulletin URLs.""" content_type = value.get("x", "") if content_type == "icon": # ignore icon urls (conditionally ignoring by accessing the value @@ -230,6 +235,7 @@ def urls_bulletin_bis(self, key, value): @model.over("custom_fields_journal", "(^916__)", override=True) def custom_fields_journal(self, key, value): + """Translates journal custom fields.""" _custom_fields = self.get("custom_fields", {}) issue = value.get("z") @@ -276,6 +282,7 @@ def bulletin_report_number(self, key, value): @model.over("custom_fields", "(^925__)") def issue_number(self, key, value): + """Translates issue number.""" _custom_fields = self.get("custom_fields", {}) issue_start = value.get("a") @@ -293,6 +300,7 @@ def issue_number(self, key, value): @model.over("bull_related_identifiers_1", "(^941__)") @for_each_value def bull_related_identifiers(self, key, value): + """Translates bulletin related identifiers.""" id = value.get("a") resource_type = value.get("t", "other") scheme = "other" diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/cms_notes.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/cms_notes.py index 3f7dc78a..6cea8302 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/cms_notes.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/cms_notes.py @@ -7,7 +7,6 @@ """CDS-RDM migration rules module.""" - from cds_migrator_kit.transform.xml_processing.quality.decorators import require from ...models.note import cms_note_model diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/courier.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/courier.py index 8ab273b5..42bf8d0c 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/courier.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/courier.py @@ -1,3 +1,5 @@ +"""CDS-RDM Courier rules.""" + import re from urllib.parse import ParseResult, urlparse @@ -138,6 +140,7 @@ def imprint_info(self, key, value): @model.over("additional_descriptions", "(^500__)") @for_each_value def additional_descriptions(self, key, value): + """Translates additional descriptions.""" description = value.get("a", "").strip() if len(description) < 3: raise IgnoreKey("additional_descriptions") @@ -149,7 +152,6 @@ def additional_descriptions(self, key, value): @model.over("description", "^520__", override=True) def description(self, key, value): """Translates description.""" - description_text = value.get("a", "") description_text_b = value.get("b", "") description_text = description_text.replace("", "").strip() @@ -165,6 +167,7 @@ def description(self, key, value): @model.over("additional_descriptions", "(^590__)") @for_each_value def translated_description(self, key, value): + """Translates translated description.""" description_text = value.get("a", "") description_text_b = value.get("b", "") description_text = description_text.replace("", "").strip() @@ -238,6 +241,7 @@ def record_submitter(self, key, value): @model.over("url_identifiers", "^8564_", override=True) @for_each_value def urls_bulletin(self, key, value): + """Translates bulletin URLs.""" content_type = value.get("x", "") if content_type == "icon": diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/en.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/en.py index 866e8c3f..a7e09697 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/en.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/en.py @@ -1,3 +1,5 @@ +"""CDS-RDM EN migration rules.""" + import math from dateutil.parser import ParserError, parse diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/hr.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/hr.py index 46569a2a..8652a045 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/hr.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/hr.py @@ -1,3 +1,5 @@ +"""CDS-RDM HR migration rules.""" + import re from dateutil.parser import ParserError, parse @@ -80,6 +82,7 @@ def additional_desc(self, key, value): @require(["a"]) @for_each_value def hr_subjects(self, key, value): + """Translates HR subjects fields.""" keyword = value.get("a") if key == "6531_": keyword = value.get("a") @@ -136,6 +139,7 @@ def collection(self, key, value): @model.over("creators", "(^110__)") @for_each_value def corpo_author(self, key, value): + """Translates corporate author.""" author = value.get("a", "").strip() if not author: raise UnexpectedValue(subfield="a", value=value, field=key) @@ -275,6 +279,7 @@ def description(self, key, value): @model.over("additional_descriptions", "(^590__)") @for_each_value def translated_description(self, key, value): + """Translates translated description.""" description_text = value.get("a", "") if description_text: _additional_description = { diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py index dafc780c..88080e49 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py @@ -1,3 +1,5 @@ +"""CDS-RDM IT migration rules.""" + import math import re @@ -122,6 +124,7 @@ def resource_type(self, key, value): @model.over("meeting_info", "^111__") def meeting(self, key, value): + """Translates meeting fields.""" _custom_fields = self.setdefault("custom_fields", {}) meeting_fields = _custom_fields.get("meeting:meeting", {}) meeting_fields["title"] = StringValue(value.get("a", "")).parse() @@ -368,7 +371,6 @@ def imprint_dates(self, key, value): @for_each_value def conference_title(self, key, value): """Translates notes and conference meeting.""" - # --- MEETING FIELD --- conference_title = StringValue(value.get("d")).parse() if conference_title: @@ -387,6 +389,7 @@ def conference_title(self, key, value): @model.over("additional_descriptions", "(^590__)") @for_each_value def translated_description(self, key, value): + """Translates translated description.""" description_text = value.get("a", "") description_text_b = value.get("b", "") description_text = description_text.replace("", "").strip() diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it_meetings.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it_meetings.py index eb65cd92..430b2459 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it_meetings.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it_meetings.py @@ -1,3 +1,5 @@ +"""CDS-RDM IT meetings rules.""" + import os import pickle from datetime import datetime diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index cac7f0ae..96ad40b2 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -1,3 +1,5 @@ +"""CDS-RDM research migration rules.""" + import re from datetime import datetime @@ -31,6 +33,7 @@ @model.over("isbns", "^020__", override_tag=True) def isbn(self, key, value): + """Translates ISBN identifiers.""" _custom_fields = self.get("custom_fields", {}) _isbn = StringValue(value.get("a", "")).parse() _isbn_u = StringValue(value.get("u", "")).parse() @@ -66,6 +69,7 @@ def isbn(self, key, value): @model.over("related_identifiers", "(^022__)") @for_each_value def issn(self, key, value): + """Translates ISSN identifiers.""" _issn = StringValue(value.get("a", "")).parse() if _issn: try: @@ -100,6 +104,7 @@ def udc(self, key, value): @model.over("creators", "(^110__)") @for_each_value def corpo_author(self, key, value): + """Translates corporate author.""" author = value.get("a", "").strip() if not author: raise UnexpectedValue(subfield="a", value=value, field=key) @@ -182,6 +187,7 @@ def abbreviation(self, key, value): @model.over("funding", "(^536__)") def funding(self, key, value): + """Translates funding information.""" _custom_fields = self.get("custom_fields", {}) programme = value.get("a") _access_info = value.get("r", "").strip().lower() @@ -225,7 +231,7 @@ def funding(self, key, value): @model.over("_approval", "(^591__)", override=True) def status(self, key, value): - + """Translates status fields.""" val_a = value.get("a", "").lower().strip() val_b = value.get("b", "").lower().strip() @@ -246,6 +252,7 @@ def status(self, key, value): @model.over("custom_fields", "(^773__)") def journal(self, key, value): + """Translates journal fields.""" _custom_fields = self.get("custom_fields", {}) journal_fields = _custom_fields.get("journal:journal", {}) year = StringValue(value.get("y", "")).parse() @@ -398,7 +405,7 @@ def access_grants(self, key, value): @model.over("internal_notes", "^562__") @for_each_value def internal_notes(self, key, value): - """Translate internal notes""" + """Translate internal notes.""" note = value.get("c", "") return {"note": note} @@ -406,6 +413,7 @@ def internal_notes(self, key, value): @model.over("contributors", "^901__") @for_each_value def organisation(self, key, value): + """Translates organisation contributors.""" contributor = value.get("u", "") return { "person_or_org": { @@ -419,6 +427,7 @@ def organisation(self, key, value): @model.over("request_reviewers", "^906__", override=True) @for_each_value def request_reviewers(self, key, value): + """Translates request reviewers.""" name = StringValue(value.get("p", "")).parse().strip() email = StringValue(value.get("m", "")).parse().strip() diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py index fe988843..8a555a56 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py @@ -74,7 +74,7 @@ "RD": {"id": "publication-report"}, "STATUS-REPORT": {"id": "publication-report"}, "MEMO": {"id": "publication-memorandum"}, - "INTERNAL-REPORT": {"id": "publication-report"}, + "INTERNAL-REPORT": {"id": "publication-report"}, } # Longest type-token-sequence first, so "STATUS-REPORT" (2 tokens) is tried @@ -126,7 +126,7 @@ "rapport": {"id": "publication-report"}, "technical note": {"id": "publication-technicalnote"}, "note": {"id": "publication-technicalnote"}, - "decision taken at the meeting": {"id": "publication-meetingminutes"} + "decision taken at the meeting": {"id": "publication-meetingminutes"}, } @@ -145,7 +145,9 @@ def _set_resource_type_if_higher_priority(self, resource_type, rank): - """Set `resource_type` (+ its `_resource_type_rank`), but only if + """Set resource_type only if the new rank outranks the current one. + + Set `resource_type` (+ its `_resource_type_rank`), but only if `rank` outranks (is strictly lower than) whatever has already been decided for this record so far - by an earlier field in tag order, or by a higher-priority rule matching the same field. @@ -176,7 +178,9 @@ def _committee_report_type(committee, type_code): def _apply_committee_report_number(self, identifier): - """Detect a `--` report number (e.g. + """Derive resource_type from a committee report number. + + Detect a `--` report number (e.g. "SPSC-I-170") and derive the record's resource_type - and, for some types, an extra subject - from the type code. @@ -247,7 +251,9 @@ def _apply_committee_report_number(self, identifier): def _apply_generic_report_number_type(self, identifier): - """Detect one of `_UNANCHORED_REPORT_TYPES`'s tokens anywhere in a + """Derive resource_type from an unanchored report-number type token. + + Detect one of `_UNANCHORED_REPORT_TYPES`'s tokens anywhere in a report number and derive the record's resource_type from it - see the comment above `_UNANCHORED_REPORT_TYPES` for why this is safe without a committee gate, unlike `_apply_committee_report_number`. @@ -274,7 +280,9 @@ def _apply_generic_report_number_type(self, identifier): def _apply_series_resource_type(self, value_a): - """Detect a document type spelled out in 490__$a (see + """Derive resource_type from a 490__ series statement. + + Detect a document type spelled out in 490__$a (see `_SERIES_RESOURCE_TYPES`) and derive the record's resource_type from it. See `_set_resource_type_if_higher_priority` and `_RANK_SERIES` for how @@ -288,7 +296,9 @@ def _apply_series_resource_type(self, value_a): def _free_text_resource_type(text): - """Return the resource_type for text containing one of + """Return a resource_type matched from free-text phrases, or None. + + Return the resource_type for text containing one of `_SERIES_RESOURCE_TYPES`'s phrases anywhere in it (e.g. "Draft minutes of the third meeting of the EEC ...", https://cds.cern.ch/record/1015008, or "Addendum 1"), matched at a word @@ -305,7 +315,9 @@ def _free_text_resource_type(text): def _apply_edition_resource_type(self, value_a): - """Detect a document type spelled out in 250__$a (edition statement, + """Derive resource_type from a 250__ edition statement. + + Detect a document type spelled out in 250__$a (edition statement, e.g. "Addendum", "Addendum 1" - see `_free_text_resource_type`) and derive the record's resource_type from it. Matched the same way as the title (anywhere, word boundary) rather than 490__ series' exact match, @@ -324,7 +336,9 @@ def _apply_edition_resource_type(self, value_a): def _apply_title_resource_type(self, title_value): - """Detect a document type mentioned anywhere in the 245__ title (see + """Derive resource_type from a document type mentioned in the title. + + Detect a document type mentioned anywhere in the 245__ title (see `_free_text_resource_type`) and derive the record's resource_type from it. diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py index 2f19ca44..a6b3b21d 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py @@ -1,11 +1,15 @@ +"""CDS-RDM small experiments rules.""" + from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey from cds_migrator_kit.errors import UnexpectedValue +from cds_migrator_kit.rdm.records.transform.models.antares import ( + antares_research_model as model, +) from .base import normalize -from cds_migrator_kit.rdm.records.transform.models.antares import antares_research_model as model @model.over("publication_date", "(^225__)", override_tag=True) def imprint_info(self, key, value): diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/staff_association.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/staff_association.py index 094fec6f..fd84d99a 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/staff_association.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/staff_association.py @@ -12,6 +12,8 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import ( additional_titles, +) +from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import ( title as base_title, ) from cds_migrator_kit.transform.xml_processing.quality.decorators import for_each_value diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/summer_student_report.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/summer_student_report.py index 61b26b6e..ce015c0b 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/summer_student_report.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/summer_student_report.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Common RDM fields.""" + from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey from dojson.utils import force_list diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/sy.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/sy.py index 27799121..f9fab7d1 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/sy.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/sy.py @@ -1,3 +1,5 @@ +"""CDS-RDM SY migration rules.""" + from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/te.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/te.py index fe832f74..e0843996 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/te.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/te.py @@ -1,3 +1,5 @@ +"""CDS-RDM TE migration rules.""" + import math from dateutil.parser import ParserError, parse diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py index 683d7eb0..6ed0a5e2 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py @@ -1,17 +1,23 @@ +"""CDS-RDM technical support rules.""" + import math from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey from dojson.utils import force_list + from cds_migrator_kit.errors import UnexpectedValue -from cds_migrator_kit.transform.xml_processing.quality.decorators import for_each_value, \ - require +from cds_migrator_kit.transform.xml_processing.quality.decorators import ( + for_each_value, + require, +) from cds_migrator_kit.transform.xml_processing.quality.parsers import StringValue -from .base import subjects as base_subjects + from ...config import IGNORED_THESIS_COLLECTIONS from ...models.technical_support import technical_support_model as model from .base import corporate_author as base_corporate_author from .base import normalize +from .base import subjects as base_subjects @model.over("administrative_unit", "^710__", override=True) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/thesis.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/thesis.py index af24de55..97744c03 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/thesis.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/thesis.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """CDS-RDM migration rules module.""" + import re from dateutil.parser import ParserError, parse @@ -192,6 +193,7 @@ def thesis(self, key, value): @model.over("dates", "(^500__)") @for_each_value def dates(self, key, value): + """Translates date fields.""" text = value.get("a", "") source = value.get("9", "") # redundant information from arxiv @@ -237,6 +239,7 @@ def dates(self, key, value): @model.over("funding", "(^536__)", override=True) def funding(self, key, value): + """Translates funding information.""" _custom_fields = self.get("custom_fields", {}) programme = value.get("a") _access_info = value.get("r", "").strip().lower() @@ -278,6 +281,7 @@ def funding(self, key, value): @model.over("affiliations", "^901__", override_tag=True) @for_each_value def rec_affiliation(self, key, value): + """Translates record affiliation.""" affiliation = value.get("u", "") if type(affiliation) is not str: raise UnexpectedValue(f"Record affiliation has a wrong format.") @@ -293,6 +297,7 @@ def rec_affiliation(self, key, value): @model.over("resource_type", "^980__", override=True) @for_each_value def collection(self, key, value): + """Translates collection into resource type.""" col = value.get("a", "") colb = value.get("b", "") if type(col) != str or type(colb) != str: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/yellow_report.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/yellow_report.py index 9c53336c..597e627c 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/yellow_report.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/yellow_report.py @@ -1,3 +1,5 @@ +"""CDS-RDM yellow report rules.""" + from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey diff --git a/cds_migrator_kit/rdm/stats/event_generator.py b/cds_migrator_kit/rdm/stats/event_generator.py index bb980203..d8b67a84 100644 --- a/cds_migrator_kit/rdm/stats/event_generator.py +++ b/cds_migrator_kit/rdm/stats/event_generator.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration stats events generator module.""" + import json from copy import deepcopy from datetime import datetime diff --git a/cds_migrator_kit/rdm/stats/load.py b/cds_migrator_kit/rdm/stats/load.py index 3838144e..34ee8f64 100644 --- a/cds_migrator_kit/rdm/stats/load.py +++ b/cds_migrator_kit/rdm/stats/load.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration load module.""" + import json import logging import os @@ -197,7 +198,9 @@ def validate_stats_for_recid(self, recid, record, event_type): try: assert legacy_total["count"] == new_total["count"] - file_ids_msg = f" - Legacy file_ids: {file_ids}" if file_ids is not None else "" + file_ids_msg = ( + f" - Legacy file_ids: {file_ids}" if file_ids is not None else "" + ) logger.warning( f"Successfully migrated statistics for {recid} `{event_type}` in RDM: {new_total['count']}{file_ids_msg}" ) diff --git a/cds_migrator_kit/rdm/stats/search.py b/cds_migrator_kit/rdm/stats/search.py index a065bba6..238d0635 100644 --- a/cds_migrator_kit/rdm/stats/search.py +++ b/cds_migrator_kit/rdm/stats/search.py @@ -50,7 +50,11 @@ def os_search( ex = None i = 0 q = generate_query( - doc_type, identifier, legacy_to_rdm_events_map, less_than_date, file_ids=file_ids + doc_type, + identifier, + legacy_to_rdm_events_map, + less_than_date, + file_ids=file_ids, ) while i < 10: try: diff --git a/cds_migrator_kit/rdm/stats/streams.py b/cds_migrator_kit/rdm/stats/streams.py index d663fea9..23c0bc55 100644 --- a/cds_migrator_kit/rdm/stats/streams.py +++ b/cds_migrator_kit/rdm/stats/streams.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration streams module.""" + from invenio_rdm_migrator.streams import StreamDefinition from invenio_rdm_migrator.transform import IdentityTransform diff --git a/cds_migrator_kit/rdm/users/api.py b/cds_migrator_kit/rdm/users/api.py index 68f6d03d..fa5f4037 100644 --- a/cds_migrator_kit/rdm/users/api.py +++ b/cds_migrator_kit/rdm/users/api.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform user.""" + import csv import json from copy import deepcopy diff --git a/cds_migrator_kit/rdm/users/streams.py b/cds_migrator_kit/rdm/users/streams.py index a917f0c3..c680d7e2 100644 --- a/cds_migrator_kit/rdm/users/streams.py +++ b/cds_migrator_kit/rdm/users/streams.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration streams module.""" + from invenio_rdm_migrator.streams import StreamDefinition from cds_migrator_kit.extract.extract import LegacyExtract diff --git a/cds_migrator_kit/rdm/users/transform/__init__.py b/cds_migrator_kit/rdm/users/transform/__init__.py index b5efdd67..365ae7bf 100644 --- a/cds_migrator_kit/rdm/users/transform/__init__.py +++ b/cds_migrator_kit/rdm/users/transform/__init__.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration stats module.""" + from cds_dojson.overdo import OverdoBase users_migrator_marc21 = OverdoBase( diff --git a/cds_migrator_kit/rdm/users/transform/users.py b/cds_migrator_kit/rdm/users/transform/users.py index 2ab54e51..bcbdea55 100644 --- a/cds_migrator_kit/rdm/users/transform/users.py +++ b/cds_migrator_kit/rdm/users/transform/users.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform user.""" + import csv import json from copy import deepcopy @@ -68,6 +69,7 @@ class CDSRDMUserTransform(Transform): """CDSUserTransform.""" def __init__(self, workers=None, throw=False, dojson_model=None): + """Constructor.""" self.dojson_model = dojson_model super().__init__(workers, throw) diff --git a/cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py b/cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py index b44a24b3..224659cb 100644 --- a/cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py +++ b/cds_migrator_kit/rdm/users/transform/xml_processing/rules/reviewers.py @@ -21,7 +21,7 @@ @model.over("reviewers", "^906__") def record_reviewer(self, key, value): - """Translate request reviewers. + r"""Translate request reviewers. 906__m holds the reviewer's email directly, same as 859__f does for the submitter. 906__p instead holds one or more "Family name, Given diff --git a/cds_migrator_kit/transform/xml_processing/quality/contributors.py b/cds_migrator_kit/transform/xml_processing/quality/contributors.py index ba4e13bc..e9c8e46d 100644 --- a/cds_migrator_kit/transform/xml_processing/quality/contributors.py +++ b/cds_migrator_kit/transform/xml_processing/quality/contributors.py @@ -6,9 +6,9 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM contributors migration module.""" -import idutils import re +import idutils from dojson.utils import force_list from cds_migrator_kit.errors import UnexpectedValue @@ -176,7 +176,11 @@ def extract_json_contributor_ids(info, orcid_subfield="k"): if new_id not in ids: ids.append(new_id) else: - raise UnexpectedValue(message="Author has invalid orcid", value=author_orcid, stage="transform") + raise UnexpectedValue( + message="Author has invalid orcid", + value=author_orcid, + stage="transform", + ) inspire = info.get("i", "") if inspire and inspire.startswith("INSPIRE-"): diff --git a/setup.cfg b/setup.cfg index 82972026..4e098690 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,10 +45,14 @@ videos = cds @ git+https://github.com/CERNDocumentServer/cds-videos@main#egg=cds srt-to-vtt==1.0.0 +# Pin formatters so Python 3.9 and 3.14 CI install the same major versions. +# black>=26 and isort>=7 require Python>=3.10 and change formatting rules. tests = pytest-black>=0.3.0 pytest-invenio>=3.0.0,<4.0.0 pytest-mock>=3 + black>=24.0.0,<26.0.0 + isort>=5.12.0,<7.0.0 [options.entry_points] From f6637745aa9ae88357eace9e9025aa72541292cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=C3=BCbeyde=20Civelek?= Date: Fri, 11 Sep 2026 15:22:38 +0200 Subject: [PATCH 3/3] fix(videos): pydocstye --- .../videos/weblecture_migration/transform/transform_files.py | 1 - .../transform/xml_processing/quality/files.py | 1 + .../transform/xml_processing/quality/multiple_video.py | 5 +++++ .../xml_processing/rules/digitized_video_lecture.py | 3 ++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cds_migrator_kit/videos/weblecture_migration/transform/transform_files.py b/cds_migrator_kit/videos/weblecture_migration/transform/transform_files.py index ceb89c89..8d3bd030 100644 --- a/cds_migrator_kit/videos/weblecture_migration/transform/transform_files.py +++ b/cds_migrator_kit/videos/weblecture_migration/transform/transform_files.py @@ -119,7 +119,6 @@ def _get_highest_and_other_composites(self, file_list): Composite videos will always be inside the media_data folder. """ - required_resolutions = {1080, 720, 480, 360} composite_videos = {} diff --git a/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/files.py b/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/files.py index 231e03d9..692f024c 100644 --- a/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/files.py +++ b/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/files.py @@ -19,6 +19,7 @@ def get_files_by_recid(recid, directory): """ Search JSON files in a directory for a given recid. + Return a list of all "files" dicts for that recid. Stop searching after the first file that contains it. """ diff --git a/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/multiple_video.py b/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/multiple_video.py index 490d5df1..10dde4a9 100644 --- a/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/multiple_video.py +++ b/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/quality/multiple_video.py @@ -228,6 +228,7 @@ def parse_entry(entry): def grouped_values_with_code(entries): """ Group flat entries into logical MARC-like groups. + A new group starts when a new 9 subfield appears. """ groups = [] @@ -253,6 +254,8 @@ def grouped_values_with_code(entries): def get_single_selector(event_id): """ + Return the single letter selector from an event id, or None. + Examples: CERN-VIDEO-C-123-A -> "a" CERN-VIDEO-C-402-A_pt1 -> "a" @@ -272,6 +275,8 @@ def get_single_selector(event_id): def match_with_code(entries, event_id, value_code="a"): """ + Match MARC-like groups against an event id selector. + Returns: matches: matched values matched_groups: raw groups that matched this record diff --git a/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/rules/digitized_video_lecture.py b/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/rules/digitized_video_lecture.py index dd290c47..71664a1e 100644 --- a/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/rules/digitized_video_lecture.py +++ b/cds_migrator_kit/videos/weblecture_migration/transform/xml_processing/rules/digitized_video_lecture.py @@ -61,6 +61,7 @@ def add_contributor(self, name, role): def validate_copyright(self, key, holder, year): + """Validate and merge copyright holder and year into the record.""" if "copyright" in self: copyright = self["copyright"] if holder.lower() not in copyright["holder"].lower(): @@ -350,6 +351,7 @@ def subject_indicators(self, key, value): def subject_categories(self, key, value): """ Translates subject_category as keywords from tag 65017,65027. + Digitization project: EPFL_MC Categories. """ keyword = value.get("a", "").strip() @@ -397,7 +399,6 @@ def creation_date(self, key, value): - cataloguer 'a' subfield (name of the curator) - cataloguer level 'b' subfield (curator?) """ - b_value = value.get("b", "").strip().lower() if b_value and b_value != "curator": raise UnexpectedValue(field=key, subfield="b", value=b_value)