Refactoring transformation and load

cds_migrator_kit/config.py

@@ -86,7 +86,9 @@ def _(x):
 
 # Cache
 # =====
-CACHE_TYPE = "null"
+# flask caching v2.5
+# https://github.com/inveniosoftware/invenio-cache/blob/master/invenio_cache/config.py#L19C1-L20C1
+CACHE_TYPE = "flask_caching.backends.RedisCache"
 
 # JSONSchemas
 # ===========

cds_migrator_kit/rdm/README.md

@@ -152,27 +152,25 @@ Run the below command to migrate records in the created community from before:
 invenio migration run
 ```
 
-#### EP approval records (`--ep-approval`)
+#### EP approval records
 
-EP approval records must be migrated in a **separate stream**. Do not mix them with regular records.
+EP approval records are detected automatically per-record (via the `9031_:EPPHAPP` history popped during transform) and split into a public/restricted pair by the loader - no separate flag or stream is needed, they can be migrated in the same run as regular records.
 
 1. **Dump** them filtered by `9031_:EPPHAPP`, for example (FASER):
 
 ```bash
 inveniomigrator dump records -q '980__a:ARTICLE or 980__a:PREPRINT and 693:"FASER" not 980:CONFERENCEPAPER not 591__b:"Draft" 9031_:EPPHAPP -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY' --file-prefix faser-papers-cern-ep --chunk-size=1000
 ```
 
-2. Configure a dedicated collection entry in `streams.yaml` pointing at that dump.
+2. Configure a collection entry in `streams.yaml` pointing at that dump.
 
-3. **Migrate** with the `--ep-approval` flag (dry run first):
+3. **Migrate** normally (dry run first):
 
 ```shell
-invenio migration run --collection faser-ep --ep-approval --dry-run
-invenio migration run --collection faser-ep --ep-approval
+invenio migration run --collection faser-ep --dry-run
+invenio migration run --collection faser-ep
 ```
 
-Without `--ep-approval`, the loader will reject EP approval records.
-
 #### Comments migration
 
 Configure the collection under `comments` in `streams.yaml` (`dir_path`, `reviewers`). Paths and reviewers are loaded from there when you pass `--collection`.

cds_migrator_kit/rdm/cli.py

@@ -22,7 +22,6 @@
     CommentsStreamDefinition,
 )
 from cds_migrator_kit.rdm.records.streams import (  # UserStreamDefinition,
-    RecordEPApprovalStreamDefinition,
     RecordStreamDefinition,
 )
 from cds_migrator_kit.rdm.stats.runner import RecordStatsRunner
@@ -70,20 +69,12 @@ def migration():
         "Can also be set per-collection in streams.yaml under transform.workers."
     ),
 )
-@click.option(
-    "--ep-approval",
-    is_flag=True,
-    help="Use the EP approval load stream (pre-EP draft snapshots without legacy minting).",
-)
 @with_appcontext
-def run(collection, dry_run=False, keep_logs=False, workers=None, ep_approval=False):
+def run(collection, dry_run=False, keep_logs=False, workers=None):
     """Run."""
     stream_config = current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"]
-    stream_definition = (
-        RecordEPApprovalStreamDefinition if ep_approval else RecordStreamDefinition
-    )
     runner = Runner(
-        stream_definitions=[stream_definition],
+        stream_definitions=[RecordStreamDefinition],
         # stream_definitions=[UserStreamDefinition],
         config_filepath=Path(stream_config).absolute(),
         dry_run=dry_run,

cds_migrator_kit/rdm/migration_config.py

@@ -87,7 +87,6 @@ def _(x):  # needed to avoid start time failure with lazy strings
 # See https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/
 
 SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://cds-rdm:cds-rdm@localhost/cds-rdm"
-SQLALCHEMY_ENGINE_OPTIONS = {"connect_args": {"options": "-c timezone=UTC"}}
 
 # Invenio-App
 # ===========

cds_migrator_kit/rdm/records/load/__init__.py

@@ -7,7 +7,6 @@
 
 """CDS-RDM Migration load package."""
 
-from .ep_approval_load import CDSEPApprovalRecordServiceLoad
-from .load import CDSRecordServiceLoad
+from .load import CDSMigrationEntryLoad
 
-__all__ = ("CDSEPApprovalRecordServiceLoad", "CDSRecordServiceLoad")
+__all__ = ("CDSMigrationEntryLoad",)