Unify mlcd and mlca: docker_X fallback, apptainer meta key, schema validation

.github/workflows/test-mlc-apptainer-core.yml

@@ -6,6 +6,7 @@ on:
     paths:
       - '.github/workflows/test-mlc-apptainer-core.yml'
       - 'mlc/**'
+      - 'automation/**'
       - 'tests/**'
       - 'pyproject.toml'
       - '!**.md'

.github/workflows/test-unit.yml

@@ -9,13 +9,15 @@ on:
     paths:
       - 'mlc/**'
       - 'tests/**'
+      - 'automation/**'
       - '.github/workflows/test-unit.yml'
       - 'pyproject.toml'
   push:
     branches: [ "main", "dev" ]
     paths:
       - 'mlc/**'
       - 'tests/**'
+      - 'automation/**'
       - '.github/workflows/test-unit.yml'
       - 'pyproject.toml'
 

automation/script/apptainer.py

@@ -8,9 +8,10 @@
 
 def apptainerfile(self_module, input_params):
 
-    # Step 1: Prune and prepare input
+    # Step 1: Prune and prepare input (remove both apptainer_ and docker_
+    # prefixed keys)
     prune_result = prune_input(
-        {'input': input_params, 'extra_keys_starts_with': ['apptainer_']})
+        {'input': input_params, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
     if prune_result['return'] > 0:
         return prune_result
 
@@ -56,13 +57,15 @@ def apptainerfile(self_module, input_params):
 
     run_state = self_module.run_state
 
-    apptainer_settings = run_state.get('docker', {})
+    apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
+    utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
+                       'append_lists': True, 'append_unique': True})
     apptainer_settings_default_env = apptainer_settings.get('default_env', {})
     for key in apptainer_settings_default_env:
         env.setdefault(key, apptainer_settings_default_env[key])
 
     if not apptainer_settings.get('run', True) and not input_params.get(
-            'apptainer_run_override', False):
+            'apptainer_run_override', input_params.get('docker_run_override', False)):
         logger.info("Apptainer 'run' is set to False in meta.yaml")
         return {'return': 0,
                 'warning': 'Apptainer run is set to false in script meta'}
@@ -86,7 +89,9 @@ def apptainerfile(self_module, input_params):
     if update_state_result['return'] > 0:
         return update_state_result
 
-    apptainer_settings = run_state.get('docker', {})
+    apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
+    utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
+                       'append_lists': True, 'append_unique': True})
 
     # Prune temporary environment variables
     run_command = copy.deepcopy(run_command_arc)
@@ -102,7 +107,7 @@ def apptainerfile(self_module, input_params):
         'tags': script_tags,
         'fake_run': True,
         'docker_settings': apptainer_settings,
-        'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', ''))
+        'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', input_params.get('docker_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', '')))
     })
     if regenerate_result['return'] > 0:
         return regenerate_result
@@ -155,10 +160,10 @@ def apptainerfile(self_module, input_params):
     apptainer_v = False
     apptainer_s = False
     if is_true(input_params.get(
-            'apptainer_v', input_params.get('apptainer_verbose', False))):
+            'apptainer_v', input_params.get('apptainer_verbose', input_params.get('docker_v', input_params.get('docker_verbose', False))))):
         apptainer_v = True
     if is_true(input_params.get(
-            'apptainer_s', input_params.get('apptainer_silent', False))):
+            'apptainer_s', input_params.get('apptainer_silent', input_params.get('docker_s', input_params.get('docker_silent', False))))):
         apptainer_s = True
 
     if apptainer_s and apptainer_v:
@@ -213,11 +218,16 @@ def apptainer_run(self_module, i):
     if quiet:
         env['MLC_QUIET'] = 'yes'
 
-    regenerate_def_file = not i.get('apptainer_noregenerate', False)
-    rebuild_apptainer_image = i.get('apptainer_rebuild', False)
+    regenerate_def_file = not i.get(
+        'apptainer_noregenerate', i.get(
+            'docker_noregenerate', False))
+    rebuild_apptainer_image = i.get(
+        'apptainer_rebuild', i.get(
+            'docker_rebuild', False))
 
-    # Prune unnecessary Apptainer-related input keys
-    r = prune_input({'input': i, 'extra_keys_starts_with': ['apptainer_']})
+    # Prune unnecessary Apptainer- and Docker-related input keys
+    r = prune_input(
+        {'input': i, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
     f_run_cmd = r['new_input']
 
     # Save current directory and prepare to search for scripts
@@ -259,7 +269,7 @@ def apptainer_run(self_module, i):
     mounts = copy.deepcopy(
         i.get(
             'apptainer_mounts',
-            []))
+            i.get('docker_mounts', [])))
     variations = meta.get('variations', {})
 
     if not hasattr(self_module, 'run_state'):
@@ -273,7 +283,9 @@ def apptainer_run(self_module, i):
 
     run_state = self_module.run_state
 
-    apptainer_settings = run_state.get('docker', {})
+    apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
+    utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
+                       'append_lists': True, 'append_unique': True})
 
     apptainer_settings_default_env = apptainer_settings.get('default_env', {})
     for key in apptainer_settings_default_env:
@@ -297,7 +309,7 @@ def apptainer_run(self_module, i):
 
     # Skip scripts marked as non-runnable
     if not apptainer_settings.get('run', True) and not i.get(
-            'apptainer_run_override', False):
+            'apptainer_run_override', i.get('docker_run_override', False)):
         logger.info("apptainer.run set to False in meta.yaml")
         return {'return': 0,
                 'warning': 'Apptainer run is set to false in script meta'}
@@ -443,14 +455,15 @@ def prepare_apptainer_inputs(input_params, apptainer_settings,
             "network", "security_opt"
         ]
 
-    # Collect inputs
-    apptainer_inputs = {
-        key: input_params.get(
-            f"apptainer_{key}", apptainer_settings.get(
-                key, get_apptainer_default(key)))
-        for key in keys
-        if (value := input_params.get(f"apptainer_{key}", apptainer_settings.get(key, get_apptainer_default(key)))) is not None
-    }
+    # Collect inputs: apptainer_X > docker_X > meta settings > default
+    apptainer_inputs = {}
+    for key in keys:
+        value = input_params.get(
+            f"apptainer_{key}",
+            input_params.get(
+                f"docker_{key}", apptainer_settings.get(key, get_apptainer_default(key))))
+        if value is not None:
+            apptainer_inputs[key] = value
 
     # Convert boolean values to 'yes'/'no' strings for MLC input mapping
     for key in list(apptainer_inputs.keys()):

automation/script/meta_schema.py

@@ -86,6 +86,7 @@
 
     # Docker
     "docker": DICT,        # dict - see DOCKER_SCHEMA
+    "apptainer": DICT,     # dict - apptainer overrides; merges with docker settings
 
     # Output / debugging
     "print_env_at_the_end": DICT,        # dict[str, list[str]]
@@ -164,6 +165,7 @@
     "state": DICT,
     "const": DICT,
     "docker": DICT,
+    "apptainer": DICT,
     "alias": STR,
     "default_version": STR_OR_FLOAT,
     "required_disk_space": INT,
@@ -248,6 +250,7 @@
     "default_env": DICT,
     "default_variations": DICT,
     "docker": DICT,
+    "apptainer": DICT,
     "adr": DICT,
     "ad": DICT,
 }
@@ -403,6 +406,20 @@ def validate_meta(data, file_path=""):
                 errors.append(
                     f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
 
+    # Validate apptainer section (same schema as docker; apptainer overrides docker)
+    apptainer = data.get("apptainer")
+    if isinstance(apptainer, dict):
+        for ak, av in apptainer.items():
+            if ak not in DOCKER_SCHEMA:
+                warnings.append(
+                    f"{prefix}apptainer: unknown key '{ak}'")
+                continue
+            actual = type(av).__name__
+            allowed = DOCKER_SCHEMA[ak]
+            if actual not in allowed:
+                errors.append(
+                    f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
+
     # Validate tests section
     tests = data.get("tests")
     if isinstance(tests, dict):

automation/script/module.py

@@ -151,7 +151,7 @@ def init_run_state(self, run_state):
         for f in ['fake_deps', 'cache']:
             run_state.setdefault(f, False)
 
-        for d in ['input_mapping', 'docker', 'remote_run']:
+        for d in ['input_mapping', 'docker', 'remote_run', 'apptainer']:
             run_state.setdefault(d, {})
 
         for l in ['deps', 'post_deps', 'prehook_deps', 'posthook_deps',
@@ -5768,6 +5768,14 @@ def _apply_conditional_meta_updates(update_meta_if_env, default_env, env, const,
                                'append_lists': True,
                                'append_unique': True})
 
+        if c_meta.get('apptainer', {}):
+            if not run_state.get('apptainer', {}):
+                run_state['apptainer'] = {}
+            utils.merge_dicts({'dict1': run_state['apptainer'],
+                               'dict2': c_meta['apptainer'],
+                               'append_lists': True,
+                               'append_unique': True})
+
         c_add_deps_info = c_meta.get('ad', {})
         if not c_add_deps_info:
             c_add_deps_info = c_meta.get('add_deps', {})
@@ -5975,6 +5983,13 @@ def update_state_from_meta(meta, env, state, const, const_state, run_state, i):
         if remote_run_settings.get('deps', []):
             update_deps(remote_run_settings['deps'], add_deps_info, False, env)
 
+    new_apptainer_settings = meta.get('apptainer')
+    if new_apptainer_settings:
+        utils.merge_dicts({'dict1': run_state['apptainer'],
+                           'dict2': new_apptainer_settings,
+                           'append_lists': True,
+                           'append_unique': True})
+
     new_env_keys_from_meta = meta.get('new_env_keys', [])
     if new_env_keys_from_meta:
         run_state['new_env_keys'] += new_env_keys_from_meta

mlc/meta_schema.py

@@ -86,6 +86,7 @@
 
     # Docker
     "docker": DICT,        # dict - see DOCKER_SCHEMA
+    "apptainer": DICT,     # dict - apptainer overrides; merges with docker settings
 
     # Output / debugging
     "print_env_at_the_end": DICT,        # dict[str, list[str]]
@@ -164,6 +165,7 @@
     "state": DICT,
     "const": DICT,
     "docker": DICT,
+    "apptainer": DICT,
     "alias": STR,
     "default_version": STR_OR_FLOAT,
     "required_disk_space": INT,
@@ -248,6 +250,7 @@
     "default_env": DICT,
     "default_variations": DICT,
     "docker": DICT,
+    "apptainer": DICT,
     "adr": DICT,
     "ad": DICT,
 }
@@ -403,6 +406,20 @@ def validate_meta(data, file_path=""):
                 errors.append(
                     f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
 
+    # Validate apptainer section (same schema as docker; apptainer overrides docker)
+    apptainer = data.get("apptainer")
+    if isinstance(apptainer, dict):
+        for ak, av in apptainer.items():
+            if ak not in DOCKER_SCHEMA:
+                warnings.append(
+                    f"{prefix}apptainer: unknown key '{ak}'")
+                continue
+            actual = type(av).__name__
+            allowed = DOCKER_SCHEMA[ak]
+            if actual not in allowed:
+                errors.append(
+                    f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
+
     # Validate tests section
     tests = data.get("tests")
     if isinstance(tests, dict):

mlc/script_action.py

@@ -441,6 +441,38 @@ def apptainer_run(self, run_args):
     mlc apptainer script --tags=detect,os -j
     mlca detect,os -j
 
+    Flags Available (--apptainer_X takes priority over --docker_X for each option):
+
+    1. --apptainer_rebuild / --docker_rebuild:
+       Force rebuild of the Apptainer image even if it already exists.
+
+    2. --apptainer_noregenerate / --docker_noregenerate:
+       Skip regenerating the Apptainer definition file before running.
+
+    3. --apptainer_mounts / --docker_mounts:
+       List of bind mounts to pass to the container (host:container format).
+
+    4. --apptainer_run_cmd_prefix / --docker_run_cmd_prefix:
+       Command prefix to prepend before the mlcr command inside the container.
+
+    5. --apptainer_verbose / --apptainer_v / --docker_verbose / --docker_v:
+       Enable verbose output inside the container.
+
+    6. --apptainer_silent / --apptainer_s / --docker_silent / --docker_s:
+       Enable silent output inside the container.
+
+    7. --apptainer_run_override / --docker_run_override:
+       Force apptainer execution even if 'run' is set to False in script meta.
+
+    All --docker_X options listed above are accepted as defaults when the
+    corresponding --apptainer_X option is not provided. Docker-only options
+    (e.g. --docker_dt, --docker_cache, --docker_shm_size) are not applicable
+    to Apptainer and are ignored.
+
+    Script meta.yaml keys:
+    - ``docker``: base container settings (used by both mlcd and mlca).
+    - ``apptainer``: apptainer-specific overrides; merged over ``docker`` settings.
+
         """
         return self.call_script_module_function("apptainer", run_args)