[API] Keep execution commands in CLI

doc/reporting.rst

@@ -36,15 +36,26 @@ Unified experiment output
 
 CloudAI writes ``experiment.json`` in each scenario's results directory.
 
-The file contains scenario details, test cases, status, timing, and result paths. Standalone execution also records each
-run's process ID, status, iteration, timing, and workload metrics.
+The file contains scenario details and test cases under ``tests``. Standalone and Slurm execution records appear in each
+test case's ``runs`` list. Each record represents an iteration or DSE step and includes its number, process or Slurm job
+ID, status, timing, result path, and workload metrics.
 
-CloudAI updates the file when standalone runs start and finish, then finalizes it when scenario execution succeeds or
-fails. Timestamps use UTC; durations use seconds. Unknown timestamps are ``null``. Dry runs also produce scenario and
-test-case details without launching workloads.
+Timestamps use UTC; durations use seconds. Unknown timestamps are ``null``. A final status of ``unknown`` means the outcome
+could not be determined. Dry runs include scenario and test-case details without launching workloads.
 
-Metrics come from ``TestDefinition.metric_observations()``, independently of reporter settings. Each file update replaces
-the previous snapshot atomically. Metric extraction or write errors produce warnings without affecting execution.
+When a test case executes once successfully, ``tests[].metrics`` contains that execution's metrics. For DSE, it contains
+metrics from the successful step with the highest valid reward. The search space, selected step, and configuration appear
+in ``tests[].dse``. For example:
+
+.. code-block:: json
+
+   {
+     "space": {"extra_env_vars.NCCL_ALGO": ["Ring", "Tree"]},
+     "best_step": 2,
+     "best_config": {"extra_env_vars.NCCL_ALGO": "Tree"}
+   }
+
+Here, step 2 using ``Tree`` was selected. Its metrics appear in the test case's ``metrics`` list.
 
 
 .. _general-flow:

pyproject.toml

@@ -152,20 +152,26 @@ root_package = "cloudai"
     "cloudai.workloads",
     "cloudai.util",
     "cloudai.cli",
+    "cloudai.handlers",
     "cloudai.report_generator",
   ]
 
   [[tool.importlinter.contracts]]
   name = "Report generator is leaf dependency"
   type = "forbidden"
-  forbidden_modules = ["cloudai.workloads", "cloudai.cli"]
-  allow_indirect_imports = true                            # allow "from cloudai.core import ..."
+  forbidden_modules = ["cloudai.workloads", "cloudai.cli", "cloudai.handlers"]
+  allow_indirect_imports = true                                                # allow "from cloudai.core import ..."
   source_modules = ["cloudai.report_generator"]
 
   [[tool.importlinter.contracts]]
   name = "Util modules are leaf dependencies"
   type = "forbidden"
-  forbidden_modules = ["cloudai.systems", "cloudai.workloads", "cloudai.cli"]
+  forbidden_modules = [
+    "cloudai.systems",
+    "cloudai.workloads",
+    "cloudai.cli",
+    "cloudai.handlers",
+  ]
   allow_indirect_imports = true
   source_modules = ["cloudai.util"]
 

src/cloudai/_core/base_runner.py

@@ -113,7 +113,9 @@ def update_run_output(self, job: BaseJob, result: JobStatusResult | None = None)
         self.experiment_output.write()
 
     def finish_output(self, successful: bool) -> None:
-        status = "completed" if successful else "failed"
+        status: cloudai.models.output.Status = "completed" if successful else "failed"
+        if self.shutting_down and not any(test.status == "failed" for test in self.experiment_output.experiment.tests):
+            status = "cancelled"
         self.experiment_output.finish(status=status, finish=datetime.datetime.now(datetime.timezone.utc))
 
     def shutdown(self):