From a189b1484915a9c45ebe952d9929a54a43cb5943 Mon Sep 17 00:00:00 2001 From: Dylan Gormley Date: Mon, 24 Aug 2026 17:47:52 -0500 Subject: [PATCH] feat(ps): expose per-storage-element common paths in --json output --- docs/ps.md | 21 +++++++++++++++++++ dtcli/ps.py | 23 +++++++++++++-------- dtcli/utilities/utilities.py | 34 ++++++++++++++++++++++++++++++ tests/test_cli.py | 2 ++ tests/test_utils.py | 40 ++++++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 9 deletions(-) diff --git a/docs/ps.md b/docs/ps.md index efc6b68..fb08313 100644 --- a/docs/ps.md +++ b/docs/ps.md @@ -102,6 +102,16 @@ $ datatrail ps kko.event.baseband.raw 308892599 --json ] } }, + "common_paths": { + "minoc": { + "common_path": "data/kko/baseband/raw/2023/08/07/astro_308892599", + "files": [ + "baseband_308892599_129.h5", + "baseband_308892599_1013.h5", + ... + ] + } + }, "policies": { "replication_policy": { "preferred_storage_elements": ["chime"], @@ -127,6 +137,13 @@ $ datatrail ps kko.event.baseband.raw 308892599 --json } ``` +The `common_paths` field gives, per storage element, the deepest common +directory of its file replicas and the file names relative to it, so scripts +do not have to re-derive the split. When no common directory exists, it is +`""` and the original paths are listed. Note that `minoc` paths keep any +collection prefix (such as `cadc:CHIMEFRB`) exactly as reported in +`file_replica_locations`. + ### Usage in scripts ```python @@ -145,6 +162,10 @@ data = json.loads(result.stdout) file_locations = data["files"]["file_replica_locations"] minoc_files = file_locations.get("minoc", []) +# Compose full paths from the derived common path +minoc = data["common_paths"].get("minoc", {}) +full_paths = [f"{minoc['common_path']}/{name}" for name in minoc.get("files", [])] + # Access policies replication_policy = data["policies"]["replication_policy"] deletion_policy = data["policies"]["deletion_policy"] diff --git a/dtcli/ps.py b/dtcli/ps.py index 1d7e225..e36892f 100644 --- a/dtcli/ps.py +++ b/dtcli/ps.py @@ -2,7 +2,6 @@ import logging import os -from pathlib import Path import click from requests.exceptions import SSLError @@ -12,7 +11,12 @@ from dtcli.ls import list from dtcli.src import functions from dtcli.utilities import cadcclient -from dtcli.utilities.utilities import check_canfar_status, set_log_level, validate_scope +from dtcli.utilities.utilities import ( + check_canfar_status, + common_paths, + set_log_level, + validate_scope, +) logger = logging.getLogger("ps") @@ -107,6 +111,9 @@ def ps( # noqa: C901 "scope": scope, "files": files, "policies": policies, + "common_paths": common_paths(files.get("file_replica_locations", {})) + if isinstance(files, dict) + else {}, } print(json.dumps(result, indent=2)) return None @@ -255,15 +262,13 @@ def create_files_table(dataset: str, scope: str, files: dict): f"Datatrail: Files for {dataset} {scope}", style="bold magenta" ) - for se in files["file_replica_locations"]: - common_path = os.path.commonpath(files["file_replica_locations"][se]) - names = [ - Path(_).relative_to(common_path) for _ in files["file_replica_locations"][se] - ] - for idx, fn in enumerate(names): + for se, derived in common_paths(files["file_replica_locations"]).items(): + for idx, fn in enumerate(derived["files"]): if idx == 0: file_table.add_row(f"Storage Element: [magenta]{se}") - file_table.add_row(f"Common Path: {common_path}/", style="bold green") + file_table.add_row( + f"Common Path: {derived['common_path']}/", style="bold green" + ) file_table.add_row(f"[green]- {fn}") else: file_table.add_row(f"- {fn}", style="green") diff --git a/dtcli/utilities/utilities.py b/dtcli/utilities/utilities.py index 004f309..bf47338 100644 --- a/dtcli/utilities/utilities.py +++ b/dtcli/utilities/utilities.py @@ -2,6 +2,8 @@ import json import logging +import os +from pathlib import Path from typing import Any, Dict, List, Tuple, Union import requests @@ -19,6 +21,38 @@ REQUEST_TIMEOUT: Tuple[float, float] = (10.0, 300.0) +def common_paths( + file_replica_locations: Dict[str, List[str]], +) -> Dict[str, Dict[str, Any]]: + """Derive each storage element's common path and relative file names. + + Args: + file_replica_locations (Dict[str, List[str]]): File paths per + storage element, as reported in a dataset's file information. + + Returns: + Dict[str, Dict[str, Any]]: Per storage element, the deepest common + directory as 'common_path' and the file names relative to it as + 'files'. When no common directory exists, 'common_path' is "" + and 'files' holds the original paths. A storage element with no + valid file list is omitted. + """ + derived: Dict[str, Dict[str, Any]] = {} + for se, paths in file_replica_locations.items(): + if not paths or not all(isinstance(p, str) and p for p in paths): + continue + try: + if len(paths) == 1: + base = os.path.dirname(paths[0]) + else: + base = os.path.commonpath(paths) + except ValueError: + base = "" + names = [str(Path(p).relative_to(base)) for p in paths] if base else [*paths] + derived[se] = {"common_path": base, "files": names} + return derived + + def set_log_level(logger: logging.Logger, verbose: int = 0, quiet: bool = False) -> None: """Set log level.""" if verbose == 1: diff --git a/tests/test_cli.py b/tests/test_cli.py index 77ddee8..b0f53ee 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -707,6 +707,8 @@ def test_cli_ps_json(runner: CliRunner) -> None: assert "policies" in output_data assert output_data["dataset"] == "289007650" assert output_data["scope"] == "chime.event.baseband.raw" + # Derived per-storage-element common path and relative file names. + assert "common_paths" in output_data def test_check_version_banner_on_stderr(monkeypatch, capsys) -> None: diff --git a/tests/test_utils.py b/tests/test_utils.py index ab108b6..6d7671a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -41,3 +41,43 @@ def test_split_more_batches_than_items(): assert len(result) <= len(test_array) # every element appears exactly once assert sorted(sum(result, [])) == sorted(test_array) + + +def test_common_paths() -> None: + """Test common path derivation per storage element.""" + derived = utilities.common_paths( + { + "minoc": [ + "cadc:CHIMEFRB/data/event/1/a.h5", + "cadc:CHIMEFRB/data/event/1/b.h5", + "cadc:CHIMEFRB/data/event/1/sub/c.h5", + ], + "arc": ["/arc/projects/chime_frb/data/event/1/a.h5"], + "empty": [], + } + ) + assert derived["minoc"] == { + "common_path": "cadc:CHIMEFRB/data/event/1", + "files": ["a.h5", "b.h5", "sub/c.h5"], + } + assert derived["arc"] == { + "common_path": "/arc/projects/chime_frb/data/event/1", + "files": ["a.h5"], + } + # A storage element with no files is omitted, not invented. + assert "empty" not in derived + + +def test_common_paths_no_usable_split() -> None: + """Test elements with no common directory keep their original paths.""" + derived = utilities.common_paths( + { + "mixed": ["/abs/a.h5", "rel/b.h5"], + "no_common": ["a/b.h5", "c/d.h5"], + "bare": ["a.h5"], + } + ) + # No usable split: common_path is "" and the original paths survive. + assert derived["mixed"] == {"common_path": "", "files": ["/abs/a.h5", "rel/b.h5"]} + assert derived["no_common"] == {"common_path": "", "files": ["a/b.h5", "c/d.h5"]} + assert derived["bare"] == {"common_path": "", "files": ["a.h5"]}