diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..d31e5e4 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Filesystem integration + +on: + workflow_dispatch: + push: + branches: + - pull-request/[0-9]+ + +permissions: {} + +concurrency: + group: filesystem-integration + cancel-in-progress: false + +jobs: + integration: + name: Setup and test all (${{ matrix.architecture }}) + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runner: linux-amd64-cpu4 + - architecture: arm64 + runner: linux-arm64-cpu4 + runs-on: ${{ matrix.runner }} + timeout-minutes: 135 + permissions: + contents: read + steps: + - name: Check out source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - name: Configure the package proxy + uses: nv-gha-runners/setup-proxy-cache@eadc57871cde0c92a223d0f486ab669a2360f978 # main + with: + enable-apt: "true" + enable-pip: "true" + enable-conda: "false" + enable-pypi-anaconda: "false" + - name: Verify privileged runner prerequisites + run: | + sudo -n true + docker info >/dev/null + - name: Run the bounded integration lifecycle + run: | + timeout --kill-after=10m 105m \ + integration-tests/bin/ci-integration.sh "${{ matrix.architecture }}" + - name: Upload integration diagnostics + if: ${{ always() }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: filesystem-integration-${{ matrix.architecture }} + path: tmp/integration-diagnostics + if-no-files-found: warn + retention-days: 7 + + integration-status: + name: Filesystem integration status + if: ${{ always() }} + needs: integration + runs-on: ubuntu-latest + steps: + - name: Require every architecture to pass + env: + INTEGRATION_RESULT: ${{ needs.integration.result }} + run: test "$INTEGRATION_RESULT" = success diff --git a/AGENTS.md b/AGENTS.md index a434577..5c42e0d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,9 @@ See the README "Getting Started" section for the full quickstart and ./utils/run_ci_checks.sh ``` +After changing `.github/`, parse every `.yml` and `.yaml` file beneath it with +a YAML parser before committing. + If any required check tool is missing from the sandbox, install it into the repo's local environment and rerun the check. Do not skip required tooling just because it is not preinstalled. @@ -72,6 +75,8 @@ The script creates and reuses `.venv-ci` with the pinned tools. Pass `pylint`, or `pytest` to run one check. Set `CI_CHECK_JOBS=1` in a constrained sandbox. In a pre-provisioned, network-restricted sandbox, set `CI_BOOTSTRAP=0` and use `CI_PYTHON` or `CI_SHELLCHECK` to select installed tools. +Run tests and lint through this script or an environment populated from both +requirements files; never treat ambient Python tooling as authoritative. `black` must be 25.9.0+; `pylint` must score 10.00/10. Details and rationale: [docs/CODING_STANDARDS.md](docs/CODING_STANDARDS.md). @@ -93,6 +98,8 @@ and use `CI_PYTHON` or `CI_SHELLCHECK` to select installed tools. ## Pull requests +Wrap commit-message lines at about 72 characters. + This project is currently not accepting external contributions. For maintainer changes: keep PRs focused, ensure the checks above pass, and update `README.md`/`docs/` when behavior or configuration changes. diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index ea9d032..cfa7569 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -54,8 +54,46 @@ only: NVIDIA and the project do not publish or deliver benchmark binaries or prepared deployment tarballs. Users may create a deployment tarball locally and are responsible for every binary they place in it. -Slurm is the default execution substrate. Setting `SSH_HOST_LIST` selects -passwordless SSH instead. Kubernetes execution is not implemented. +Slurm is the default execution substrate; `SSH_HOST_LIST` selects passwordless +SSH. Kubernetes benchmark execution is not implemented. The `integration-tests/` +fixture provisions three kind nodes, RWX storage, two SSH workers, and Slinky +Slurm. Its `nfs` backend uses loop-backed NFSv4 and NFS CSI; `sbx-shared` uses +static volumes over a repository-shared path. Both test the same storage contract. + +One budget drives PVC capacity and the growable 4 GiB NFS image. Setup publishes +image tags transactionally, grows retained filesystems, checks fixture and Docker +backing capacity, and reconciles eight NFS workers. Teardown restores recorded +NFS active, enabled, and worker-count states. SBX pins kind 0.30 and +Kubernetes/kubectl 1.34; setup replaces clusters whose kubelets do not match the +node-image profile. + +Lifecycle actions run as an ordinary user, store state under +`tmp/integration-state`, and use `sudo` only for NFS host operations. Ownership +markers protect dedicated state and export leaves. A symlink-safe root-owned +global lock and owner record protect fixed NFS configuration. Cached upstream +images must match their pinned digest and runner architecture. NFS CSI misses try +`registry.k8s.io` and `gcr.io/k8s-staging-sig-storage`; its chart tags exist only +inside kind. Kind node and other image misses use bounded host-Docker retries. +Interrupted private aliases are reconciled; MariaDB and Slinky's Alpine helpers +use preloaded fixture-private tags. Cleanup recovers partial bootstrap, removes +only owned resources, restores prior NFS state, verifies unmounts, and does not +depend on writable diagnostics. + +The test CLI selects substrates and scenarios independently. Its planner batches +shared-home SSH cases behind a crash-recoverable transition; separate homes are +canonical. Pod work uses `tester` UID/GID 2000, matching the all-squashed NFS +export and Slurm account. + +The harness builds the ordinary deployment archive from an immutable tracked +snapshot, caches it by snapshot, architecture, fixed recipe, and seeded +Elbencho/runtime identity, and extracts isolated scenario workspaces. Real cases +cover baseline and default I/O, failure/resume, retained data, live capture, +Cartesian sweeps, single-file and weighted-root behavior, shared SSH homes, and +Slurm scheduling. Fast tests cover parsing, precedence, path and workload safety, +sizing, scheduler boundaries, failure contracts, and reporting. CI runs the full +NFS-backed catalog concurrently on amd64 and arm64 with repeatable-teardown +headroom; SBX is a supported local backend. + GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. @@ -79,6 +117,7 @@ alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python | `utils/build_tarball.sh` | User-local deployment-tarball builder | | `utils/build/` | Helpers for building Warp and the in-tree s3test program | | `tests/` | Python and shell-behavior regression tests collected by `pytest` | +| `integration-tests/` | Single-host kind, RWX storage, SSH, and Slinky fixture | The checked-in benchmark entry points are: @@ -119,6 +158,9 @@ Important configuration relationships: inclusive ranges, and `+step` increments. Entry points validate the complete expanded list before dispatch. - `TEST_DIRS` is an associative array of filesystem test roots and weights. + `validate_env.sh` compares each path's `stat -c %d` device number with `/` + through both dispatch interfaces; this establishes distinct backing storage, + not that the configured path is itself the exact mountpoint. Object tests use a dedicated `OBJ_BUCKET`, endpoint settings, and credentials sourced from `OBJ_AUTH_FILE`. @@ -161,7 +203,12 @@ In Slurm mode, `dispatch_slurm_executions` allocates the largest node count needed by any non-successful cell and submits `storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh`. The coordinator starts elbencho services once across the allocation and processes cells sequentially, -using the first requested number of allocation hosts for each cell. +using the first requested number of allocation hosts for each cell. Initial +service health failure preserves its log and gets one bounded restart attempt; +phase-level checks can also restart unhealthy services. Signals cancel and +wait for the exact allocation before restoring the caller's traps. Cancellation +is armed immediately after submission, before dispatch-lock handoff; an +unverified cancellation retains the lock. In SSH mode, `dispatch_ssh_executions` starts services once across the usable host pool. It selects a fresh host subset for each cell, runs cells sequentially, diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 0a47fb0..7aa9b3b 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -210,7 +210,7 @@ edit-validate loop: | Slurm connectivity | `sinfo`, `sbatch` a test job, wait for completion | | SSH connectivity | `ssh` command execution + scriptlet execution on each host | | Binary architecture match | `file` on binary vs. `uname -m` on remote | - | Filesystem paths are mountpoints | `mountpoint` on compute nodes (via Slurm/SSH) | + | Filesystem paths use storage distinct from `/` | Compare `stat -c %d` device IDs on compute nodes (via Slurm/SSH) | | Filesystem paths are writable | Touch test on compute nodes | | S3 credentials and bucket access | `s3test` binary | | S3 bucket emptiness | Object count check (warning if non-empty) | @@ -692,7 +692,10 @@ All analysis scripts support filtering to reduce clutter: Intermediate data can be cached (`--to-csv`, `--to-json`) and reloaded (`--from-csv`, `--from-json`) to avoid re-parsing raw files, enabling fast -iterative report refinement. +iterative report refinement. For Elbencho, `--from-csv` is an alternative input +source and cannot be combined with raw result directories. Malformed filters +and filters that match neither aggregate nor live metrics fail instead of +silently producing an unfiltered or empty report. ### 9.5 Plot Design Principles diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index ba0a66d..ded5fa9 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -219,7 +219,7 @@ These requirements address the infrastructure and runtime constraints the tool m | CV-2.3 | Validation shall verify that benchmark binaries are present, executable, and compiled for the correct architecture. | Yes | | CV-2.4 | Validation shall verify Slurm connectivity (if Slurm mode is enabled): partition access, account, reservation, sbatch and srun functionality. | Yes | | CV-2.5 | Validation shall verify SSH connectivity (if SSH mode is enabled): ability to run commands and scriptlets on remote hosts. | Yes | -| CV-2.6 | Validation shall verify that filesystem test paths are mountpoints and writable on compute nodes. | Yes | +| CV-2.6 | Validation shall verify that filesystem test paths reside on a filesystem device distinct from `/` and are writable on compute nodes. | Yes | | CV-2.7 | Validation shall verify S3 object storage credentials and bucket accessibility (if object tests are enabled). | Yes | | CV-2.8 | Validation shall warn if the target S3 bucket contains existing objects (warp deletes all objects). | Yes | | CV-2.9 | Validation shall validate elbencho configuration parameters (thread list contains integers, IO sizes are valid, duration is valid). | Yes | diff --git a/env.sh.template b/env.sh.template index 19ce1eb..0108fbb 100644 --- a/env.sh.template +++ b/env.sh.template @@ -272,4 +272,6 @@ export NETBENCH_ITERATIONS=3 # any specific cluster easier). # shellcheck disable=SC1091 +# Integration harnesses may replace this marker with bounded overrides. +# STORAGE_SCALE_TEST_INTEGRATION_OVERRIDES source "${SCALE_TEST_BASE}/lib/env_base.sh" diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 0000000..7f5b74d --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,212 @@ + + +# Single-host integration environment + +`bin/integration-test.py` provisions the lightweight, three-node kind fixture +used to exercise Kubernetes, SSH, and Slurm storage substrates on one Linux +server. The control-plane node is also the Slinky login node and negative +control for the storage-worker label. The two worker nodes run storage clients. + +The current setup target is Ubuntu 24.04 on x86-64 or ARM64 with at least two +CPUs, 8 GiB total RAM, 6 GiB available RAM, and 20 GiB free on the selected +backend's filesystem. Python 3.12 and an accessible rootful Docker daemon are +prerequisites. The driver installs its other host packages and pinned client +tools when needed. It also builds small derived Slinky login and compute images +containing the fixed integration workload account; the login image additionally +provides the standard `file` package required by `validate_env.sh`. + +Run setup (or its exact synonym, start) as the ordinary test user. The NFS +profile invokes passwordless `sudo` itself only for package installation and +the dedicated export, loop-device, firewall, and systemd operations: + +```bash +sudo -v +integration-tests/bin/integration-test.py setup +``` + +`--storage-backend auto` is the default. It selects `nfs` when the host has the +required loop, mount, systemd, and kernel NFS facilities. It selects +`sbx-shared` only when a recognized capability needed by that profile is +absent. An explicit backend never falls back, and setup records the selection; +changing it requires teardown first. Arbitrary download, image, Kubernetes, +manifest, storage-visibility, SSH, and Slurm failures remain fatal. + +The `nfs` backend uses a loop-backed NFSv4 export and NFS CSI. The +`sbx-shared` backend is specifically for Docker SBX: it mounts one +repository-backed directory into every kind node and binds static RWX claims to +separate test-data and shared-home subdirectories. Both implement the same +in-scope integration contract: cross-node and host read/write visibility, +shared-home behavior, and successful SSH and Slurm filesystem sweeps. NFS and +CSI provisioning themselves are infrastructure details outside this +repository's test scope; the backend difference is environmental fidelity, not +repository feature coverage. + +Select Docker SBX explicitly with: + +```bash +integration-tests/bin/integration-test.py \ + --storage-backend sbx-shared setup +``` + +The SBX profile requires Docker's private engine to bind-mount the checked-out +repository path. It uses the tested kind v0.30.0/Kubernetes v1.34.0 profile and +maps `/dev/null` to `/dev/kmsg` in kind nodes only when the SBX environment +lacks that device. When Docker SBX exposes its proxy CA, setup installs that CA +in the disposable kind nodes so containerd can pull the fixture images. The +default shared root is `tmp/integration-sbx-shared`; an alternate path may be +set with `--sbx-shared-root`, but must remain below the repository's `tmp/` +directory. This profile never invokes `sudo`; required host packages and an +accessible Docker engine must already be present. Its two disposable backing +directories deliberately use non-sticky mode `0777`. Docker SBX can map the +host caller and UID 2000 workloads to different owners, so omitting the sticky +bit lets either side create and remove scenario data. This is safe only for +these marker-owned, disposable leaves below the repository's `tmp/` directory. + +Every lifecycle action runs as the ordinary test account and refuses root. +Kubeconfig, keys, downloaded clients, cached deployments, rendered manifests, +logs, and test runs remain user-owned from creation under the default +`tmp/integration-state` directory. Setup never recursively changes that +tree's ownership. It verifies that the caller can use Docker, kind, kubectl, +and the state directory after provisioning. + +The host account and in-cluster workload account are deliberately independent. +LoginSet coordination, Slurm jobs, SSH workers, and shared-storage staging run +as `tester` with UID/GID 2000; setup verifies that identity on the coordinator +and on both real `srun` tasks. This matches the NFS export's anonymous mapping, +so clients create scenario data directly with a restrictive umask and never +attempt to change ownership through an all-squashed mount. + +SSH workers normally use separate `emptyDir` homes. A scenario that requires +the RWX shared-home claim owns a bounded StatefulSet transition and restores +separate homes afterward. Setup and SSH test preflight recover an interrupted +transition before allowing more SSH work; the kind and Slurm fixtures remain +running throughout. + +The generated host SSH key, strict known-hosts file, and two worker addresses +are kept under `tmp/integration-state/`. Re-running setup +reconciles and validates the environment without replacing those credentials +or retained backend data. + +After setup succeeds, run the bounded filesystem regression cases with: + +```bash +integration-tests/bin/integration-test.py test +integration-tests/bin/integration-test.py test --substrate ssh +integration-tests/bin/integration-test.py test --scenario baseline +integration-tests/bin/integration-test.py test --list-scenarios +``` + +With no options, `test` runs every available scenario on each applicable +substrate. `--substrate` accepts `all`, `ssh`, or `slurm`; repeatable +`--scenario` options select named cases independently. Scenario listing needs +no setup state or privileges. Actual tests refuse root execution, require the +saved non-root identity, validate the live topology, generate environments +from the packaged `env.sh.template`, and run `validate_env.sh` before a sweep. + +The real scenario catalog covers buffered and direct I/O, one- and two-node +selection, failure and resume, retained write/read/delete data, extended live +CSV capture, and result reporting on both SSH and Slurm. Focused cases add a +multidimensional Slurm sweep, Slurm include/exclude and exclusive-user +allocation behavior, SSH weighted roots, generated and staged single-file +work, and shared SSH homes. Workloads stay deliberately small; assertions +check execution coordinates and state transitions, phase and workload +evidence, dataset totals, required native flags, relevant scheduling evidence, +and semantic report rows and plot families without treating incidental output +or performance values as contracts. + +The harness materializes one immutable tracked-source snapshot and builds a +real deployment archive from it with the zero-argument +`utils/build_tarball.sh`. It caches the validated archive by snapshot manifest, +architecture, fixed integration recipe, and seeded Elbencho/runtime identity. +Every scenario extracts that artifact into an isolated workspace, adds only +its own environment and inputs, and cleans its remote data afterward; host-side +results and diagnostics remain under the state directory. The SSH cases launch +`validate_env.sh` and +`nv-elbencho-sweep.sh` on the host and reach the two worker pods over SSH. The +Slurm cases stream the same archive to the LoginSet, extract it in shared +storage, and launch both commands there. + +The NFS profile uses a size-limited, checksum-verified upstream benchmark +archive. Docker SBX, where GitHub release assets may be unavailable, extracts +the binary and runtime libraries from the digest-pinned upstream +`breuner/elbencho:v3.1-11` image and includes them only in the generated test +deployment. Timestamped build and step logs are retained below the state +directory's `test-runs/` directory. The test also requires successful execution +records, exact one- and two-node workload totals, ordered worker selection, +nonempty benchmark output, environment snapshots, and cleanup of generated +data directories. It runs `utils/extract-elbencho.sh` on host-side result +copies and checks the semantic report content applicable to each scenario. + +## On-demand CI + +The `Filesystem integration` GitHub Actions workflow runs independent amd64 +and arm64 jobs concurrently. Each job runs setup twice, stops and restarts the +fixture, proves that root lifecycle execution is rejected, runs `test` as the +ordinary runner account, and tears down twice. A final status job requires both +architectures to pass. The workflow is deliberately absent from ordinary +pull-request and default-branch events. + +For a pull request, use the repository's existing PR authorization control—the +same control used to start the regular PR checks. Authorization copies the +reviewed PR commit to the trusted `pull-request/` branch. A push to +that narrowly matched branch starts the integration workflow. Updating a PR +requires authorizing its new head before a new integration run can start. An +existing run can instead be repeated with **Re-run jobs** in GitHub Actions. + +Before this workflow file is present on the default branch, that authorized PR +branch is the way to run it. After the workflow is merged, a maintainer can also +open **Actions**, choose **Filesystem integration**, select **Run workflow**, +and choose an authorized branch or the default branch. + +Delete the disposable kind cluster and, for the NFS backend when owned +exclusively by the harness, stop NFS with: + +```bash +integration-tests/bin/integration-test.py stop +``` + +Stop preserves packages, downloaded charts, Docker images, generated keys and +passwords, rendered state, and backend data. Kind containers, Kubernetes +objects, and MariaDB's node-local volume are disposable and are deleted. A +subsequent start therefore creates and validates a fresh cluster and takes +longer than an idempotent setup against an already-running cluster. Timestamped +logs and rendered manifests are retained in the state directory. Add +`--verbose` for command-level logging. The failure path captures host, Docker, +backend, Kubernetes node, pod, and event diagnostics without printing +Kubernetes Secrets. + +For CI workers or any host where retained fixture data is not wanted, run: + +```bash +integration-tests/bin/integration-test.py teardown +``` + +Teardown is idempotent. It performs the disposable stop and deletes the +fixture's generated data, keys, logs, and locally built image tags. For NFS it +also removes the dedicated export and configuration, removes any harness-owned +UFW rule, and unmounts the verified loop-backed filesystem. It disables and +stops `nfs-server` only when setup started it and no unrelated exports remain; +a pre-existing service is left running. For Docker SBX it removes only the +marker-owned shared root and never invokes NFS, systemd, firewall, loop, or +mount operations. It refuses destructive cleanup when the applicable ownership +and path checks do not match the fixture. Operating-system packages, +pre-existing client tools, and reusable upstream Docker image layers are not +uninstalled. Checksum-verified client copies that the harness downloaded into +its own state tree are removed with that tree. If a locally built tag existed +before setup, teardown restores that exact prior image ID instead of deleting +it. diff --git a/integration-tests/bin/ci-integration.sh b/integration-tests/bin/ci-integration.sh new file mode 100755 index 0000000..ef057db --- /dev/null +++ b/integration-tests/bin/ci-integration.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +expected_arch=${1:?usage: ci-integration.sh [nfs|sbx-shared]} +storage_backend=${2:-nfs} +case "${expected_arch}:$(uname -m)" in + amd64:x86_64|arm64:aarch64) ;; + *) + echo "runner architecture mismatch: expected $expected_arch, found $(uname -m)" >&2 + exit 1 + ;; +esac +case "$storage_backend" in + nfs|sbx-shared) ;; + *) + echo "unsupported integration storage backend: $storage_backend" >&2 + exit 1 + ;; +esac + +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P) +readonly repo_root +readonly driver=${INTEGRATION_DRIVER:-$repo_root/integration-tests/bin/integration-test.py} +readonly state_dir=${INTEGRATION_STATE_DIR:-$repo_root/tmp/integration-state} +readonly diagnostics=${INTEGRATION_DIAGNOSTICS:-$repo_root/tmp/integration-diagnostics} +python_bin=${INTEGRATION_PYTHON:-} +if [[ -z "$python_bin" ]]; then + python_bin=$(command -v python3) || { + echo "python3 is required for the integration lifecycle" >&2 + exit 1 + } +fi +readonly python_bin +readonly privilege_command=${INTEGRATION_PRIVILEGE_COMMAND:-sudo} +cleanup_started=0 + +run_teardown() { + local log_path=${1:-} + local teardown_rc=0 + local teardown_log_fd + if [[ -n "$log_path" ]]; then + if exec {teardown_log_fd}>"$log_path"; then + "$python_bin" "$driver" --storage-backend "$storage_backend" teardown \ + >&"$teardown_log_fd" 2>&1 || teardown_rc=$? + exec {teardown_log_fd}>&- + return "$teardown_rc" + fi + echo "Warning: cannot write teardown log $log_path; using stderr" >&2 + fi + "$python_bin" "$driver" --storage-backend "$storage_backend" teardown \ + || teardown_rc=$? + return "$teardown_rc" +} + +cleanup() { + local original_rc=$1 + local first_rc=0 + local second_rc=0 + local diagnostics_ready=0 + local first_log="" + local second_log="" + (( cleanup_started )) && return "$original_rc" + cleanup_started=1 + trap - EXIT INT TERM + set +e + if mkdir -p -- "$diagnostics" && [[ -d "$diagnostics" ]]; then + diagnostics_ready=1 + first_log="$diagnostics/teardown-1.log" + second_log="$diagnostics/teardown-2.log" + else + echo "Warning: diagnostics path is unavailable: $diagnostics" >&2 + fi + if (( diagnostics_ready )) && [[ -d "$state_dir/logs" ]]; then + cp -a -- "$state_dir/logs" "$diagnostics/pre-teardown-logs" \ + || echo "Warning: could not preserve pre-teardown logs" >&2 + fi + if (( diagnostics_ready )) && [[ -d "$state_dir/test-runs" ]]; then + cp -a -- "$state_dir/test-runs" "$diagnostics/test-runs" \ + || echo "Warning: could not preserve test-run diagnostics" >&2 + fi + run_teardown "$first_log" || first_rc=$? + run_teardown "$second_log" || second_rc=$? + if (( original_rc == 0 && (first_rc != 0 || second_rc != 0) )); then + return 1 + fi + return "$original_rc" +} + +finish() { + local original_rc=$? + local final_rc + trap - EXIT INT TERM + cleanup "$original_rc" || final_rc=$? + final_rc=${final_rc:-0} + exit "$final_rc" +} + +trap finish EXIT +trap 'exit 124' TERM +trap 'exit 130' INT + +cd -- "$repo_root" +"$python_bin" "$driver" --storage-backend "$storage_backend" setup +"$python_bin" "$driver" --storage-backend "$storage_backend" setup +"$python_bin" "$driver" --storage-backend "$storage_backend" stop +"$python_bin" "$driver" --storage-backend "$storage_backend" start +for action in setup test; do + if "$privilege_command" env INTEGRATION_EXPECT_ROOT_REJECTION=1 \ + "$python_bin" "$driver" --storage-backend "$storage_backend" "$action"; then + echo "integration $action action unexpectedly accepted root" >&2 + exit 1 + fi +done +"$python_bin" "$driver" --storage-backend "$storage_backend" test diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py new file mode 100755 index 0000000..4b4b355 --- /dev/null +++ b/integration-tests/bin/integration-test.py @@ -0,0 +1,4777 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provision the single-host storage-scale integration environment.""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import logging +import os +import platform +import pwd +import re +import secrets +import shlex +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +from collections.abc import Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import IO + +INTEGRATION_LIB = Path(__file__).resolve().parents[1] / "lib" +sys.path.insert(0, str(INTEGRATION_LIB)) + +from filesystem_integration import ( # pylint: disable=wrong-import-position + IntegrationTestError, + run_filesystem_tests, +) +from fixture_capacity import ( # pylint: disable=wrong-import-position + GIB, + NFS_BUDGET_BYTES, + NFS_IMAGE_BYTES, + SSH_HOME_CAPACITY, + STORAGE_TEST_CAPACITY, +) +from scenario_planner import ( # pylint: disable=wrong-import-position + SUBSTRATES, + format_scenario_listing, +) +from ssh_home_transition import ( # pylint: disable=wrong-import-position + CANONICAL_HOME_MODE, + SHARED_HOME_MODE, + PoolObservation, + SshHomeTransitionError, + SshHomeTransitionManager, + statefulset_checksum, +) + +KIND_VERSION = "v0.33.0" +SBX_KIND_VERSION = "v0.30.0" +KUBERNETES_VERSION = "v1.37.0" +SBX_KUBERNETES_VERSION = "v1.34.0" +KUBECTL_VERSION = "v1.37.0" +SBX_KUBECTL_VERSION = "v1.34.0" +HELM_VERSION = "v3.22.0" +KIND_NODE_IMAGE = ( + f"kindest/node:{KUBERNETES_VERSION}@" + "sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5" +) +SBX_KIND_NODE_IMAGE = ( + f"kindest/node:{SBX_KUBERNETES_VERSION}@" + "sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a" +) +NFS_CSI_VERSION = "4.13.4" +NFS_CSI_SOURCE_SHA256 = ( + "ded6ffba8b1600d4c723ce1ecb1fd91721ef48e732ce7ca30c0efeeecbb0b900" +) +NFS_CSI_CHART_SHA256 = ( + "815ac441a2dd0e48c82fa92d043e96caac4dd8ac422fbba91ed76892ed32da54" +) +SLINKY_VERSION = "1.2.0" +SLINKY_LOGIN_BASE_IMAGE = ( + "ghcr.io/slinkyproject/login:26.05-ubuntu26.04@" + "sha256:9578f6773891a9eee75395c60c7f5acad1385ceb6e715c8c76f606ccfd89968d" +) +SLINKY_LOGIN_IMAGE = "storage-scale-integration-login:slinky-26.05-user" +SLINKY_SLURMD_BASE_IMAGE = ( + "ghcr.io/slinkyproject/slurmd:26.05-ubuntu26.04@" + "sha256:f06ab7b1ce18b3d59c54c8c6f6b03698a28973d4d8fb30985bd0dfd673bd6dd9" +) +SLINKY_SLURMD_IMAGE = "storage-scale-integration-slurmd:slinky-26.05-user" +MARIADB_BASE_IMAGE = ( + "mariadb:11.4@" + "sha256:70cc072b29b4a89ae07abb2d4da2c64678a7f2dfe092751bb51c87d67dc1338b" +) +MARIADB_IMAGE = "storage-scale-integration-mariadb:11.4" +SLINKY_HELPER_BASE_IMAGE = ( + "alpine:3.22@" + "sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8" +) +SLINKY_HELPER_IMAGE_REPOSITORY = "storage-scale-integration-alpine" +SLINKY_HELPER_IMAGE_TAG = "3.22" +SLINKY_HELPER_IMAGE = f"{SLINKY_HELPER_IMAGE_REPOSITORY}:{SLINKY_HELPER_IMAGE_TAG}" +SSH_IMAGE = "storage-scale-integration-ssh:ubuntu-24.04" +SSH_BASE_IMAGE = ( + "ubuntu:24.04@" + "sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3" +) +STATE_SCHEMA = 1 +TARGET_LABEL = "storage-scale-test/target=true" +LOGIN_LABEL = "storage-scale-test/login=true" +WORKLOAD_USER = "tester" +WORKLOAD_UID = 2000 +WORKLOAD_GID = 2000 +WORKLOAD_ACCOUNT = "storage-test" +NFS_UID = WORKLOAD_UID +NFS_GID = WORKLOAD_GID +NFS_SERVER_THREADS = 8 +NFS_THREADS_PATH = Path("/proc/fs/nfsd/threads") +NFS_SERVICE_STATE_SCHEMA = 2 +STORAGE_BACKENDS = ("nfs", "sbx-shared") +SYSTEM_ADMIN_PATHS = ("/usr/local/sbin", "/usr/sbin", "/sbin") +SBX_SHARED_DIRECTORY_MODE = 0o777 +SSH_STORAGE_VISIBILITY_TIMEOUT_SECONDS = 30 +TEMPORARY_UNMOUNT_ATTEMPTS = 10 +TEMPORARY_UNMOUNT_RETRY_SECONDS = 1 +IMAGE_PULL_ATTEMPTS = 4 +IMAGE_PULL_TIMEOUT_SECONDS = 60 +IMAGE_PULL_DEADLINE_SECONDS = 240 +IMAGE_PULL_INITIAL_BACKOFF_SECONDS = 10 +IMAGE_PULL_JITTER_MILLISECONDS = 5000 +IMAGE_INSPECT_TIMEOUT_SECONDS = 30 +DEFAULT_STATE_DIR = Path(__file__).resolve().parents[2] / "tmp" / "integration-state" +DEFAULT_EXPORT_DIR = Path("/srv/storage-scale-test-integration") +DEFAULT_SBX_SHARED_ROOT = ( + Path(__file__).resolve().parents[2] / "tmp" / "integration-sbx-shared" +) +NFS_EXPORT_CONFIG = Path("/etc/exports.d/storage-scale-test-integration.exports") +NFS_DAEMON_CONFIG = Path("/etc/nfs.conf.d/storage-scale-test-integration.conf") +NFS_HOST_LOCK_DIR = Path("/run/lock/storage-scale-test-integration") +NFS_HOST_LOCK = NFS_HOST_LOCK_DIR / "nfs.lock" +NFS_HOST_OWNER = Path("/var/lib/storage-scale-test-integration-owner.json") +EXPORT_MARKER = ".storage-scale-test-integration.json" +STATE_MARKER = "state-owner.json" +SSH_HOME_ANNOTATION = "storage-scale-test/ssh-home-mode" +SSH_CONFIG_ANNOTATION = "storage-scale-test/ssh-config-checksum" +UFW_COMMENT = "storage-scale-test integration NFSv4" +LOG = logging.getLogger("storage-scale-integration") + +# Digests identify upstream multi-architecture indexes. Docker selects and the +# harness separately verifies the runner's amd64 or arm64 platform image. +CSI_IMAGES = ( + ( + "csi-node-driver-registrar", + "v2.17.0", + "sha256:f9de845b170155199f2a2a3f9531cf13d78e31235e9db6b6582a8b0db0a50dad", + ), + ( + "csi-provisioner", + "v6.3.0", + "sha256:a4b0b1a37605b7b04a293e136edf7006ec1786a8eb3f4e5a945f81d667dcc371", + ), + ( + "csi-resizer", + "v2.2.0", + "sha256:a2d40c1c3ccb0c48b467125a6652c4dd5dcbf0d295641c9989581cfc690f6cf3", + ), + ( + "livenessprobe", + "v2.19.0", + "sha256:06da0d5b8908072f2e4522692aee8dc119fba7247a9658497e1153992cd777e9", + ), + ( + "nfsplugin", + "v4.13.4", + "sha256:1eb5a85180a4ad0193a31d319b163f35c8c1857794ebaac71d8abcdd5a0516d3", + ), +) +CSI_REPOSITORIES = ( + "registry.k8s.io/sig-storage", + "gcr.io/k8s-staging-sig-storage", +) + + +class ProvisionError(RuntimeError): + """An actionable provisioning failure.""" + + +@dataclass(frozen=True) +class Config: + """Resolved integration environment configuration.""" + + cluster_name: str + namespace: str + state_dir: Path + export_dir: Path + storage_backend: str + sbx_shared_root: Path + test_user: str + test_uid: int + test_gid: int + verbose: bool + + @property + def kubeconfig(self) -> Path: + """Return the private kubeconfig path.""" + return self.state_dir / "kubeconfig" + + @property + def manifests_dir(self) -> Path: + """Return the rendered-manifest state directory.""" + return self.state_dir / "manifests" + + @property + def keys_dir(self) -> Path: + """Return the persistent test-key directory.""" + return self.state_dir / "keys" + + @property + def nfs_image(self) -> Path: + """Return the sparse backing image for the dedicated NFS export.""" + return self.state_dir / "nfs-export.ext4" + + +class Runner: + """Run commands with bounded execution and consistent diagnostics.""" + + def run( + self, + args: Sequence[str | Path], + *, + timeout: float = 300, + check: bool = True, + sensitive: bool = False, + stdin: IO[bytes] | None = None, + cwd: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + """Run *args* and return its completed process.""" + command = [str(item) for item in args] + display = ( + command[0] + " [redacted arguments]" if sensitive else shlex.join(command) + ) + LOG.debug("Running: %s", display) + result = subprocess.run( + command, + check=False, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=stdin is None, + timeout=timeout, + cwd=cwd, + ) + stdout = _output_text(result.stdout) + stderr = _output_text(result.stderr) + if stdout and not sensitive: + LOG.debug("stdout from %s:\n%s", command[0], stdout.rstrip()) + if stderr and not sensitive: + LOG.debug("stderr from %s:\n%s", command[0], stderr.rstrip()) + if check and result.returncode: + detail = _failure_detail(stdout, stderr, sensitive) + raise ProvisionError( + f"command failed ({result.returncode}): {display}{detail}" + ) + return result + + +def _output_text(output: str | bytes | None) -> str: + """Return subprocess output as text.""" + if output is None: + return "" + if isinstance(output, bytes): + return output.decode(errors="replace") + return output + + +def _failure_detail(stdout: str, stderr: str, sensitive: bool) -> str: + """Return bounded non-secret command failure output.""" + if sensitive: + return " (output redacted)" + detail = stderr.strip() or stdout.strip() + if not detail: + return "" + return f"\n{detail[-8000:]}" + + +def _repository_root() -> Path: + """Return the repository root from this script location.""" + return Path(__file__).resolve().parents[2] + + +def _resource_path(name: str) -> Path: + """Return a checked-in integration resource path.""" + return _repository_root() / "integration-tests" / name + + +def _sudo_prefix() -> list[str]: + """Return a sudo prefix when the caller is not root.""" + return [] if os.geteuid() == 0 else ["sudo"] + + +def _bootstrap_state_dir(config: Config) -> None: + """Create user-owned state before file logging or privileged work.""" + create_marker = not config.state_dir.exists() + config.state_dir.mkdir(mode=0o750, parents=True, exist_ok=True) + config.state_dir.chmod(0o750) + for directory in ( + config.manifests_dir, + config.keys_dir, + config.state_dir / "logs", + config.state_dir / "bin", + config.state_dir / "tool-state" / "helm" / "cache", + config.state_dir / "tool-state" / "helm" / "config", + config.state_dir / "tool-state" / "helm" / "data", + ): + directory.mkdir(mode=0o750, parents=True, exist_ok=True) + directory.chmod(0o750) + if create_marker: + _write_text( + config.state_dir / STATE_MARKER, + json.dumps( + {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name}, + sort_keys=True, + ) + + "\n", + ) + + +def _configure_user_tool_path(config: Config) -> None: + """Confine installed clients and their mutable state to user-owned paths.""" + tool_dir = str(config.state_dir / "bin") + current = os.environ.get("PATH", "") + entries = [entry for entry in current.split(os.pathsep) if entry] + entries = [entry for entry in entries if entry != tool_dir] + entries.insert(0, tool_dir) + entries.extend(path for path in SYSTEM_ADMIN_PATHS if path not in entries) + os.environ["PATH"] = os.pathsep.join(entries) + helm_root = config.state_dir / "tool-state" / "helm" + os.environ.update( + { + "HELM_CACHE_HOME": str(helm_root / "cache"), + "HELM_CONFIG_HOME": str(helm_root / "config"), + "HELM_DATA_HOME": str(helm_root / "data"), + } + ) + + +def _configure_logging(config: Config, action: str) -> Path: + """Configure console and timestamped file logging.""" + timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + log_path = config.state_dir / "logs" / f"{action}-{timestamp}.log" + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + console = logging.StreamHandler() + console.setLevel(logging.DEBUG if config.verbose else logging.INFO) + console.setFormatter(formatter) + file_handler = logging.FileHandler(log_path, encoding="utf-8") + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(formatter) + LOG.handlers.clear() + LOG.setLevel(logging.DEBUG) + LOG.addHandler(console) + LOG.addHandler(file_handler) + return log_path + + +def _write_text(path: Path, text: str, mode: int = 0o640) -> None: + """Atomically write *text* with an explicit file mode.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, delete=False + ) as handle: + handle.write(text) + temporary = Path(handle.name) + temporary.chmod(mode) + temporary.replace(path) + + +def _write_bytes(path: Path, content: bytes, mode: int = 0o640) -> None: + """Atomically write binary *content* with an explicit file mode.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as handle: + handle.write(content) + temporary = Path(handle.name) + temporary.chmod(mode) + temporary.replace(path) + + +def _render_resource_text(name: str, replacements: dict[str, str]) -> str: + """Render and validate one checked-in template as text.""" + source = _resource_path(name) + text = source.read_text(encoding="utf-8") + for token, value in replacements.items(): + text = text.replace(f"@@{token}@@", value) + unresolved = [word for word in text.split() if "@@" in word] + if unresolved: + raise ProvisionError(f"unresolved template token in {source}: {unresolved[0]}") + return text + + +def _render_resource(config: Config, name: str, replacements: dict[str, str]) -> Path: + """Render a checked-in template into protected state.""" + text = _render_resource_text(name, replacements) + source = _resource_path(name) + destination = config.manifests_dir / source.name.removesuffix(".tmpl") + _write_text(destination, text) + return destination + + +def _acquire_lock(config: Config) -> IO[str]: + """Acquire the exclusive lifecycle lock.""" + lock_path = _lifecycle_lock_path(config) + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(lock_path, flags, 0o600) + except OSError as error: + raise ProvisionError( + f"cannot safely open lifecycle lock {lock_path}" + ) from error + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.geteuid(): + os.close(descriptor) + raise ProvisionError(f"refusing unsafe lifecycle lock {lock_path}") + handle = os.fdopen(descriptor, "r+", encoding="utf-8") + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + handle.close() + raise ProvisionError( + f"another integration lifecycle command holds {lock_path}" + ) from error + handle.seek(0) + handle.truncate() + handle.write(f"pid={os.getpid()}\n") + handle.flush() + return handle + + +def _lifecycle_lock_path(config: Config) -> Path: + """Return a stable user lock outside the teardown-owned state tree.""" + runtime_parent = Path(os.environ.get("XDG_RUNTIME_DIR", tempfile.gettempdir())) + lock_root = runtime_parent / f"storage-scale-test-integration-{os.geteuid()}" + try: + lock_root.mkdir(mode=0o700, exist_ok=True) + except OSError as error: + raise ProvisionError( + f"cannot create lifecycle lock root {lock_root}" + ) from error + metadata = lock_root.lstat() + if ( + not stat.S_ISDIR(metadata.st_mode) + or metadata.st_uid != os.geteuid() + or stat.S_IMODE(metadata.st_mode) != 0o700 + ): + raise ProvisionError(f"refusing unsafe lifecycle lock root {lock_root}") + identity = hashlib.sha256(str(config.state_dir.resolve()).encode()).hexdigest()[:20] + return lock_root / f"{identity}.lock" + + +@contextmanager +def _nfs_host_lock(runner: Runner): + """Serialize access to host-global NFS files across state directories.""" + directory_probe = runner.run( + [*_sudo_prefix(), "test", "-e", NFS_HOST_LOCK_DIR], check=False + ) + if directory_probe.returncode == 1: + symlink_probe = runner.run( + [*_sudo_prefix(), "test", "-L", NFS_HOST_LOCK_DIR], check=False + ) + if symlink_probe.returncode == 0: + raise ProvisionError( + f"refusing symlinked NFS lock directory {NFS_HOST_LOCK_DIR}" + ) + if symlink_probe.returncode != 1: + raise ProvisionError( + f"cannot inspect NFS lock directory {NFS_HOST_LOCK_DIR}" + ) + runner.run([*_sudo_prefix(), "mkdir", "--", NFS_HOST_LOCK_DIR]) + runner.run([*_sudo_prefix(), "chown", "root:root", NFS_HOST_LOCK_DIR]) + runner.run([*_sudo_prefix(), "chmod", "0755", NFS_HOST_LOCK_DIR]) + elif directory_probe.returncode != 0: + raise ProvisionError(f"cannot inspect NFS lock directory {NFS_HOST_LOCK_DIR}") + runner.run([*_sudo_prefix(), "test", "-d", NFS_HOST_LOCK_DIR]) + runner.run([*_sudo_prefix(), "test", "!", "-L", NFS_HOST_LOCK_DIR]) + directory_stat = runner.run( + [*_sudo_prefix(), "stat", "-c", "%u:%g:%a", NFS_HOST_LOCK_DIR] + ).stdout.strip() + if directory_stat != "0:0:755": + raise ProvisionError( + f"refusing unsafe NFS lock directory {NFS_HOST_LOCK_DIR}: " + f"{directory_stat!r}" + ) + lock_probe = runner.run([*_sudo_prefix(), "test", "-e", NFS_HOST_LOCK], check=False) + if lock_probe.returncode == 1: + symlink_probe = runner.run( + [*_sudo_prefix(), "test", "-L", NFS_HOST_LOCK], check=False + ) + if symlink_probe.returncode == 0: + raise ProvisionError(f"refusing symlinked NFS host lock {NFS_HOST_LOCK}") + if symlink_probe.returncode != 1: + raise ProvisionError(f"cannot inspect NFS host lock {NFS_HOST_LOCK}") + runner.run( + [ + *_sudo_prefix(), + "install", + "-o", + "root", + "-g", + "root", + "-m", + "0666", + "/dev/null", + NFS_HOST_LOCK, + ] + ) + elif lock_probe.returncode != 0: + raise ProvisionError(f"cannot inspect NFS host lock {NFS_HOST_LOCK}") + runner.run([*_sudo_prefix(), "test", "-f", NFS_HOST_LOCK]) + runner.run([*_sudo_prefix(), "test", "!", "-L", NFS_HOST_LOCK]) + lock_stat = runner.run( + [*_sudo_prefix(), "stat", "-c", "%u:%g:%a", NFS_HOST_LOCK] + ).stdout.strip() + if lock_stat != "0:0:666": + raise ProvisionError( + f"refusing unsafe NFS host lock {NFS_HOST_LOCK}: {lock_stat!r}" + ) + flags = os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(NFS_HOST_LOCK, flags) + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != 0 + or metadata.st_gid != 0 + or stat.S_IMODE(metadata.st_mode) != 0o666 + ): + os.close(descriptor) + raise ProvisionError(f"NFS host lock changed while opening: {NFS_HOST_LOCK}") + handle = os.fdopen(descriptor, "r+", encoding="utf-8") + try: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise ProvisionError( + "another integration lifecycle owns the host-global NFS lock" + ) from error + yield + finally: + handle.close() + + +def _require_python() -> None: + """Require the repository's supported Python baseline.""" + if sys.version_info < (3, 12): + raise ProvisionError("integration-test.py requires Python 3.12 or newer") + + +def _nearest_existing_parent(path: Path) -> Path: + """Return the path itself or its nearest existing parent.""" + candidate = path.resolve(strict=False) + while not candidate.exists() and candidate != candidate.parent: + candidate = candidate.parent + return candidate + + +def _check_disk_capacity(disk_path: Path, label: str) -> None: + """Require free space on the filesystem backing one concrete path.""" + disk = shutil.disk_usage(disk_path) + if disk.free < 20 * GIB: + raise ProvisionError( + f"host capacity check failed: need at least 20 GiB free on " + f"{label} ({disk_path})" + ) + LOG.info( + "Host capacity accepted: %.1f GiB free on %s (%s)", + disk.free / GIB, + label, + disk_path, + ) + + +def _check_host_capacity(disk_path: Path) -> None: + """Fail before provisioning an undersized host.""" + cpu_count = os.cpu_count() or 0 + memory = _meminfo() + failures: list[str] = [] + if cpu_count < 2: + failures.append(f"need at least 2 CPUs; found {cpu_count}") + if memory.get("MemTotal", 0) < 8 * GIB: + failures.append("need at least 8 GiB total memory") + if memory.get("MemAvailable", 0) < 6 * GIB: + failures.append("need at least 6 GiB available memory") + if failures: + raise ProvisionError("host capacity check failed: " + "; ".join(failures)) + LOG.info( + "Host capacity accepted: %s CPUs, %.1f GiB available RAM", + cpu_count, + memory["MemAvailable"] / GIB, + ) + _check_disk_capacity(_nearest_existing_parent(disk_path), "fixture storage") + + +def _nfs_capability_failures() -> list[str]: + """Return recognized reasons that the full NFS backend cannot run.""" + failures = [] + for path in (Path("/dev/kmsg"), Path("/dev/loop-control")): + if not path.exists(): + failures.append(f"{path} is absent") + # exportfs is supplied by nfs-kernel-server, which setup installs after + # selection; its pre-setup absence is not a host capability failure. + for command in ("losetup", "mount", "systemctl"): + if not shutil.which(command): + failures.append(f"{command} is unavailable") + return failures + + +def _storage_backend_document(config: Config, backend: str) -> dict[str, object]: + """Return the persistent storage-backend selection document.""" + document: dict[str, object] = { + "schema": STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "backend": backend, + "profile": _fixture_profile(config, backend), + } + if backend == "sbx-shared": + document["shared_root"] = str(config.sbx_shared_root) + else: + document["export_dir"] = str(config.export_dir) + return document + + +def _storage_backend_identity(config: Config, backend: str) -> dict[str, object]: + """Return immutable lifecycle identity without the replaceable profile.""" + document = _storage_backend_document(config, backend) + document.pop("profile") + return document + + +def _nfs_host_owner_document(config: Config) -> dict[str, object]: + """Return ownership identity for fixed host-global NFS artifacts.""" + return { + "schema": STATE_SCHEMA, + "state_dir": str(config.state_dir), + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + } + + +def _read_nfs_host_owner(runner: Runner) -> dict[str, object] | None: + """Read the root-owned host-global NFS owner record.""" + text = _read_system_file(runner, NFS_HOST_OWNER) + if text is None: + return None + try: + document = json.loads(text) + except json.JSONDecodeError as error: + raise ProvisionError( + f"invalid NFS host ownership record: {NFS_HOST_OWNER}" + ) from error + if not isinstance(document, dict): + raise ProvisionError(f"invalid NFS host ownership record: {NFS_HOST_OWNER}") + return document + + +def _claim_nfs_host_owner(runner: Runner, config: Config) -> None: + """Claim fixed NFS paths before their first mutation.""" + expected = _nfs_host_owner_document(config) + current = _read_nfs_host_owner(runner) + if current is not None: + if current != expected: + raise ProvisionError( + f"host-global NFS artifacts are owned by another fixture: {current}" + ) + return + occupied = [ + path + for path in (NFS_EXPORT_CONFIG, NFS_DAEMON_CONFIG) + if _read_system_file(runner, path) is not None + ] + if occupied: + raise ProvisionError( + "refusing to overwrite unowned host NFS configuration: " + + ", ".join(str(path) for path in occupied) + ) + source = config.manifests_dir / "nfs-host-owner.json" + _write_text(source, json.dumps(expected, sort_keys=True) + "\n", mode=0o600) + runner.run([*_sudo_prefix(), "install", "-m", "0600", source, NFS_HOST_OWNER]) + + +def _validate_nfs_host_owner(runner: Runner, config: Config) -> bool: + """Validate the fixed owner record when it exists.""" + current = _read_nfs_host_owner(runner) + if current is None: + return False + expected = _nfs_host_owner_document(config) + if current != expected: + raise ProvisionError( + f"refusing host NFS cleanup with mismatched owner: {NFS_HOST_OWNER}" + ) + return True + + +def _validate_retained_state_summary(config: Config) -> None: + """Reject immutable option changes after a setup has completed.""" + path = config.state_dir / "state.json" + if not path.exists(): + return + state = json.loads(path.read_text(encoding="utf-8")) + backend = state.get("storage_backend") + if backend not in STORAGE_BACKENDS: + raise ProvisionError(f"invalid retained setup state: {path}") + expected: dict[str, object] = { + "schema": STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + } + if backend == "nfs": + expected["export_dir"] = str(config.export_dir) + mismatches = [name for name, value in expected.items() if state.get(name) != value] + backend_changed = config.storage_backend not in {"auto", backend} + if mismatches or backend_changed: + changed = ", ".join(mismatches or ["storage_backend"]) + raise ProvisionError( + f"retained setup differs in immutable fields ({changed}); " + "use the original options to teardown first" + ) + + +def _select_storage_backend(config: Config) -> str: + """Select once, persist, and validate the integration storage backend.""" + _validate_retained_state_summary(config) + path = config.state_dir / "storage-backend.json" + if path.exists(): + document = json.loads(path.read_text(encoding="utf-8")) + backend = str(document.get("backend", "")) + if backend not in STORAGE_BACKENDS: + raise ProvisionError(f"invalid retained storage backend state: {path}") + retained_identity = dict(document) + retained_identity.pop("profile", None) + expected_identity = _storage_backend_identity(config, backend) + if retained_identity != expected_identity: + raise ProvisionError(f"invalid retained storage backend state: {path}") + if config.storage_backend not in {"auto", backend}: + raise ProvisionError( + f"retained setup uses storage backend {backend}; teardown is " + f"required before selecting {config.storage_backend}" + ) + return backend + + failures = _nfs_capability_failures() + backend = config.storage_backend + if backend == "auto": + backend = "sbx-shared" if failures else "nfs" + if backend == "nfs" and failures: + raise ProvisionError( + "NFS storage backend requirements are unavailable: " + "; ".join(failures) + ) + _write_text( + path, + json.dumps(_storage_backend_document(config, backend), sort_keys=True) + "\n", + ) + if backend == "sbx-shared": + LOG.info( + "Using the Docker SBX shared-path backend for the required RWX " + "storage contract" + ) + else: + LOG.info("Using full-fidelity NFS CSI storage backend") + return backend + + +def _record_storage_backend_profile(config: Config, backend: str) -> None: + """Persist the exact profile after cluster reconciliation succeeds.""" + _write_text( + config.state_dir / "storage-backend.json", + json.dumps(_storage_backend_document(config, backend), sort_keys=True) + "\n", + ) + + +def _meminfo() -> dict[str, int]: + """Read selected Linux memory counters in bytes.""" + result: dict[str, int] = {} + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + name, value = line.split(":", maxsplit=1) + result[name] = int(value.strip().split()[0]) * 1024 + return result + + +def _check_platform() -> str: + """Validate Linux and return the download architecture.""" + if sys.platform != "linux": + raise ProvisionError( + "the integration environment currently supports Linux only" + ) + architectures = {"x86_64": "amd64", "aarch64": "arm64"} + try: + return architectures[platform.machine()] + except KeyError as error: + raise ProvisionError( + f"unsupported architecture: {platform.machine()}" + ) from error + + +def _ensure_apt_packages(runner: Runner, backend: str) -> None: + """Install the narrow tested Ubuntu/Debian package set when absent.""" + packages = [ + "ca-certificates", + "curl", + "file", + "jq", + "openssh-client", + "openssl", + "python3-venv", + ] + if backend == "nfs": + packages.extend(("e2fsprogs", "nfs-common", "nfs-kernel-server")) + elif not shutil.which("kind"): + packages.append("kind") + missing = [ + package + for package in packages + if runner.run( + ["dpkg-query", "-W", "-f=${db:Status-Abbrev}", package], check=False + ).stdout.strip() + != "ii" + ] + if not missing: + LOG.info("Required operating-system packages are already installed") + return + if backend == "sbx-shared": + raise ProvisionError( + "sbx-shared requires its host packages to be preinstalled and " + "never invokes sudo; missing: " + ", ".join(missing) + ) + if not shutil.which("apt-get"): + raise ProvisionError(f"missing packages and apt-get is unavailable: {missing}") + LOG.info("Installing required packages: %s", ", ".join(missing)) + runner.run([*_sudo_prefix(), "apt-get", "update"], timeout=600) + runner.run( + [ + *_sudo_prefix(), + "env", + "DEBIAN_FRONTEND=noninteractive", + "apt-get", + "install", + "-y", + "--no-install-recommends", + *missing, + ], + timeout=600, + ) + + +def _prepare_host_dependencies(runner: Runner, config: Config, backend: str) -> None: + """Install host packages after capturing service ownership.""" + if backend == "nfs": + _record_nfs_service_state(runner, config) + _ensure_apt_packages(runner, backend) + + +def _ensure_docker(runner: Runner) -> None: + """Require a working rootful Docker daemon.""" + if not shutil.which("docker"): + raise ProvisionError( + "Docker is required but absent; install a supported rootful Docker Engine first" + ) + result = runner.run( + ["docker", "info", "--format", "{{json .SecurityOptions}}"], timeout=60 + ) + if "rootless" in result.stdout.lower(): + raise ProvisionError( + "rootless Docker is not supported by this integration fixture" + ) + LOG.info("Rootful Docker is available") + + +def _check_docker_capacity(runner: Runner) -> None: + """Check Docker's data filesystem when it is visible to this process.""" + result = runner.run( + ["docker", "info", "--format", "{{.DockerRootDir}}"], timeout=60 + ) + docker_root = Path(result.stdout.strip()) + if not result.stdout.strip() or not docker_root.exists(): + LOG.info( + "Docker data root %s is not visible locally; skipping host-side " + "free-space validation for it", + docker_root if result.stdout.strip() else "", + ) + return + _check_disk_capacity(docker_root, "Docker data root") + + +def _command_version(runner: Runner, command: str) -> str: + """Return normalized version output, or an empty string if unavailable.""" + if not shutil.which(command): + return "" + arguments = { + "kind": ["kind", "version"], + "kubectl": ["kubectl", "version", "--client"], + "helm": ["helm", "version", "--short"], + }[command] + return runner.run(arguments, check=False, timeout=30).stdout + + +def _ensure_client_tools( + runner: Runner, + config: Config, + architecture: str, + storage_backend: str, +) -> None: + """Install checksum-verified clients into the user-owned state tree.""" + expected = { + "kind": KIND_VERSION, + "kubectl": ( + SBX_KUBECTL_VERSION if storage_backend == "sbx-shared" else KUBECTL_VERSION + ), + "helm": HELM_VERSION, + } + for command, version in expected.items(): + if command == "kind" and storage_backend == "sbx-shared": + installed = _command_version(runner, command) + if SBX_KIND_VERSION not in installed: + raise ProvisionError( + "sbx-shared compatibility profile requires kind " + f"{SBX_KIND_VERSION}; found {installed.strip() or 'nothing'}" + ) + LOG.info( + "Using Docker SBX compatibility profile with %s", installed.strip() + ) + continue + if version in _command_version(runner, command): + LOG.info("Using %s %s", command, version) + continue + _install_client_tool(runner, config, command, version, architecture) + + +def _install_client_tool( + runner: Runner, + config: Config, + command: str, + version: str, + architecture: str, +) -> None: + """Download, verify, and install one user-local client binary.""" + LOG.info("Installing %s %s", command, version) + with tempfile.TemporaryDirectory(prefix="storage-scale-tool-") as directory: + target = Path(directory) + if command == "kind": + binary = _download_kind(runner, target, version, architecture) + elif command == "kubectl": + binary = _download_kubectl(runner, target, version, architecture) + else: + binary = _download_helm(runner, target, version, architecture) + runner.run( + ["install", "-m", "0755", binary, config.state_dir / "bin" / command] + ) + + +def _curl(runner: Runner, url: str, destination: Path) -> None: + """Download one URL with bounded retries.""" + runner.run( + [ + "curl", + "--fail", + "--location", + "--retry", + "3", + "--retry-all-errors", + "--output", + destination, + url, + ], + timeout=300, + ) + + +def _verify_sha256(path: Path, expected: str) -> None: + """Verify one downloaded file against an expected SHA-256.""" + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected.lower(): + raise ProvisionError( + f"SHA-256 mismatch for {path.name}: {actual} != {expected}" + ) + + +def _download_kind( + runner: Runner, directory: Path, version: str, architecture: str +) -> Path: + """Download and verify kind.""" + name = f"kind-linux-{architecture}" + base = f"https://github.com/kubernetes-sigs/kind/releases/download/{version}" + binary = directory / "kind" + checksum = directory / "kind.sha256sum" + _curl(runner, f"{base}/{name}", binary) + _curl(runner, f"{base}/{name}.sha256sum", checksum) + _verify_sha256(binary, checksum.read_text(encoding="utf-8").split()[0]) + return binary + + +def _download_kubectl( + runner: Runner, directory: Path, version: str, architecture: str +) -> Path: + """Download and verify kubectl.""" + base = f"https://dl.k8s.io/release/{version}/bin/linux/{architecture}/kubectl" + binary = directory / "kubectl" + checksum = directory / "kubectl.sha256" + _curl(runner, base, binary) + _curl(runner, f"{base}.sha256", checksum) + _verify_sha256(binary, checksum.read_text(encoding="utf-8").strip()) + return binary + + +def _download_helm( + runner: Runner, directory: Path, version: str, architecture: str +) -> Path: + """Download and verify Helm 3.""" + archive_name = f"helm-{version}-linux-{architecture}.tar.gz" + base = f"https://get.helm.sh/{archive_name}" + archive = directory / archive_name + checksum = directory / f"{archive_name}.sha256sum" + _curl(runner, base, archive) + _curl(runner, f"{base}.sha256sum", checksum) + _verify_sha256(archive, checksum.read_text(encoding="utf-8").split()[0]) + with tarfile.open(archive, "r:gz") as tar: + member = tar.getmember(f"linux-{architecture}/helm") + member.name = "helm" + tar.extract(member, directory, filter="data") + return directory / "helm" + + +def _kubectl(config: Config, *arguments: str | Path) -> list[str | Path]: + """Build a kubectl command using the private kubeconfig.""" + return ["kubectl", "--kubeconfig", config.kubeconfig, *arguments] + + +def _kind_clusters(runner: Runner) -> set[str]: + """Return running kind cluster names.""" + if not shutil.which("kind"): + LOG.info("kind is unavailable; relying on Docker cluster discovery") + return set() + result = runner.run(["kind", "get", "clusters"], check=False, timeout=30) + if result.returncode: + raise ProvisionError( + "kind could not list clusters: " + f"{(result.stderr or result.stdout).strip()}" + ) + return { + line.strip() + for line in result.stdout.splitlines() + if line.strip() and "No kind clusters" not in line + } + + +def _kind_containers(runner: Runner, config: Config, running_only: bool) -> list[str]: + """Return kind node container IDs owned by this cluster.""" + arguments = ["docker", "ps"] + if not running_only: + arguments.append("--all") + arguments.extend( + [ + "--filter", + f"label=io.x-k8s.kind.cluster={config.cluster_name}", + "--format", + "{{.ID}}", + ] + ) + output = runner.run(arguments, timeout=30).stdout + return [line for line in output.splitlines() if line] + + +def _render_kind_config(config: Config, backend: str) -> Path: + """Render the immutable three-node topology.""" + name, replacements = _kind_config_inputs(config, backend) + return _render_resource(config, name, replacements) + + +def _kind_config_inputs(config: Config, backend: str) -> tuple[str, dict[str, str]]: + """Return the exact kind template and interpolation inputs.""" + if backend != "sbx-shared": + return "manifests/kind.yaml.tmpl", {"CLUSTER_NAME": config.cluster_name} + kmsg_mount = "" + if not Path("/dev/kmsg").exists(): + kmsg_mount = "\n".join( + (" - hostPath: /dev/null", " containerPath: /dev/kmsg") + ) + return ( + "manifests/kind-sbx-shared.yaml.tmpl", + { + "CLUSTER_NAME": config.cluster_name, + "SHARED_ROOT": str(config.sbx_shared_root), + "KMSG_MOUNT": kmsg_mount, + }, + ) + + +def _fixture_profile(config: Config, backend: str) -> dict[str, str]: + """Return the cluster inputs that require disposable-cluster replacement.""" + name, replacements = _kind_config_inputs(config, backend) + rendered = _render_resource_text(name, replacements) + return { + "kind_version": SBX_KIND_VERSION if backend == "sbx-shared" else KIND_VERSION, + "node_image": ( + SBX_KIND_NODE_IMAGE if backend == "sbx-shared" else KIND_NODE_IMAGE + ), + "kind_config_sha256": hashlib.sha256(rendered.encode()).hexdigest(), + } + + +def _create_cluster(runner: Runner, config: Config, backend: str) -> None: + """Create a new owned kind cluster.""" + manifest = _render_kind_config(config, backend) + node_image = SBX_KIND_NODE_IMAGE if backend == "sbx-shared" else KIND_NODE_IMAGE + _ensure_pinned_image(runner, node_image) + LOG.info("Creating three-node kind cluster %s", config.cluster_name) + runner.run( + [ + "kind", + "create", + "cluster", + "--name", + config.cluster_name, + "--image", + node_image, + "--config", + manifest, + "--kubeconfig", + config.kubeconfig, + "--wait", + "180s", + ], + timeout=600, + ) + + +def _validate_sbx_shared_root(config: Config) -> None: + """Require the Docker SBX root to be a narrow path inside repository tmp.""" + allowed_parent = (_repository_root() / "tmp").resolve() + root = config.sbx_shared_root + if root == allowed_parent or allowed_parent not in root.parents: + raise ProvisionError( + f"Docker SBX shared root must be below {allowed_parent}: {root}" + ) + + +def _sbx_shared_marker(config: Config) -> dict[str, object]: + """Return the exact marker for the repository-backed shared root.""" + return { + **_owner_document(config), + "backend": "sbx-shared", + "shared_root": str(config.sbx_shared_root), + } + + +def _prepare_sbx_shared(runner: Runner, config: Config) -> None: + """Create and prove the repository-backed Docker-shared directory.""" + _validate_sbx_shared_root(config) + _ensure_pinned_image(runner, SBX_KIND_NODE_IMAGE) + root = config.sbx_shared_root + marker = root / EXPORT_MARKER + if root.exists() and not marker.exists() and any(root.iterdir()): + raise ProvisionError(f"refusing nonempty unowned SBX shared root: {root}") + root.mkdir(parents=True, exist_ok=True) + if marker.exists(): + document = json.loads(marker.read_text(encoding="utf-8")) + if document != _sbx_shared_marker(config): + raise ProvisionError(f"SBX shared marker does not match: {marker}") + else: + _write_text( + marker, + json.dumps(_sbx_shared_marker(config), sort_keys=True) + "\n", + ) + for directory in (root / "storage-test", root / "ssh-home"): + directory.mkdir(exist_ok=True) + # Docker SBX can remap one pod's file owner when the same bind mount is + # observed from a replacement pod. A sticky directory would then stop + # the fixed workload identity from deleting its own prior files. + directory.chmod(SBX_SHARED_DIRECTORY_MODE) + token = secrets.token_hex(16) + source = root / "agent-probe" + source.write_text(token + "\n", encoding="utf-8") + try: + runner.run( + [ + "docker", + "run", + "--rm", + "--entrypoint", + "sh", + "--mount", + f"type=bind,src={root},dst=/probe", + SBX_KIND_NODE_IMAGE, + "-ec", + 'test "$(cat /probe/agent-probe)" = "$1"; ' + 'printf "%s\\n" "$1" >/probe/engine-probe', + "sbx-shared-probe", + token, + ], + timeout=300, + ) + if (root / "engine-probe").read_text(encoding="utf-8").strip() != token: + raise ProvisionError("Docker shared-path probe returned the wrong token") + finally: + source.unlink(missing_ok=True) + (root / "engine-probe").unlink(missing_ok=True) + + +def _configure_sbx_node_trust(runner: Runner, config: Config) -> None: + """Install Docker SBX's proxy CA in kind nodes when it is present.""" + source = Path("/usr/local/share/ca-certificates/proxy-ca.crt") + if not source.is_file(): + LOG.info("Docker SBX proxy CA is absent; leaving kind trust unchanged") + return + expected = hashlib.sha256(source.read_bytes()).hexdigest() + destination = "/usr/local/share/ca-certificates/docker-sbx-proxy-ca.crt" + nodes = _kind_containers(runner, config, running_only=True) + if len(nodes) != 3: + raise ProvisionError( + f"cannot configure Docker SBX trust: expected 3 nodes, found {len(nodes)}" + ) + for node in nodes: + current = runner.run( + ["docker", "exec", node, "sha256sum", destination], check=False + ) + if current.returncode == 0 and current.stdout.split()[0] == expected: + continue + runner.run(["docker", "cp", source, f"{node}:{destination}"]) + runner.run(["docker", "exec", node, "update-ca-certificates"]) + runner.run(["docker", "exec", node, "systemctl", "restart", "containerd"]) + runner.run(["docker", "exec", node, "systemctl", "restart", "kubelet"]) + LOG.info("Configured kind nodes to trust the Docker SBX proxy CA") + + +def _ensure_cluster_ownership(config: Config, cluster_exists: bool) -> None: + """Claim a new cluster name or validate its persistent ownership marker.""" + marker = config.state_dir / "cluster-owner.json" + expected = {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name} + if marker.exists(): + if json.loads(marker.read_text(encoding="utf-8")) != expected: + raise ProvisionError(f"cluster ownership marker does not match: {marker}") + return + if cluster_exists: + raise ProvisionError( + f"refusing to adopt existing unowned kind cluster {config.cluster_name}; " + f"use another --cluster-name or restore {marker}" + ) + _write_text(marker, json.dumps(expected, sort_keys=True) + "\n") + + +def _export_kubeconfig(runner: Runner, config: Config) -> None: + """Refresh the private kubeconfig for an existing cluster.""" + runner.run( + [ + "kind", + "export", + "kubeconfig", + "--name", + config.cluster_name, + "--kubeconfig", + config.kubeconfig, + ], + timeout=60, + ) + + +def _delete_cluster(runner: Runner, config: Config) -> None: + """Delete the exact marker-owned disposable kind cluster.""" + LOG.info("Deleting disposable kind cluster %s", config.cluster_name) + runner.run( + ["kind", "delete", "cluster", "--name", config.cluster_name], timeout=300 + ) + leftovers = _kind_containers(runner, config, running_only=False) + if leftovers: + raise ProvisionError( + "kind reported successful deletion, but cluster containers remain: " + + ", ".join(leftovers) + ) + + +def _wait_for_cluster(runner: Runner, config: Config) -> None: + """Wait for exactly three Ready nodes and enforce fixture labels.""" + _wait_for_kube_api(runner, config) + runner.run( + _kubectl( + config, + "wait", + "--for=condition=Ready", + "nodes", + "--all", + "--timeout=180s", + ), + timeout=210, + ) + runner.run( + _kubectl( + config, + "taint", + "nodes", + f"{config.cluster_name}-control-plane", + "node-role.kubernetes.io/control-plane:NoSchedule-", + ), + check=False, + ) + nodes = json.loads( + runner.run(_kubectl(config, "get", "nodes", "-o", "json")).stdout + ) + if len(nodes["items"]) != 3: + raise ProvisionError( + f"expected exactly 3 Kubernetes nodes; found {len(nodes['items'])}" + ) + _validate_node_labels(nodes) + + +def _expected_kubernetes_version(backend: str) -> str: + """Return the kubelet version selected by one fixture profile.""" + return SBX_KUBERNETES_VERSION if backend == "sbx-shared" else KUBERNETES_VERSION + + +def _observed_kubernetes_version(runner: Runner, config: Config) -> str: + """Return the one kubelet version observed on every fixture node.""" + nodes = json.loads( + runner.run(_kubectl(config, "get", "nodes", "-o", "json")).stdout + ).get("items", []) + versions = { + node.get("status", {}).get("nodeInfo", {}).get("kubeletVersion") + for node in nodes + } + versions.discard(None) + if len(nodes) != 3 or len(versions) != 1: + raise ProvisionError( + "retained cluster does not have three nodes at one kubelet version: " + f"nodes={len(nodes)}, versions={sorted(versions)}" + ) + return str(versions.pop()) + + +def _retained_cluster_matches_profile( + runner: Runner, config: Config, backend: str +) -> bool: + """Return whether a running retained cluster matches the selected profile.""" + state_path = config.state_dir / "storage-backend.json" + if not state_path.exists(): + LOG.warning("Replacing retained kind cluster without a fixture profile") + return False + retained = json.loads(state_path.read_text(encoding="utf-8")) + if retained.get("profile") != _fixture_profile(config, backend): + LOG.warning("Replacing retained kind cluster after fixture profile drift") + return False + _wait_for_kube_api(runner, config) + observed = _observed_kubernetes_version(runner, config) + expected = _expected_kubernetes_version(backend) + if observed == expected: + return True + LOG.warning( + "Replacing retained kind cluster with kubelet %s; profile requires %s", + observed, + expected, + ) + return False + + +def _wait_for_kube_api(runner: Runner, config: Config) -> None: + """Wait for the Kubernetes API to become usable.""" + deadline = time.monotonic() + 90 + while True: + request_timeout, process_timeout = _poll_timeouts(deadline, 5) + if process_timeout <= 0: + break + probe = runner.run( + _kubectl( + config, + "get", + "nodes", + f"--request-timeout={request_timeout}s", + ), + check=False, + timeout=process_timeout, + ) + if probe.returncode == 0: + return + LOG.info("Waiting for the Kubernetes API") + time.sleep(min(3, max(0, deadline - time.monotonic()))) + raise ProvisionError("Kubernetes API did not become usable within 90 seconds") + + +def _poll_timeouts(deadline: float, request_limit: int) -> tuple[int, float]: + """Return API and process bounds that cannot cross an overall deadline.""" + remaining = deadline - time.monotonic() + if remaining <= 0: + return 0, 0 + request_timeout = max(1, min(request_limit, int(remaining))) + return request_timeout, min(request_timeout + 5, remaining) + + +def _validate_node_labels(nodes: dict[str, object]) -> None: + """Validate two targets and one target-negative login node.""" + target_count = 0 + login_count = 0 + for node in nodes["items"]: # type: ignore[index] + labels = node["metadata"]["labels"] # type: ignore[index] + target_count += labels.get(TARGET_LABEL.split("=")[0]) == "true" + login_count += labels.get(LOGIN_LABEL.split("=")[0]) == "true" + if target_count != 2 or login_count != 1: + raise ProvisionError( + f"node label invariant failed: target nodes={target_count}, login nodes={login_count}" + ) + + +def _kind_ipv4_network(runner: Runner) -> tuple[str, str]: + """Return the kind Docker network's IPv4 subnet and gateway.""" + data = json.loads(runner.run(["docker", "network", "inspect", "kind"]).stdout) + for entry in data[0]["IPAM"]["Config"]: + subnet = entry.get("Subnet", "") + if "." in subnet: + return subnet, entry["Gateway"] + raise ProvisionError("kind Docker network has no IPv4 IPAM entry") + + +def _ensure_export_marker(runner: Runner, config: Config) -> None: + """Create or validate ownership of the dedicated host export.""" + marker_path = config.export_dir / EXPORT_MARKER + expected = json.dumps( + {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name}, sort_keys=True + ) + export_probe = runner.run( + [*_sudo_prefix(), "test", "-e", config.export_dir], + check=False, + timeout=30, + ) + if export_probe.returncode not in {0, 1}: + raise ProvisionError( + f"could not determine whether export exists: {config.export_dir}" + ) + export_exists = export_probe.returncode == 0 + existing = _read_system_file(runner, marker_path) + if export_exists and existing is None: + raise ProvisionError( + f"refusing to modify unowned export directory: {config.export_dir}" + ) + if not export_exists: + parent_exists = runner.run( + [*_sudo_prefix(), "test", "-d", config.export_dir.parent], + check=False, + timeout=30, + ) + if parent_exists.returncode not in {0, 1}: + raise ProvisionError( + f"could not inspect export parent: {config.export_dir.parent}" + ) + if parent_exists.returncode == 1: + raise ProvisionError( + "export directory must be a leaf below an existing directory: " + f"{config.export_dir}" + ) + if existing is not None and existing.strip() != expected: + raise ProvisionError( + f"refusing export with mismatched ownership marker: {marker_path}" + ) + runner.run([*_sudo_prefix(), "install", "-d", "-m", "0770", config.export_dir]) + runner.run([*_sudo_prefix(), "chown", f"+{NFS_UID}:+{NFS_GID}", config.export_dir]) + marker_source = config.state_dir / "export-marker.json" + _write_text(marker_source, expected + "\n") + runner.run( + [ + *_sudo_prefix(), + "install", + "-m", + "0644", + marker_source, + marker_path, + ] + ) + + +def _export_mount_type(runner: Runner, config: Config) -> str: + """Return the export mount filesystem type, or an empty string.""" + result = runner.run( + [ + "findmnt", + "--noheadings", + "--output", + "FSTYPE", + "--mountpoint", + config.export_dir, + ], + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + if result.returncode == 1: + return "" + raise ProvisionError( + f"findmnt could not inspect {config.export_dir}: " + f"{(result.stderr or result.stdout).strip()}" + ) + + +def _unmount_temporary_filesystem(runner: Runner, mountpoint: Path) -> None: + """Unmount a temporary filesystem and verify it is no longer mounted.""" + last_result: subprocess.CompletedProcess[str] | None = None + for attempt in range(1, TEMPORARY_UNMOUNT_ATTEMPTS + 1): + last_result = runner.run([*_sudo_prefix(), "umount", mountpoint], check=False) + mounted = runner.run( + [ + "findmnt", + "--noheadings", + "--mountpoint", + mountpoint, + ], + check=False, + ) + if mounted.returncode == 1: + return + if mounted.returncode != 0: + raise ProvisionError( + f"could not verify temporary mount was removed at {mountpoint}: " + f"findmnt exited {mounted.returncode}" + ) + if attempt < TEMPORARY_UNMOUNT_ATTEMPTS: + LOG.warning( + "Temporary mount %s remains active after unmount attempt %d/%d; " + "retrying", + mountpoint, + attempt, + TEMPORARY_UNMOUNT_ATTEMPTS, + ) + time.sleep(TEMPORARY_UNMOUNT_RETRY_SECONDS) + + detail = (last_result.stderr or last_result.stdout).strip() if last_result else "" + suffix = f": {detail}" if detail else "" + raise ProvisionError( + f"temporary mount remains active at {mountpoint} after " + f"{TEMPORARY_UNMOUNT_ATTEMPTS} unmount attempts{suffix}" + ) + + +def _loop_devices_for_file(runner: Runner, path: Path) -> list[str]: + """Return loop devices associated with one exact backing file.""" + if not path.exists(): + return [] + result = runner.run([*_sudo_prefix(), "losetup", "--associated", path], check=False) + if result.returncode: + raise ProvisionError( + f"losetup could not inspect {path}: " + f"{(result.stderr or result.stdout).strip()}" + ) + return [line.split(":", maxsplit=1)[0] for line in result.stdout.splitlines()] + + +def _migrate_export_data_to_image(runner: Runner, config: Config) -> None: + """Transactionally create an NFS image containing existing export data.""" + LOG.info("Creating sparse %s-byte NFS backing filesystem", NFS_IMAGE_BYTES) + temporary_image = config.nfs_image.with_suffix(".ext4.new") + stale_loops = _loop_devices_for_file(runner, temporary_image) + if stale_loops: + raise ProvisionError( + f"temporary NFS image remains attached to {', '.join(stale_loops)}; " + "unmount it before retrying setup" + ) + temporary_image.unlink(missing_ok=True) + runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), temporary_image]) + runner.run(["/usr/sbin/mkfs.ext4", "-F", "-q", "-m", "0", temporary_image]) + + migration_mount = Path( + tempfile.mkdtemp(prefix="nfs-migration-", dir=config.state_dir) + ) + try: + try: + runner.run( + [ + *_sudo_prefix(), + "mount", + "-o", + "loop", + temporary_image, + migration_mount, + ] + ) + runner.run( + [ + *_sudo_prefix(), + "cp", + "-a", + f"{config.export_dir}/.", + f"{migration_mount}/", + ] + ) + finally: + # A failed mount command can still leave a live mount. Always + # verify unmount before the path or temporary image can be removed. + _unmount_temporary_filesystem(runner, migration_mount) + check = runner.run( + [*_sudo_prefix(), "e2fsck", "-pf", temporary_image], + check=False, + timeout=300, + ) + if check.returncode not in {0, 1}: + raise ProvisionError( + "temporary NFS filesystem validation failed with exit " + f"{check.returncode}: {(check.stderr or check.stdout).strip()}" + ) + temporary_image.replace(config.nfs_image) + except Exception: + mounted = runner.run( + ["findmnt", "--noheadings", "--mountpoint", migration_mount], + check=False, + ) + if mounted.returncode == 1 and not _loop_devices_for_file( + runner, temporary_image + ): + temporary_image.unlink(missing_ok=True) + migration_mount.rmdir() + raise + migration_mount.rmdir() + + +def _ensure_nfs_image_capacity( + runner: Runner, config: Config, loop_device: str | None = None +) -> None: + """Grow the sparse NFS image and its ext4 filesystem when required.""" + current_size = config.nfs_image.stat().st_size + if current_size < NFS_IMAGE_BYTES: + LOG.info( + "Growing sparse NFS backing image from %s to %s bytes", + current_size, + NFS_IMAGE_BYTES, + ) + runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), config.nfs_image]) + if loop_device: + runner.run( + [*_sudo_prefix(), "losetup", "--set-capacity", loop_device], + timeout=60, + ) + runner.run( + [*_sudo_prefix(), "resize2fs", loop_device or config.nfs_image], + timeout=300, + ) + + +def _validate_nfs_filesystem_capacity(runner: Runner, config: Config) -> None: + """Require the mounted export to satisfy the shared fixture budget.""" + result = runner.run( + [ + *_sudo_prefix(), + "stat", + "-f", + "-c", + "%b %S %a", + config.export_dir, + ], + timeout=30, + ) + fields = result.stdout.split() + if len(fields) != 3 or not all(field.isdigit() for field in fields): + raise ProvisionError( + f"could not parse NFS filesystem capacity: {result.stdout!r}" + ) + blocks, fragment_size, available_blocks = map(int, fields) + total_bytes = blocks * fragment_size + available_bytes = available_blocks * fragment_size + if total_bytes < NFS_BUDGET_BYTES or available_bytes < NFS_BUDGET_BYTES: + raise ProvisionError( + "NFS backing filesystem is below the fixture capacity budget: " + f"total={total_bytes}, available={available_bytes}, " + f"required_available={NFS_BUDGET_BYTES}" + ) + + +def _ensure_export_filesystem(runner: Runner, config: Config) -> None: + """Mount a sized persistent filesystem for realistic mount validation.""" + _ensure_export_marker(runner, config) + mounted_type = _export_mount_type(runner, config) + if mounted_type: + if mounted_type != "ext4": + raise ProvisionError( + f"refusing non-ext4 mount at dedicated export {config.export_dir}: " + f"{mounted_type}" + ) + loop_device = _verified_export_loop(runner, config) + _ensure_nfs_image_capacity(runner, config, loop_device) + _validate_nfs_filesystem_capacity(runner, config) + return + + if not config.nfs_image.exists(): + _migrate_export_data_to_image(runner, config) + _ensure_nfs_image_capacity(runner, config) + runner.run([*_sudo_prefix(), "exportfs", "-u", config.export_dir], check=False) + runner.run( + [*_sudo_prefix(), "mount", "-o", "loop", config.nfs_image, config.export_dir] + ) + _ensure_export_marker(runner, config) + _validate_nfs_filesystem_capacity(runner, config) + + +def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> None: + """Reconcile the narrow NFSv4 export and firewall rule.""" + LOG.info("Configuring NFSv4 export for kind subnet %s", subnet) + _record_nfs_service_state(runner, config) + _ensure_export_filesystem(runner, config) + _ensure_export_marker(runner, config) + export_line = ( + f"{config.export_dir} {subnet}(rw,sync,no_subtree_check,fsid=0," + f"all_squash,anonuid={NFS_UID},anongid={NFS_GID})\n" + ) + export_source = config.manifests_dir / "storage-scale-test.exports" + nfs_source = config.manifests_dir / "storage-scale-test-nfs.conf" + desired = ( + (NFS_EXPORT_CONFIG, export_source, export_line), + ( + NFS_DAEMON_CONFIG, + nfs_source, + f"[nfsd]\nvers3 = n\nvers4 = y\nthreads = {NFS_SERVER_THREADS}\n", + ), + ) + for entry in desired: + installed, source = entry[:2] + retained = source.read_text(encoding="utf-8") if source.exists() else None + existing = _read_system_file(runner, installed) + if existing is not None and existing != retained: + raise ProvisionError( + f"refusing to overwrite changed host NFS configuration: {installed}" + ) + for entry in desired: + source, content = entry[1:] + _write_text(source, content) + runner.run([*_sudo_prefix(), "install", "-d", "/etc/exports.d", "/etc/nfs.conf.d"]) + for entry in desired: + installed, source = entry[:2] + runner.run( + [ + *_sudo_prefix(), + "install", + "-m", + "0644", + source, + installed, + ] + ) + _ensure_nfs_firewall(runner, config, subnet) + runner.run([*_sudo_prefix(), "exportfs", "-rav"]) + runner.run([*_sudo_prefix(), "systemctl", "enable", "--now", "nfs-server"]) + _set_live_nfs_threads(runner, NFS_SERVER_THREADS) + state = {"subnet": subnet, "gateway": gateway} + _write_text(config.state_dir / "network.json", json.dumps(state, indent=2) + "\n") + + +def _live_nfs_threads(runner: Runner) -> int | None: + """Return the active kernel NFS worker count when it is available.""" + result = runner.run( + [*_sudo_prefix(), "cat", NFS_THREADS_PATH], + check=False, + timeout=30, + ) + value = result.stdout.strip() + if result.returncode or not value.isdigit() or int(value) < 1: + return None + return int(value) + + +def _set_live_nfs_threads(runner: Runner, count: int) -> None: + """Apply and verify a kernel NFS worker count without restarting NFS.""" + current = _live_nfs_threads(runner) + if current == count: + return + rpc_nfsd = shutil.which("rpc.nfsd") + if not rpc_nfsd: + raise ProvisionError("rpc.nfsd is unavailable; cannot set NFS worker count") + runner.run([*_sudo_prefix(), rpc_nfsd, str(count)], timeout=60) + actual = _live_nfs_threads(runner) + if actual != count: + raise ProvisionError( + f"NFS worker reconciliation requested {count}, found {actual!r}" + ) + + +def _systemd_unit_exists(runner: Runner, unit: str) -> bool: + """Return whether systemd currently knows about one unit.""" + result = runner.run( + [*_sudo_prefix(), "systemctl", "show", "--property=LoadState", "--value", unit], + check=False, + ) + if result.returncode: + raise ProvisionError( + f"systemctl could not inspect {unit}: " + f"{(result.stderr or result.stdout).strip()}" + ) + return result.stdout.strip() not in {"", "not-found"} + + +def _systemd_unit_state(runner: Runner, unit: str) -> tuple[bool, bool, bool]: + """Return unit existence plus independent active and enabled states.""" + exists = _systemd_unit_exists(runner, unit) + if not exists: + return False, False, False + active = ( + runner.run( + [*_sudo_prefix(), "systemctl", "is-active", unit], check=False + ).returncode + == 0 + ) + enabled = ( + runner.run( + [*_sudo_prefix(), "systemctl", "is-enabled", unit], check=False + ).returncode + == 0 + ) + return True, active, enabled + + +def _record_nfs_service_state(runner: Runner, config: Config) -> None: + """Remember independent NFS installation, runtime, and boot states.""" + path = config.state_dir / "nfs-service.json" + if path.exists(): + state = json.loads(path.read_text(encoding="utf-8")) + if state.get("schema") != NFS_SERVICE_STATE_SCHEMA: + raise ProvisionError( + "retained NFS service state predates independent active/enabled " + "tracking; run teardown before setup" + ) + return + existed, active, enabled = _systemd_unit_state(runner, "nfs-server") + state: dict[str, object] = { + "schema": NFS_SERVICE_STATE_SCHEMA, + "service_existed": existed, + "was_active": active, + "was_enabled": enabled, + "previous_threads": None, + } + if active: + previous_threads = _live_nfs_threads(runner) + if previous_threads is None: + raise ProvisionError( + "cannot record the pre-existing NFS worker count before setup" + ) + state["previous_threads"] = previous_threads + _write_text(path, json.dumps(state, sort_keys=True) + "\n") + + +def _ensure_nfs_firewall(runner: Runner, config: Config, subnet: str) -> None: + """Allow NFS only from kind when UFW is active.""" + state_path = config.state_dir / "ufw-rule.json" + previous: dict[str, object] = {} + if state_path.exists(): + previous = json.loads(state_path.read_text(encoding="utf-8")) + if not shutil.which("ufw"): + LOG.warning("ufw is absent; verify an equivalent TCP-2049 restriction") + return + status = runner.run([*_sudo_prefix(), "ufw", "status"]) + if not status.stdout.startswith("Status: active"): + LOG.info("ufw is inactive; exportfs remains restricted to %s", subnet) + if not state_path.exists(): + _write_text( + state_path, + json.dumps({"added_by_harness": False, "subnet": subnet}) + "\n", + ) + return + if previous.get("added_by_harness") and previous.get("subnet") != subnet: + _delete_nfs_firewall_rule(runner, str(previous["subnet"])) + state_path.unlink(missing_ok=True) + previous = {} + status = runner.run([*_sudo_prefix(), "ufw", "status"]) + rule_exists = _nfs_firewall_rule_exists(runner, subnet) + added_by_harness = bool(previous.get("added_by_harness")) + if rule_exists: + LOG.info("The dedicated UFW NFS rule is already present") + else: + _write_text( + state_path, + json.dumps({"added_by_harness": True, "subnet": subnet, "status": "adding"}) + + "\n", + ) + runner.run( + [ + *_sudo_prefix(), + "ufw", + "allow", + "from", + subnet, + "to", + "any", + "port", + "2049", + "proto", + "tcp", + "comment", + UFW_COMMENT, + ] + ) + added_by_harness = True + _write_text( + state_path, + json.dumps( + { + "added_by_harness": added_by_harness, + "subnet": subnet, + "status": "present" if added_by_harness else "external", + } + ) + + "\n", + ) + + +def _delete_nfs_firewall_rule(runner: Runner, subnet: str) -> None: + """Idempotently delete and verify the exact harness UFW rule.""" + if not _nfs_firewall_rule_exists(runner, subnet): + return + runner.run( + [ + *_sudo_prefix(), + "ufw", + "delete", + "allow", + "from", + subnet, + "to", + "any", + "port", + "2049", + "proto", + "tcp", + "comment", + UFW_COMMENT, + ], + ) + if _nfs_firewall_rule_exists(runner, subnet): + raise ProvisionError(f"UFW rule remains configured for {subnet}") + + +def _nfs_firewall_rule_exists(runner: Runner, subnet: str) -> bool: + """Check UFW's stored rules even when its runtime firewall is inactive.""" + added = runner.run([*_sudo_prefix(), "ufw", "show", "added"]) + return any( + subnet in line and "2049" in line and "tcp" in line and UFW_COMMENT in line + for line in added.stdout.splitlines() + ) + + +def _probe_nfs(runner: Runner, config: Config, gateway: str) -> None: + """Prove a kind node can mount and write the export.""" + node = f"{config.cluster_name}-control-plane" + script = ( + "set -eu; mkdir -p /tmp/storage-scale-nfs-probe; " + f"mount -t nfs4 -o vers=4.1 {gateway}:/ /tmp/storage-scale-nfs-probe; " + "touch /tmp/storage-scale-nfs-probe/.provisioner-probe; " + "rm /tmp/storage-scale-nfs-probe/.provisioner-probe; " + "umount /tmp/storage-scale-nfs-probe" + ) + runner.run(["docker", "exec", node, "sh", "-c", script], timeout=90) + + +def _image_id(runner: Runner, image: str) -> str | None: + """Return a local Docker image ID, or None when its tag is absent.""" + result = runner.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", image], check=False + ) + if result.returncode == 0: + return result.stdout.strip() + detail = (result.stderr or result.stdout).lower() + if result.returncode == 1 and ( + "no such image" in detail or "no such object" in detail + ): + return None + raise ProvisionError( + f"Docker could not inspect image {image}: " + f"{(result.stderr or result.stdout).strip()}" + ) + + +def _inspect_pinned_image( + runner: Runner, + reference: str, + digest: str, + architecture: str, + timeout: float = IMAGE_INSPECT_TIMEOUT_SECONDS, +) -> bool: + """Return whether one local reference has the required digest and platform.""" + try: + result = runner.run( + ["docker", "image", "inspect", "--format", "{{json .}}", reference], + check=False, + timeout=timeout, + ) + except subprocess.TimeoutExpired as error: + raise ProvisionError( + f"Docker timed out inspecting image {reference} after {timeout:.1f} seconds" + ) from error + if result.returncode: + detail = (result.stderr or result.stdout).lower() + if result.returncode == 1 and ( + "no such image" in detail or "no such object" in detail + ): + return False + raise ProvisionError( + f"Docker could not inspect image {reference}: " + f"{(result.stderr or result.stdout).strip()}" + ) + try: + document = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise ProvisionError( + f"Docker returned invalid metadata for {reference}" + ) from error + repo_digests = document.get("RepoDigests", []) + if document.get("Architecture") != architecture or not any( + isinstance(item, str) and item.endswith(f"@{digest}") for item in repo_digests + ): + return False + return True + + +def _transient_image_pull_failure(detail: str) -> bool: + """Return whether a registry failure is suitable for bounded retry.""" + lowered = detail.lower() + phrases = ( + "too many requests", + "toomanyrequests", + "timeout", + "timed out", + "deadline exceeded", + "connection", + "network is unreachable", + "no route to host", + "temporary failure", + "temporarily unavailable", + "unexpected eof", + ) + status_failure = re.search( + r"\b(?:http(?: status)?|status(?: code)?|response code)" + r"[ :=]+(?:429|5\d\d)\b", + lowered, + ) or re.search( + r"\b5\d\d\s+(?:internal server error|bad gateway|service unavailable|gateway timeout)\b", + lowered, + ) + return any(phrase in lowered for phrase in phrases) or bool(status_failure) + + +def _pull_pinned_image( + runner: Runner, + reference: str, + digest: str, + architecture: str, + deadline: float, +) -> str | None: + """Pull and verify one image, returning its final error when unavailable.""" + for attempt in range(1, IMAGE_PULL_ATTEMPTS + 1): + remaining = deadline - time.monotonic() + if remaining <= 0: + return "image pull retry budget expired" + pull_timeout = min(float(IMAGE_PULL_TIMEOUT_SECONDS), remaining) + try: + pulled = runner.run( + [ + "docker", + "pull", + "--platform", + f"linux/{architecture}", + reference, + ], + check=False, + timeout=pull_timeout, + ) + if pulled.returncode == 0: + remaining = deadline - time.monotonic() + if remaining <= 0: + return ( + f"image acquisition exceeded " + f"{IMAGE_PULL_DEADLINE_SECONDS} seconds" + ) + if _inspect_pinned_image( + runner, + reference, + digest, + architecture, + timeout=min(float(IMAGE_INSPECT_TIMEOUT_SECONDS), remaining), + ): + return None + raise ProvisionError( + "Docker pulled the wrong digest or architecture for " f"{reference}" + ) + detail = (pulled.stderr or pulled.stdout).strip() or ( + f"docker pull exited {pulled.returncode}" + ) + except subprocess.TimeoutExpired: + detail = f"timed out after {pull_timeout:.1f} seconds" + if attempt == IMAGE_PULL_ATTEMPTS or not _transient_image_pull_failure(detail): + return detail + remaining = deadline - time.monotonic() + delay = IMAGE_PULL_INITIAL_BACKOFF_SECONDS * (2 ** (attempt - 1)) + ( + secrets.randbelow(IMAGE_PULL_JITTER_MILLISECONDS) / 1000 + ) + if delay >= remaining: + return ( + f"{detail}; insufficient retry time within the " + "image acquisition deadline" + ) + LOG.warning( + "Image pull attempt %d/%d failed for %s: %s; retrying in %.2fs", + attempt, + IMAGE_PULL_ATTEMPTS, + reference, + detail, + delay, + ) + time.sleep(delay) + raise AssertionError("image pull retry loop exhausted without a result") + + +def _acquire_pinned_image( + runner: Runner, references: Sequence[str], digest: str +) -> str: + """Reuse or pull one verified platform image from ordered references.""" + architecture = _check_platform() + deadline = time.monotonic() + IMAGE_PULL_DEADLINE_SECONDS + for reference in references: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ProvisionError( + f"image acquisition exceeded {IMAGE_PULL_DEADLINE_SECONDS} seconds" + ) + if _inspect_pinned_image( + runner, + reference, + digest, + architecture, + timeout=min(float(IMAGE_INSPECT_TIMEOUT_SECONDS), remaining), + ): + return reference + pull_references: list[str] = [] + for reference in references: + repository, separator, reference_digest = reference.rpartition("@") + if not separator: + continue + if not repository or reference_digest != digest: + raise ProvisionError( + f"fixture image reference does not match {digest}: {reference}" + ) + pull_references.append(reference) + failures: list[str] = [] + reserve_per_reference = IMAGE_PULL_TIMEOUT_SECONDS + IMAGE_INSPECT_TIMEOUT_SECONDS + for index, reference in enumerate(pull_references): + later_references = len(pull_references) - index - 1 + reference_deadline = deadline - (later_references * reserve_per_reference) + failure = _pull_pinned_image( + runner, + reference, + digest, + architecture, + reference_deadline, + ) + if failure is None: + return reference + failures.append(f"{reference}: {failure}") + raise ProvisionError("could not obtain pinned image; " + " | ".join(failures)) + + +def _ensure_pinned_image(runner: Runner, reference: str) -> str: + """Reuse or pull one digest-qualified image for the current architecture.""" + repository, separator, digest = reference.rpartition("@") + if not separator or not repository or not digest.startswith("sha256:"): + raise ProvisionError(f"fixture base image is not digest-qualified: {reference}") + return _acquire_pinned_image(runner, (reference,), digest) + + +def _image_ownership_state(config: Config) -> dict[str, dict[str, str | None]]: + """Read the journal for fixture-owned Docker tags.""" + state_path = config.state_dir / "built-images.json" + if not state_path.exists(): + return {} + return json.loads(state_path.read_text(encoding="utf-8")) + + +def _write_image_ownership_state( + config: Config, state: dict[str, dict[str, str | None]] +) -> None: + """Atomically persist the Docker-tag ownership journal.""" + state_path = config.state_dir / "built-images.json" + _write_text(state_path, json.dumps(state, indent=2, sort_keys=True) + "\n") + + +def _recover_pending_image_build( + runner: Runner, + config: Config, + image: str, + ownership: dict[str, str | None], +) -> None: + """Reconcile an interrupted uniquely tagged build before reuse.""" + temporary_tag = ownership.get("pending_tag") + if not temporary_tag: + return + pending_id = ownership.get("pending_id") or _image_id(runner, temporary_tag) + current = _image_id(runner, image) + allowed = { + ownership.get("previous_id"), + ownership.get("built_id"), + pending_id, + None, + } + if current not in allowed: + raise ProvisionError( + f"refusing to alter Docker tag changed outside the fixture: {image}" + ) + if pending_id and current == pending_id: + ownership["built_id"] = pending_id + temporary_id = _image_id(runner, temporary_tag) + if temporary_id is not None: + if pending_id is not None and temporary_id != pending_id: + raise ProvisionError( + f"refusing to remove changed pending Docker tag: {temporary_tag}" + ) + runner.run(["docker", "image", "rm", temporary_tag]) + ownership["pending_tag"] = None + ownership["pending_id"] = None + state = _image_ownership_state(config) + state[image] = ownership + _write_image_ownership_state(config, state) + + +def _begin_image_build(runner: Runner, config: Config, image: str) -> str: + """Journal and return a unique tag without mutating the published tag.""" + state = _image_ownership_state(config) + ownership = state.get(image) + if ownership is None: + ownership = { + "previous_id": _image_id(runner, image), + "built_id": None, + "pending_tag": None, + "pending_id": None, + } + state[image] = ownership + elif ownership.get("pending_tag"): + _recover_pending_image_build(runner, config, image, ownership) + state = _image_ownership_state(config) + ownership = state[image] + current = _image_id(runner, image) + if current not in {ownership.get("previous_id"), ownership.get("built_id"), None}: + raise ProvisionError( + f"refusing to overwrite Docker tag changed outside the fixture: {image}" + ) + temporary_tag = f"{image}-pending-{secrets.token_hex(6)}" + ownership["pending_tag"] = temporary_tag + ownership["pending_id"] = None + _write_image_ownership_state(config, state) + return temporary_tag + + +def _publish_image_build( + runner: Runner, config: Config, image: str, temporary_tag: str +) -> None: + """Publish a completed temporary image and finish its ownership journal.""" + state = _image_ownership_state(config) + ownership = state[image] + if ownership.get("pending_tag") != temporary_tag: + raise ProvisionError(f"Docker build journal changed unexpectedly for {image}") + pending_id = _image_id(runner, temporary_tag) + if pending_id is None: + raise ProvisionError(f"Docker build did not produce {temporary_tag}") + ownership["pending_id"] = pending_id + _write_image_ownership_state(config, state) + current = _image_id(runner, image) + if current not in {ownership.get("previous_id"), ownership.get("built_id"), None}: + raise ProvisionError( + f"refusing to overwrite Docker tag changed outside the fixture: {image}" + ) + runner.run(["docker", "image", "tag", temporary_tag, image]) + ownership["built_id"] = pending_id + _write_image_ownership_state(config, state) + runner.run(["docker", "image", "rm", temporary_tag]) + ownership["pending_tag"] = None + ownership["pending_id"] = None + _write_image_ownership_state(config, state) + + +def _build_owned_image( + runner: Runner, + config: Config, + image: str, + dockerfile: Path, + base_image: str, + build_arguments: Sequence[str] = (), +) -> None: + """Build one fixture image without exposing an unjournaled fixed tag.""" + _ensure_pinned_image(runner, base_image) + temporary_tag = _begin_image_build(runner, config, image) + runner.run( + [ + "docker", + "build", + "--tag", + temporary_tag, + *build_arguments, + "--file", + dockerfile, + _resource_path("."), + ], + timeout=600, + ) + _publish_image_build(runner, config, image, temporary_tag) + + +def _prepare_csi_images(runner: Runner, config: Config) -> None: + """Reuse or pull pinned CSI images and load fixture-only node tags.""" + LOG.info("Preparing pinned NFS CSI images") + nodes = _kind_containers(runner, config, running_only=True) + if len(nodes) != 3: + raise ProvisionError( + f"cannot preload CSI images: expected 3 nodes, found {len(nodes)}" + ) + _remove_stale_csi_host_aliases(runner) + _remove_stale_node_image_aliases( + runner, + nodes, + "docker.io/storage-scale-integration-csi/", + ) + for name, tag, digest in CSI_IMAGES: + destination = f"registry.k8s.io/sig-storage/{name}:{tag}" + repositories = tuple(f"{repository}/{name}" for repository in CSI_REPOSITORIES) + candidates = ( + destination, + *(f"{repository}@{digest}" for repository in repositories), + ) + selected = _acquire_pinned_image(runner, candidates, digest) + temporary_prefix = f"storage-scale-integration-csi/{name}:{tag}-" + temporary = f"{temporary_prefix}{secrets.token_hex(6)}" + runner.run(["docker", "image", "tag", selected, temporary]) + try: + _load_image_into_nodes(runner, temporary, nodes, destination=destination) + except (ProvisionError, subprocess.SubprocessError, OSError): + try: + runner.run(["docker", "image", "rm", temporary]) + except (ProvisionError, subprocess.SubprocessError, OSError): + LOG.exception( + "Could not remove temporary CSI alias %s after staging failed", + temporary, + ) + raise + else: + runner.run(["docker", "image", "rm", temporary]) + + +def _remove_stale_host_image_aliases(runner: Runner, prefix: str) -> None: + """Remove exact fixture-private Docker aliases left by interruption.""" + result = runner.run( + [ + "docker", + "image", + "ls", + "--format", + "{{.Repository}}:{{.Tag}}", + "--filter", + f"reference={prefix}*", + ] + ) + aliases = sorted(set(result.stdout.splitlines())) + if any(not alias.startswith(prefix) for alias in aliases): + raise ProvisionError( + f"Docker returned an unexpected temporary alias: {aliases}" + ) + for alias in aliases: + runner.run(["docker", "image", "rm", alias]) + + +def _remove_stale_csi_host_aliases(runner: Runner) -> None: + """Remove all fixture-private CSI aliases left by interrupted setup.""" + _remove_stale_host_image_aliases(runner, "storage-scale-integration-csi/") + + +def _remove_stale_node_image_aliases( + runner: Runner, nodes: list[str], prefix: str +) -> None: + """Remove exact fixture-private containerd aliases left by interruption.""" + for node in nodes: + result = runner.run( + [ + "docker", + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "list", + "--quiet", + ], + timeout=60, + ) + aliases = sorted( + alias + for alias in set(result.stdout.splitlines()) + if alias.startswith(prefix) + ) + for alias in aliases: + runner.run( + [ + "docker", + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "remove", + alias, + ], + timeout=60, + ) + + +def _load_image_into_nodes( + runner: Runner, + image: str, + nodes: list[str], + *, + destination: str | None = None, +) -> None: + """Import an image and optionally publish a node-local chart tag.""" + with tempfile.TemporaryDirectory(prefix="storage-scale-image-") as directory: + archive_path = Path(directory) / "image.tar" + runner.run(["docker", "save", "--output", archive_path, image], timeout=300) + for node in nodes: + with archive_path.open("rb") as archive: + runner.run( + [ + "docker", + "exec", + "-i", + node, + "ctr", + "-n", + "k8s.io", + "images", + "import", + "--snapshotter=overlayfs", + "-", + ], + stdin=archive, + timeout=300, + ) + if destination: + registry = image.split("/", maxsplit=1)[0] + source = ( + image + if "." in registry or ":" in registry or registry == "localhost" + else f"docker.io/{image}" + ) + runner.run( + [ + "docker", + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "tag", + "--force", + source, + destination, + ], + timeout=60, + ) + runner.run( + [ + "docker", + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "remove", + source, + ], + timeout=60, + ) + + +def _install_nfs_csi(runner: Runner, config: Config, gateway: str) -> None: + """Install NFS CSI and bind the two RWX claims.""" + _prepare_csi_images(runner, config) + chart = _ensure_nfs_csi_chart(runner, config) + values = _resource_path("manifests/nfs-csi-values.yaml") + runner.run( + [ + "helm", + "upgrade", + "--install", + "csi-driver-nfs", + chart, + "--namespace", + "kube-system", + "--values", + values, + "--wait", + "--timeout", + "5m", + "--kubeconfig", + config.kubeconfig, + ], + timeout=420, + ) + _ensure_namespace(runner, config) + storage = _render_resource( + config, + "manifests/nfs-storage.yaml.tmpl", + { + "NAMESPACE": config.namespace, + "NFS_SERVER": gateway, + "STORAGE_TEST_CAPACITY": STORAGE_TEST_CAPACITY, + "SSH_HOME_CAPACITY": SSH_HOME_CAPACITY, + }, + ) + runner.run(_kubectl(config, "apply", "-f", storage)) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "wait", + "--for=jsonpath={.status.phase}=Bound", + "pvc/storage-test-rwx", + "pvc/ssh-home-rwx", + "--timeout=180s", + ), + timeout=210, + ) + + +def _install_sbx_shared_storage(runner: Runner, config: Config) -> None: + """Bind the shared kind-node paths to the fixture's stable RWX claims.""" + _ensure_namespace(runner, config) + storage = _render_resource( + config, + "manifests/sbx-storage.yaml.tmpl", + { + "NAMESPACE": config.namespace, + "STORAGE_TEST_CAPACITY": STORAGE_TEST_CAPACITY, + "SSH_HOME_CAPACITY": SSH_HOME_CAPACITY, + }, + ) + runner.run(_kubectl(config, "apply", "-f", storage)) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "wait", + "--for=jsonpath={.status.phase}=Bound", + "pvc/storage-test-rwx", + "pvc/ssh-home-rwx", + "--timeout=60s", + ), + timeout=90, + ) + + +def _ensure_nfs_csi_chart(runner: Runner, config: Config) -> Path: + """Cache the pinned NFS CSI chart from a checksum-verified source archive.""" + chart = config.state_dir / "charts" / f"csi-driver-nfs-{NFS_CSI_VERSION}.tgz" + if chart.exists(): + _verify_sha256(chart, NFS_CSI_CHART_SHA256) + return chart + url = ( + "https://codeload.github.com/kubernetes-csi/csi-driver-nfs/tar.gz/" + f"refs/tags/v{NFS_CSI_VERSION}" + ) + with tempfile.TemporaryDirectory(prefix="storage-scale-nfs-chart-") as directory: + source = Path(directory) / "source.tar.gz" + _curl(runner, url, source) + _verify_sha256(source, NFS_CSI_SOURCE_SHA256) + member_name = ( + f"csi-driver-nfs-{NFS_CSI_VERSION}/charts/latest/" + f"csi-driver-nfs-{NFS_CSI_VERSION}.tgz" + ) + with tarfile.open(source, "r:gz") as archive: + member = archive.extractfile(member_name) + if member is None: + raise ProvisionError(f"NFS CSI chart is absent from {source.name}") + content = member.read() + if hashlib.sha256(content).hexdigest() != NFS_CSI_CHART_SHA256: + raise ProvisionError("SHA-256 mismatch for embedded NFS CSI chart") + _write_bytes(chart, content) + return chart + + +def _ensure_namespace(runner: Runner, config: Config) -> None: + """Create the integration namespace when absent.""" + probe = runner.run( + _kubectl(config, "get", "namespace", config.namespace), check=False + ) + if probe.returncode: + runner.run(_kubectl(config, "create", "namespace", config.namespace)) + + +def _ensure_ssh_key(runner: Runner, config: Config) -> tuple[Path, Path]: + """Create or reuse the dedicated integration-test SSH key.""" + private_key = config.keys_dir / "id_ed25519" + public_key = config.keys_dir / "id_ed25519.pub" + if private_key.exists() and public_key.exists(): + return private_key, public_key + if private_key.exists() or public_key.exists(): + raise ProvisionError(f"incomplete SSH identity in {config.keys_dir}") + runner.run( + [ + "ssh-keygen", + "-q", + "-t", + "ed25519", + "-N", + "", + "-C", + "storage-scale-integration", + "-f", + private_key, + ], + sensitive=True, + ) + private_key.chmod(0o600) + public_key.chmod(0o644) + return private_key, public_key + + +def _ensure_file_secret( + runner: Runner, + config: Config, + name: str, + files: dict[str, Path], +) -> None: + """Create an immutable-input file Secret when absent.""" + probe = runner.run( + _kubectl(config, "-n", config.namespace, "get", "secret", name), check=False + ) + if probe.returncode == 0: + return + arguments: list[str | Path] = [ + *_kubectl(config, "-n", config.namespace, "create", "secret", "generic", name) + ] + arguments.extend(f"--from-file={key}={value}" for key, value in files.items()) + runner.run(arguments, sensitive=True) + + +def _install_ssh_workers(runner: Runner, config: Config) -> None: + """Build, deploy, and validate the two SSH workers.""" + private_key, public_key = _ensure_ssh_key(runner, config) + _build_owned_image( + runner, + config, + SSH_IMAGE, + _resource_path("ssh-image.Dockerfile"), + SSH_BASE_IMAGE, + ("--build-arg", f"BASE_IMAGE={SSH_BASE_IMAGE}"), + ) + nodes = _kind_containers(runner, config, running_only=True) + _load_image_into_nodes(runner, SSH_IMAGE, nodes) + _ensure_file_secret( + runner, + config, + "storage-ssh-identity", + {"id_ed25519": private_key, "authorized_keys": public_key}, + ) + _ensure_ssh_home_mode( + runner, + config, + CANONICAL_HOME_MODE, + run_id="setup", + scenario_id="setup-preflight", + ) + + +def _ssh_home_volume(mode: str) -> str: + """Return the manifest fragment for one supported SSH home mode.""" + if mode == SHARED_HOME_MODE: + return "persistentVolumeClaim:\n claimName: ssh-home-rwx" + if mode == CANONICAL_HOME_MODE: + return "emptyDir:\n sizeLimit: 64Mi" + raise ProvisionError(f"unsupported SSH home mode: {mode}") + + +def _render_ssh_workers(config: Config, mode: str) -> tuple[Path, str]: + """Render one canonical SSH StatefulSet form and return its checksum.""" + source = _resource_path("manifests/ssh-workers.yaml.tmpl") + rendered = source.read_text(encoding="utf-8") + replacements = { + "NAMESPACE": config.namespace, + "SSH_HOME_MODE": mode, + "SSH_HOME_VOLUME": _ssh_home_volume(mode), + } + for token, value in replacements.items(): + rendered = rendered.replace(f"@@{token}@@", value) + checksum = statefulset_checksum(rendered.replace("@@SSH_CONFIG_CHECKSUM@@", "")) + rendered = rendered.replace("@@SSH_CONFIG_CHECKSUM@@", checksum) + unresolved = [word for word in rendered.split() if "@@" in word] + if unresolved: + raise ProvisionError( + f"unresolved SSH manifest token in {source}: {unresolved[0]}" + ) + destination = config.manifests_dir / f"ssh-workers-{mode}.yaml" + _write_text(destination, rendered) + return destination, checksum + + +def _inspect_ssh_home_pool(runner: Runner, config: Config) -> PoolObservation: + """Return the live SSH StatefulSet form and rollout state.""" + statefulset = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "statefulset/ssh-worker", + "-o", + "json", + ), + check=False, + timeout=30, + ) + if statefulset.returncode: + return PoolObservation("absent", "", 0, 0, statefulset.stderr.strip()) + document = json.loads(statefulset.stdout) + metadata = document.get("metadata", {}) + annotations = metadata.get("annotations", {}) + status = document.get("status", {}) + generation = metadata.get("generation") + observed_generation = status.get("observedGeneration") + revision_ready = ( + isinstance(generation, int) + and isinstance(observed_generation, int) + and observed_generation >= generation + and status.get("currentRevision") == status.get("updateRevision") + and status.get("updatedReplicas", 0) == 2 + and status.get("readyReplicas", 0) == 2 + ) + pods = _ssh_pods(runner, config) + terminating = sum( + bool(pod.get("metadata", {}).get("deletionTimestamp")) for pod in pods + ) + ready = sum( + _pod_ready(pod) and not pod.get("metadata", {}).get("deletionTimestamp") + for pod in pods + ) + names = ", ".join(str(pod.get("metadata", {}).get("name", "?")) for pod in pods) + details = ( + f"pods=[{names}], generation={generation}, " + f"observedGeneration={observed_generation}, " + f"currentRevision={status.get('currentRevision')}, " + f"updateRevision={status.get('updateRevision')}" + ) + return PoolObservation( + str(annotations.get(SSH_HOME_ANNOTATION, "unknown")), + str(annotations.get(SSH_CONFIG_ANNOTATION, "")), + ready if revision_ready else 0, + terminating, + details, + ) + + +def _pod_ready(pod: dict[str, object]) -> bool: + """Return whether every container in a running pod is ready.""" + status = pod.get("status", {}) + if not isinstance(status, dict) or status.get("phase") != "Running": + return False + containers = status.get("containerStatuses", []) + return bool(containers) and all( + isinstance(item, dict) and item.get("ready") for item in containers + ) + + +def _wait_for_no_ssh_pods(runner: Runner, config: Config) -> None: + """Wait until the host-network SSH port has no owning pod.""" + deadline = time.monotonic() + 180 + pods: list[dict[str, object]] = [] + while True: + request_timeout, process_timeout = _poll_timeouts(deadline, 10) + if process_timeout <= 0: + break + pods = _ssh_pods( + runner, + config, + request_timeout, + process_timeout=process_timeout, + ) + if not pods: + return + time.sleep(min(2, max(0, deadline - time.monotonic()))) + names = [str(pod["metadata"]["name"]) for pod in pods] + raise ProvisionError(f"SSH pods did not terminate before transition: {names}") + + +def _reconcile_ssh_home_pool( + runner: Runner, config: Config, mode: str, expected_checksum: str +) -> None: + """Replace only the SSH StatefulSet with one validated home mode.""" + manifest, checksum = _render_ssh_workers(config, mode) + if checksum != expected_checksum: + raise ProvisionError( + "SSH StatefulSet checksum changed during reconciliation: " + f"expected {expected_checksum}, rendered {checksum}" + ) + existing = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "statefulset/ssh-worker", + ), + check=False, + timeout=30, + ) + if existing.returncode == 0: + _scale_ssh(runner, config, replicas=0) + _wait_for_no_ssh_pods(runner, config) + runner.run( + _kubectl( + config, + "apply", + "--server-side", + "--force-conflicts", + "--field-manager=storage-scale-integration", + "-f", + manifest, + ) + ) + _scale_ssh(runner, config, replicas=2) + _wait_for_ssh(runner, config) + _validate_ssh_workers(runner, config, config.keys_dir / "id_ed25519", mode) + + +def _ssh_transition_manager(runner: Runner, config: Config) -> SshHomeTransitionManager: + """Return a transition manager backed by the live Kubernetes fixture.""" + return SshHomeTransitionManager( + config.state_dir, + lambda: _inspect_ssh_home_pool(runner, config), + lambda mode, checksum: _reconcile_ssh_home_pool(runner, config, mode, checksum), + ) + + +def _ensure_ssh_home_mode( + runner: Runner, + config: Config, + mode: str, + *, + run_id: str, + scenario_id: str, +) -> None: + """Enter a validated SSH home mode through persistent transition state.""" + separate_checksum = _render_ssh_workers(config, CANONICAL_HOME_MODE)[1] + shared_checksum = _render_ssh_workers(config, SHARED_HOME_MODE)[1] + manager = _ssh_transition_manager(runner, config) + if mode == CANONICAL_HOME_MODE: + manager.preflight(separate_checksum, run_id=run_id, scenario_id=scenario_id) + return + if mode == SHARED_HOME_MODE: + manager.enter_shared( + run_id=run_id, + scenario_id=scenario_id, + separate_checksum=separate_checksum, + shared_checksum=shared_checksum, + ) + return + raise ProvisionError(f"unsupported SSH home mode: {mode}") + + +def _restore_ssh_home_mode( + runner: Runner, config: Config, *, run_id: str, scenario_id: str +) -> None: + """Restore canonical SSH homes after a shared-home scenario batch.""" + checksum = _render_ssh_workers(config, CANONICAL_HOME_MODE)[1] + _ssh_transition_manager(runner, config).restore_separate( + checksum, run_id=run_id, scenario_id=scenario_id + ) + + +def _ssh_pods( + runner: Runner, + config: Config, + request_timeout: int = 10, + *, + process_timeout: float | None = None, +) -> list[dict[str, object]]: + """Return SSH pod objects in ordinal order.""" + result = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "pods", + "-l", + "app.kubernetes.io/name=storage-ssh-worker", + "-o", + "json", + f"--request-timeout={request_timeout}s", + ), + timeout=process_timeout or request_timeout + 5, + ) + return sorted( + json.loads(result.stdout)["items"], key=lambda item: item["metadata"]["name"] + ) + + +def _validate_ssh_workers( + runner: Runner, config: Config, private_key: Path, home_mode: str +) -> None: + """Validate placement, host SSH, home semantics, and RWX visibility.""" + pods = _ssh_pods(runner, config) + if len(pods) != 2: + raise ProvisionError(f"expected 2 SSH pods; found {len(pods)}") + nodes = {pod["spec"]["nodeName"] for pod in pods} + if len(nodes) != 2 or f"{config.cluster_name}-control-plane" in nodes: + raise ProvisionError(f"SSH placement invariant failed: {sorted(nodes)}") + known_hosts = config.state_dir / "ssh_known_hosts" + scan_lines: list[str] = [] + addresses: list[str] = [] + for pod in pods: + address = str(pod["status"]["podIP"]) + addresses.append(address) + scan = runner.run(["ssh-keyscan", "-T", "10", str(address)], timeout=30) + scan_lines.extend(scan.stdout.splitlines()) + _write_text(known_hosts, "\n".join(scan_lines) + "\n", mode=0o600) + _write_text(config.state_dir / "ssh_hosts", "\n".join(addresses) + "\n") + for address in addresses: + runner.run( + [ + "ssh", + "-i", + private_key, + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=yes", + "-o", + f"UserKnownHostsFile={known_hosts}", + f"tester@{address}", + "test $(id -u) -eq 2000 && test $(stat -f -c %T /root) = overlayfs", + ], + timeout=30, + ) + root_probe = runner.run( + [ + "ssh", + "-i", + private_key, + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=yes", + "-o", + f"UserKnownHostsFile={known_hosts}", + f"root@{addresses[0]}", + "true", + ], + check=False, + timeout=30, + ) + if root_probe.returncode == 0: + raise ProvisionError("SSH root-login rejection validation failed") + for pod in pods: + _pod_exec( + runner, + config, + str(pod["metadata"]["name"]), + "rm -f /home/tester/.ssh/known_hosts", + ) + for source, destination in ((0, 1), (1, 0)): + _pod_exec( + runner, + config, + str(pods[source]["metadata"]["name"]), + "runuser -u tester -- ssh -o BatchMode=yes " + "-o StrictHostKeyChecking=accept-new " + f"tester@{addresses[destination]} true", + ) + _validate_ssh_storage(runner, config, pods, home_mode) + + +def _pod_exec( + runner: Runner, + config: Config, + pod: str, + script: str, + *, + check: bool = True, + timeout: float = 60, +) -> subprocess.CompletedProcess[str]: + """Run a bounded shell command in an SSH worker pod.""" + return runner.run( + _kubectl( + config, + "-n", + config.namespace, + "exec", + pod, + "--", + "bash", + "-c", + script, + ), + check=check, + timeout=timeout, + ) + + +def _validate_ssh_storage( + runner: Runner, + config: Config, + pods: list[dict[str, object]], + home_mode: str, +) -> None: + """Validate the selected home mode and shared storage claim.""" + names = [str(pod["metadata"]["name"]) for pod in pods] + token = secrets.token_hex(16) + home_probe_path = f"/home/tester/.integration-home-probe-{token}" + storage_probe_path = f"/mnt/storage-test/.integration-rwx-probe-{token}" + storage_check = f'test "$(cat {shlex.quote(storage_probe_path)})" = ' + shlex.quote( + token + ) + backend = _select_storage_backend(config) + if backend == "nfs": + storage_check += ( + " && case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac" + ) + expected_home_rc = 0 if home_mode == "shared" else 1 + cleanup = f"rm -f {shlex.quote(home_probe_path)} {shlex.quote(storage_probe_path)}" + host_visible = backend != "sbx-shared" + home_rc: int | None = None + rwx_rc: int | None = None + try: + _pod_exec( + runner, + config, + names[0], + f"touch {shlex.quote(home_probe_path)}; " + f"printf '%s\\n' {shlex.quote(token)} " + f">{shlex.quote(storage_probe_path)}", + ) + deadline = time.monotonic() + SSH_STORAGE_VISIBILITY_TIMEOUT_SECONDS + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + home_probe = _pod_exec( + runner, + config, + names[1], + f"test -e {shlex.quote(home_probe_path)}", + check=False, + timeout=min(10, remaining), + ) + home_rc = home_probe.returncode + remaining = deadline - time.monotonic() + if remaining <= 0: + break + rwx_probe = _pod_exec( + runner, + config, + names[1], + storage_check, + check=False, + timeout=min(10, remaining), + ) + rwx_rc = rwx_probe.returncode + if backend == "sbx-shared": + host_probe = ( + config.sbx_shared_root + / "storage-test" + / Path(storage_probe_path).name + ) + host_visible = ( + host_probe.is_file() + and host_probe.read_text(encoding="utf-8").strip() == token + ) + if ( + home_probe.returncode == expected_home_rc + and rwx_probe.returncode == 0 + and host_visible + ): + return + time.sleep(min(1, max(0, deadline - time.monotonic()))) + raise ProvisionError( + "SSH home or RWX visibility did not converge: " + f"home_rc={home_rc}, expected_home_rc={expected_home_rc}, " + f"rwx_rc={rwx_rc}, host_visible={host_visible}" + ) + finally: + for name in names: + _pod_exec(runner, config, name, cleanup, check=False) + + +def _load_or_create_db_credentials(config: Config) -> dict[str, str]: + """Return stable protected MariaDB credentials.""" + path = config.keys_dir / "mariadb.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + credentials = { + "password": secrets.token_hex(24), + "root_password": secrets.token_hex(24), + } + _write_text(path, json.dumps(credentials) + "\n", mode=0o600) + return credentials + + +def _ensure_mariadb_secret(runner: Runner, config: Config) -> None: + """Create the stable MariaDB Secret when absent.""" + name = "mariadb-password" + probe = runner.run( + _kubectl(config, "-n", config.namespace, "get", "secret", name), check=False + ) + if probe.returncode == 0: + return + credentials = _load_or_create_db_credentials(config) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "create", + "secret", + "generic", + name, + f"--from-literal=password={credentials['password']}", + f"--from-literal=root-password={credentials['root_password']}", + ), + sensitive=True, + ) + + +def _install_slurm(runner: Runner, config: Config) -> None: + """Install MariaDB, Slinky, and the two-node Slurm fixture.""" + _prepare_slurm_images(runner, config) + _ensure_mariadb_secret(runner, config) + mariadb = _render_resource( + config, + "manifests/mariadb-accounting.yaml.tmpl", + { + "MARIADB_IMAGE": MARIADB_IMAGE, + "NAMESPACE": config.namespace, + }, + ) + runner.run(_kubectl(config, "apply", "-f", mariadb)) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "statefulset/mariadb-accounting", + "--timeout=240s", + ), + timeout=270, + ) + _helm_slinky(runner, config) + _wait_for_slurm(runner, config) + _restart_slinky_login(runner, config) + _ensure_slurm_workload_account(runner, config) + _validate_slurm(runner, config) + + +def _build_slinky_image( + runner: Runner, + config: Config, + *, + image: str, + base_image: str, + dockerfile: str, + nodes: list[str], +) -> None: + """Build, record, and preload one fixture-owned Slinky image.""" + _build_owned_image( + runner, + config, + image, + _resource_path(dockerfile), + base_image, + ( + "--build-arg", + f"BASE_IMAGE={base_image}", + ), + ) + _load_image_into_nodes(runner, image, nodes) + + +def _stage_pinned_fixture_image( + runner: Runner, + config: Config, + *, + upstream: str, + image: str, + nodes: list[str], +) -> None: + """Acquire, transactionally tag, and preload one pinned workload image.""" + selected = _ensure_pinned_image(runner, upstream) + temporary_tag = _begin_image_build(runner, config, image) + runner.run(["docker", "image", "tag", selected, temporary_tag]) + _publish_image_build(runner, config, image, temporary_tag) + _load_image_into_nodes(runner, image, nodes) + + +def _prepare_slurm_images(runner: Runner, config: Config) -> None: + """Build or acquire every image that the Slurm fixture runs in kind.""" + control_plane = f"{config.cluster_name}-control-plane" + worker_nodes = [ + f"{config.cluster_name}-worker", + f"{config.cluster_name}-worker2", + ] + all_nodes = [control_plane, *worker_nodes] + _stage_pinned_fixture_image( + runner, + config, + upstream=MARIADB_BASE_IMAGE, + image=MARIADB_IMAGE, + nodes=[control_plane], + ) + _stage_pinned_fixture_image( + runner, + config, + upstream=SLINKY_HELPER_BASE_IMAGE, + image=SLINKY_HELPER_IMAGE, + nodes=all_nodes, + ) + _build_slinky_image( + runner, + config, + image=SLINKY_LOGIN_IMAGE, + base_image=SLINKY_LOGIN_BASE_IMAGE, + dockerfile="slinky-login-image.Dockerfile", + nodes=[control_plane], + ) + _build_slinky_image( + runner, + config, + image=SLINKY_SLURMD_IMAGE, + base_image=SLINKY_SLURMD_BASE_IMAGE, + dockerfile="slinky-slurmd-image.Dockerfile", + nodes=worker_nodes, + ) + + +def _helm_slinky(runner: Runner, config: Config) -> None: + """Reconcile the three pinned Slinky releases.""" + slurm_values = _render_resource( + config, + "manifests/slinky-slurm-values.yaml.tmpl", + { + "SLINKY_HELPER_IMAGE_REPOSITORY": SLINKY_HELPER_IMAGE_REPOSITORY, + "SLINKY_HELPER_IMAGE_TAG": SLINKY_HELPER_IMAGE_TAG, + }, + ) + releases = ( + ( + "slurm-operator-crds", + "slurm-operator-crds", + None, + ), + ( + "slurm-operator", + "slurm-operator", + _resource_path("manifests/slinky-operator-values.yaml"), + ), + ( + "slurm", + "slurm", + slurm_values, + ), + ) + for release, chart, values in releases: + arguments: list[str | Path] = [ + "helm", + "upgrade", + "--install", + release, + f"oci://ghcr.io/slinkyproject/charts/{chart}", + "--version", + SLINKY_VERSION, + "--namespace", + config.namespace, + "--kubeconfig", + config.kubeconfig, + "--wait", + "--timeout", + "8m", + ] + if values: + arguments.extend(["--values", values]) + _install_slinky_release(runner, arguments, release) + if release == "slurm-operator": + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "restart", + "deployment/slurm-operator-webhook", + ) + ) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "deployment/slurm-operator-webhook", + "--timeout=180s", + ), + timeout=210, + ) + + +def _install_slinky_release( + runner: Runner, arguments: list[str | Path], release: str +) -> None: + """Install a release, retrying the operator webhook startup race once.""" + for attempt in range(2): + result = runner.run(arguments, check=False, timeout=600) + if result.returncode == 0: + return + detail = result.stdout + result.stderr + webhook_race = "failed calling webhook" in detail + if attempt == 0 and webhook_race: + LOG.warning( + "Slinky release %s reached its webhook before it was responsive; " + "retrying once", + release, + ) + time.sleep(10) + continue + raise ProvisionError( + f"Helm failed to install Slinky release {release!r}" + f"{_failure_detail(result.stdout, result.stderr, False)}" + ) + + +def _wait_for_slurm(runner: Runner, config: Config) -> None: + """Wait for Slinky child resources not covered by Helm's wait.""" + expected = {"slurmdbd": 1, "slurmctld": 1, "slurmrestd": 1, "login": 1, "slurmd": 2} + ready = {container: 0 for container in expected} + deadline = time.monotonic() + 300 + while True: + request_timeout, process_timeout = _poll_timeouts(deadline, 10) + if process_timeout <= 0: + break + pods = _namespace_pods( + runner, + config, + request_timeout, + process_timeout=process_timeout, + ) + ready = { + container: sum( + _pod_is_ready(pod) for pod in _pods_with_container(pods, container) + ) + for container in expected + } + if ready == expected: + return + LOG.info("Waiting for Slinky child pods: ready=%s expected=%s", ready, expected) + time.sleep(min(5, max(0, deadline - time.monotonic()))) + raise ProvisionError(f"Slinky child pods did not become ready: {ready}") + + +def _namespace_pods( + runner: Runner, + config: Config, + request_timeout: int = 10, + *, + process_timeout: float | None = None, +) -> list[dict[str, object]]: + """Return all pod objects in the integration namespace.""" + result = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "pods", + "-o", + "json", + f"--request-timeout={request_timeout}s", + ), + timeout=process_timeout or request_timeout + 5, + ) + return json.loads(result.stdout)["items"] + + +def _pods_with_container( + pods: list[dict[str, object]], container: str +) -> list[dict[str, object]]: + """Select nonterminating pods containing a Slinky workload container.""" + return [ + pod + for pod in pods + if not pod["metadata"].get("deletionTimestamp") # type: ignore[index] + and container + in {entry["name"] for entry in pod["spec"]["containers"]} # type: ignore[index] + ] + + +def _pod_is_ready(pod: dict[str, object]) -> bool: + """Return whether a pod is running with all containers ready.""" + status = pod["status"] # type: ignore[index] + containers = status.get("containerStatuses", []) + return ( + status.get("phase") == "Running" + and bool(containers) + and all(container.get("ready", False) for container in containers) + ) + + +def _login_pod(runner: Runner, config: Config) -> str: + """Wait for and return the single ready, nonterminating LoginSet pod.""" + deadline = time.monotonic() + 180 + while True: + request_timeout, process_timeout = _poll_timeouts(deadline, 10) + if process_timeout <= 0: + break + pods = [ + pod + for pod in _pods_with_container( + _namespace_pods( + runner, + config, + request_timeout, + process_timeout=process_timeout, + ), + "login", + ) + if not pod["metadata"].get("deletionTimestamp") # type: ignore[index] + and _pod_is_ready(pod) + ] + if len(pods) == 1: + return str(pods[0]["metadata"]["name"]) # type: ignore[index] + LOG.info( + "Waiting for one ready, nonterminating LoginSet pod; found %s", len(pods) + ) + time.sleep(min(3, max(0, deadline - time.monotonic()))) + raise ProvisionError( + "LoginSet did not converge to one ready pod within 180 seconds" + ) + + +def _restart_slinky_login(runner: Runner, config: Config) -> None: + """Restart the configless login client after accounting is available.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "restart", + "deployment/slurm-login-test", + ) + ) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "deployment/slurm-login-test", + "--timeout=180s", + ), + timeout=210, + ) + + +def _sacctmgr_rows( + runner: Runner, + prefix: list[str | Path], + entity: str, + fields: tuple[str, ...], +) -> set[tuple[str, ...]]: + """Return exact pipe-delimited rows from one Slurm accounting query.""" + result = runner.run( + [ + *prefix, + "sacctmgr", + "--noheader", + "--parsable2", + "show", + entity, + f"format={','.join(fields)}", + ] + ) + return { + tuple(line.removesuffix("|").split("|")) + for line in result.stdout.splitlines() + if line.strip() + } + + +def _ensure_slurm_workload_account(runner: Runner, config: Config) -> None: + """Reconcile the fixed non-root workload identity in Slurm accounting.""" + login = _login_pod(runner, config) + prefix = _kubectl(config, "-n", config.namespace, "exec", login, "--") + if (WORKLOAD_ACCOUNT,) not in _sacctmgr_rows( + runner, prefix, "account", ("Account",) + ): + runner.run( + [ + *prefix, + "sacctmgr", + "--immediate", + "add", + "account", + WORKLOAD_ACCOUNT, + "Description=storage scale integration workloads", + "Organization=storage-scale-test", + ] + ) + expected_association = (WORKLOAD_USER, WORKLOAD_ACCOUNT) + if expected_association not in _sacctmgr_rows( + runner, prefix, "association", ("User", "Account") + ): + runner.run( + [ + *prefix, + "sacctmgr", + "--immediate", + "add", + "user", + WORKLOAD_USER, + f"Account={WORKLOAD_ACCOUNT}", + f"DefaultAccount={WORKLOAD_ACCOUNT}", + ] + ) + expected_user = (WORKLOAD_USER, WORKLOAD_ACCOUNT) + if expected_user not in _sacctmgr_rows( + runner, prefix, "user", ("User", "DefaultAccount") + ): + runner.run( + [ + *prefix, + "sacctmgr", + "--immediate", + "modify", + "user", + "where", + f"Name={WORKLOAD_USER}", + "set", + f"DefaultAccount={WORKLOAD_ACCOUNT}", + ] + ) + account_ready = (WORKLOAD_ACCOUNT,) in _sacctmgr_rows( + runner, prefix, "account", ("Account",) + ) + association_ready = expected_association in _sacctmgr_rows( + runner, prefix, "association", ("User", "Account") + ) + user_ready = expected_user in _sacctmgr_rows( + runner, prefix, "user", ("User", "DefaultAccount") + ) + if not account_ready or not association_ready or not user_ready: + raise ProvisionError( + "Slurm workload accounting reconciliation did not produce the " + f"required {WORKLOAD_USER}/{WORKLOAD_ACCOUNT} association" + ) + + +def _validate_slurm(runner: Runner, config: Config) -> None: + """Validate LoginSet placement, storage, two-node fan-out, and accounting.""" + login = _login_pod(runner, config) + login_node = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "pod", + login, + "-o", + "jsonpath={.spec.nodeName}", + ) + ).stdout.strip() + expected_login_node = f"{config.cluster_name}-control-plane" + if login_node != expected_login_node: + raise ProvisionError( + f"LoginSet placement failed: {login} is on {login_node}, " + f"expected {expected_login_node}" + ) + pods = _namespace_pods(runner, config) + workers = _pods_with_container(pods, "slurmd") + worker_nodes = {str(pod["spec"]["nodeName"]) for pod in workers} # type: ignore[index] + target_result = runner.run( + _kubectl( + config, + "get", + "nodes", + "-l", + TARGET_LABEL, + "-o", + "jsonpath={.items[*].metadata.name}", + ) + ) + target_nodes = set(target_result.stdout.split()) + if worker_nodes != target_nodes or len(worker_nodes) != 2: + raise ProvisionError( + f"Slurm worker placement failed: pods={sorted(worker_nodes)}, " + f"targets={sorted(target_nodes)}" + ) + prefix = _kubectl(config, "-n", config.namespace, "exec", login, "--") + workload_prefix = [*prefix, "runuser", "-u", WORKLOAD_USER, "--"] + coordinator_uid = runner.run([*workload_prefix, "id", "-u"]).stdout.strip() + if coordinator_uid != str(WORKLOAD_UID) or coordinator_uid == "0": + raise ProvisionError( + "Slurm coordinator workload identity is invalid: " + f"expected {WORKLOAD_UID}, found {coordinator_uid!r}" + ) + backend = _select_storage_backend(config) + token = secrets.token_hex(16) + storage_probe = ( + f"printf '%s\\n' {shlex.quote(token)} >/mnt/storage-test/.login-probe" + ) + if backend == "nfs": + storage_probe += ( + "; case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac" + ) + runner.run([*workload_prefix, "bash", "-c", storage_probe]) + if backend == "sbx-shared": + host_probe = config.sbx_shared_root / "storage-test" / ".login-probe" + if ( + not host_probe.is_file() + or host_probe.read_text(encoding="utf-8").strip() != token + ): + raise ProvisionError("LoginSet SBX data is not visible from the agent") + runner.run([*workload_prefix, "rm", "-f", "/mnt/storage-test/.login-probe"]) + fanout = runner.run( + [ + *workload_prefix, + "srun", + "--account", + WORKLOAD_ACCOUNT, + "-p", + "all", + "-N2", + "-n2", + "--ntasks-per-node=1", + "sh", + "-c", + 'printf "%s %s\\n" "$(hostname)" "$(id -u)"', + ], + timeout=120, + ).stdout.splitlines() + identities = [line.split() for line in fanout if line.strip()] + hostnames = {fields[0] for fields in identities if len(fields) == 2} + task_uids = {fields[1] for fields in identities if len(fields) == 2} + if len(identities) != 2 or len(hostnames) != 2 or task_uids != {str(WORKLOAD_UID)}: + raise ProvisionError( + "Slurm fan-out did not reach two distinct nodes as the fixed " + f"workload identity {WORKLOAD_UID}: {fanout}" + ) + job = runner.run( + [ + *workload_prefix, + "sbatch", + "--account", + WORKLOAD_ACCOUNT, + "--wait", + "--parsable", + "-p", + "all", + "-N2", + "-n2", + "--ntasks-per-node=1", + "--output=/mnt/storage-test/integration-accounting-%j.out", + "--wrap=srun hostname", + ], + timeout=180, + ).stdout.strip() + accounting = runner.run( + [ + *workload_prefix, + "sacct", + "-X", + "-j", + job, + "--format=State,ExitCode", + "-n", + "-P", + ] + ).stdout + if "COMPLETED|0:0" not in accounting: + raise ProvisionError( + f"Slurm accounting validation failed for job {job}: {accounting}" + ) + + +def _write_state_summary( + config: Config, + backend: str, + kubernetes_version: str, + subnet: str = "", + gateway: str = "", +) -> None: + """Persist non-secret desired state for later diagnostics.""" + state = { + "schema": STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + "ssh_home_mode": "separate", + "storage_backend": backend, + "sbx_shared_root": ( + str(config.sbx_shared_root) if backend == "sbx-shared" else None + ), + "test_user": config.test_user, + "test_uid": config.test_uid, + "test_gid": config.test_gid, + "workload_user": WORKLOAD_USER, + "workload_uid": WORKLOAD_UID, + "workload_gid": WORKLOAD_GID, + "workload_account": WORKLOAD_ACCOUNT, + "kind_version": (SBX_KIND_VERSION if backend == "sbx-shared" else KIND_VERSION), + "kubectl_version": ( + SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION + ), + "kubernetes_version": kubernetes_version, + "nfs_csi_version": NFS_CSI_VERSION if backend == "nfs" else None, + "slinky_version": SLINKY_VERSION, + "kind_subnet": subnet, + "kind_gateway": gateway, + } + _write_text(config.state_dir / "state.json", json.dumps(state, indent=2) + "\n") + + +def setup_environment(runner: Runner, config: Config) -> None: + """Provision while serializing host-global NFS ownership.""" + architecture = _check_platform() + backend = _select_storage_backend(config) + if backend == "nfs": + with _nfs_host_lock(runner): + _claim_nfs_host_owner(runner, config) + _setup_environment_locked(runner, config, architecture, backend) + return + _setup_environment_locked(runner, config, architecture, backend) + + +def _setup_environment_locked( + runner: Runner, config: Config, architecture: str, backend: str +) -> None: + """Idempotently provision after backend ownership is established.""" + capacity_path = ( + config.sbx_shared_root if backend == "sbx-shared" else config.state_dir + ) + _check_host_capacity(capacity_path) + _prepare_host_dependencies(runner, config, backend) + _ensure_docker(runner) + _check_docker_capacity(runner) + _ensure_client_tools(runner, config, architecture, backend) + running_clusters = _kind_clusters(runner) + containers = _kind_containers(runner, config, running_only=False) + running_containers = _kind_containers(runner, config, running_only=True) + cluster_exists = config.cluster_name in running_clusters or bool(containers) + _ensure_cluster_ownership(config, cluster_exists) + if backend == "nfs" and cluster_exists and not _export_mount_type(runner, config): + LOG.warning( + "Replacing the disposable cluster before initializing the NFS " + "backing filesystem" + ) + _delete_cluster(runner, config) + running_clusters = set() + running_containers = set() + cluster_exists = False + if backend == "nfs": + _ensure_export_filesystem(runner, config) + else: + _prepare_sbx_shared(runner, config) + if config.cluster_name in running_clusters and len(running_containers) == 3: + _export_kubeconfig(runner, config) + if not _retained_cluster_matches_profile(runner, config, backend): + _delete_cluster(runner, config) + _create_cluster(runner, config, backend) + elif cluster_exists: + LOG.warning("Replacing incomplete or stopped disposable kind cluster") + _delete_cluster(runner, config) + _create_cluster(runner, config, backend) + else: + _create_cluster(runner, config, backend) + if backend == "sbx-shared": + _configure_sbx_node_trust(runner, config) + _wait_for_cluster(runner, config) + kubernetes_version = _observed_kubernetes_version(runner, config) + _record_storage_backend_profile(config, backend) + subnet = "" + gateway = "" + if backend == "nfs": + subnet, gateway = _kind_ipv4_network(runner) + _configure_nfs(runner, config, subnet, gateway) + _probe_nfs(runner, config, gateway) + _install_nfs_csi(runner, config, gateway) + else: + _install_sbx_shared_storage(runner, config) + _install_ssh_workers(runner, config) + _scale_ssh(runner, config, replicas=0) + _install_slurm(runner, config) + _scale_ssh(runner, config, replicas=2) + _wait_for_ssh(runner, config) + private_key = config.keys_dir / "id_ed25519" + _validate_ssh_workers(runner, config, private_key, "separate") + _write_state_summary(config, backend, kubernetes_version, subnet, gateway) + LOG.info("Integration environment is provisioned and running") + + +def _verify_user_access(runner: Runner, config: Config) -> None: + """Verify the current user can use the provisioned cluster and state.""" + probe = config.state_dir / "test-runs" / ".access-probe" + runner.run(["mkdir", "-p", probe.parent]) + runner.run(["touch", probe]) + runner.run(["rm", "--", probe]) + runner.run(["docker", "info"], timeout=60) + runner.run(["kind", "get", "clusters"], timeout=30) + runner.run( + [ + "kubectl", + "--kubeconfig", + config.kubeconfig, + "get", + "nodes", + ], + timeout=30, + ) + LOG.info("Verified integration access for current user %s", config.test_user) + + +def _scale_ssh(runner: Runner, config: Config, replicas: int) -> None: + """Scale SSH workers to conserve host capacity between checks.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "scale", + "statefulset/ssh-worker", + f"--replicas={replicas}", + ) + ) + + +def _wait_for_ssh(runner: Runner, config: Config) -> None: + """Wait for exactly two ready, nonterminating SSH workers.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "statefulset/ssh-worker", + "--timeout=180s", + ), + timeout=210, + ) + deadline = time.monotonic() + 180 + pods: list[dict[str, object]] = [] + while True: + request_timeout, process_timeout = _poll_timeouts(deadline, 10) + if process_timeout <= 0: + break + pods = _ssh_pods( + runner, + config, + request_timeout, + process_timeout=process_timeout, + ) + ready = [ + pod + for pod in pods + if _pod_ready(pod) and not pod.get("metadata", {}).get("deletionTimestamp") + ] + terminating = [ + pod for pod in pods if pod.get("metadata", {}).get("deletionTimestamp") + ] + if len(ready) == 2 and not terminating and len(pods) == 2: + return + time.sleep(min(2, max(0, deadline - time.monotonic()))) + details = [ + { + "name": pod.get("metadata", {}).get("name"), + "ready": _pod_ready(pod), + "terminating": bool(pod.get("metadata", {}).get("deletionTimestamp")), + } + for pod in pods + ] + raise ProvisionError(f"SSH worker rollout did not become healthy: {details}") + + +def _stop_environment_locked(runner: Runner, config: Config, backend: str) -> None: + """Delete the disposable cluster and stop owned host services.""" + if (config.state_dir / "cluster-owner.json").exists(): + clusters = _kind_clusters(runner) + containers = _kind_containers(runner, config, running_only=False) + cluster_exists = config.cluster_name in clusters or bool(containers) + if cluster_exists: + _ensure_cluster_ownership(config, cluster_exists=True) + _delete_cluster(runner, config) + else: + LOG.info( + "Disposable kind cluster %s is already absent", config.cluster_name + ) + else: + LOG.info("No owned disposable kind cluster was established") + if backend == "nfs": + _stop_owned_nfs(runner, config) + LOG.info( + "Integration environment stopped; host packages, caches, keys, and %s " + "data were preserved", + "NFS" if backend == "nfs" else "SBX shared", + ) + + +def stop_environment(runner: Runner, config: Config) -> None: + """Stop the fixture while serializing host-global NFS state.""" + backend = _select_storage_backend(config) + if backend == "nfs": + with _nfs_host_lock(runner): + host_owned = _validate_nfs_host_owner(runner, config) + if (config.state_dir / "nfs-service.json").exists() and not host_owned: + raise ProvisionError( + "refusing to alter NFS service without the host-global owner record" + ) + _stop_environment_locked(runner, config, backend) + return + _stop_environment_locked(runner, config, backend) + + +def teardown_environment(runner: Runner, config: Config) -> None: + """Remove the fixture while serializing host-global NFS state.""" + backend = _select_storage_backend(config) + if backend == "nfs": + with _nfs_host_lock(runner): + _teardown_environment_locked(runner, config, backend) + return + _teardown_environment_locked(runner, config, backend) + + +def _teardown_environment_locked(runner: Runner, config: Config, backend: str) -> None: + """Stop and remove all validated harness-owned resources.""" + if backend == "sbx-shared": + state_owned, setup_owned, shared_owned = _validate_sbx_teardown_ownership( + runner, config + ) + _stop_environment_locked(runner, config, backend) + if setup_owned: + _remove_harness_images(runner, config) + if shared_owned: + _remove_sbx_shared(runner, config) + _remove_owned_directories( + runner, config, remove_state=state_owned, remove_export=False + ) + LOG.info( + "Docker SBX integration environment torn down; installed host " + "packages were preserved" + ) + return + state_owned, setup_owned, export_owned, nfs_configured = ( + _validate_teardown_ownership(runner, config) + ) + host_owned = _validate_nfs_host_owner(runner, config) + _stop_environment_locked(runner, config, backend) + if nfs_configured: + _remove_nfs_configuration(runner, config) + if export_owned: + _unmount_export_filesystem(runner, config) + if setup_owned: + _remove_harness_images(runner, config) + _remove_owned_directories( + runner, config, remove_state=state_owned, remove_export=export_owned + ) + if host_owned: + runner.run([*_sudo_prefix(), "rm", "--force", "--", NFS_HOST_OWNER]) + LOG.info( + "Integration environment torn down; installed host packages were preserved" + ) + + +def _validate_sbx_teardown_ownership( + runner: Runner, config: Config +) -> tuple[bool, bool, bool]: + """Validate owned state and shared paths before Docker SBX cleanup.""" + _validate_cleanup_paths(config) + _validate_sbx_shared_root(config) + expected_owner = _owner_document(config) + state_marker = config.state_dir / STATE_MARKER + state_owned = state_marker.is_file() + if ( + state_owned + and json.loads(state_marker.read_text(encoding="utf-8")) != expected_owner + ): + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {state_marker}" + ) + if config.state_dir.exists() and not state_owned: + raise ProvisionError( + f"refusing to remove unowned setup state: {config.state_dir}" + ) + + cluster_marker = config.state_dir / "cluster-owner.json" + setup_owned = cluster_marker.is_file() + if ( + setup_owned + and json.loads(cluster_marker.read_text(encoding="utf-8")) != expected_owner + ): + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {cluster_marker}" + ) + + root = config.sbx_shared_root + shared_owned = root.exists() + if shared_owned: + marker = root / EXPORT_MARKER + if not marker.is_file() or json.loads(marker.read_text(encoding="utf-8")) != ( + _sbx_shared_marker(config) + ): + raise ProvisionError( + f"refusing teardown of unowned SBX shared root: {root}" + ) + allowed = {EXPORT_MARKER, "storage-test", "ssh-home"} + unexpected = sorted( + path.name for path in root.iterdir() if path.name not in allowed + ) + if unexpected: + raise ProvisionError( + f"refusing unexpected entries in SBX shared root: {unexpected}" + ) + if setup_owned: + _validate_image_ownership(runner, config) + return state_owned, setup_owned, shared_owned + + +def _remove_sbx_shared(runner: Runner, config: Config) -> None: + """Remove only the validated marker-owned Docker SBX shared root.""" + root = config.sbx_shared_root + _ensure_pinned_image(runner, SBX_KIND_NODE_IMAGE) + runner.run( + [ + "docker", + "run", + "--rm", + "--entrypoint", + "sh", + "--mount", + f"type=bind,src={root},dst=/owned", + SBX_KIND_NODE_IMAGE, + "-ec", + f"find /owned -mindepth 1 -xdev -depth ! -name {EXPORT_MARKER} -delete", + ], + timeout=120, + ) + (root / EXPORT_MARKER).unlink() + root.rmdir() + if root.exists(): + raise ProvisionError(f"cleanup did not remove SBX shared root: {root}") + + +def _owner_document(config: Config) -> dict[str, object]: + """Return the exact ownership document used by persistent markers.""" + return {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name} + + +def _read_system_file(runner: Runner, path: Path) -> str | None: + """Read a root-owned file, returning None when it is absent.""" + probe = runner.run([*_sudo_prefix(), "test", "-e", path], check=False, timeout=30) + if probe.returncode == 1: + return None + if probe.returncode != 0: + raise ProvisionError(f"could not inspect system path: {path}") + return runner.run([*_sudo_prefix(), "cat", path], timeout=30).stdout + + +def _validate_teardown_ownership( + runner: Runner, config: Config +) -> tuple[bool, bool, bool, bool]: + """Validate every persistent artifact before destructive cleanup.""" + _validate_cleanup_paths(config) + expected_owner = _owner_document(config) + state_marker = config.state_dir / STATE_MARKER + state_owned = False + if state_marker.exists(): + if json.loads(state_marker.read_text(encoding="utf-8")) != expected_owner: + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {state_marker}" + ) + state_owned = True + cluster_marker = config.state_dir / "cluster-owner.json" + cluster_owned = False + if cluster_marker.exists(): + if json.loads(cluster_marker.read_text(encoding="utf-8")) != expected_owner: + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {cluster_marker}" + ) + cluster_owned = True + state_owned = True + elif config.state_dir.exists() and not state_owned: + raise ProvisionError( + f"refusing to remove unowned setup state: {config.state_dir}" + ) + + export_marker = config.export_dir / EXPORT_MARKER + marker_text = _read_system_file(runner, export_marker) + export_owned = False + export_directory: subprocess.CompletedProcess[str] | None = None + if marker_text is not None: + try: + marker = json.loads(marker_text) + except json.JSONDecodeError as error: + raise ProvisionError( + f"refusing teardown with invalid export marker: {export_marker}" + ) from error + if marker != expected_owner: + raise ProvisionError( + f"refusing teardown with mismatched export marker: {export_marker}" + ) + export_owned = True + else: + export_directory = runner.run( + [*_sudo_prefix(), "test", "-d", config.export_dir], check=False + ) + if export_directory.returncode not in {0, 1}: + raise ProvisionError( + f"could not inspect export directory: {config.export_dir}" + ) + if export_directory is not None and export_directory.returncode == 0: + contents = runner.run( + [ + *_sudo_prefix(), + "find", + config.export_dir, + "-mindepth", + "1", + "-maxdepth", + "1", + "-print", + ] + ).stdout.strip() + if contents: + raise ProvisionError( + f"refusing to remove nonempty unowned export: {config.export_dir}" + ) + + export_config = _read_system_file(runner, NFS_EXPORT_CONFIG) + daemon_config = _read_system_file(runner, NFS_DAEMON_CONFIG) + service_state = config.state_dir / "nfs-service.json" + host_owned = _validate_nfs_host_owner(runner, config) + if not host_owned and ( + service_state.exists() or export_owned or config.nfs_image.exists() + ): + raise ProvisionError( + "refusing to remove NFS artifacts without the host-global owner record" + ) + nfs_configured = host_owned + if host_owned: + _validate_installed_config( + export_config, + config.manifests_dir / "storage-scale-test.exports", + NFS_EXPORT_CONFIG, + ) + _validate_installed_config( + daemon_config, + config.manifests_dir / "storage-scale-test-nfs.conf", + NFS_DAEMON_CONFIG, + ) + if ( + export_config is not None or daemon_config is not None + ) and not export_owned: + raise ProvisionError( + "refusing to remove NFS configuration without the matching export marker" + ) + if _export_mount_type(runner, config): + if not export_owned: + raise ProvisionError( + f"refusing to unmount unowned export directory: {config.export_dir}" + ) + _verified_export_loop(runner, config) + _validate_loop_associations(runner, config) + if cluster_owned: + _validate_image_ownership(runner, config) + + return state_owned, cluster_owned, export_owned, nfs_configured + + +def _validate_cleanup_paths(config: Config) -> None: + """Reject broad or overlapping lifecycle paths.""" + forbidden = {Path("/"), Path("/var"), Path("/srv"), Path("/etc")} + if config.state_dir in forbidden or config.export_dir in forbidden: + raise ProvisionError( + "refusing lifecycle action with a broad state or export path" + ) + if config.state_dir == config.export_dir: + raise ProvisionError("state and export directories must be different") + if config.state_dir in config.export_dir.parents: + raise ProvisionError("export directory must not be inside the state directory") + if config.export_dir in config.state_dir.parents: + raise ProvisionError("state directory must not be inside the export directory") + + +def _validate_lifecycle_paths(config: Config) -> None: + """Validate state ownership before bootstrap can mutate its path.""" + _validate_cleanup_paths(config) + state_dir = config.state_dir + if not state_dir.exists(): + if not state_dir.parent.is_dir() and state_dir != DEFAULT_STATE_DIR: + raise ProvisionError( + f"state directory must be a leaf below an existing directory: {state_dir}" + ) + return + if not state_dir.is_dir(): + raise ProvisionError(f"state path is not a directory: {state_dir}") + if state_dir.stat().st_uid != os.getuid(): + raise ProvisionError( + f"setup state is not owned by the current user: {state_dir}" + ) + marker = state_dir / STATE_MARKER + if not marker.is_file(): + raise ProvisionError(f"refusing to modify unowned setup state: {state_dir}") + try: + owner = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ProvisionError( + f"invalid setup state ownership marker: {marker}" + ) from error + if owner != _owner_document(config): + raise ProvisionError(f"setup state ownership marker does not match: {marker}") + + +def _validate_installed_config( + installed: str | None, source: Path, destination: Path +) -> None: + """Require a host config file to match its harness-rendered source.""" + if installed is None: + return + if not source.exists() or installed != source.read_text(encoding="utf-8"): + raise ProvisionError( + f"refusing to remove modified or unowned host configuration: {destination}" + ) + + +def _export_paths(runner: Runner) -> list[str]: + """Return currently exported local paths.""" + if not shutil.which("exportfs"): + return [] + exports = runner.run([*_sudo_prefix(), "exportfs", "-v"]).stdout + return [line.split()[0] for line in exports.splitlines() if line.startswith("/")] + + +def _verified_export_loop(runner: Runner, config: Config) -> str: + """Return the export loop device after verifying its exact backing image.""" + mounted = runner.run( + [ + "findmnt", + "--noheadings", + "--output", + "SOURCE,FSTYPE", + "--mountpoint", + config.export_dir, + ] + ).stdout.split() + if ( + len(mounted) != 2 + or mounted[1] != "ext4" + or not mounted[0].startswith("/dev/loop") + ): + raise ProvisionError( + f"refusing unexpected mount at dedicated export {config.export_dir}" + ) + backing = runner.run( + [ + *_sudo_prefix(), + "losetup", + "--noheadings", + "--output", + "BACK-FILE", + mounted[0], + ] + ).stdout.strip() + if Path(backing).resolve() != config.nfs_image.resolve(): + raise ProvisionError( + f"refusing loop device {mounted[0]} backed by unexpected file {backing}" + ) + return mounted[0] + + +def _associated_loop_devices(runner: Runner, config: Config) -> list[str]: + """Return loop devices associated with the exact NFS backing image.""" + return _loop_devices_for_file(runner, config.nfs_image) + + +def _validate_loop_associations(runner: Runner, config: Config) -> None: + """Reject an owned loop device mounted anywhere except the export path.""" + for device in _associated_loop_devices(runner, config): + result = runner.run( + ["findmnt", "--noheadings", "--output", "TARGET", "--source", device], + check=False, + ) + if result.returncode == 1: + continue + if result.returncode != 0: + raise ProvisionError( + f"findmnt could not inspect loop device {device}: " + f"{(result.stderr or result.stdout).strip()}" + ) + mounts = result.stdout.splitlines() + unexpected = [ + target for target in mounts if Path(target).resolve() != config.export_dir + ] + if unexpected: + raise ProvisionError( + f"refusing loop device {device} mounted outside the fixture: " + + ", ".join(unexpected) + ) + + +def _validate_image_ownership(runner: Runner, config: Config) -> None: + """Reject fixture tags that no longer identify images built by setup.""" + state_path = config.state_dir / "built-images.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + for image, ownership in state.items(): + previous_id = ownership.get("previous_id") + if previous_id and _image_id(runner, previous_id) != previous_id: + raise ProvisionError( + f"cannot restore prior Docker image for fixture tag {image}: " + f"{previous_id} is absent" + ) + current = _image_id(runner, image) + pending_id = ownership.get("pending_id") + pending_tag = ownership.get("pending_tag") + if pending_tag: + temporary_id = _image_id(runner, pending_tag) + if pending_id is not None and temporary_id not in {pending_id, None}: + raise ProvisionError( + f"pending Docker tag changed outside the fixture: {pending_tag}" + ) + allowed = {previous_id, ownership.get("built_id"), pending_id, None} + if current not in allowed: + raise ProvisionError( + f"refusing to alter Docker tag changed outside the fixture: {image}" + ) + + +def _remove_nfs_configuration(runner: Runner, config: Config) -> None: + """Unexport storage and remove only the fixture's host configuration.""" + LOG.info("Removing the dedicated NFS export and host configuration") + exportfs = shutil.which("exportfs") + export_source = config.manifests_dir / "storage-scale-test.exports" + if exportfs and export_source.exists(): + client = export_source.read_text(encoding="utf-8").split()[1].split("(", 1)[0] + runner.run( + [ + *_sudo_prefix(), + exportfs, + "-u", + f"{client}:{config.export_dir}", + ], + check=False, + ) + for path in (NFS_EXPORT_CONFIG, NFS_DAEMON_CONFIG): + runner.run([*_sudo_prefix(), "rm", "--force", "--", path]) + if exportfs: + runner.run([*_sudo_prefix(), exportfs, "-ra"]) + else: + LOG.info("exportfs is unavailable; skipping partial-bootstrap reload") + if str(config.export_dir) in _export_paths(runner): + raise ProvisionError(f"NFS export is still active: {config.export_dir}") + _restore_nfs_service_state(runner, config) + firewall_state = config.state_dir / "ufw-rule.json" + if firewall_state.exists(): + state = json.loads(firewall_state.read_text(encoding="utf-8")) + if state.get("added_by_harness"): + _delete_nfs_firewall_rule(runner, str(state["subnet"])) + firewall_state.unlink(missing_ok=True) + + +def _unmount_export_filesystem(runner: Runner, config: Config) -> None: + """Unmount and detach only the verified fixture backing filesystem.""" + if _export_mount_type(runner, config): + _verified_export_loop(runner, config) + runner.run([*_sudo_prefix(), "umount", config.export_dir]) + _validate_loop_associations(runner, config) + for device in _associated_loop_devices(runner, config): + runner.run([*_sudo_prefix(), "losetup", "--detach", device]) + if _export_mount_type(runner, config): + raise ProvisionError(f"export remains mounted: {config.export_dir}") + if _associated_loop_devices(runner, config): + raise ProvisionError(f"loop devices remain attached to {config.nfs_image}") + + +def _remove_harness_images(runner: Runner, config: Config) -> None: + """Remove owned image tags or restore the tags that setup replaced.""" + state_path = config.state_dir / "built-images.json" + state = ( + json.loads(state_path.read_text(encoding="utf-8")) + if state_path.exists() + else {} + ) + for image, ownership in state.items(): + built_id = ownership.get("built_id") + pending_id = ownership.get("pending_id") + pending_tag = ownership.get("pending_tag") + previous_id = ownership.get("previous_id") + current = _image_id(runner, image) + if current in {built_id, pending_id} and current is not None: + if previous_id: + runner.run(["docker", "image", "tag", previous_id, image]) + else: + runner.run(["docker", "image", "rm", image]) + if pending_tag: + temporary_id = _image_id(runner, pending_tag) + if temporary_id is not None: + if pending_id is not None and temporary_id != pending_id: + raise ProvisionError( + f"refusing to remove changed pending Docker tag: {pending_tag}" + ) + runner.run(["docker", "image", "rm", pending_tag]) + _remove_stale_csi_host_aliases(runner) + + +def _remove_owned_directories( + runner: Runner, + config: Config, + *, + remove_state: bool, + remove_export: bool, +) -> None: + """Remove validated user and privileged roots without crossing mounts.""" + paths: list[tuple[Path, bool]] = [] + if remove_state: + paths.append((config.state_dir, False)) + if remove_export: + paths.insert(0, (config.export_dir, True)) + for path, privileged in paths: + prefix = _sudo_prefix() if privileged else [] + marker_name = EXPORT_MARKER if privileged else STATE_MARKER + probe = runner.run([*prefix, "test", "-e", path], check=False) + if probe.returncode == 1: + continue + if probe.returncode != 0: + raise ProvisionError(f"could not inspect cleanup path: {path}") + runner.run( + [ + *prefix, + "find", + path, + "-mindepth", + "1", + "-xdev", + "-depth", + "!", + "-name", + marker_name, + "-delete", + ], + timeout=120, + ) + runner.run([*prefix, "rm", "--force", "--", path / marker_name]) + runner.run([*prefix, "rmdir", "--", path]) + removed = runner.run([*prefix, "test", "-e", path], check=False) + if removed.returncode == 0: + raise ProvisionError(f"cleanup did not remove {path}") + if removed.returncode != 1: + raise ProvisionError(f"could not verify cleanup of {path}") + + +def _stop_owned_nfs(runner: Runner, config: Config) -> None: + """Restore an initially inactive NFS service to its runtime state.""" + if not (config.state_dir / "nfs-service.json").exists(): + LOG.info("NFS ownership state is absent; leaving nfs-server unchanged") + return + if not _nfs_service_started_by_harness(config): + LOG.info("nfs-server predated this fixture; leaving it running") + return + service = runner.run( + [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], check=False + ) + if service.returncode == 0: + runner.run([*_sudo_prefix(), "systemctl", "stop", "nfs-server"]) + LOG.info("Stopped harness-owned nfs-server") + else: + LOG.info("nfs-server is already stopped") + + +def _nfs_service_started_by_harness(config: Config) -> bool: + """Return whether setup recorded ownership of the NFS service lifecycle.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + return False + state = json.loads(state_path.read_text(encoding="utf-8")) + if state.get("schema") == NFS_SERVICE_STATE_SCHEMA: + return not bool(state.get("was_active")) + return bool(state.get("started_by_harness", False)) + + +def _restore_preexisting_nfs_threads(runner: Runner, config: Config) -> None: + """Restore the worker count of an NFS service that predates the fixture.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + previous = state.get("previous_threads") + was_active = ( + bool(state.get("was_active")) + if state.get("schema") == NFS_SERVICE_STATE_SCHEMA + else not bool(state.get("started_by_harness")) + ) + if not was_active or not isinstance(previous, int): + return + active = runner.run( + [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], + check=False, + ) + if active.returncode: + LOG.info("Pre-existing nfs-server is inactive; worker restoration skipped") + return + _set_live_nfs_threads(runner, previous) + LOG.info("Restored pre-existing NFS worker count to %d", previous) + + +def _restore_nfs_service_state(runner: Runner, config: Config) -> None: + """Restore the pre-setup NFS runtime and boot-policy state.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + _restore_preexisting_nfs_threads(runner, config) + if not _systemd_unit_exists(runner, "nfs-server"): + LOG.info("nfs-server is unavailable; no boot policy needs restoration") + return + if state.get("schema") != NFS_SERVICE_STATE_SCHEMA: + if state.get("started_by_harness") and not _export_paths(runner): + runner.run([*_sudo_prefix(), "systemctl", "disable", "nfs-server"]) + return + was_enabled = bool(state.get("was_enabled")) + action = "enable" if was_enabled else "disable" + runner.run([*_sudo_prefix(), "systemctl", action, "nfs-server"]) + + +def _diagnostic_backend(config: Config) -> str | None: + """Return the configured or retained backend without probing or mutation.""" + if config.storage_backend != "auto": + return config.storage_backend + path = config.state_dir / "storage-backend.json" + if not path.is_file(): + return None + try: + backend = json.loads(path.read_text(encoding="utf-8")).get("backend") + except (OSError, json.JSONDecodeError): + return None + return backend if backend in STORAGE_BACKENDS else None + + +def _collect_diagnostics(runner: Runner, config: Config) -> None: + """Collect bounded troubleshooting state after a setup failure.""" + LOG.error("Collecting troubleshooting diagnostics") + commands: list[Sequence[str | Path]] = [ + ("free", "-h"), + ("df", "-h", "/"), + ("docker", "ps", "--all"), + ("kind", "get", "clusters"), + ] + if _diagnostic_backend(config) == "nfs": + commands.extend( + ( + ( + *_sudo_prefix(), + "systemctl", + "status", + "nfs-server", + "--no-pager", + ), + (*_sudo_prefix(), "exportfs", "-v"), + ) + ) + for command in commands: + if not shutil.which(str(command[0])): + LOG.error("diagnostic command is unavailable: %s", command[0]) + continue + result = runner.run(command, check=False, timeout=30) + output = (result.stdout + result.stderr).strip() + if output: + LOG.error("diagnostic %s:\n%s", command[0], output[-12000:]) + if config.kubeconfig.exists(): + for arguments in ( + ("get", "nodes", "-o", "wide"), + ("get", "pods", "--all-namespaces", "-o", "wide"), + ("get", "events", "--all-namespaces", "--sort-by=.lastTimestamp"), + ): + result = runner.run(_kubectl(config, *arguments), check=False, timeout=30) + output = (result.stdout + result.stderr).strip() + if output: + LOG.error("kubectl diagnostic:\n%s", output[-16000:]) + + +def _parser() -> argparse.ArgumentParser: + """Build the command-line parser.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cluster-name", default="storage-scale-integration") + parser.add_argument("--namespace", default="storage-scale-integration") + parser.add_argument("--state-dir", type=Path, default=DEFAULT_STATE_DIR) + parser.add_argument("--export-dir", type=Path, default=DEFAULT_EXPORT_DIR) + parser.add_argument( + "--storage-backend", + choices=("auto", *STORAGE_BACKENDS), + default="auto", + ) + parser.add_argument( + "--sbx-shared-root", + type=Path, + default=DEFAULT_SBX_SHARED_ROOT, + ) + parser.add_argument("--verbose", action="store_true") + actions = parser.add_subparsers(dest="action", required=True) + for action in ("setup", "start", "stop", "teardown"): + actions.add_parser(action) + test_parser = actions.add_parser("test") + test_parser.add_argument( + "--substrate", + choices=SUBSTRATES, + default="all", + help="execution substrate to test (default: all)", + ) + test_parser.add_argument( + "--scenario", + dest="scenarios", + action="append", + default=[], + metavar="NAME", + help="scenario to run; repeat to select more than one", + ) + test_parser.add_argument( + "--list-scenarios", + action="store_true", + help="list scenarios without inspecting or changing fixture state", + ) + return parser + + +def _config(arguments: argparse.Namespace) -> Config: + """Convert parsed arguments into immutable configuration.""" + account = _test_account() + return Config( + cluster_name=arguments.cluster_name, + namespace=arguments.namespace, + state_dir=arguments.state_dir.resolve(), + export_dir=arguments.export_dir.resolve(), + storage_backend=arguments.storage_backend, + sbx_shared_root=arguments.sbx_shared_root.resolve(), + test_user=account.pw_name, + test_uid=account.pw_uid, + test_gid=account.pw_gid, + verbose=arguments.verbose, + ) + + +def _test_account() -> pwd.struct_passwd: + """Resolve the ordinary account running every lifecycle action.""" + try: + account = pwd.getpwuid(os.getuid()) + except KeyError as error: + raise ProvisionError( + f"current uid has no password-database entry: {os.getuid()}" + ) from error + if account.pw_uid == 0: + raise ProvisionError( + "integration lifecycle actions must run as an ordinary user; " + "the driver invokes sudo only for narrow host-system operations" + ) + return account + + +def _run_filesystem_action( + runner: Runner, config: Config, arguments: argparse.Namespace +) -> None: + """Run selected scenarios with crash-recoverable SSH home transitions.""" + run_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{os.getpid()}" + + def transition(mode: str, scenario_id: str) -> None: + if mode == CANONICAL_HOME_MODE and scenario_id != "preflight": + _restore_ssh_home_mode( + runner, config, run_id=run_id, scenario_id=scenario_id + ) + return + _ensure_ssh_home_mode( + runner, + config, + mode, + run_id=run_id, + scenario_id=scenario_id, + ) + + run_filesystem_tests( + runner, + config, + _repository_root(), + arguments.substrate, + arguments.scenarios, + transition, + ) + + +def main() -> int: + """Run one integration environment lifecycle action.""" + _require_python() + arguments = _parser().parse_args() + if arguments.action == "test" and arguments.list_scenarios: + print(format_scenario_listing()) + return 0 + try: + if os.geteuid() == 0: + raise ProvisionError( + "refusing to run the integration lifecycle as root; rerun as " + "an ordinary user with sudo available for NFS host operations" + ) + config = _config(arguments) + _validate_lifecycle_paths(config) + if arguments.action == "test" and not config.state_dir.is_dir(): + raise ProvisionError( + f"setup state directory is absent at {config.state_dir}; run setup first" + ) + _bootstrap_state_dir(config) + _configure_user_tool_path(config) + log_path = _configure_logging(config, arguments.action) + LOG.info("Detailed log: %s", log_path) + with _acquire_lock(config): + runner = Runner() + if arguments.action == "stop": + stop_environment(runner, config) + elif arguments.action == "teardown": + teardown_environment(runner, config) + elif arguments.action == "test": + _run_filesystem_action(runner, config, arguments) + else: + setup_environment(runner, config) + _verify_user_access(runner, config) + return 0 + except ( + ProvisionError, + IntegrationTestError, + SshHomeTransitionError, + OSError, + subprocess.TimeoutExpired, + json.JSONDecodeError, + ) as error: + LOG.error("%s", error) + if ( + "runner" in locals() + and "config" in locals() + and arguments.action not in ("stop", "teardown") + ): + _collect_diagnostics(runner, config) + return 1 + except KeyboardInterrupt: + LOG.error("Interrupted") + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integration-tests/lib/deployment_cache.py b/integration-tests/lib/deployment_cache.py new file mode 100644 index 0000000..18132a5 --- /dev/null +++ b/integration-tests/lib/deployment_cache.py @@ -0,0 +1,396 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Content-addressed deployment archives for the integration harness.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import tempfile +import uuid +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +CACHE_SCHEMA = 1 +DEFAULT_RECIPE = 1 +ARCHIVE_NAME = "storage-scale-test.tar.gz" +MANIFEST_NAME = "manifest.json" +BUILD_LOG_NAME = "build-tarball.log" + + +class DeploymentCacheError(RuntimeError): + """A deployment snapshot or cache entry is unsafe or inconsistent.""" + + +@dataclass(frozen=True) +class DeploymentCacheRequest: + """Inputs needed to build one content-addressed deployment archive.""" + + repo_root: Path + cache_root: Path + architecture: str + binary: Path + binary_name: str + runtime: Path | None = None + recipe: int = DEFAULT_RECIPE + build_timeout: int = 600 + + +@dataclass(frozen=True) +class CachedDeployment: + """One verified archive returned from the deployment cache.""" + + key: str + archive: Path + manifest: Path + build_log: Path + cache_hit: bool + + +def _sha256(path: Path) -> str: + """Return the SHA-256 digest of one regular file.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_digest(document: object) -> str: + """Hash one JSON-compatible document in a stable representation.""" + encoded = json.dumps( + document, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _safe_relative(name: str) -> Path: + """Validate and convert a Git-provided repository-relative path.""" + relative = PurePosixPath(name) + if relative.is_absolute() or not relative.parts or ".." in relative.parts: + raise DeploymentCacheError(f"unsafe tracked path from git: {name!r}") + return Path(*relative.parts) + + +def _tracked_names(runner: Any, repo_root: Path) -> tuple[str, ...]: + """Return tracked paths in a deterministic order.""" + result = runner.run( + ["git", "ls-files", "-z", "--cached"], cwd=repo_root, timeout=30 + ) + names = tuple(name for name in result.stdout.split("\0") if name) + if not names: + raise DeploymentCacheError(f"repository has no tracked files: {repo_root}") + return tuple(sorted(names)) + + +def _copy_tracked_file(source: Path, target: Path) -> None: + """Copy one regular file or safe relative symbolic link.""" + mode = source.lstat().st_mode + target.parent.mkdir(parents=True, exist_ok=True) + if stat.S_ISREG(mode): + shutil.copy2(source, target, follow_symlinks=False) + return + if stat.S_ISLNK(mode): + link = os.readlink(source) + link_path = PurePosixPath(link) + if link_path.is_absolute() or ".." in link_path.parts: + raise DeploymentCacheError( + f"unsafe tracked symbolic link: {source} -> {link}" + ) + target.symlink_to(link) + return + raise DeploymentCacheError( + f"deployment snapshot requires a regular file or relative symlink: {source}" + ) + + +def _manifest_entry(root: Path, path: Path) -> dict[str, object]: + """Describe one file-system entry below a snapshot root.""" + mode = path.lstat().st_mode + relative = path.relative_to(root).as_posix() + if stat.S_ISREG(mode): + kind = "file" + digest = _sha256(path) + elif stat.S_ISDIR(mode): + kind = "directory" + digest = hashlib.sha256(b"").hexdigest() + elif stat.S_ISLNK(mode): + kind = "symlink" + digest = hashlib.sha256(os.readlink(path).encode("utf-8")).hexdigest() + else: + raise DeploymentCacheError(f"unsupported snapshot entry: {path}") + return { + "path": relative, + "mode": stat.S_IMODE(mode), + "type": kind, + "sha256": digest, + } + + +def _manifest_for_paths(root: Path, paths: tuple[Path, ...]) -> dict[str, object]: + """Describe an explicit ordered set of paths.""" + entries = [_manifest_entry(root, path) for path in paths] + document: dict[str, object] = {"entries": entries} + document["sha256"] = _canonical_digest(entries) + return document + + +def _copy_tracked_snapshot( + runner: Any, repo_root: Path, destination: Path +) -> dict[str, object]: + """Materialize and describe the exact tracked working-tree snapshot.""" + names = _tracked_names(runner, repo_root) + destination.mkdir(parents=True) + copied: list[Path] = [] + for name in names: + relative = _safe_relative(name) + source = repo_root / relative + if not source.exists() and not source.is_symlink(): + continue + target = destination / relative + _copy_tracked_file(source, target) + copied.append(target) + for directory in sorted( + (path for path in destination.rglob("*") if stat.S_ISDIR(path.lstat().st_mode)), + reverse=True, + ): + directory.chmod(0o755) + destination.chmod(0o755) + return _manifest_for_paths(destination, tuple(copied)) + + +def _tree_paths(root: Path) -> tuple[Path, ...]: + """Return all entries in a tree without following symbolic links.""" + paths: list[Path] = [] + for directory, dirnames, filenames in os.walk(root, followlinks=False): + base = Path(directory) + linked_directories = [name for name in dirnames if (base / name).is_symlink()] + dirnames[:] = sorted( + name for name in dirnames if name not in linked_directories + ) + paths.extend(base / name for name in sorted(linked_directories)) + paths.extend(base / name for name in dirnames) + paths.extend(base / name for name in sorted(filenames)) + return tuple(sorted(paths, key=lambda item: item.relative_to(root).as_posix())) + + +def _stage_binary_and_runtime( + request: DeploymentCacheRequest, snapshot: Path +) -> tuple[dict[str, object], dict[str, object] | None]: + """Seed external build inputs and return their staged identities.""" + if not request.binary.is_file() or request.binary.is_symlink(): + raise DeploymentCacheError( + f"deployment binary is not a regular file: {request.binary}" + ) + binary_name = PurePosixPath(request.binary_name) + if ( + binary_name.is_absolute() + or len(binary_name.parts) != 1 + or binary_name.name in ("", ".", "..") + ): + raise DeploymentCacheError( + f"deployment binary name is not a basename: {request.binary_name!r}" + ) + binary_target = snapshot / "utils" / binary_name.name + binary_target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(request.binary, binary_target) + binary_target.chmod(0o755) + binary_identity = _manifest_entry(snapshot, binary_target) + + if request.runtime is None: + return binary_identity, None + if not request.runtime.is_dir() or request.runtime.is_symlink(): + raise DeploymentCacheError( + f"deployment runtime is not a directory: {request.runtime}" + ) + runtime_target = snapshot / "utils" / "elbencho-runtime" + shutil.copytree(request.runtime, runtime_target) + return binary_identity, _manifest_for_paths( + runtime_target, _tree_paths(runtime_target) + ) + + +def _identity_document( + request: DeploymentCacheRequest, + source_manifest: dict[str, object], + binary_identity: dict[str, object], + runtime_identity: dict[str, object] | None, +) -> dict[str, object]: + """Build the cache-key document for one immutable snapshot.""" + return { + "schema": CACHE_SCHEMA, + "recipe": request.recipe, + "architecture": request.architecture, + "source": source_manifest, + "binary": binary_identity, + "runtime": runtime_identity, + } + + +def _entry_paths(cache_root: Path, key: str) -> tuple[Path, Path, Path, Path]: + """Return the entry and its fixed files.""" + entry = cache_root / key + return ( + entry, + entry / ARCHIVE_NAME, + entry / MANIFEST_NAME, + entry / BUILD_LOG_NAME, + ) + + +def _load_json(path: Path) -> dict[str, object] | None: + """Read a JSON object, returning None for malformed input.""" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + return document if isinstance(document, dict) else None + + +def _valid_entry(cache_root: Path, key: str, identity: dict[str, object]) -> bool: + """Return whether a cache entry exactly matches its expected identity.""" + entry, archive, manifest_path, build_log = _entry_paths(cache_root, key) + if not entry.is_dir() or entry.is_symlink(): + return False + if any( + path.is_symlink() or not path.is_file() + for path in (archive, manifest_path, build_log) + ): + return False + manifest = _load_json(manifest_path) + if manifest is None or manifest.get("identity") != identity: + return False + archive_record = manifest.get("archive") + if not isinstance(archive_record, dict) or archive.stat().st_size <= 0: + return False + return ( + archive_record.get("bytes") == archive.stat().st_size + and archive_record.get("sha256") == _sha256(archive) + and manifest.get("input_digest") == key + ) + + +def _run_builder( + runner: Any, request: DeploymentCacheRequest, staging: Path, snapshot: Path +) -> tuple[Path, str]: + """Invoke the existing builder from an exact disposable snapshot copy.""" + build_parent = staging / "builder" + build_tree = build_parent / "source" + shutil.copytree(snapshot, build_tree, symlinks=True) + builder = build_tree / "utils" / "build_tarball.sh" + if not builder.is_file() or builder.is_symlink(): + raise DeploymentCacheError(f"deployment builder is absent: {builder}") + result = runner.run([builder], cwd=build_tree, timeout=request.build_timeout) + archive = build_parent / ARCHIVE_NAME + if not archive.is_file() or archive.is_symlink() or archive.stat().st_size <= 0: + raise DeploymentCacheError(f"deployment builder did not create {archive}") + return archive, result.stdout + result.stderr + + +def _publish( + cache_root: Path, + key: str, + identity: dict[str, object], + archive: Path, + build_log: str, +) -> bool: + """Atomically publish one completed entry; return whether ours won.""" + publish: Path | None = cache_root / f".publish-{key}-{uuid.uuid4().hex}" + publish.mkdir(mode=0o700) + try: + published_archive = publish / ARCHIVE_NAME + shutil.copy2(archive, published_archive) + (publish / BUILD_LOG_NAME).write_text(build_log, encoding="utf-8") + manifest = { + "schema": CACHE_SCHEMA, + "input_digest": key, + "identity": identity, + "archive": { + "bytes": published_archive.stat().st_size, + "sha256": _sha256(published_archive), + }, + } + (publish / MANIFEST_NAME).write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if _valid_entry(cache_root, key, identity): + return False + _replace_invalid_entry(cache_root, key, publish) + publish = None + return True + finally: + if publish is not None: + shutil.rmtree(publish, ignore_errors=True) + + +def _replace_invalid_entry(cache_root: Path, key: str, publish: Path) -> None: + """Replace an invalid entry while keeping the new entry atomically visible.""" + target = cache_root / key + quarantine = cache_root / f".invalid-{key}-{uuid.uuid4().hex}" + moved_invalid = False + if target.exists() or target.is_symlink(): + os.replace(target, quarantine) + moved_invalid = True + try: + os.replace(publish, target) + except OSError: + if moved_invalid and not target.exists(): + os.replace(quarantine, target) + raise + finally: + if moved_invalid: + if quarantine.is_dir() and not quarantine.is_symlink(): + shutil.rmtree(quarantine, ignore_errors=True) + else: + quarantine.unlink(missing_ok=True) + + +def get_or_build_deployment( + runner: Any, request: DeploymentCacheRequest +) -> CachedDeployment: + """Return a verified cached deployment built from one exact snapshot.""" + request.cache_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".deployment-staging-", dir=request.cache_root + ) as temporary: + staging = Path(temporary) + snapshot = staging / "source" + source_manifest = _copy_tracked_snapshot(runner, request.repo_root, snapshot) + binary_identity, runtime_identity = _stage_binary_and_runtime(request, snapshot) + build_input_manifest = _manifest_for_paths(snapshot, _tree_paths(snapshot)) + identity = _identity_document( + request, source_manifest, binary_identity, runtime_identity + ) + key = _canonical_digest(identity) + entry, archive, manifest, build_log = _entry_paths(request.cache_root, key) + if _valid_entry(request.cache_root, key, identity): + return CachedDeployment(key, archive, manifest, build_log, True) + + built_archive, output = _run_builder(runner, request, staging, snapshot) + if _manifest_for_paths(snapshot, _tree_paths(snapshot)) != build_input_manifest: + raise DeploymentCacheError( + "deployment builder modified its immutable source snapshot" + ) + published = _publish(request.cache_root, key, identity, built_archive, output) + if not _valid_entry(request.cache_root, key, identity): + raise DeploymentCacheError( + f"published deployment cache is invalid: {entry}" + ) + return CachedDeployment(key, archive, manifest, build_log, not published) diff --git a/integration-tests/lib/failure_injection.py b/integration-tests/lib/failure_injection.py new file mode 100644 index 0000000..40de57c --- /dev/null +++ b/integration-tests/lib/failure_injection.py @@ -0,0 +1,369 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scenario-owned Elbencho failure injection for integration tests. + +The generated wrapper is deliberately independent of production deployment +code. Callers stage its real-binary delegate through injected operations, set +``ELBENCHO`` to the wrapper for a scenario, run the failed attempt and resume +inside one :func:`staged_failure_injection` context, and copy diagnostics before +leaving that outer context. +""" + +from __future__ import annotations + +import re +import shlex +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +DEFAULT_INJECTED_EXIT_CODE = 97 +MAX_SCENARIO_ID_LENGTH = 64 +_SCENARIO_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z") + + +class FailureInjectionError(RuntimeError): + """A failure-injection plan is unsafe or internally inconsistent.""" + + +@dataclass(frozen=True) +class FailureInjectionTarget: + """One machine or shared filesystem on which artifacts are staged. + + ``endpoint`` is opaque to this module. An SSH adapter can use a hostname, + while a Slurm adapter can name its shared login/compute filesystem. + ``install_wrapper`` is false for SSH workers because production code copies + the selected coordinator wrapper to them; they only need its delegate and + optional runtime at the same absolute paths. + """ + + endpoint: str + install_wrapper: bool + + +@dataclass(frozen=True) +class FailureInjectionLayout: + """Exact POSIX paths used on every staging endpoint.""" + + root: PurePosixPath + wrapper: PurePosixPath + delegate: PurePosixPath + runtime: PurePosixPath + marker: PurePosixPath + + +@dataclass(frozen=True) +class FailureInjectionPlan: + """Complete immutable staging and wrapper plan for one scenario.""" + + scenario_id: str + source_binary: Path + source_runtime: Path | None + target_argument: str + injected_exit_code: int + layout: FailureInjectionLayout + targets: tuple[FailureInjectionTarget, ...] + wrapper_script: str + + +MakeDirectory = Callable[[str, PurePosixPath], None] +CopyFile = Callable[[Path, str, PurePosixPath, int], None] +CopyTree = Callable[[Path, str, PurePosixPath], None] +WriteText = Callable[[str, PurePosixPath, str, int], None] +RemoveTree = Callable[[str, PurePosixPath], None] + + +@dataclass(frozen=True) +class FailureInjectionOperations: + """Caller-provided local, SSH, or Slurm staging operations.""" + + make_directory: MakeDirectory + copy_file: CopyFile + copy_tree: CopyTree + write_text: WriteText + remove_tree: RemoveTree + + +def _validated_root(root: str | PurePosixPath, scenario_id: str) -> PurePosixPath: + """Return a safe scenario-specific absolute staging root.""" + if ( + not scenario_id + or len(scenario_id) > MAX_SCENARIO_ID_LENGTH + or _SCENARIO_ID.fullmatch(scenario_id) is None + ): + raise FailureInjectionError(f"invalid scenario identifier: {scenario_id!r}") + base = PurePosixPath(root) + if not base.is_absolute() or base == PurePosixPath("/") or ".." in base.parts: + raise FailureInjectionError( + f"failure-injection root must be a dedicated absolute path: {root!s}" + ) + return base / scenario_id / "failure-injection" + + +def _layout(root: PurePosixPath) -> FailureInjectionLayout: + """Derive fixed wrapper, delegate, runtime, and marker paths.""" + delegate_root = root / "delegate" + return FailureInjectionLayout( + root=root, + wrapper=root / "elbencho-inject", + delegate=delegate_root / "elbencho", + runtime=delegate_root / "elbencho-runtime", + marker=root / "failure-injected.marker", + ) + + +def _validate_targets(targets: Sequence[FailureInjectionTarget]) -> None: + """Require unique endpoints and exactly one wrapper-capable coordinator.""" + if not targets: + raise FailureInjectionError("failure-injection plan has no staging targets") + endpoints = [target.endpoint for target in targets] + if any(not endpoint or "\n" in endpoint for endpoint in endpoints): + raise FailureInjectionError("failure-injection endpoint is empty or multiline") + if len(set(endpoints)) != len(endpoints): + raise FailureInjectionError("failure-injection endpoints must be unique") + wrappers = sum(target.install_wrapper for target in targets) + if wrappers != 1: + raise FailureInjectionError( + "failure-injection plan requires exactly one coordinator wrapper target" + ) + + +def _wrapper_script( + layout: FailureInjectionLayout, + target_argument: str, + injected_exit_code: int, +) -> str: + """Generate the exact Bash wrapper staged for one scenario.""" + delegate = shlex.quote(str(layout.delegate)) + marker = shlex.quote(str(layout.marker)) + target = shlex.quote(target_argument) + return f"""#!/usr/bin/env bash +set -euo pipefail + +readonly delegate={delegate} +readonly marker={marker} +readonly trace=$(dirname "$marker")/invocations.log +readonly target_argument={target} +readonly injected_exit_code={injected_exit_code} + +service_mode=0 +target_invocation=0 +for argument in "$@"; do + [[ "$argument" == "--service" ]] && service_mode=1 + [[ "$argument" == *"$target_argument"* ]] && target_invocation=1 +done + +if (( service_mode )); then + exec "$delegate" "$@" +fi +printf '%q ' "$@" >>"$trace" +printf '\n' >>"$trace" +if (( ! target_invocation )); then + exec "$delegate" "$@" +fi +if ! mkdir -- "$marker" 2>/dev/null; then + exec "$delegate" "$@" +fi + +child_pid="" +forward_signal() {{ + local signal=$1 + [[ -z "$child_pid" ]] || kill -s "$signal" "$child_pid" 2>/dev/null || true +}} +trap 'forward_signal HUP' HUP +trap 'forward_signal INT' INT +trap 'forward_signal TERM' TERM + +"$delegate" "$@" & +child_pid=$! +set +e +while true; do + wait "$child_pid" + delegate_rc=$? + if ! kill -0 "$child_pid" 2>/dev/null; then + break + fi +done +set -e +trap - HUP INT TERM + +if (( delegate_rc != 0 )); then + rm -rf -- "$marker" + exit "$delegate_rc" +fi +printf '%s\n' injected >"$marker/state" +exit "$injected_exit_code" +""" + + +def build_failure_injection_plan( + *, + scenario_id: str, + staging_root: str | PurePosixPath, + source_binary: Path, + source_runtime: Path | None, + target_argument: str, + targets: Sequence[FailureInjectionTarget], + injected_exit_code: int = DEFAULT_INJECTED_EXIT_CODE, +) -> FailureInjectionPlan: + """Build a substrate-neutral failure-injection plan. + + ``target_argument`` must be a distinctive argument fragment unique to the + cell that should fail, normally its execution-specific JSON path. Service + startup is handled separately before fragment matching. + """ + if not target_argument or "\x00" in target_argument: + raise FailureInjectionError( + "target argument must be nonempty and contain no NUL" + ) + if not 1 <= injected_exit_code <= 255: + raise FailureInjectionError("injected exit code must be between 1 and 255") + _validate_targets(targets) + plan_layout = _layout(_validated_root(staging_root, scenario_id)) + return FailureInjectionPlan( + scenario_id=scenario_id, + source_binary=source_binary, + source_runtime=source_runtime, + target_argument=target_argument, + injected_exit_code=injected_exit_code, + layout=plan_layout, + targets=tuple(targets), + wrapper_script=_wrapper_script( + plan_layout, target_argument, injected_exit_code + ), + ) + + +def build_ssh_failure_injection_plan( + *, + scenario_id: str, + staging_root: str | PurePosixPath, + source_binary: Path, + source_runtime: Path | None, + target_argument: str, + coordinator_endpoint: str, + worker_endpoints: Sequence[str], + injected_exit_code: int = DEFAULT_INJECTED_EXIT_CODE, +) -> FailureInjectionPlan: + """Build a plan that stages delegates on the coordinator and SSH workers.""" + targets = [FailureInjectionTarget(coordinator_endpoint, True)] + targets.extend( + FailureInjectionTarget(endpoint, False) for endpoint in worker_endpoints + ) + return build_failure_injection_plan( + scenario_id=scenario_id, + staging_root=staging_root, + source_binary=source_binary, + source_runtime=source_runtime, + target_argument=target_argument, + targets=targets, + injected_exit_code=injected_exit_code, + ) + + +def build_slurm_failure_injection_plan( + *, + scenario_id: str, + staging_root: str | PurePosixPath, + source_binary: Path, + source_runtime: Path | None, + target_argument: str, + shared_endpoint: str, + injected_exit_code: int = DEFAULT_INJECTED_EXIT_CODE, +) -> FailureInjectionPlan: + """Build a plan whose artifacts are visible to login and compute nodes.""" + return build_failure_injection_plan( + scenario_id=scenario_id, + staging_root=staging_root, + source_binary=source_binary, + source_runtime=source_runtime, + target_argument=target_argument, + targets=(FailureInjectionTarget(shared_endpoint, True),), + injected_exit_code=injected_exit_code, + ) + + +def stage_failure_injection( + plan: FailureInjectionPlan, operations: FailureInjectionOperations +) -> None: + """Stage a plan without assuming local, SSH, or Slurm command transport.""" + if not plan.source_binary.is_file() or plan.source_binary.is_symlink(): + raise FailureInjectionError( + f"real Elbencho delegate is not a regular file: {plan.source_binary}" + ) + if plan.source_runtime is not None and ( + not plan.source_runtime.is_dir() or plan.source_runtime.is_symlink() + ): + raise FailureInjectionError( + f"Elbencho runtime is not a directory: {plan.source_runtime}" + ) + for target in plan.targets: + operations.make_directory(target.endpoint, plan.layout.root) + operations.make_directory(target.endpoint, plan.layout.delegate.parent) + operations.copy_file( + plan.source_binary, target.endpoint, plan.layout.delegate, 0o755 + ) + if plan.source_runtime is not None: + operations.copy_tree( + plan.source_runtime, target.endpoint, plan.layout.runtime + ) + if target.install_wrapper: + operations.write_text( + target.endpoint, + plan.layout.wrapper, + plan.wrapper_script, + 0o755, + ) + + +def cleanup_failure_injection( + plan: FailureInjectionPlan, operations: FailureInjectionOperations +) -> None: + """Attempt removal on every endpoint and report aggregate failures.""" + failures = [] + for target in reversed(plan.targets): + try: + operations.remove_tree(target.endpoint, plan.layout.root) + except Exception as error: # pylint: disable=broad-exception-caught + failures.append(f"{target.endpoint}: {error!r}") + if failures: + raise FailureInjectionError( + "failure-injection cleanup failed: " + "; ".join(failures) + ) + + +@contextmanager +def staged_failure_injection( + plan: FailureInjectionPlan, operations: FailureInjectionOperations +) -> Iterator[FailureInjectionPlan]: + """Stage once and clean only after the whole failed-attempt/resume scenario. + + Callers should execute the initial expected failure, copy intermediate + diagnostics, execute resume, and copy final diagnostics inside the context. + Cleanup also runs if the overall scenario aborts. + """ + try: + stage_failure_injection(plan, operations) + yield plan + except BaseException as primary: + try: + cleanup_failure_injection(plan, operations) + except FailureInjectionError as cleanup_error: + primary.add_note(str(cleanup_error)) + raise + else: + cleanup_failure_injection(plan, operations) diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py new file mode 100644 index 0000000..acdb417 --- /dev/null +++ b/integration-tests/lib/filesystem_integration.py @@ -0,0 +1,2871 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run bounded filesystem integration tests against the provisioned fixture.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import platform +import re +import shlex +import shutil +import subprocess +import tarfile +import tempfile +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath +from typing import Any + +from deployment_cache import ( + DeploymentCacheError, + DeploymentCacheRequest, + get_or_build_deployment, +) +from failure_injection import ( + FailureInjectionOperations, + build_slurm_failure_injection_plan, + build_ssh_failure_injection_plan, + staged_failure_injection, +) +from fixture_capacity import ( + MAX_DEPLOYMENT_CONTENT_BYTES, + MAX_LIVE_CAPTURE_DATASET_BYTES, +) +from filesystem_scenario_specs import ( + SCENARIO_SPECS_BY_NAME, + CommandKind, + DatasetExpectation, + ExecutionStatus, + FilesystemScenarioSpec, + ScenarioStep, + WorkloadPhase, +) +from scenario_planner import ( + ScenarioPlanningError, + SshHomeTransition, + WorkItem, + plan_scenarios, +) + +LOG = logging.getLogger("storage-scale-integration") + +ELBENCHO_VERSION = "v3.1-11" +ELBENCHO_RELEASE_API = ( + "https://api.github.com/repos/breuner/elbencho/releases/tags/" + ELBENCHO_VERSION +) +ELBENCHO_CONTAINER = ( + "breuner/elbencho:v3.1-11@" + "sha256:719fba92cab57c773ddf7a2776414b358aeb8126a15fbc8e3c52469ce3a5b8b2" +) +SBX_ELBENCHO_BUNDLE_RECIPE = 1 +MAX_ARCHIVE_BYTES = 32 * 1024 * 1024 +MAX_DEPLOYMENT_ARCHIVE_BYTES = 128 * 1024 * 1024 +MAX_DEPLOYMENT_FILES = 10_000 +SLURM_CLEANUP_MARGIN_SECONDS = 120 +ELBENCHO_ARCHIVES = { + "x86_64": ( + "elbencho-static-x86_64.tar.gz", + "8d7cf885481dbd8f39908b7f4ff588d9e80cbc0fd26eeaba0b764c77587884d2", + "elbencho", + ), + "aarch64": ( + "elbencho-static-aarch64.tar.gz", + "a744c82ab4e15d8cf4023f7f5053c2008e148f77349e2ba53d2352c4f7508683", + "elbencho.aarch64", + ), +} +REMOTE_BASE = "/mnt/storage-test/integration-regression" +SSH_FAILURE_STAGING_BASE = "/tmp/storage-scale-test-integration-failure" +VALIDATION_SUCCESS = "All validation checks passed successfully" +WORKLOAD_USER = "tester" +WORKLOAD_UID = 2000 +WORKLOAD_GID = 2000 + + +class IntegrationTestError(RuntimeError): + """An actionable filesystem integration test failure.""" + + +@dataclass(frozen=True) +class Fixture: + """Live fixture details discovered from Kubernetes and Slurm.""" + + login_pod: str + login_container: str + ssh_addresses: tuple[str, ...] + slurm_nodes: tuple[str, ...] + slurm_addresses: tuple[str, ...] + architecture: str + ssh_home_mode: str + storage_backend: str + + +@dataclass +class ScenarioRuntime: + """Mutable state shared by a scenario's ordered command steps.""" + + scenario: FilesystemScenarioSpec + selector: str + workspace: str + local_workspace: Path + artifact_root: Path + data_root: str + values: dict[str, str] + copied_results: list[Path] + + +@dataclass(frozen=True) +class StepOutcome: + """Result locations and output for one scenario command.""" + + output: str + remote_result: str | None + local_result: Path | None + + +def _kubectl(config: Any, *arguments: str | Path) -> list[str | Path]: + """Build a kubectl command using the fixture's private kubeconfig.""" + return ["kubectl", "--kubeconfig", config.kubeconfig, *arguments] + + +def _pod_command( + config: Any, + pod: str, + container: str, + command: str, + *, + as_user: str | None = None, + timeout: int = 600, + kill_after: int = 10, +) -> list[str | Path]: + """Build a remotely bounded command for one fixture pod.""" + prefix: list[str | Path] = _kubectl( + config, + "-n", + config.namespace, + "exec", + pod, + "-c", + container, + "--", + ) + if as_user: + prefix.extend(("runuser", "-u", as_user, "--")) + prefix.extend( + ( + "timeout", + f"--kill-after={kill_after}s", + f"{timeout}s", + "bash", + "-lc", + command, + ) + ) + return prefix + + +def _ready(pod: dict[str, Any]) -> bool: + """Return whether all containers in a running pod are ready.""" + status = pod.get("status", {}) + containers = status.get("containerStatuses", []) + return ( + status.get("phase") == "Running" + and bool(containers) + and all(item.get("ready", False) for item in containers) + ) + + +def _pods_with_container( + pods: list[dict[str, Any]], container: str +) -> list[dict[str, Any]]: + """Return ready pods that contain *container*.""" + return [ + pod + for pod in pods + if _ready(pod) + and not pod.get("metadata", {}).get("deletionTimestamp") + and container + in {item["name"] for item in pod.get("spec", {}).get("containers", [])} + ] + + +def _load_state(config: Any) -> dict[str, Any]: + """Load and validate the successful-setup marker.""" + path = config.state_dir / "state.json" + if not path.is_file(): + raise IntegrationTestError( + f"setup state is absent at {path}; run integration-test.py setup first" + ) + state = json.loads(path.read_text(encoding="utf-8")) + expected = { + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + "test_user": config.test_user, + "test_uid": config.test_uid, + "test_gid": config.test_gid, + } + mismatches = [ + f"{name}={state.get(name)!r} (expected {value!r})" + for name, value in expected.items() + if state.get(name) != value + ] + if mismatches: + raise IntegrationTestError( + "setup state does not match the requested fixture: " + ", ".join(mismatches) + ) + mode = state.get("ssh_home_mode") + if mode not in {"separate", "shared"}: + raise IntegrationTestError(f"invalid ssh_home_mode in {path}: {mode!r}") + backend = state.get("storage_backend") + if backend not in {"nfs", "sbx-shared"}: + raise IntegrationTestError(f"invalid storage_backend in {path}: {backend!r}") + requested_backend = getattr(config, "storage_backend", "auto") + if requested_backend not in {"auto", backend}: + raise IntegrationTestError( + f"requested storage backend {requested_backend} does not match " + f"retained backend {backend}" + ) + if backend == "sbx-shared": + expected_root = str(config.sbx_shared_root) + if state.get("sbx_shared_root") != expected_root: + raise IntegrationTestError( + "retained SBX shared root does not match the requested fixture: " + f"{state.get('sbx_shared_root')!r} != {expected_root!r}" + ) + return state + + +def _require_nodes(runner: Any, config: Any) -> None: + """Require the exact ready three-node topology and labels.""" + clusters = runner.run(["kind", "get", "clusters"], timeout=30).stdout.split() + if config.cluster_name not in clusters: + raise IntegrationTestError( + f"kind cluster {config.cluster_name!r} is not running; run setup first" + ) + result = runner.run(_kubectl(config, "get", "nodes", "-o", "json"), timeout=30) + nodes = json.loads(result.stdout)["items"] + if len(nodes) != 3: + raise IntegrationTestError(f"expected 3 Kubernetes nodes; found {len(nodes)}") + if not all( + any( + condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in node.get("status", {}).get("conditions", []) + ) + for node in nodes + ): + raise IntegrationTestError("all three Kubernetes nodes must be Ready") + target = sum( + node["metadata"].get("labels", {}).get("storage-scale-test/target") == "true" + for node in nodes + ) + login = sum( + node["metadata"].get("labels", {}).get("storage-scale-test/login") == "true" + for node in nodes + ) + if (target, login) != (2, 1): + raise IntegrationTestError( + f"node label invariant failed: target={target}, login={login}" + ) + + +def _require_storage(runner: Any, config: Any) -> None: + """Require both integration PVCs to be bound.""" + result = runner.run( + _kubectl(config, "-n", config.namespace, "get", "pvc", "-o", "json"), + timeout=30, + ) + claims = { + item["metadata"]["name"]: item.get("status", {}).get("phase") + for item in json.loads(result.stdout)["items"] + } + expected = {"storage-test-rwx", "ssh-home-rwx"} + if any(claims.get(name) != "Bound" for name in expected): + raise IntegrationTestError(f"integration PVCs are not Bound: {claims}") + + +def _pod_inventory(runner: Any, config: Any) -> list[dict[str, Any]]: + """Return the namespace pod inventory.""" + result = runner.run( + _kubectl(config, "-n", config.namespace, "get", "pods", "-o", "json"), + timeout=30, + ) + return json.loads(result.stdout)["items"] + + +def _require_pods( + pods: list[dict[str, Any]], + substrates: set[str], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Require only the live workloads needed by selected substrates.""" + login = _pods_with_container(pods, "login") + ssh = sorted( + _pods_with_container(pods, "sshd"), key=lambda item: item["metadata"]["name"] + ) + slurmd = _pods_with_container(pods, "slurmd") + invalid = len(login) != 1 + invalid = invalid or ("ssh" in substrates and len(ssh) != 2) + invalid = invalid or ("slurm" in substrates and len(slurmd) != 2) + if invalid: + raise IntegrationTestError( + "fixture workloads are incomplete: " + f"required={sorted(substrates)}, login={len(login)}, ssh={len(ssh)}, " + f"slurmd={len(slurmd)}; run setup" + ) + return login[0], ssh + + +def _probe_pod( + runner: Any, + config: Any, + pod: str, + container: str, + command: str, + *, + as_user: str | None = None, +) -> str: + """Run a short fixture probe and return stripped stdout.""" + result = runner.run( + _pod_command( + config, + pod, + container, + command, + as_user=as_user, + timeout=30, + ), + timeout=45, + ) + return result.stdout.strip() + + +def _require_fixture( + runner: Any, + config: Any, + substrates: set[str], + ssh_home_mode: str = "separate", +) -> Fixture: + """Validate setup without reconciling or installing anything.""" + state = _load_state(config) + required_host_tools = { + "bash", + "file", + "find", + "git", + "tar", + "timeout", + } + if "ssh" in substrates: + required_host_tools.update(("ssh-add", "ssh-agent")) + missing_host_tools = [ + tool for tool in sorted(required_host_tools) if shutil.which(tool) is None + ] + if missing_host_tools: + raise IntegrationTestError( + "required host tools are absent: " + ", ".join(missing_host_tools) + ) + if not config.kubeconfig.is_file(): + raise IntegrationTestError( + f"private kubeconfig is absent at {config.kubeconfig}; run setup first" + ) + _require_nodes(runner, config) + _require_storage(runner, config) + login, ssh = _require_pods(_pod_inventory(runner, config), substrates) + login_name = str(login["metadata"]["name"]) + addresses: tuple[str, ...] = () + if "ssh" in substrates: + addresses = tuple(str(item["status"]["podIP"]) for item in ssh) + if len(set(addresses)) != 2: + raise IntegrationTestError( + f"SSH workers lack distinct addresses: {addresses}" + ) + tools = ( + "for tool in bash file find tar timeout; do " + 'command -v "$tool" >/dev/null || exit 1; done' + ) + _probe_pod( + runner, + config, + login_name, + "login", + tools, + as_user=WORKLOAD_USER, + ) + if "ssh" in substrates: + _probe_pod( + runner, + config, + str(ssh[0]["metadata"]["name"]), + "sshd", + tools, + as_user="tester", + ) + mount_probe = "test -w /mnt/storage-test" + if state["storage_backend"] == "nfs": + mount_probe += ( + " && case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac" + ) + _probe_pod( + runner, + config, + login_name, + "login", + mount_probe, + as_user=WORKLOAD_USER, + ) + if "ssh" in substrates: + _probe_pod( + runner, + config, + str(ssh[0]["metadata"]["name"]), + "sshd", + mount_probe, + as_user="tester", + ) + login_arch = _probe_pod( + runner, + config, + login_name, + "login", + "uname -m", + as_user=WORKLOAD_USER, + ) + host_arch = platform.machine() + fixture_architectures = {"host": host_arch, "login": login_arch} + if "ssh" in substrates: + fixture_architectures["ssh"] = _probe_pod( + runner, + config, + str(ssh[0]["metadata"]["name"]), + "sshd", + "uname -m", + as_user="tester", + ) + if len(set(fixture_architectures.values())) != 1: + raise IntegrationTestError( + "fixture architecture mismatch: " + + ", ".join( + f"{name}={architecture}" + for name, architecture in fixture_architectures.items() + ) + ) + if host_arch not in ELBENCHO_ARCHIVES: + raise IntegrationTestError(f"unsupported fixture architecture: {host_arch}") + slurm_nodes: tuple[str, ...] = () + slurm_addresses: tuple[str, ...] = () + if "slurm" in substrates: + slurm_output = _probe_pod( + runner, + config, + login_name, + "login", + "sinfo -N -h -o %N | sort -u", + as_user=WORKLOAD_USER, + ) + slurm_nodes = tuple(line for line in slurm_output.splitlines() if line) + if len(slurm_nodes) != 2: + raise IntegrationTestError( + f"expected two Slurm compute nodes; found {slurm_nodes}" + ) + slurm_address_output = _probe_pod( + runner, + config, + login_name, + "login", + "for node in " + + " ".join(shlex.quote(node) for node in slurm_nodes) + + "; do getent ahostsv4 \"$node\" | awk 'NR == 1 {print $1}'; done", + as_user=WORKLOAD_USER, + ) + slurm_addresses = tuple( + line for line in slurm_address_output.splitlines() if line + ) + if len(slurm_addresses) != 2 or len(set(slurm_addresses)) != 2: + raise IntegrationTestError( + f"Slurm workers lack distinct IPv4 addresses: {slurm_addresses}" + ) + return Fixture( + login_pod=login_name, + login_container="login", + ssh_addresses=addresses, + slurm_nodes=slurm_nodes, + slurm_addresses=slurm_addresses, + architecture=host_arch, + ssh_home_mode=ssh_home_mode, + storage_backend=str(state["storage_backend"]), + ) + + +def _sha256(path: Path) -> str: + """Return the SHA-256 digest of *path*.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _request(url: str, accept: str = "application/vnd.github+json") -> Any: + """Open a bounded upstream request with retries.""" + request = urllib.request.Request( + url, + headers={ + "Accept": accept, + "User-Agent": "storage-scale-test-integration", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + last_error: Exception | None = None + for attempt in range(1, 4): + try: + return urllib.request.urlopen(request, timeout=60) + except (OSError, urllib.error.URLError) as error: + last_error = error + if attempt < 3: + time.sleep(attempt * 2) + raise IntegrationTestError(f"download failed after 3 attempts: {url}: {last_error}") + + +def _asset_url(name: str) -> str: + """Resolve a pinned release asset through the public GitHub API.""" + with _request(ELBENCHO_RELEASE_API) as response: + release = json.load(response) + matches = [ + asset for asset in release.get("assets", []) if asset.get("name") == name + ] + if len(matches) != 1: + raise IntegrationTestError( + f"expected one {name!r} asset in {ELBENCHO_VERSION}; found {len(matches)}" + ) + return str(matches[0]["url"]) + + +def _download_archive(destination: Path, name: str, expected: str) -> None: + """Download and verify one pinned elbencho archive atomically.""" + destination.parent.mkdir(parents=True, exist_ok=True) + url = _asset_url(name) + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as handle: + temporary = Path(handle.name) + try: + with _request(url, "application/octet-stream") as response: + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > MAX_ARCHIVE_BYTES: + raise IntegrationTestError( + f"refusing oversized archive {name}: {content_length} bytes" + ) + copied = 0 + while chunk := response.read(1024 * 1024): + copied += len(chunk) + if copied > MAX_ARCHIVE_BYTES: + raise IntegrationTestError( + f"refusing archive {name} larger than " + f"{MAX_ARCHIVE_BYTES} bytes" + ) + handle.write(chunk) + except Exception: + temporary.unlink(missing_ok=True) + raise + actual = _sha256(temporary) + if actual != expected: + temporary.unlink(missing_ok=True) + raise IntegrationTestError( + f"checksum mismatch for {name}: expected {expected}, got {actual}" + ) + temporary.chmod(0o640) + temporary.replace(destination) + + +def _extract_container_elbencho( + runner: Any, cache: Path, binary: Path, runtime: Path, architecture: str +) -> None: + """Build a portable wrapper from the digest-pinned upstream image.""" + runner.run(["docker", "pull", ELBENCHO_CONTAINER], timeout=300) + container = runner.run( + ["docker", "create", "--entrypoint", "sleep", ELBENCHO_CONTAINER, "infinity"] + ).stdout.strip() + if not container: + raise IntegrationTestError("Docker did not return an elbencho container ID") + archive = cache / f".{binary.name}.runtime.tar" + try: + bundle = """ +set -eu +rm -rf /tmp/elbencho-runtime /tmp/elbencho-runtime.tar +mkdir -p /tmp/elbencho-runtime +libraries=$(ldd /usr/bin/elbencho | awk '$3 ~ /^\\// {print $3} $1 ~ /^\\// {print $1}') +loader=$(ldd /usr/bin/elbencho | awk '{for (i=1; i<=NF; i++) if ($i ~ /^\\/.*ld-linux.*\\.so/) {print $i; exit}}') +test -n "$loader" +cp -L --parents /usr/bin/elbencho $libraries /tmp/elbencho-runtime +printf '%s\n' "$loader" > /tmp/elbencho-runtime/.loader-path +cd /tmp/elbencho-runtime +tar -cf /tmp/elbencho-runtime.tar . +""".strip() + runner.run(["docker", "start", container]) + runner.run(["docker", "exec", container, "sh", "-ec", bundle]) + runner.run(["docker", "cp", f"{container}:/tmp/elbencho-runtime.tar", archive]) + temporary_runtime = cache / f".{runtime.name}.new" + shutil.rmtree(temporary_runtime, ignore_errors=True) + temporary_runtime.mkdir() + with tarfile.open(archive) as tar: + tar.extractall(temporary_runtime, filter="data") + loader_marker = temporary_runtime / ".loader-path" + loader_path = Path(loader_marker.read_text(encoding="utf-8").strip()) + if not loader_path.is_absolute() or ".." in loader_path.parts: + raise IntegrationTestError( + f"container reported an invalid dynamic loader path: {loader_path}" + ) + loader_relative = loader_path.relative_to("/") + if not (temporary_runtime / loader_relative).is_file(): + raise IntegrationTestError( + f"container runtime is missing its dynamic loader: {loader_path}" + ) + loader_marker.unlink() + shutil.rmtree(runtime, ignore_errors=True) + temporary_runtime.replace(runtime) + library_arch = ( + "aarch64-linux-gnu" if architecture == "aarch64" else "x86_64-linux-gnu" + ) + wrapper = "\n".join( + ( + "#!/usr/bin/env bash", + "set -euo pipefail", + 'runtime_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/' + 'elbencho-runtime" && pwd)', + f'exec "$runtime_dir/{loader_relative.as_posix()}" \\', + f' --library-path "$runtime_dir/usr/lib/{library_arch}:' + f'$runtime_dir/lib/{library_arch}" \\', + ' "$runtime_dir/usr/bin/elbencho" "$@"', + "", + ) + ) + binary.write_text(wrapper, encoding="utf-8") + binary.chmod(0o755) + finally: + archive.unlink(missing_ok=True) + runner.run(["docker", "rm", "--force", container], check=False) + + +def _sbx_bundle_document(architecture: str, binary_name: str) -> dict[str, object]: + """Return the exact recipe identity for a cached SBX Elbencho bundle.""" + return { + "schema": 1, + "recipe": SBX_ELBENCHO_BUNDLE_RECIPE, + "container": ELBENCHO_CONTAINER, + "architecture": architecture, + "binary_name": binary_name, + } + + +def _write_sbx_bundle_marker(path: Path, document: dict[str, object]) -> None: + """Atomically record a successfully built SBX Elbencho bundle.""" + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, delete=False + ) as handle: + json.dump(document, handle, sort_keys=True) + handle.write("\n") + temporary = Path(handle.name) + temporary.chmod(0o640) + temporary.replace(path) + + +def _sbx_bundle_is_current( + binary: Path, + runtime: Path, + marker: Path, + expected: dict[str, object], +) -> bool: + """Return whether all cached bundle artifacts match the current recipe.""" + if not binary.is_file() or not runtime.is_dir() or not marker.is_file(): + return False + try: + document = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return document == expected + + +def _ensure_elbencho( + runner: Any, config: Any, architecture: str, storage_backend: str +) -> tuple[Path, str, Path | None]: + """Return a verified, extracted pinned elbencho binary and staged name.""" + archive_name, expected, binary_name = ELBENCHO_ARCHIVES[architecture] + cache = config.state_dir / "test-cache" + cache.mkdir(parents=True, exist_ok=True) + binary = cache / f"{ELBENCHO_VERSION}-{binary_name}" + if storage_backend == "sbx-shared": + runtime = cache / f"{ELBENCHO_VERSION}-{binary_name}.runtime" + marker = cache / f"{ELBENCHO_VERSION}-{binary_name}.bundle.json" + bundle_document = _sbx_bundle_document(architecture, binary_name) + if not _sbx_bundle_is_current(binary, runtime, marker, bundle_document): + LOG.info("Extracting pinned elbencho %s container", ELBENCHO_VERSION) + _extract_container_elbencho(runner, cache, binary, runtime, architecture) + _write_sbx_bundle_marker(marker, bundle_document) + else: + LOG.info("Using cached elbencho from the pinned upstream container") + return binary, binary_name, runtime + archive = cache / f"{ELBENCHO_VERSION}-{archive_name}" + if not archive.is_file() or _sha256(archive) != expected: + LOG.info( + "Downloading pinned elbencho %s for %s", ELBENCHO_VERSION, architecture + ) + _download_archive(archive, archive_name, expected) + else: + LOG.info("Using cached pinned elbencho archive for %s", architecture) + with tarfile.open(archive, "r:gz") as tar: + members = [ + member + for member in tar.getmembers() + if member.isfile() and Path(member.name).name == "elbencho" + ] + if len(members) != 1: + raise IntegrationTestError( + f"expected one elbencho binary in {archive}; found {len(members)}" + ) + source = tar.extractfile(members[0]) + if source is None: + raise IntegrationTestError(f"cannot extract elbencho from {archive}") + with tempfile.NamedTemporaryFile(dir=cache, delete=False) as handle: + temporary = Path(handle.name) + shutil.copyfileobj(source, handle) + temporary.chmod(0o755) + temporary.replace(binary) + return binary, binary_name, None + + +def _shell(value: str | Path) -> str: + """Quote one value for the generated Bash environment.""" + return shlex.quote(str(value)) + + +def _slurm_run_time(timeout_seconds: int) -> str: + """Return an allocation limit that outlives the harness deadline.""" + total = timeout_seconds + SLURM_CLEANUP_MARGIN_SECONDS + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + + +def _override_block( + selector: str, + remote_root: str, + fixture: Fixture, + *, + results_dir: str | None = None, + logs_dir: str | None = None, + extra_env: str = "", + timeout_seconds: int = 300, +) -> tuple[str, dict[str, str]]: + """Return template overrides and small support-file contents.""" + data = "/mnt/storage-test" + lines = [ + "# Bounded integration regression overrides.", + f"export RESULTS_DIR={_shell(results_dir or remote_root + '/results')}", + f"export LOGS_DIR={_shell(logs_dir or remote_root + '/logs')}", + "ORDER_NODES=1", + 'client_type="cpu"', + f"client_arch={_shell(fixture.architecture)}", + "unset TEST_DIRS", + f"declare -A TEST_DIRS=([{_shell(data)}]=1)", + "export FS_MAX_AGG_THROUGHPUT=1", + "export FS_MAX_NODE_THROUGHPUT_GBPS=1", + "export FS_MAX_NODE_IOPS=100", + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + "export ELBENCHO_FILE_SIZE_MULTIPLIER=1", + 'export ELBENCHO_FILE_LAYOUT="shared-directory"', + "export ELBENCHO_FILES_PER_NODE=1", + 'export ELBENCHO_FILE_SIZE="16M"', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_READ_AFTER_WRITE_PAUSE=0", + "export ELBENCHO_LIVE_CSV_EXTENDED=0", + "export ELBENCHO_SINGLE_BIG_FILE=0", + 'export OBJ_BUCKET=""', + ] + support: dict[str, str] = {} + if selector == "ssh": + host_file = f"{remote_root}/ssh-hosts" + lines.extend( + ( + f"export SSH_HOST_LIST={_shell(host_file)}", + 'export SSH_USER="tester"', + "unset SLURM_NODE_INCLUDES SLURM_NODE_IGNORES", + ) + ) + if fixture.ssh_home_mode == "shared": + lines.append("export SSH_HOMEDIR_SHARED=1") + else: + lines.append("unset SSH_HOMEDIR_SHARED") + support["ssh-hosts"] = "\n".join(fixture.ssh_addresses) + "\n" + else: + include_file = f"{remote_root}/slurm-nodes" + ignore_file = f"{remote_root}/slurm-ignore" + lines.extend( + ( + "unset SSH_HOST_LIST SSH_USER SSH_HOMEDIR_SHARED", + 'account="storage-test"', + 'reservation=""', + 'partition="all"', + f"run_time={_shell(_slurm_run_time(timeout_seconds))}", + "SLURM_EXCLUSIVE_USER=0", + f"export SLURM_NODE_INCLUDES={_shell(include_file)}", + f"export SLURM_NODE_IGNORES={_shell(ignore_file)}", + ) + ) + support["slurm-nodes"] = "\n".join(fixture.slurm_nodes) + "\n" + support["slurm-ignore"] = "" + if extra_env: + lines.extend(("# Scenario-specific integration overrides.", extra_env.rstrip())) + return "\n".join(lines) + "\n", support + + +def _render_env( + template: Path, + selector: str, + remote_root: str, + fixture: Fixture, + *, + results_dir: str | None = None, + logs_dir: str | None = None, + extra_env: str = "", + timeout_seconds: int = 300, +) -> tuple[str, dict[str, str]]: + """Render one runtime env from the repository's real user template.""" + text = template.read_text(encoding="utf-8") + anchor = "# STORAGE_SCALE_TEST_INTEGRATION_OVERRIDES" + if text.count(anchor) != 1: + raise IntegrationTestError( + f"expected exactly one integration override marker in {template}" + ) + overrides, support = _override_block( + selector, + remote_root, + fixture, + results_dir=results_dir, + logs_dir=logs_dir, + extra_env=extra_env, + timeout_seconds=timeout_seconds, + ) + rendered = text.replace(anchor, overrides + "\n" + anchor) + return rendered, support + + +def _validate_deployment_archive( + archive: Path, + destination: Path, + binary_name: str, + binary_digest: str, + architecture: str, + runner: Any, + bundled_runtime: bool, +) -> Path: + """Validate and safely extract the deployment tarball.""" + if not archive.is_file() or archive.stat().st_size > MAX_DEPLOYMENT_ARCHIVE_BYTES: + raise IntegrationTestError( + f"deployment archive is absent or oversized: {archive}" + ) + required = { + "storage-scale-test/NOTICE", + "storage-scale-test/env.sh.template", + "storage-scale-test/validate_env.sh", + "storage-scale-test/lib/env_base.sh", + "storage-scale-test/storage-tests/fs/nv-elbencho-sweep.sh", + f"storage-scale-test/utils/{binary_name}", + } + if bundled_runtime: + required.add("storage-scale-test/utils/elbencho-runtime/usr/bin/elbencho") + names: set[str] = set() + content_bytes = 0 + with tarfile.open(archive, "r:gz") as tar: + members = tar.getmembers() + if len(members) > MAX_DEPLOYMENT_FILES: + raise IntegrationTestError( + f"deployment archive has too many members: {len(members)}" + ) + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or not path.parts + or path.parts[0] != "storage-scale-test" + or ".." in path.parts + or member.name in names + or member.issym() + or member.islnk() + or member.isdev() + or member.isfifo() + or not (member.isfile() or member.isdir()) + ): + raise IntegrationTestError( + f"unsafe deployment archive member: {member.name!r}" + ) + if "env.sh" == path.name or ".obj_auth" in path.parts: + raise IntegrationTestError( + f"unexpected private deployment member: {member.name!r}" + ) + names.add(member.name) + content_bytes += member.size + if content_bytes > MAX_DEPLOYMENT_CONTENT_BYTES: + raise IntegrationTestError("deployment archive content is oversized") + absent = sorted(required - names) + if absent: + raise IntegrationTestError( + "deployment archive lacks required files: " + ", ".join(absent) + ) + destination.mkdir(parents=True) + tar.extractall(destination, filter="data") + root = destination / "storage-scale-test" + packaged_binary = root / "utils" / binary_name + if ( + not packaged_binary.is_file() + or packaged_binary.is_symlink() + or not os.access(packaged_binary, os.X_OK) + or _sha256(packaged_binary) != binary_digest + ): + raise IntegrationTestError( + f"packaged elbencho failed identity checks: {packaged_binary}" + ) + inspected_binary = packaged_binary + if bundled_runtime: + inspected_binary = root / "utils" / "elbencho-runtime" / "usr/bin/elbencho" + description = runner.run(["file", inspected_binary], timeout=30).stdout + expected_arch = "x86-64" if architecture == "x86_64" else "aarch64" + if expected_arch not in description: + raise IntegrationTestError( + f"packaged elbencho architecture mismatch: {description.strip()}" + ) + runner.run([packaged_binary, "--help"], timeout=30) + return root + + +def _build_deployment_archive( + runner: Any, + config: Any, + repo_root: Path, + build_root: Path, + binary: Path, + binary_name: str, + architecture: str, + runtime: Path | None, +) -> tuple[Path, Path]: + """Reuse or build and inspect one filesystem-only deployment tarball.""" + request = DeploymentCacheRequest( + repo_root=repo_root, + cache_root=config.state_dir / "test-cache" / "deployments", + architecture=architecture, + binary=binary, + binary_name=binary_name, + runtime=runtime, + ) + try: + cached = get_or_build_deployment(runner, request) + except DeploymentCacheError as error: + raise IntegrationTestError(str(error)) from error + LOG.info( + "%s deployment cache entry %s", + "Reusing" if cached.cache_hit else "Built", + cached.key, + ) + shutil.copy2(cached.build_log, build_root / "build-tarball.log") + archive = cached.archive + extracted = _validate_deployment_archive( + archive, + build_root / "extracted", + binary_name, + _sha256(binary), + architecture, + runner, + runtime is not None, + ) + return archive, extracted + + +def _write_runtime_files( + workspace: Path, + selector: str, + runtime_root: str, + fixture: Fixture, + *, + template: Path | None = None, + results_dir: str | None = None, + logs_dir: str | None = None, + extra_env: str = "", + extra_support: dict[str, str] | None = None, + timeout_seconds: int = 300, +) -> None: + """Add the generated environment and support files to a deployment.""" + rendered, support = _render_env( + template or workspace / "env.sh.template", + selector, + runtime_root, + fixture, + results_dir=results_dir, + logs_dir=logs_dir, + extra_env=extra_env, + timeout_seconds=timeout_seconds, + ) + (workspace / "env.sh").write_text(rendered, encoding="utf-8") + (workspace / "env.sh").chmod(0o640) + for name, content in support.items(): + (workspace / name).write_text(content, encoding="utf-8") + (workspace / name).chmod(0o640) + for name, content in (extra_support or {}).items(): + path = workspace / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + path.chmod(0o640) + + +def _stream_to_login( + runner: Any, + config: Any, + fixture: Fixture, + source: Path, + command: list[str | Path], + *, + as_user: str | None = None, + timeout: int = 180, +) -> None: + """Stream one local archive to a command in the Slinky LoginSet.""" + with source.open("rb") as stream: + pod_command: list[str | Path] = _kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + fixture.login_pod, + "-c", + fixture.login_container, + "--", + ) + if as_user: + pod_command.extend(("runuser", "-u", as_user, "--")) + pod_command.extend(command) + runner.run(pod_command, stdin=stream, timeout=timeout) + + +def _stage_ssh_runtime( + runner: Any, config: Any, runtime: Path, pods: list[dict[str, Any]] +) -> None: + """Install the container-derived runtime beside each SSH wrapper target.""" + ssh_pods = sorted( + _pods_with_container(pods, "sshd"), + key=lambda item: item["metadata"]["name"], + ) + with tempfile.NamedTemporaryFile(suffix=".tar") as stream: + with tarfile.open(fileobj=stream, mode="w") as archive: + archive.add(runtime, arcname="elbencho-runtime") + stream.flush() + for pod in ssh_pods: + stream.seek(0) + command = [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + pod["metadata"]["name"], + "-c", + "sshd", + "--", + ), + "runuser", + "-u", + WORKLOAD_USER, + "--", + "bash", + "-ec", + "umask 0007; rm -rf -- /home/tester/elbencho-runtime && " + "tar --no-same-owner --no-same-permissions -xf - -C /home/tester", + ] + runner.run(command, stdin=stream, timeout=180) + + +def _stage_workspace( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + archive: Path, + extracted: Path, + build_root: Path, + workspace_id: str, +) -> str: + """Stage a packaged deployment for host SSH or LoginSet Slurm execution.""" + if selector == "ssh": + workspace = build_root / "ssh" / "storage-scale-test" + shutil.copytree(extracted, workspace) + _write_runtime_files(workspace, selector, str(workspace), fixture) + return str(workspace) + + remote_base = f"{REMOTE_BASE}/workspaces/{workspace_id}" + remote_root = f"{remote_base}/storage-scale-test" + reset = ( + f"umask 0007; rm -rf -- {_shell(remote_base)} && " + f"mkdir -p -- {_shell(remote_base)}" + ) + runner.run( + _pod_command( + config, + fixture.login_pod, + fixture.login_container, + reset, + as_user=WORKLOAD_USER, + timeout=60, + ), + timeout=75, + ) + extract_command = [ + "bash", + "-ec", + "umask 0007; tar --no-same-owner --no-same-permissions " + f"-xzf - -C {_shell(remote_base)}", + ] + _stream_to_login( + runner, + config, + fixture, + archive, + extract_command, + as_user=WORKLOAD_USER, + ) + support_stage = build_root / "slurm-runtime" + support_stage.mkdir() + _write_runtime_files( + support_stage, + selector, + remote_root, + fixture, + template=extracted / "env.sh.template", + ) + support_archive = build_root / "slurm-runtime.tar" + with tarfile.open(support_archive, "w") as tar: + for child in sorted(support_stage.iterdir()): + tar.add(child, arcname=child.name) + support_command = [ + "bash", + "-ec", + "umask 0007; tar --no-same-owner --no-same-permissions " + f"-xf - -C {_shell(remote_root)}", + ] + _stream_to_login( + runner, + config, + fixture, + support_archive, + support_command, + as_user=WORKLOAD_USER, + ) + return remote_root + + +def _test_command( + config: Any, fixture: Fixture, selector: str, command: str, timeout: int +) -> list[str | Path]: + """Build one substrate test command.""" + if selector == "ssh": + private_key = config.keys_dir / "id_ed25519" + host_command = "\n".join( + ( + "set -euo pipefail", + 'eval "$(ssh-agent -s)" >/dev/null', + "trap 'ssh-agent -k >/dev/null 2>&1 || true' EXIT", + f"ssh-add -- {_shell(private_key)} >/dev/null 2>&1", + command, + ) + ) + return [ + "timeout", + "--kill-after=10s", + f"{timeout}s", + "bash", + "-c", + host_command, + ] + return _pod_command( + config, + fixture.login_pod, + fixture.login_container, + command, + as_user=WORKLOAD_USER, + timeout=timeout, + kill_after=105, + ) + + +def _run_step( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + name: str, + command: str, + log_dir: Path, + timeout: int, + *, + expected_failure: bool = False, +) -> str: + """Run one bounded substrate step and preserve diagnostic output.""" + LOG.info("Running %s filesystem step: %s", selector, name) + outer_grace = 120 if selector == "slurm" else 30 + result = runner.run( + _test_command(config, fixture, selector, command, timeout), + check=False, + timeout=timeout + outer_grace, + ) + output = result.stdout + result.stderr + log_path = log_dir / f"{selector}-{name}.log" + log_path.write_text(output, encoding="utf-8") + log_path.chmod(0o640) + if expected_failure and result.returncode == 0: + raise IntegrationTestError( + f"{selector} {name} unexpectedly succeeded; full output: {log_path}" + ) + if result.returncode and not expected_failure: + detail = output.strip()[-8000:] + raise IntegrationTestError( + f"{selector} {name} failed with exit code {result.returncode}; " + f"full output: {log_path}\n{detail}" + ) + LOG.info( + "%s %s filesystem step: %s", + "Observed expected failure for" if expected_failure else "Passed", + selector, + name, + ) + return output + + +def _ordered_worker_evidence(selector: str, sweep_output: str, result: Path) -> str: + """Return substrate-appropriate durable worker-selection evidence.""" + if selector == "ssh": + return sweep_output + logs = sorted((result / "executions").glob("*.log")) + logs.extend(sorted(result.glob("coordinator-*.log"))) + return "\n".join( + path.read_text(encoding="utf-8", errors="replace") for path in logs + ) + + +def _assert_ordered_workers( + fixture: Fixture, selector: str, sweep_output: str, result: Path +) -> None: + """Prove increasing cells use the configured workers in prefix order.""" + workers = fixture.ssh_addresses if selector == "ssh" else fixture.slurm_addresses + expected = { + "1": workers[0], + "2": ",".join(workers), + } + selected: dict[str, str] = {} + evidence = _ordered_worker_evidence(selector, sweep_output, result) + for line in evidence.splitlines(): + match = re.search(r"starting execution \d+:.*nodes=(\d+).*hosts=([^ ]+)", line) + if match: + selected[match.group(1)] = match.group(2) + if selected != expected: + raise IntegrationTestError( + f"{selector} ordered worker selection was {selected!r}; " + f"expected {expected!r}" + ) + + +def _copy_result_for_reporting( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + result_dir: str, + destination: Path, +) -> Path: + """Bring one completed result tree to the host for report validation.""" + destination.mkdir(parents=True, exist_ok=True) + if selector == "ssh": + shutil.copytree(result_dir, destination / Path(result_dir).name) + else: + runner.run( + [ + *_kubectl(config, "-n", config.namespace, "cp"), + "-c", + fixture.login_container, + f"{fixture.login_pod}:{result_dir}", + destination / Path(result_dir).name, + ], + timeout=120, + ) + return destination / Path(result_dir).name + + +def _login_shell( + runner: Any, + config: Any, + fixture: Fixture, + command: str, + *, + timeout: int = 60, +) -> str: + """Run a storage-inspection command in the PVC-mounted login pod.""" + return runner.run( + _pod_command( + config, + fixture.login_pod, + fixture.login_container, + command, + as_user=WORKLOAD_USER, + timeout=timeout, + ), + timeout=timeout + 15, + ).stdout + + +def _prepare_scenario_data( + runner: Any, + config: Any, + fixture: Fixture, + data_root: str, +) -> None: + """Reset one marker-owned scenario subtree on the shared test storage.""" + command = f""" +set -euo pipefail +umask 0007 +rm -rf -- {_shell(data_root)} +mkdir -p -- {_shell(data_root + '/primary')} {_shell(data_root + '/secondary')} +printf 'scenario-owned\n' > {_shell(data_root + '/primary/.integration-sentinel')} +printf 'scenario-owned\n' > {_shell(data_root + '/secondary/.integration-sentinel')} +""".strip() + _login_shell(runner, config, fixture, command) + + +def _sync_step_runtime( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + template: Path, + result_base: str, +) -> None: + """Render and install one step's env and support files.""" + extra_env = step.render_env(runtime.values) + extra_support = { + item.relative_path: item.content + for item in step.render_support_files(runtime.values) + } + if runtime.selector == "ssh": + _write_runtime_files( + Path(runtime.workspace), + runtime.selector, + runtime.workspace, + fixture, + template=template, + results_dir=result_base, + logs_dir=f"{runtime.workspace}/logs/{step.name}", + extra_env=extra_env, + extra_support=extra_support, + timeout_seconds=step.timeout_seconds, + ) + return + stage = runtime.local_workspace / f"runtime-{step.name}" + if stage.exists(): + shutil.rmtree(stage) + stage.mkdir() + _write_runtime_files( + stage, + runtime.selector, + runtime.workspace, + fixture, + template=template, + results_dir=result_base, + logs_dir=f"{runtime.workspace}/logs/{step.name}", + extra_env=extra_env, + extra_support=extra_support, + timeout_seconds=step.timeout_seconds, + ) + archive_path = runtime.local_workspace / f"runtime-{step.name}.tar" + with tarfile.open(archive_path, "w") as archive: + for child in sorted(stage.rglob("*")): + archive.add(child, arcname=child.relative_to(stage)) + _stream_to_login( + runner, + config, + fixture, + archive_path, + [ + "bash", + "-ec", + "umask 0007; tar --no-same-owner --no-same-permissions " + f"-xf - -C {_shell(runtime.workspace)}", + ], + as_user=WORKLOAD_USER, + ) + + +def _create_generated_inputs( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, +) -> None: + """Create bounded scenario inputs through the shared PVC mount.""" + for generated in step.generated_inputs: + path = f"{runtime.values['test_root']}/{generated.relative_path}" + command = f""" +set -euo pipefail +umask 0007 +mkdir -p -- {_shell(str(PurePosixPath(path).parent))} +truncate -s {generated.size_bytes} -- {_shell(path)} +test "$(stat -c %s -- {_shell(path)})" -eq {generated.size_bytes} +""".strip() + _login_shell(runner, config, fixture, command) + + +def _discover_result( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + result_base: str, + log_dir: Path, +) -> str: + """Find the single normal sweep result below an invocation-owned base.""" + output = _run_step( + runner, + config, + fixture, + runtime.selector, + f"{step.name}-discover", + f"find {_shell(result_base)} -mindepth 1 -maxdepth 1 -type d " + "-name 'elbencho-*' -printf '%p\\n'", + log_dir, + 30, + ) + directories = [line for line in output.splitlines() if line.strip()] + if len(directories) != 1: + raise IntegrationTestError( + f"{runtime.selector} {step.name} produced {len(directories)} " + f"normal result directories under {result_base}: {directories}" + ) + return directories[0] + + +def _copy_scenario_result( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + remote_result: str, +) -> Path: + """Copy one immutable result snapshot into the retained test-run log.""" + destination = runtime.artifact_root / "copied-results" / step.name + destination.mkdir(parents=True, exist_ok=True) + if runtime.selector == "ssh": + target = destination / Path(remote_result).name + if target.exists(): + shutil.rmtree(target) + shutil.copytree(remote_result, target) + else: + target = _copy_result_for_reporting( + runner, + config, + fixture, + runtime.selector, + remote_result, + destination, + ) + runtime.copied_results.append(target) + return target + + +def _shell_assignments(path: Path) -> dict[str, str]: + """Read simple exported scalar assignments from a reified execution.""" + assignments: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"export ([A-Za-z_][A-Za-z0-9_]*)=(.*)", line) + if not match: + continue + values = shlex.split(match.group(2), posix=True) + assignments[match.group(1)] = values[0] if values else "" + return assignments + + +def _coordinate_from_execution(path: Path) -> tuple[int, str, int, int]: + """Return a reified execution's stable sweep coordinate.""" + values = _shell_assignments(path) + try: + return ( + int(values["nodes"]), + values["io_size"], + int(values["thread_count"]), + int(values["io_depth"]), + ) + except (KeyError, ValueError) as error: + raise IntegrationTestError(f"invalid execution metadata in {path}") from error + + +def _workload_values(path: Path) -> dict[str, str]: + """Load a workload TSV while rejecting duplicate or malformed keys.""" + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + fields = line.split("\t") + if len(fields) != 2 or fields[0] in values: + raise IntegrationTestError(f"malformed workload metadata: {path}") + values[fields[0]] = fields[1] + return values + + +def _expected_dataset_totals( + scenario: str, step: ScenarioStep, nodes: int +) -> tuple[int, int] | None: + """Return exact bounded totals for workloads with a metadata contract.""" + if scenario in {"baseline", "ssh-shared-home", "failure-resume"}: + return nodes, nodes * 16 * 1024 * 1024 + if scenario == "live-capture": + return nodes * 2, nodes * (MAX_LIVE_CAPTURE_DATASET_BYTES // 2) + if scenario == "slurm-cartesian": + return nodes * 2, nodes * 2 * 1024 * 1024 + if scenario == "retained-lifecycle" and step.kind is not CommandKind.DELETE: + return 1, 16 * 1024 * 1024 + return None + + +def _assert_phase_artifacts( + execution_root: Path, + execution_id: str, + step: ScenarioStep, + status: ExecutionStatus, +) -> None: + """Require phase evidence without freezing complete native arguments.""" + phase_files = { + WorkloadPhase.WRITE: "write.json", + WorkloadPhase.READ: "read.json", + WorkloadPhase.REMOVE_FILES: "delete.json", + } + for phase, suffix in phase_files.items(): + if phase not in step.required_phases: + continue + if status is ExecutionStatus.FAILED and phase is WorkloadPhase.REMOVE_FILES: + continue + path = execution_root / f"{execution_id}.{suffix}" + if not path.is_file() or path.stat().st_size == 0: + raise IntegrationTestError(f"missing {phase.value} evidence: {path}") + + +def _assert_execution_contract( + scenario: FilesystemScenarioSpec, + step: ScenarioStep, + result: Path, +) -> None: + """Validate exact coordinates, states, totals, and required phases.""" + execution_root = result / "executions" + scripts = sorted(execution_root.glob("[0-9][0-9][0-9][0-9].sh")) + if len(scripts) != len(step.executions): + raise IntegrationTestError( + f"{scenario.name}/{step.name}: expected {len(step.executions)} " + f"executions, found {len(scripts)}" + ) + actual_coordinates = [_coordinate_from_execution(path) for path in scripts] + expected_coordinates = [ + ( + item.coordinate.nodes, + item.coordinate.io_size, + item.coordinate.threads, + item.coordinate.io_depth, + ) + for item in step.executions + ] + if actual_coordinates != expected_coordinates: + raise IntegrationTestError( + f"{scenario.name}/{step.name}: coordinates {actual_coordinates!r} " + f"do not match {expected_coordinates!r}" + ) + for index, expected in enumerate(step.executions, start=1): + execution_id = f"{index:04d}" + status_path = execution_root / f"{execution_id}.status" + status = status_path.read_text(encoding="utf-8").strip() + if status != expected.status.value: + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: expected " + f"{expected.status.value}, found {status}" + ) + if expected.status is ExecutionStatus.PENDING: + continue + exitcode = ( + (execution_root / f"{execution_id}.exitcode") + .read_text(encoding="utf-8") + .strip() + ) + expected_exit = "97" if expected.status is ExecutionStatus.FAILED else "0" + if exitcode != expected_exit: + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: expected exit " + f"{expected_exit}, found {exitcode}" + ) + if scenario.name not in { + "default-dio", + "ssh-single-big-file", + "ssh-weighted-roots", + } and not ( + scenario.name == "retained-lifecycle" + and step.name.startswith("read-cache-") + ): + _assert_phase_artifacts(execution_root, execution_id, step, expected.status) + workload_path = execution_root / f"{execution_id}.workload.tsv" + totals = _expected_dataset_totals( + scenario.name, step, expected.coordinate.nodes + ) + if totals is not None: + if not workload_path.is_file(): + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: missing " + f"required workload metadata {workload_path}" + ) + workload = _workload_values(workload_path) + if workload.get("dataset_files_total") != str(totals[0]) or workload.get( + "dataset_bytes_total" + ) != str(totals[1]): + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: invalid " + f"dataset totals in {workload_path}" + ) + valid_completion = {"completed"} + if scenario.name == "retained-lifecycle" and step.name.startswith( + "read-cache-" + ): + valid_completion.add("not_applicable_time_based") + if ( + expected.status is ExecutionStatus.SUCCESS + and workload.get("completion_state") not in valid_completion + ): + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: workload did " + "not complete" + ) + + +def _execution_targets(result: Path) -> tuple[str, ...]: + """Return every exact generated target recorded by a result tree.""" + targets: list[str] = [] + for script in sorted((result / "executions").glob("[0-9][0-9][0-9][0-9].sh")): + value = _shell_assignments(script).get( + "ELBENCHO_RUN_GENERATED_TEST_DIRS_CSV", "" + ) + targets.extend(item for item in value.split(",") if item) + return tuple(targets) + + +def _assert_dataset_state( + runner: Any, + config: Any, + fixture: Fixture, + step: ScenarioStep, + result: Path | None, + retained_path: str | None, +) -> None: + """Validate cleanup or retention only within scenario-owned paths.""" + if step.dataset is DatasetExpectation.PRESERVED: + if retained_path: + _login_shell(runner, config, fixture, f"test -e {_shell(retained_path)}") + return + paths = list(_execution_targets(result)) if result is not None else [] + if step.dataset is DatasetExpectation.REMOVED and retained_path: + paths.append(retained_path) + if not paths: + return + probes = "\n".join(f"test ! -e {_shell(path)}" for path in paths) + _login_shell(runner, config, fixture, f"set -euo pipefail\n{probes}") + + +def _assert_semantic_flags(scenario: str, step: ScenarioStep, result: Path) -> None: + """Check required command semantics while allowing future optional flags.""" + command_lines: list[str] = [] + for path in result.glob("executions/*.log"): + command_lines.extend( + line + for line in path.read_text(encoding="utf-8", errors="replace").splitlines() + if line.startswith("# elbencho ") + ) + evidence = "\n".join(command_lines) + if not evidence: + raise IntegrationTestError( + f"{scenario}/{step.name}: execution logs omitted native commands" + ) + required: tuple[str, ...] = () + forbidden: tuple[str, ...] = () + if scenario == "baseline" or scenario == "ssh-shared-home": + required = ("--norandalign",) + elif scenario == "default-dio": + required, forbidden = ("--direct",), ("--norandalign",) + elif scenario == "live-capture": + required = ("--livecsv", "--livecsvex", "--liveint=10") + elif scenario == "ssh-single-big-file" and step.name == "inferred-extent-read": + required, forbidden = ("--nosvcshare", "--read"), ( + "--size", + "--treescan", + "--treefile", + ) + elif scenario == "ssh-weighted-roots": + required = ("--files=1",) + missing = [flag for flag in required if flag not in evidence] + present = [ + flag + for flag in forbidden + if re.search(rf"(? None: + """Exercise reporting and require semantic rows and plot families.""" + report_dir = log_dir / f"report-{step.name}" + report_dir.mkdir() + command: list[str | Path] = [ + report_workspace / "utils" / "extract-elbencho.sh", + "--markdown", + ] + if runtime.scenario.name == "live-capture": + command.extend( + ( + "--per-client-plots", + "--client-min-underperform-segments", + "2", + "--client-max-timeseries-lines", + "2", + "--client-max-heatmap-rows", + "2", + ) + ) + command.append(result) + completed = runner.run( + command, cwd=report_dir, timeout=step.timeout_seconds, check=False + ) + output = completed.stdout + completed.stderr + log_path = report_dir / "extract-elbencho.log" + log_path.write_text(output, encoding="utf-8") + if completed.returncode: + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: reporting " + f"failed; full output: {log_path}\n{output[-8000:]}" + ) + node_counts = sorted({item.coordinate.nodes for item in step.executions}) + missing_rows = [ + nodes + for nodes in node_counts + if not re.search(rf"^\|\s*{nodes}\s*\|", output, re.MULTILINE) + ] + required_operations = [] + if WorkloadPhase.WRITE in step.required_phases: + required_operations.append("WRITE Operation") + if WorkloadPhase.READ in step.required_phases: + required_operations.append("READ Operation") + missing_operations = [item for item in required_operations if item not in output] + if missing_rows or missing_operations: + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: report " + f"missing node rows {missing_rows} or operations {missing_operations}" + ) + if not any(path.stat().st_size for path in result.glob("*.png")): + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: no " + "nonempty report plot was generated" + ) + + +def _reset_result_base( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + result_base: str, +) -> None: + """Create an empty invocation-owned result base.""" + if runtime.selector == "ssh": + path = Path(result_base) + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True) + return + _login_shell( + runner, + config, + fixture, + f"umask 0007; rm -rf -- {_shell(result_base)} && " + f"mkdir -p -- {_shell(result_base)}", + ) + + +def _validate_step_environment( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + log_dir: Path, +) -> None: + """Run the repository's validator for a rendered scenario environment.""" + output = _run_step( + runner, + config, + fixture, + runtime.selector, + f"{step.name}-validate-env", + f"cd -- {_shell(runtime.workspace)} && ./validate_env.sh", + log_dir, + 180, + ) + if VALIDATION_SUCCESS not in output: + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: " + "validate_env.sh omitted its success marker" + ) + + +def _retained_path(result: Path) -> str: + """Return the sole generated path from a one-execution retained result.""" + targets = _execution_targets(result) + if len(targets) != 1: + raise IntegrationTestError( + f"expected one retained generated target, found {targets!r}" + ) + return targets[0] + + +def _assert_read_cache(step: ScenarioStep, result: Path) -> None: + """Validate the cache miss/hit contract for retained read invocations.""" + if step.name not in {"read-cache-miss", "read-cache-hit"}: + return + workload = _workload_values(result / "executions" / "0001.workload.tsv") + expected = ( + ("cache_miss_scan", "created") + if step.name == "read-cache-miss" + else ("cache_hit", "reused") + ) + actual = ( + workload.get("treefile_source"), + workload.get("treefile_cache_publish_outcome"), + ) + if actual != expected: + raise IntegrationTestError( + f"{step.name}: expected treefile cache {expected!r}, found {actual!r}" + ) + + +def _assert_live_capture(result: Path) -> None: + """Require a stable live counter series without comparing performance.""" + files = [path for path in result.rglob("*.live.csv") if path.stat().st_size] + if not files: + raise IntegrationTestError(f"live-capture result has no live CSV: {result}") + stable_series = False + for path in files: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + if len(lines) < 3: + continue + header = lines[0] + if "DoneBytes" not in header: + raise IntegrationTestError(f"live CSV lacks DoneBytes column: {path}") + stable_series = True + if not stable_series: + raise IntegrationTestError( + "live capture has no service/phase series with at least two samples" + ) + + +def _assert_slurm_scheduling( + runner: Any, + config: Any, + fixture: Fixture, + result: Path, +) -> None: + """Verify the real allocation's name, node constraint, CPUs, and state.""" + job_id = (result / "executions" / "0001.jobid").read_text(encoding="utf-8").strip() + command = ( + f"sacct -X -j {_shell(job_id)} -n -P " + "--format=JobName,NodeList,AllocCPUS,State" + ) + output = _login_shell(runner, config, fixture, command) + records = [line.split("|") for line in output.splitlines() if line.strip()] + matches = [record for record in records if len(record) >= 4 and record[0]] + if not matches: + raise IntegrationTestError(f"Slurm accounting omitted job {job_id}") + name, nodes, cpus, state = matches[0][:4] + if ( + not name.startswith("itest-elbencho-") + or fixture.slurm_nodes[0] not in nodes + or fixture.slurm_nodes[1] in nodes + or not cpus.isdigit() + or int(cpus) < 1 + or not state.startswith("COMPLETED") + ): + raise IntegrationTestError( + f"unexpected Slurm scheduling evidence for {job_id}: {matches[0]!r}" + ) + + +def _assert_step_specials( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + result: Path, +) -> None: + """Apply focused assertions unique to individual real scenarios.""" + _assert_read_cache(step, result) + if runtime.scenario.name == "live-capture": + _assert_live_capture(result) + if runtime.scenario.name == "slurm-scheduling": + _assert_slurm_scheduling(runner, config, fixture, result) + if runtime.scenario.name == "ssh-single-big-file" and step.name == ( + "inferred-extent-read" + ): + path = f"{runtime.values['test_root']}/staged-input/integration-bigfile" + output = _login_shell( + runner, + config, + fixture, + f"stat -c %s -- {_shell(path)} && sha256sum -- {_shell(path)}", + ) + if not output.startswith(f"{16 * 1024 * 1024}\n"): + raise IntegrationTestError("staged single-file input changed size") + + +def _run_regular_step( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + template: Path, + report_workspace: Path, + log_dir: Path, +) -> StepOutcome: + """Render, execute, collect, and validate one ordinary scenario step.""" + if step.kind is CommandKind.RESUME: + raise IntegrationTestError("resume steps require the failure scenario runner") + result_base = f"{runtime.workspace}/results/{step.name}" + _reset_result_base(runner, config, fixture, runtime, result_base) + _sync_step_runtime( + runner, + config, + fixture, + runtime, + step, + template, + result_base, + ) + _create_generated_inputs(runner, config, fixture, runtime, step) + _validate_step_environment(runner, config, fixture, runtime, step, log_dir) + arguments = shlex.join(step.render_arguments(runtime.values)) + command = ( + f"cd -- {_shell(runtime.workspace)} && " + f"./storage-tests/fs/nv-elbencho-sweep.sh {arguments}" + ) + output = _run_step( + runner, + config, + fixture, + runtime.selector, + step.name, + command, + log_dir, + step.timeout_seconds, + ) + if step.kind is CommandKind.DELETE: + _assert_dataset_state( + runner, + config, + fixture, + step, + None, + runtime.values.get("retained_data_dir") + or f"{runtime.values['test_root']}/staged-input", + ) + return StepOutcome(output, None, None) + remote_result = _discover_result( + runner, + config, + fixture, + runtime, + step, + result_base, + log_dir, + ) + local_result = _copy_scenario_result( + runner, config, fixture, runtime, step, remote_result + ) + _assert_execution_contract(runtime.scenario, step, local_result) + _assert_semantic_flags(runtime.scenario.name, step, local_result) + if "retained_data_dir" in step.exports: + runtime.values["retained_data_dir"] = _retained_path(local_result) + _assert_dataset_state( + runner, + config, + fixture, + step, + local_result, + runtime.values.get("retained_data_dir"), + ) + _assert_step_specials(runner, config, fixture, runtime, step, local_result) + _assert_scenario_report( + runner, + report_workspace, + local_result, + runtime, + step, + log_dir, + ) + if {item.coordinate.nodes for item in step.executions} == {1, 2}: + _assert_ordered_workers(fixture, runtime.selector, output, local_result) + return StepOutcome(output, remote_result, local_result) + + +def _make_scenario_runtime( + fixture: Fixture, + scenario: FilesystemScenarioSpec, + selector: str, + build_root: Path, + artifact_root: Path, + run_id: str, +) -> ScenarioRuntime: + """Create cleanup-capable scenario identity before remote mutation.""" + workspace_id = f"{run_id}-{scenario.name}-{selector}" + workspace = ( + str(build_root / "ssh" / "storage-scale-test") + if selector == "ssh" + else f"{REMOTE_BASE}/workspaces/{workspace_id}/storage-scale-test" + ) + data_root = f"{REMOTE_BASE}/test-data/{workspace_id}" + values = { + "workspace": workspace, + "test_root": f"{data_root}/primary", + "test_root_secondary": f"{data_root}/secondary", + "slurm_node_1": fixture.slurm_nodes[0] if fixture.slurm_nodes else "", + "slurm_node_2": fixture.slurm_nodes[1] if fixture.slurm_nodes else "", + } + return ScenarioRuntime( + scenario, + selector, + workspace, + build_root, + artifact_root, + data_root, + values, + [], + ) + + +def _prepare_scenario_runtime( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + archive: Path, + extracted: Path, +) -> None: + """Stage deployment and data after cleanup identity exists.""" + workspace_id = PurePosixPath(runtime.data_root).name + workspace = _stage_workspace( + runner, + config, + fixture, + runtime.selector, + archive, + extracted, + runtime.local_workspace, + workspace_id, + ) + if workspace != runtime.workspace: + raise IntegrationTestError( + f"scenario workspace drifted: expected {runtime.workspace}, found {workspace}" + ) + _prepare_scenario_data(runner, config, fixture, runtime.data_root) + + +def _run_regular_scenario( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + template: Path, + report_workspace: Path, + log_dir: Path, +) -> None: + """Run all ordered steps for a non-failure scenario.""" + for step in runtime.scenario.steps: + _run_regular_step( + runner, + config, + fixture, + runtime, + step, + template, + report_workspace, + log_dir, + ) + + +def _cleanup_ssh_remote_results(runner: Any, config: Any) -> None: + """Remove retrieved per-scenario output trees from bounded SSH homes.""" + failures = [] + for pod in _pods_with_container(_pod_inventory(runner, config), "sshd"): + result = runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + pod["metadata"]["name"], + "-c", + "sshd", + "--", + ), + "runuser", + "-u", + WORKLOAD_USER, + "--", + "bash", + "-c", + "rm -rf -- /home/tester/elbencho-[0-9]*", + ], + check=False, + timeout=60, + ) + if result.returncode: + failures.append( + f"{pod['metadata']['name']}: " + f"{(result.stderr or result.stdout).strip()}" + ) + if failures: + raise IntegrationTestError( + "could not clean SSH scenario results: " + "; ".join(failures) + ) + + +def _preserve_scenario_failure_diagnostics( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + log_dir: Path, + error: Exception, +) -> None: + """Copy bounded scenario logs before its disposable workspace is removed.""" + destination = log_dir / "failure-diagnostics" + destination.mkdir(parents=True, exist_ok=True) + (destination / "failure.txt").write_text(f"{error!r}\n", encoding="utf-8") + failures: list[str] = [] + for name in ("logs", "results"): + source = f"{runtime.workspace}/{name}" + target = destination / name + try: + if runtime.selector == "ssh": + local_source = Path(source) + if local_source.exists(): + shutil.copytree(local_source, target, dirs_exist_ok=True) + else: + failures.append(f"missing local diagnostic path: {source}") + continue + result = runner.run( + [ + *_kubectl(config, "-n", config.namespace, "cp"), + "-c", + fixture.login_container, + f"{fixture.login_pod}:{source}", + target, + ], + check=False, + timeout=180, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + failures.append( + f"could not copy {source} (exit {result.returncode}): {detail}" + ) + except (OSError, subprocess.SubprocessError) as diagnostic_error: + failures.append(f"could not copy {source}: {diagnostic_error!r}") + if failures: + (destination / "collection-errors.txt").write_text( + "\n".join(failures) + "\n", encoding="utf-8" + ) + LOG.warning( + "Some %s/%s failure diagnostics could not be retained; see %s", + runtime.scenario.name, + runtime.selector, + destination / "collection-errors.txt", + ) + + +def _cleanup_scenario_storage( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, +) -> None: + """Remove one scenario's isolated PVC data and Slurm deployment.""" + data_root = PurePosixPath(runtime.data_root) + data_base = PurePosixPath(REMOTE_BASE) / "test-data" + targets = [data_root] + if runtime.selector == "slurm": + workspace_root = PurePosixPath(runtime.workspace).parent + workspace_base = PurePosixPath(REMOTE_BASE) / "workspaces" + if workspace_root.parent != workspace_base: + raise IntegrationTestError( + f"refusing to remove unexpected Slurm workspace: {workspace_root}" + ) + targets.append(workspace_root) + if data_root.parent != data_base: + raise IntegrationTestError( + f"refusing to remove unexpected scenario data path: {data_root}" + ) + command = "rm -rf -- " + " ".join(_shell(path) for path in targets) + result = runner.run( + _pod_command( + config, + fixture.login_pod, + fixture.login_container, + command, + as_user=WORKLOAD_USER, + timeout=120, + ), + check=False, + timeout=135, + ) + if result.returncode: + raise IntegrationTestError( + "could not clean scenario-owned remote storage for " + f"{runtime.scenario.name}/{runtime.selector}: " + f"{(result.stderr or result.stdout).strip()}" + ) + + +def _cleanup_scenario_resources( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, +) -> None: + """Attempt every scenario cleanup and report all failures together.""" + failures = [] + operations = [] + if runtime.selector == "ssh": + operations.append(lambda: _cleanup_ssh_remote_results(runner, config)) + operations.append( + lambda: _cleanup_scenario_storage(runner, config, fixture, runtime) + ) + operations.append(lambda: _cleanup_local_scenario_workspace(config, runtime)) + for cleanup in operations: + try: + cleanup() + except Exception as error: # pylint: disable=broad-exception-caught + failures.append(f"{type(error).__name__}: {error}") + if failures: + raise IntegrationTestError("; ".join(failures)) + + +def _cleanup_local_scenario_workspace(config: Any, runtime: ScenarioRuntime) -> None: + """Remove only a scenario workspace below the harness test-run root.""" + workspace = runtime.local_workspace + test_runs = (config.state_dir / "test-runs").resolve() + resolved = workspace.resolve(strict=False) + if workspace.is_symlink() or test_runs not in resolved.parents: + raise IntegrationTestError( + f"refusing to remove unexpected local scenario workspace: {workspace}" + ) + if workspace.exists(): + shutil.rmtree(workspace) + + +def _record_secondary_cleanup_failure(log_dir: Path, error: Exception) -> None: + """Retain cleanup failure without replacing the scenario's primary error.""" + try: + destination = log_dir / "failure-diagnostics" + destination.mkdir(parents=True, exist_ok=True) + path = destination / "cleanup-error.txt" + path.write_text(f"{type(error).__name__}: {error}\n", encoding="utf-8") + LOG.error("Scenario cleanup also failed; retained details in %s", path) + except OSError as diagnostic_error: + LOG.error( + "Scenario cleanup also failed (%r), and its diagnostic could not " + "be retained: %r", + error, + diagnostic_error, + ) + + +def _remote_staging_operations( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + local_wrapper: Path | None, + restore_binary: Path | None, + remote_wrapper: PurePosixPath | None = None, +) -> FailureInjectionOperations: + """Return concrete PVC or SSH-pod operations for failure staging.""" + + def _container(_endpoint: str) -> str: + return fixture.login_container if selector == "slurm" else "sshd" + + def _command(endpoint: str, command: str, *, stdin: Any = None) -> None: + if endpoint == "local": + return + runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + endpoint, + "-c", + _container(endpoint), + "--", + ), + "runuser", + "-u", + WORKLOAD_USER, + "--", + "bash", + "-ec", + f"umask 0007; {command}", + ], + stdin=stdin, + timeout=180, + ) + + def make_directory(endpoint: str, path: PurePosixPath) -> None: + if endpoint == "local": + Path(path).mkdir(parents=True, exist_ok=True) + return + _command( + endpoint, + f"mkdir -p -- {_shell(path)}", + ) + + def copy_file( + source: Path, endpoint: str, destination: PurePosixPath, mode: int + ) -> None: + if endpoint == "local": + local_destination = Path(destination) + local_destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, local_destination) + local_destination.chmod(mode) + return + with tempfile.TemporaryFile() as stream: + stream.write(source.read_bytes()) + stream.seek(0) + _command( + endpoint, + f"cat > {_shell(destination)} && chmod {mode:o} -- " + f"{_shell(destination)}", + stdin=stream, + ) + + def copy_tree(source: Path, endpoint: str, destination: PurePosixPath) -> None: + if endpoint == "local": + local_destination = Path(destination) + if local_destination.exists(): + shutil.rmtree(local_destination) + shutil.copytree(source, local_destination) + return + if selector == "ssh": + _command( + endpoint, + f"rm -rf -- {_shell(destination)} && ln -s -- " + f"/home/tester/elbencho-runtime {_shell(destination)}", + ) + return + with tempfile.TemporaryFile() as stream: + with tarfile.open(fileobj=stream, mode="w") as archive: + for child in sorted(source.rglob("*")): + archive.add(child, arcname=child.relative_to(source)) + stream.seek(0) + _command( + endpoint, + f"rm -rf -- {_shell(destination)} && mkdir -p -- " + f"{_shell(destination)} && tar -xf - -C {_shell(destination)} " + f"&& chmod -R u+rwX,g+rX,o-rwx -- {_shell(destination)}", + stdin=stream, + ) + + def write_text( + endpoint: str, destination: PurePosixPath, content: str, mode: int + ) -> None: + if endpoint == "local": + if local_wrapper is None: + raise IntegrationTestError("local failure wrapper path is absent") + local_wrapper.write_text(content, encoding="utf-8") + local_wrapper.chmod(mode) + return + if remote_wrapper is not None: + destination = remote_wrapper + with tempfile.TemporaryFile() as stream: + stream.write(content.encode()) + stream.seek(0) + _command( + endpoint, + f"cat > {_shell(destination)} && chmod {mode:o} -- " + f"{_shell(destination)}", + stdin=stream, + ) + + def remove_tree(endpoint: str, path: PurePosixPath) -> None: + if endpoint == "local": + if local_wrapper is not None and restore_binary is not None: + shutil.copy2(restore_binary, local_wrapper) + shutil.rmtree(Path(path), ignore_errors=True) + return + if remote_wrapper is not None: + _command( + endpoint, + f"cp -- {_shell(path / 'delegate' / 'elbencho')} " + f"{_shell(remote_wrapper)}", + ) + _command(endpoint, f"rm -rf -- {_shell(path)}") + + return FailureInjectionOperations( + make_directory, copy_file, copy_tree, write_text, remove_tree + ) + + +def _failure_plan( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + binary_name: str, + source_binary: Path, + bundled_runtime: Path | None, +) -> tuple[Any, FailureInjectionOperations]: + """Build a feasible substrate-specific failure staging plan.""" + packaged_binary = ( + Path(runtime.workspace) / "utils" / binary_name + if runtime.selector == "ssh" + else runtime.local_workspace / "source-elbencho" + ) + if runtime.selector == "slurm": + raise IntegrationTestError("internal Slurm failure source was not prepared") + pods = sorted( + _pods_with_container(_pod_inventory(runner, config), "sshd"), + key=lambda item: item["metadata"]["name"], + ) + plan = build_ssh_failure_injection_plan( + scenario_id=f"{runtime.scenario.name}-{os.getpid()}", + staging_root=SSH_FAILURE_STAGING_BASE, + source_binary=source_binary, + source_runtime=bundled_runtime, + target_argument="executions/0002.write.json", + coordinator_endpoint="local", + worker_endpoints=[item["metadata"]["name"] for item in pods], + ) + operations = _remote_staging_operations( + runner, + config, + fixture, + runtime.selector, + packaged_binary, + source_binary, + ) + return plan, operations + + +def _slurm_failure_plan( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + source_binary: Path, + binary_name: str, + bundled_runtime: Path | None, +) -> tuple[Any, FailureInjectionOperations]: + """Build failure staging on the Slurm deployment's shared PVC.""" + plan = build_slurm_failure_injection_plan( + scenario_id=f"{runtime.scenario.name}-{os.getpid()}", + staging_root=f"{runtime.workspace}/.storage-scale-test-failure", + source_binary=source_binary, + source_runtime=bundled_runtime, + target_argument="executions/0002.write.json", + shared_endpoint=fixture.login_pod, + ) + operations = _remote_staging_operations( + runner, + config, + fixture, + runtime.selector, + None, + None, + PurePosixPath(runtime.workspace) / "utils" / binary_name, + ) + return plan, operations + + +def _hash_execution_contract(result: Path, execution_id: str) -> dict[str, str]: + """Hash stable evidence that resume must not replace for successful cells.""" + evidence: dict[str, str] = {} + for path in sorted((result / "executions").glob(f"{execution_id}.*")): + if path.is_file(): + evidence[path.name] = _sha256(path) + return evidence + + +def _capture_failure_diagnostics( + runner: Any, + config: Any, + fixture: Fixture, + plan: Any, + selector: str, + log_dir: Path, +) -> None: + """Copy wrapper invocation traces before scenario-level cleanup.""" + chunks: list[str] = [] + container = fixture.login_container if selector == "slurm" else "sshd" + for target in plan.targets: + if target.endpoint == "local": + continue + result = runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + target.endpoint, + "-c", + container, + "--", + ), + "bash", + "-c", + f"cat -- {_shell(plan.layout.root / 'invocations.log')} " + "2>/dev/null || true", + ], + check=False, + timeout=30, + ) + chunks.append(f"## {target.endpoint}\n{result.stdout}") + (log_dir / "failure-wrapper-invocations.log").write_text( + "\n".join(chunks), encoding="utf-8" + ) + + +def _run_failure_resume( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + template: Path, + report_workspace: Path, + log_dir: Path, + binary_name: str, + source_binary: Path, + bundled_runtime: Path | None, +) -> None: + """Run one real injected failure and resume without restaging the marker.""" + first, resume = runtime.scenario.steps + if runtime.selector == "ssh": + plan, operations = _failure_plan( + runner, + config, + fixture, + runtime, + binary_name, + source_binary, + bundled_runtime, + ) + injected_first = first + else: + plan, operations = _slurm_failure_plan( + runner, + config, + fixture, + runtime, + source_binary, + binary_name, + bundled_runtime, + ) + injected_first = first + result_base = f"{runtime.workspace}/results/{first.name}" + _reset_result_base(runner, config, fixture, runtime, result_base) + _sync_step_runtime( + runner, + config, + fixture, + runtime, + injected_first, + template, + result_base, + ) + _validate_step_environment( + runner, config, fixture, runtime, injected_first, log_dir + ) + with staged_failure_injection(plan, operations): + arguments = shlex.join(first.render_arguments(runtime.values)) + command = ( + f"cd -- {_shell(runtime.workspace)} && " + f"./storage-tests/fs/nv-elbencho-sweep.sh {arguments}" + ) + _run_step( + runner, + config, + fixture, + runtime.selector, + first.name, + command, + log_dir, + first.timeout_seconds, + expected_failure=True, + ) + remote_result = _discover_result( + runner, + config, + fixture, + runtime, + first, + result_base, + log_dir, + ) + failed_result = _copy_scenario_result( + runner, config, fixture, runtime, first, remote_result + ) + _capture_failure_diagnostics( + runner, config, fixture, plan, runtime.selector, log_dir + ) + _assert_execution_contract(runtime.scenario, first, failed_result) + runtime.values["failed_results_dir"] = remote_result + _assert_dataset_state(runner, config, fixture, first, failed_result, None) + preserved = _hash_execution_contract(failed_result, "0001") + resume_command = ( + f"cd -- {_shell(runtime.workspace)} && " + "./storage-tests/fs/nv-elbencho-sweep.sh " + f"{shlex.join(resume.render_arguments(runtime.values))}" + ) + _run_step( + runner, + config, + fixture, + runtime.selector, + resume.name, + resume_command, + log_dir, + resume.timeout_seconds, + ) + resumed_result = _copy_scenario_result( + runner, config, fixture, runtime, resume, remote_result + ) + _assert_execution_contract(runtime.scenario, resume, resumed_result) + if _hash_execution_contract(resumed_result, "0001") != preserved: + raise IntegrationTestError("resume replaced successful execution 0001") + _assert_dataset_state(runner, config, fixture, resume, resumed_result, None) + _assert_scenario_report( + runner, + report_workspace, + resumed_result, + runtime, + resume, + log_dir, + ) + + +def run_filesystem_tests( + runner: Any, + config: Any, + repo_root: Path, + substrate: str, + scenarios: list[str], + transition_ssh_home: Any | None = None, +) -> None: + """Run selected filesystem regression cases against an existing setup.""" + try: + plan = plan_scenarios(substrate=substrate, requested=scenarios) + except ScenarioPlanningError as error: + raise IntegrationTestError(str(error)) from error + work = [step for step in plan if isinstance(step, WorkItem)] + selected = {step.substrate.value for step in work} + if "ssh" in selected: + if transition_ssh_home is None: + raise IntegrationTestError("SSH scenarios require a home transition hook") + transition_ssh_home("separate", "preflight") + LOG.info("Requiring an already-running integration setup") + fixture = _require_fixture(runner, config, selected) + binary, binary_name, runtime = _ensure_elbencho( + runner, config, fixture.architecture, fixture.storage_backend + ) + if runtime is not None and "ssh" in selected: + LOG.info("Staging the pinned Elbencho container runtime in SSH worker homes") + _stage_ssh_runtime(runner, config, runtime, _pod_inventory(runner, config)) + run_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{os.getpid()}" + log_dir = config.state_dir / "test-runs" / run_id + log_dir.mkdir(parents=True, exist_ok=False) + LOG.info("Filesystem integration artifacts: %s", log_dir) + scratch_parent = config.state_dir / "test-runs" + with tempfile.TemporaryDirectory(dir=scratch_parent) as temporary: + build_root = Path(temporary) + archive, extracted = _build_deployment_archive( + runner, + config, + repo_root, + build_root, + binary, + binary_name, + fixture.architecture, + runtime, + ) + shutil.copy2(build_root / "build-tarball.log", log_dir / "build-tarball.log") + report_workspace = build_root / "report-workspace" + shutil.copytree(extracted, report_workspace) + report_fixture = fixture + if not report_fixture.ssh_addresses: + report_fixture = replace( + report_fixture, ssh_addresses=report_fixture.slurm_addresses + ) + _write_runtime_files( + report_workspace, + "ssh", + str(report_workspace), + report_fixture, + template=extracted / "env.sh.template", + ) + home_mode = "separate" + suite_error: Exception | None = None + try: + for step in plan: + if isinstance(step, SshHomeTransition): + transition_ssh_home(step.target.value, "ssh-shared-home") + home_mode = step.target.value + fixture = _require_fixture( + runner, config, selected, ssh_home_mode=home_mode + ) + if runtime is not None: + _stage_ssh_runtime( + runner, + config, + runtime, + _pod_inventory(runner, config), + ) + continue + scenario = step.scenario.name + scenario_root = build_root / f"{scenario}-{step.substrate.value}" + scenario_root.mkdir() + scenario_logs = log_dir / f"{scenario}-{step.substrate.value}" + scenario_logs.mkdir() + specification = SCENARIO_SPECS_BY_NAME[scenario] + runtime_state = _make_scenario_runtime( + fixture, + specification, + step.substrate.value, + scenario_root, + scenario_logs, + run_id, + ) + primary_error: Exception | None = None + try: + _prepare_scenario_runtime( + runner, + config, + fixture, + runtime_state, + archive, + extracted, + ) + if scenario == "failure-resume": + _run_failure_resume( + runner, + config, + fixture, + runtime_state, + extracted / "env.sh.template", + report_workspace, + scenario_logs, + binary_name, + binary, + runtime, + ) + else: + _run_regular_scenario( + runner, + config, + fixture, + runtime_state, + extracted / "env.sh.template", + report_workspace, + scenario_logs, + ) + except Exception as error: + primary_error = error + try: + _preserve_scenario_failure_diagnostics( + runner, + config, + fixture, + runtime_state, + scenario_logs, + error, + ) + except (OSError, subprocess.SubprocessError) as diagnostic_error: + LOG.warning( + "Could not retain %s/%s failure diagnostics: %r", + scenario, + step.substrate.value, + diagnostic_error, + ) + raise + finally: + try: + _cleanup_scenario_resources( + runner, config, fixture, runtime_state + ) + # Cleanup must never replace an active scenario exception. + # pylint: disable-next=broad-exception-caught + except Exception as cleanup_error: + if primary_error is None: + raise + _record_secondary_cleanup_failure(scenario_logs, cleanup_error) + except Exception as error: + suite_error = error + raise + finally: + if home_mode == "shared" and transition_ssh_home is not None: + try: + transition_ssh_home("separate", "ssh-shared-home-restore") + # pylint: disable-next=broad-exception-caught + except Exception as restore_error: + if suite_error is None: + raise + _record_secondary_cleanup_failure(log_dir, restore_error) + names = ", ".join(f"{step.scenario.name}/{step.substrate.value}" for step in work) + LOG.info("Filesystem integration tests passed: %s", names) diff --git a/integration-tests/lib/filesystem_scenario_specs.py b/integration-tests/lib/filesystem_scenario_specs.py new file mode 100644 index 0000000..1f4b434 --- /dev/null +++ b/integration-tests/lib/filesystem_scenario_specs.py @@ -0,0 +1,717 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declarative workload contracts for filesystem integration scenarios. + +The driver owns path rendering, substrate setup, and semantic assertions. This +module deliberately describes only stable workload intent. Template fields in +environment lines, command arguments, and support files are resolved from the +scenario workspace by the driver. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from itertools import product +from typing import Iterable, Mapping + + +class ScenarioSpecError(ValueError): + """A scenario specification is internally inconsistent.""" + + +class CommandKind(StrEnum): + """Kinds of sweep command represented by a scenario step.""" + + SWEEP = "sweep" + RESUME = "resume" + DELETE = "delete" + + +class DatasetExpectation(StrEnum): + """Expected state of scenario-owned benchmark data after a step.""" + + CLEANED = "cleaned" + PRESERVED = "preserved" + REMOVED = "removed" + + +class ExecutionStatus(StrEnum): + """Expected terminal or interrupted state of one reified execution.""" + + SUCCESS = "SUCCESS" + FAILED = "FAILED" + PENDING = "PENDING" + + +class WorkloadPhase(StrEnum): + """Semantic phases whose evidence must be present for a step.""" + + DIRECTORY_CREATE = "directory-create" + WRITE = "write" + TREE_SCAN = "tree-scan" + READ = "read" + REMOVE_FILES = "remove-files" + DELETE_PATH = "delete-path" + + +class FailureInjection(StrEnum): + """Integration-only overlays used by deliberately failing scenarios.""" + + NONE = "none" + FAIL_AFTER_WRITE_ONCE = "fail-after-write-once" + + +@dataclass(frozen=True, order=True) +class ExecutionCoordinate: + """One expected point in an Elbencho sweep.""" + + nodes: int + io_size: str + threads: int + io_depth: int + + +@dataclass(frozen=True) +class ExpectedExecution: + """Expected state of one execution after a command finishes.""" + + coordinate: ExecutionCoordinate + status: ExecutionStatus = ExecutionStatus.SUCCESS + + +@dataclass(frozen=True) +class SupportFile: + """A small text file rendered relative to the scenario workspace.""" + + relative_path: str + content: str + + +@dataclass(frozen=True) +class GeneratedInput: + """A bounded input file the harness creates without storing its contents.""" + + relative_path: str + size_bytes: int + + +@dataclass(frozen=True) +class ScenarioStep: + """One command and its stable semantic expectations.""" + + name: str + kind: CommandKind + arguments: tuple[str, ...] + env_lines: tuple[str, ...] + support_files: tuple[SupportFile, ...] + generated_inputs: tuple[GeneratedInput, ...] + timeout_seconds: int + executions: tuple[ExpectedExecution, ...] + required_phases: tuple[WorkloadPhase, ...] + dataset: DatasetExpectation + requires: tuple[str, ...] = () + exports: tuple[str, ...] = () + failure_injection: FailureInjection = FailureInjection.NONE + preserve_failure_staging: bool = False + + def render_arguments(self, values: Mapping[str, str]) -> tuple[str, ...]: + """Render driver-controlled placeholders in command arguments.""" + return tuple(argument.format_map(values) for argument in self.arguments) + + def render_env(self, values: Mapping[str, str]) -> str: + """Render the environment override block for this step.""" + return "\n".join(line.format_map(values) for line in self.env_lines) + "\n" + + def render_support_files( + self, values: Mapping[str, str] + ) -> tuple[SupportFile, ...]: + """Render paths and contents of small scenario support files.""" + return tuple( + SupportFile( + item.relative_path.format_map(values), item.content.format_map(values) + ) + for item in self.support_files + ) + + +@dataclass(frozen=True) +class FilesystemScenarioSpec: + """All command steps for one named real-infrastructure scenario.""" + + name: str + substrates: frozenset[str] + steps: tuple[ScenarioStep, ...] + + +_SHARED_ENV = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=1)', + 'export ELBENCHO_FILE_LAYOUT="shared-directory"', + "export ELBENCHO_FILES_PER_NODE=1", + 'export ELBENCHO_FILE_SIZE="16M"', + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_LIVE_CSV_EXTENDED=0", + "export ELBENCHO_SINGLE_BIG_FILE=0", +) + +_WORKER_ENV = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=2)', + 'export ELBENCHO_FILE_LAYOUT="worker-directories"', + "export ELBENCHO_FILES_PER_NODE=", + "export ELBENCHO_FILE_SIZE=", + "export ELBENCHO_FILE_SIZE_MULTIPLIER=4096", + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_LIVE_CSV_EXTENDED=0", + "export ELBENCHO_SINGLE_BIG_FILE=0", +) + +_NORMAL_PHASES = ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.READ, + WorkloadPhase.REMOVE_FILES, +) + +_BOTH_SUBSTRATES = frozenset({"ssh", "slurm"}) +_SSH_ONLY = frozenset({"ssh"}) +_SLURM_ONLY = frozenset({"slurm"}) + + +def _coordinates( + nodes: Iterable[int], + io_sizes: Iterable[str], + threads: Iterable[int], + depths: Iterable[int], + *, + statuses: Mapping[ExecutionCoordinate, ExecutionStatus] | None = None, +) -> tuple[ExpectedExecution, ...]: + """Create expected coordinates in the sweep's documented nesting order.""" + status_by_coordinate = statuses or {} + return tuple( + ExpectedExecution( + coordinate, + status_by_coordinate.get(coordinate, ExecutionStatus.SUCCESS), + ) + for values in product(nodes, io_sizes, threads, depths) + for coordinate in (ExecutionCoordinate(*values),) + ) + + +def _env_name(line: str) -> str: + """Return the variable named by one supported environment statement.""" + statement = line.strip() + for prefix in ("export ", "unset ", "declare -A "): + if statement.startswith(prefix): + statement = statement.removeprefix(prefix) + break + return statement.partition("=")[0].strip() + + +def _override_env( + lines: tuple[str, ...], replacements: Mapping[str, str] +) -> tuple[str, ...]: + """Replace uniquely named variables without depending on shell formatting.""" + compound = { + name + for name, replacement in replacements.items() + if "\n" in replacement or "\r" in replacement + } + if compound: + raise ScenarioSpecError( + "environment overrides require one statement: " + + ", ".join(sorted(compound)) + ) + indexes: dict[str, list[int]] = {} + for index, line in enumerate(lines): + indexes.setdefault(_env_name(line), []).append(index) + invalid = { + name: indexes.get(name, []) + for name in replacements + if len(indexes.get(name, [])) != 1 + } + if invalid: + details = ", ".join( + f"{name} ({len(matches)} matches)" + for name, matches in sorted(invalid.items()) + ) + raise ScenarioSpecError(f"environment overrides require one match: {details}") + mismatched = { + name: _env_name(replacement) + for name, replacement in replacements.items() + if _env_name(replacement) != name + } + if mismatched: + details = ", ".join( + f"{name} names {replacement_name or ''}" + for name, replacement_name in sorted(mismatched.items()) + ) + raise ScenarioSpecError(f"environment override name mismatch: {details}") + rendered = list(lines) + for name, replacement in replacements.items(): + rendered[indexes[name][0]] = replacement + return tuple(rendered) + + +def _step( + name: str, + arguments: tuple[str, ...], + env_lines: tuple[str, ...], + executions: tuple[ExpectedExecution, ...], + phases: tuple[WorkloadPhase, ...], + dataset: DatasetExpectation = DatasetExpectation.CLEANED, + *, + timeout_seconds: int = 300, + requires: tuple[str, ...] = (), + exports: tuple[str, ...] = (), + failure_injection: FailureInjection = FailureInjection.NONE, + preserve_failure_staging: bool = False, +) -> ScenarioStep: + """Build a sweep step with bounded defaults.""" + return ScenarioStep( + name=name, + kind=CommandKind.SWEEP, + arguments=arguments, + env_lines=env_lines, + support_files=(), + generated_inputs=(), + timeout_seconds=timeout_seconds, + executions=executions, + required_phases=phases, + dataset=dataset, + requires=requires, + exports=exports, + failure_injection=failure_injection, + preserve_failure_staging=preserve_failure_staging, + ) + + +def _baseline() -> FilesystemScenarioSpec: + return FilesystemScenarioSpec( + "baseline", + _BOTH_SUBSTRATES, + ( + _step( + "buffered-sweep", + ("--bio", "--nodes", "1,2"), + _SHARED_ENV, + _coordinates((1, 2), ("4K",), (1,), (1,)), + _NORMAL_PHASES, + ), + ), + ) + + +def _default_dio() -> FilesystemScenarioSpec: + return FilesystemScenarioSpec( + "default-dio", + _BOTH_SUBSTRATES, + ( + _step( + "direct-worker-directories", + ("--nodes", "1,2"), + _WORKER_ENV, + _coordinates((1, 2), ("4K",), (1,), (1,)), + ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.TREE_SCAN, + WorkloadPhase.READ, + WorkloadPhase.REMOVE_FILES, + ), + ), + ), + ) + + +def _failure_resume() -> FilesystemScenarioSpec: + coordinates = tuple( + execution.coordinate + for execution in _coordinates((1, 2), ("4K", "8K"), (1,), (1,)) + ) + statuses = { + coordinates[1]: ExecutionStatus.FAILED, + coordinates[2]: ExecutionStatus.PENDING, + coordinates[3]: ExecutionStatus.PENDING, + } + initial = _coordinates((1, 2), ("4K", "8K"), (1,), (1,), statuses=statuses) + env_lines = _override_env( + _SHARED_ENV, + {"ELBENCHO_SCALE_IO_SIZES": ('export ELBENCHO_SCALE_IO_SIZES=("4K" "8K")')}, + ) + first = _step( + "inject-one-failure", + ("--write-no-read", "--nodes", "1,2"), + env_lines, + initial, + ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.REMOVE_FILES, + ), + exports=("failed_results_dir",), + failure_injection=FailureInjection.FAIL_AFTER_WRITE_ONCE, + preserve_failure_staging=True, + ) + resume = ScenarioStep( + name="resume", + kind=CommandKind.RESUME, + arguments=("--resume", "{failed_results_dir}"), + env_lines=(), + support_files=(), + generated_inputs=(), + timeout_seconds=300, + executions=_coordinates((1, 2), ("4K", "8K"), (1,), (1,)), + required_phases=(WorkloadPhase.WRITE, WorkloadPhase.REMOVE_FILES), + dataset=DatasetExpectation.CLEANED, + requires=("failed_results_dir",), + preserve_failure_staging=True, + ) + return FilesystemScenarioSpec("failure-resume", _BOTH_SUBSTRATES, (first, resume)) + + +def _retained_lifecycle() -> FilesystemScenarioSpec: + coordinate = _coordinates((1,), ("4K",), (1,), (1,)) + write = _step( + "write-only", + ("--write-only", "--nodes", "1"), + _SHARED_ENV, + coordinate, + (WorkloadPhase.DIRECTORY_CREATE, WorkloadPhase.WRITE), + DatasetExpectation.PRESERVED, + exports=("retained_data_dir",), + ) + read_miss = _step( + "read-cache-miss", + ("--read-from", "{retained_data_dir}", "--nodes", "1"), + _SHARED_ENV, + coordinate, + (WorkloadPhase.TREE_SCAN, WorkloadPhase.READ), + DatasetExpectation.PRESERVED, + requires=("retained_data_dir",), + ) + read_hit = _step( + "read-cache-hit", + ("--read-from", "{retained_data_dir}", "--nodes", "1"), + _SHARED_ENV, + coordinate, + (WorkloadPhase.READ,), + DatasetExpectation.PRESERVED, + requires=("retained_data_dir",), + ) + delete = ScenarioStep( + name="delete-only", + kind=CommandKind.DELETE, + arguments=("--delete-only", "{retained_data_dir}"), + env_lines=_SHARED_ENV, + support_files=(), + generated_inputs=(), + timeout_seconds=180, + executions=(), + required_phases=(WorkloadPhase.DELETE_PATH,), + dataset=DatasetExpectation.REMOVED, + requires=("retained_data_dir",), + ) + return FilesystemScenarioSpec( + "retained-lifecycle", _BOTH_SUBSTRATES, (write, read_miss, read_hit, delete) + ) + + +def _live_capture() -> FilesystemScenarioSpec: + env_lines = _override_env( + _SHARED_ENV, + { + "ELBENCHO_SCALE_READ_WRITE_DURATION": ( + "export ELBENCHO_SCALE_READ_WRITE_DURATION=3" + ), + "ELBENCHO_FILES_PER_NODE": "export ELBENCHO_FILES_PER_NODE=2", + "ELBENCHO_LIVE_CSV_EXTENDED": "export ELBENCHO_LIVE_CSV_EXTENDED=1", + }, + ) + ("export ELBENCHO_LIVEINT=10",) + return FilesystemScenarioSpec( + "live-capture", + _BOTH_SUBSTRATES, + ( + _step( + "extended-live-csv", + ("--nodes", "2"), + env_lines, + _coordinates((2,), ("4K",), (1,), (1,)), + ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.READ, + WorkloadPhase.REMOVE_FILES, + ), + ), + ), + ) + + +def _slurm_cartesian() -> FilesystemScenarioSpec: + env_lines = _override_env( + _SHARED_ENV, + { + "ELBENCHO_FILES_PER_NODE": "export ELBENCHO_FILES_PER_NODE=2", + "ELBENCHO_FILE_SIZE": 'export ELBENCHO_FILE_SIZE="1M"', + "ELBENCHO_SCALE_THREAD_LIST": ( + 'export ELBENCHO_SCALE_THREAD_LIST=("1" "2")' + ), + "ELBENCHO_SCALE_IO_SIZES": ( + 'export ELBENCHO_SCALE_IO_SIZES=("4K" "4K,r8K")' + ), + "ELBENCHO_IODEPTH_LIST": ('export ELBENCHO_IODEPTH_LIST=("1" "2")'), + }, + ) + return FilesystemScenarioSpec( + "slurm-cartesian", + _SLURM_ONLY, + ( + _step( + "representative-matrix", + ("--nodes", "1,2"), + env_lines, + _coordinates((1, 2), ("4K", "4K,r8K"), (1, 2), (1, 2)), + _NORMAL_PHASES, + timeout_seconds=600, + ), + ), + ) + + +def _ssh_single_big_file() -> FilesystemScenarioSpec: + env_lines = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=1)', + 'export ELBENCHO_FILE_LAYOUT="worker-directories"', + "export ELBENCHO_FILES_PER_NODE=", + "export ELBENCHO_FILE_SIZE=", + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_SINGLE_BIG_FILE=1", + 'export ELBENCHO_SINGLE_BIG_FILE_BASENAME="integration-bigfile"', + 'export ELBENCHO_SINGLE_BIG_FILE_SIZE="16M"', + "export ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0", + ) + coordinate = _coordinates((2,), ("4K",), (1,), (1,)) + generated = _step( + "cooperative-readwrite", + ("--bio", "--nodes", "1,2"), + env_lines, + _coordinates((1, 2), ("4K",), (1,), (1,)), + (WorkloadPhase.WRITE, WorkloadPhase.READ, WorkloadPhase.REMOVE_FILES), + ) + read_env = _override_env( + env_lines, + { + "ELBENCHO_ALL_NODES_ACCESS_ALL_DATA": ( + "export ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1" + ) + }, + ) + read = ScenarioStep( + name="inferred-extent-read", + kind=CommandKind.SWEEP, + arguments=( + "--read-from", + "{test_root}/staged-input/integration-bigfile", + "--bio", + "--nodes", + "2", + ), + env_lines=read_env, + support_files=(), + generated_inputs=( + GeneratedInput("staged-input/integration-bigfile", 16 * 1024 * 1024), + ), + timeout_seconds=300, + executions=coordinate, + required_phases=(WorkloadPhase.READ,), + dataset=DatasetExpectation.PRESERVED, + ) + delete = ScenarioStep( + name="delete-retained-directory", + kind=CommandKind.DELETE, + arguments=("--delete-only", "{test_root}/staged-input"), + env_lines=env_lines, + support_files=(), + generated_inputs=(), + timeout_seconds=180, + executions=(), + required_phases=(WorkloadPhase.DELETE_PATH,), + dataset=DatasetExpectation.REMOVED, + ) + return FilesystemScenarioSpec( + "ssh-single-big-file", _SSH_ONLY, (generated, read, delete) + ) + + +def _ssh_weighted_roots() -> FilesystemScenarioSpec: + env_lines = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=1 ["{test_root_secondary}"]=2)', + 'export ELBENCHO_FILE_LAYOUT="worker-directories"', + "export ELBENCHO_FILES_PER_NODE=", + 'export ELBENCHO_FILE_SIZE="1M"', + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export FS_MAX_AGG_THROUGHPUT=1", + "export FS_MAX_NODE_THROUGHPUT_GBPS=1", + "export FS_MAX_NODE_IOPS=100", + "export ELBENCHO_SINGLE_BIG_FILE=0", + ) + return FilesystemScenarioSpec( + "ssh-weighted-roots", + _SSH_ONLY, + ( + _step( + "active-single-sizing", + ("--bio", "--single", "--nodes", "1"), + env_lines, + _coordinates((1,), ("4K",), (1,), (1,)), + _NORMAL_PHASES, + ), + ), + ) + + +def _ssh_shared_home() -> FilesystemScenarioSpec: + return FilesystemScenarioSpec( + "ssh-shared-home", + _SSH_ONLY, + ( + _step( + "shared-home-smoke", + ("--bio", "--nodes", "1,2"), + _SHARED_ENV + ("export SSH_HOMEDIR_SHARED=1",), + _coordinates((1, 2), ("4K",), (1,), (1,)), + _NORMAL_PHASES, + ), + ), + ) + + +def _slurm_scheduling() -> FilesystemScenarioSpec: + support = ( + SupportFile("slurm-scenario-includes", "{slurm_node_1}\n{slurm_node_2}\n"), + SupportFile("slurm-scenario-ignores", "{slurm_node_2}\n"), + ) + env_lines = _override_env( + _SHARED_ENV, + {"ELBENCHO_FILE_SIZE": 'export ELBENCHO_FILE_SIZE="4M"'}, + ) + ( + "SLURM_EXCLUSIVE_USER=1", + 'SLURM_JOB_NAME_PREFIX="itest-"', + 'export SLURM_NODE_INCLUDES="{workspace}/slurm-scenario-includes"', + 'export SLURM_NODE_IGNORES="{workspace}/slurm-scenario-ignores"', + ) + step = ScenarioStep( + name="allocation-options", + kind=CommandKind.SWEEP, + arguments=("--nodes", "1"), + env_lines=env_lines, + support_files=support, + generated_inputs=(), + timeout_seconds=300, + executions=_coordinates((1,), ("4K",), (1,), (1,)), + required_phases=_NORMAL_PHASES, + dataset=DatasetExpectation.CLEANED, + ) + return FilesystemScenarioSpec("slurm-scheduling", _SLURM_ONLY, (step,)) + + +SCENARIO_SPECS = ( + _baseline(), + _default_dio(), + _failure_resume(), + _retained_lifecycle(), + _live_capture(), + _slurm_cartesian(), + _ssh_single_big_file(), + _ssh_weighted_roots(), + _ssh_shared_home(), + _slurm_scheduling(), +) + +SCENARIO_SPECS_BY_NAME = {spec.name: spec for spec in SCENARIO_SPECS} + + +def validate_scenario_specs( + specifications: Iterable[FilesystemScenarioSpec] = SCENARIO_SPECS, +) -> None: + """Reject inconsistent catalog entries before a real fixture is touched.""" + specs = tuple(specifications) + names = [spec.name for spec in specs] + if len(names) != len(set(names)): + raise ScenarioSpecError("scenario specification names must be unique") + for spec in specs: + _validate_spec(spec) + + +def _validate_spec(spec: FilesystemScenarioSpec) -> None: + """Validate one scenario and its sequence dependencies.""" + if not spec.name or not spec.steps: + raise ScenarioSpecError("scenario names and step sequences must be nonempty") + if not spec.substrates or not spec.substrates <= _BOTH_SUBSTRATES: + raise ScenarioSpecError(f"{spec.name}: invalid substrates") + available: set[str] = set() + step_names: set[str] = set() + for step in spec.steps: + if step.name in step_names: + raise ScenarioSpecError(f"{spec.name}: duplicate step {step.name}") + step_names.add(step.name) + missing = set(step.requires) - available + if missing: + raise ScenarioSpecError( + f"{spec.name}/{step.name}: unavailable values: {sorted(missing)}" + ) + _validate_step(spec.name, step) + available.update(step.exports) + + +def _validate_step(scenario_name: str, step: ScenarioStep) -> None: + """Validate bounded execution and expectation invariants for one step.""" + label = f"{scenario_name}/{step.name}" + if not 1 <= step.timeout_seconds <= 600: + raise ScenarioSpecError(f"{label}: timeout must be between 1 and 600 seconds") + coordinates = [execution.coordinate for execution in step.executions] + if len(coordinates) != len(set(coordinates)): + raise ScenarioSpecError(f"{label}: duplicate execution coordinates") + if step.kind is CommandKind.DELETE and step.executions: + raise ScenarioSpecError(f"{label}: delete steps cannot reify executions") + if step.kind is not CommandKind.DELETE and not step.executions: + raise ScenarioSpecError(f"{label}: sweep and resume steps require executions") + if step.failure_injection is not FailureInjection.NONE: + if not any( + execution.status is ExecutionStatus.FAILED for execution in step.executions + ): + raise ScenarioSpecError(f"{label}: failure injection lacks failed state") + if len(step.required_phases) != len(set(step.required_phases)): + raise ScenarioSpecError(f"{label}: duplicate required phases") + + +validate_scenario_specs() diff --git a/integration-tests/lib/fixture_capacity.py b/integration-tests/lib/fixture_capacity.py new file mode 100644 index 0000000..4db92aa --- /dev/null +++ b/integration-tests/lib/fixture_capacity.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One capacity budget for the single-host integration fixture.""" + +MIB = 1024**2 +GIB = 1024**3 + +STORAGE_TEST_CAPACITY_BYTES = 2 * GIB +SSH_HOME_CAPACITY_BYTES = 64 * MIB +MAX_DEPLOYMENT_CONTENT_BYTES = 512 * MIB +MAX_LIVE_CAPTURE_DATASET_BYTES = 64 * MIB +RESULT_AND_REPORT_HEADROOM_BYTES = 448 * MIB + +NFS_BUDGET_BYTES = ( + STORAGE_TEST_CAPACITY_BYTES + + SSH_HOME_CAPACITY_BYTES + + MAX_DEPLOYMENT_CONTENT_BYTES + + MAX_LIVE_CAPTURE_DATASET_BYTES + + RESULT_AND_REPORT_HEADROOM_BYTES +) +NFS_IMAGE_BYTES = ((NFS_BUDGET_BYTES + GIB - 1) // GIB) * GIB + +STORAGE_TEST_CAPACITY = f"{STORAGE_TEST_CAPACITY_BYTES // GIB}Gi" +SSH_HOME_CAPACITY = f"{SSH_HOME_CAPACITY_BYTES // MIB}Mi" diff --git a/integration-tests/lib/scenario_planner.py b/integration-tests/lib/scenario_planner.py new file mode 100644 index 0000000..a2af45b --- /dev/null +++ b/integration-tests/lib/scenario_planner.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declare and schedule filesystem integration scenarios.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, StrEnum +from typing import Iterable, TypeAlias + + +class ScenarioPlanningError(ValueError): + """An invalid scenario selection or registry definition.""" + + +class Substrate(StrEnum): + """Supported integration execution substrates.""" + + ALL = "all" + SSH = "ssh" + SLURM = "slurm" + + +SUBSTRATES = tuple(item.value for item in Substrate) + + +class SshHomeMode(StrEnum): + """SSH worker home configurations used by scenarios.""" + + SEPARATE = "separate" + SHARED = "shared" + + +class SchedulePhase(IntEnum): + """Stable phases surrounding the single shared-home SSH batch.""" + + BEFORE_SHARED_HOME = 10 + SHARED_HOME = 20 + AFTER_SHARED_HOME = 30 + + +@dataclass(frozen=True) +class Scenario: + """Stable metadata for one filesystem integration scenario.""" + + name: str + description: str + substrates: frozenset[Substrate] + order: int + phase: SchedulePhase = SchedulePhase.BEFORE_SHARED_HOME + ssh_home_mode: SshHomeMode | None = None + + def metadata(self) -> dict[str, object]: + """Return the stable machine-readable scenario metadata.""" + return { + "name": self.name, + "substrates": sorted(item.value for item in self.substrates), + "ssh_home_mode": ( + self.ssh_home_mode.value if self.ssh_home_mode is not None else None + ), + "schedule_phase": self.phase.name.lower(), + "order": self.order, + } + + +@dataclass(frozen=True) +class WorkItem: + """One scenario execution on one concrete substrate.""" + + scenario: Scenario + substrate: Substrate + + +@dataclass(frozen=True) +class SshHomeTransition: + """A required transition of the singleton SSH worker pool.""" + + target: SshHomeMode + + +PlanStep: TypeAlias = WorkItem | SshHomeTransition + + +SCENARIO_CATALOG = ( + Scenario( + "baseline", + "Shared-directory buffered-I/O baseline and report extraction", + frozenset({Substrate.SSH, Substrate.SLURM}), + 10, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "default-dio", + "Default worker-directory direct-I/O lifecycle", + frozenset({Substrate.SSH, Substrate.SLURM}), + 20, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "failure-resume", + "Real failure, cleanup, and resume lifecycle", + frozenset({Substrate.SSH, Substrate.SLURM}), + 30, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "retained-lifecycle", + "Write-only, repeated read-from, and delete-only lifecycle", + frozenset({Substrate.SSH, Substrate.SLURM}), + 40, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "live-capture", + "Extended live-data collection and reporting", + frozenset({Substrate.SSH, Substrate.SLURM}), + 50, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "slurm-cartesian", + "Representative multidimensional Slurm sweep", + frozenset({Substrate.SLURM}), + 60, + ), + Scenario( + "ssh-single-big-file", + "Cooperative and staged single-file SSH workloads", + frozenset({Substrate.SSH}), + 70, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "ssh-weighted-roots", + "Weighted-root SSH workload with active sizing", + frozenset({Substrate.SSH}), + 80, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "ssh-shared-home", + "SSH deployment and sweep using a shared worker home", + frozenset({Substrate.SSH}), + 90, + phase=SchedulePhase.SHARED_HOME, + ssh_home_mode=SshHomeMode.SHARED, + ), + Scenario( + "slurm-scheduling", + "Slurm allocation, include/exclude, and exclusive-user behavior", + frozenset({Substrate.SLURM}), + 100, + phase=SchedulePhase.AFTER_SHARED_HOME, + ), +) + +SCENARIOS = SCENARIO_CATALOG + + +def _concrete_substrates(substrate: Substrate) -> frozenset[Substrate]: + """Return concrete substrates selected by *substrate*.""" + if substrate is Substrate.ALL: + return frozenset({Substrate.SSH, Substrate.SLURM}) + return frozenset({substrate}) + + +def _validate_scenario(scenario: Scenario) -> None: + """Validate one registry entry.""" + if not scenario.name or scenario.name.strip() != scenario.name: + raise ScenarioPlanningError(f"invalid scenario name: {scenario.name!r}") + if not scenario.substrates or Substrate.ALL in scenario.substrates: + raise ScenarioPlanningError( + f"scenario {scenario.name!r} must declare concrete substrates" + ) + supports_ssh = Substrate.SSH in scenario.substrates + if supports_ssh != (scenario.ssh_home_mode is not None): + raise ScenarioPlanningError( + f"scenario {scenario.name!r} must declare an SSH home mode exactly " + "when it supports SSH" + ) + if scenario.ssh_home_mode is SshHomeMode.SHARED: + if scenario.substrates != frozenset({Substrate.SSH}): + raise ScenarioPlanningError( + f"shared-home scenario {scenario.name!r} must be SSH-only" + ) + if scenario.phase is not SchedulePhase.SHARED_HOME: + raise ScenarioPlanningError( + f"shared-home scenario {scenario.name!r} must use the shared phase" + ) + elif supports_ssh and scenario.phase is not SchedulePhase.BEFORE_SHARED_HOME: + raise ScenarioPlanningError( + f"separate-home scenario {scenario.name!r} must run before shared home" + ) + elif scenario.phase is SchedulePhase.SHARED_HOME: + raise ScenarioPlanningError( + f"scenario {scenario.name!r} cannot enter the shared phase" + ) + + +def _registry_by_name(registry: Iterable[Scenario]) -> dict[str, Scenario]: + """Validate *registry* and index it by scenario name.""" + indexed: dict[str, Scenario] = {} + for scenario in registry: + _validate_scenario(scenario) + if scenario.name in indexed: + raise ScenarioPlanningError( + f"duplicate scenario in registry: {scenario.name!r}" + ) + indexed[scenario.name] = scenario + return indexed + + +def _requested_names(requested: Iterable[str]) -> tuple[str, ...]: + """Validate requested scenario names while preserving diagnostics.""" + names = tuple(requested) + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ScenarioPlanningError( + "duplicate requested scenario(s): " + ", ".join(duplicates) + ) + return names + + +def select_scenarios( + *, + substrate: Substrate | str = Substrate.ALL, + requested: Iterable[str] = (), + registry: Iterable[Scenario] = SCENARIOS, +) -> tuple[Scenario, ...]: + """Return selected scenarios in deterministic metadata order.""" + try: + selected_substrate = Substrate(substrate) + except ValueError as error: + choices = ", ".join(item.value for item in Substrate) + raise ScenarioPlanningError( + f"unknown substrate {substrate!r}; choose one of: {choices}" + ) from error + indexed = _registry_by_name(registry) + names = _requested_names(requested) + unknown = sorted(set(names) - set(indexed)) + if unknown: + raise ScenarioPlanningError("unknown scenario(s): " + ", ".join(unknown)) + candidates = [indexed[name] for name in names] if names else list(indexed.values()) + concrete = _concrete_substrates(selected_substrate) + incompatible = [ + scenario.name + for scenario in candidates + if names and scenario.substrates.isdisjoint(concrete) + ] + if incompatible: + raise ScenarioPlanningError( + f"scenario(s) incompatible with {selected_substrate.value}: " + + ", ".join(sorted(incompatible)) + ) + selected = [ + scenario + for scenario in candidates + if not scenario.substrates.isdisjoint(concrete) + ] + return tuple(sorted(selected, key=lambda item: (item.phase, item.order, item.name))) + + +def _work_items( + scenarios: Iterable[Scenario], substrate: Substrate +) -> tuple[WorkItem, ...]: + """Expand scenarios across selected concrete substrates.""" + selected_substrates = _concrete_substrates(substrate) + items = [ + WorkItem(scenario, concrete) + for scenario in scenarios + for concrete in scenario.substrates & selected_substrates + ] + return tuple( + sorted( + items, + key=lambda item: ( + item.scenario.phase, + item.scenario.order, + item.scenario.name, + item.substrate.value, + ), + ) + ) + + +def plan_scenarios( + *, + substrate: Substrate | str = Substrate.ALL, + requested: Iterable[str] = (), + registry: Iterable[Scenario] = SCENARIOS, +) -> tuple[PlanStep, ...]: + """Build a deterministic execution plan with one shared-home batch.""" + scenarios = select_scenarios( + substrate=substrate, requested=requested, registry=registry + ) + selected_substrate = Substrate(substrate) + items = _work_items(scenarios, selected_substrate) + before = [ + item + for item in items + if item.scenario.phase is SchedulePhase.BEFORE_SHARED_HOME + ] + shared = [ + item for item in items if item.scenario.phase is SchedulePhase.SHARED_HOME + ] + after = [ + item for item in items if item.scenario.phase is SchedulePhase.AFTER_SHARED_HOME + ] + steps: list[PlanStep] = list(before) + if shared: + steps.append(SshHomeTransition(SshHomeMode.SHARED)) + steps.extend(shared) + steps.append(SshHomeTransition(SshHomeMode.SEPARATE)) + steps.extend(after) + return tuple(steps) + + +def scenario_metadata( + registry: Iterable[Scenario] = SCENARIOS, +) -> tuple[dict[str, object], ...]: + """Return deterministic machine-readable metadata for scenario listing.""" + indexed = _registry_by_name(registry) + return tuple( + scenario.metadata() + for scenario in sorted( + indexed.values(), key=lambda item: (item.phase, item.order, item.name) + ) + ) + + +def format_scenario_listing(registry: Iterable[Scenario] = SCENARIOS) -> str: + """Format a concise human-readable scenario listing.""" + indexed = _registry_by_name(registry) + scenarios = sorted( + indexed.values(), key=lambda item: (item.phase, item.order, item.name) + ) + return "\n".join( + f"{scenario.name}\t{','.join(sorted(item.value for item in scenario.substrates))}" + f"\t{scenario.description}" + for scenario in scenarios + ) diff --git a/integration-tests/lib/ssh_home_transition.py b/integration-tests/lib/ssh_home_transition.py new file mode 100644 index 0000000..9bdd607 --- /dev/null +++ b/integration-tests/lib/ssh_home_transition.py @@ -0,0 +1,465 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Crash-recoverable state for temporary SSH shared-home transitions.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Callable, Mapping +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +CANONICAL_HOME_MODE = "separate" +SHARED_HOME_MODE = "shared" +UNKNOWN_HOME_MODE = "unknown" +STATE_FILENAME = "ssh-home-transition.json" +STATE_SCHEMA = 1 +EXPECTED_POD_COUNT = 2 +MAX_FAILURE_DIAGNOSTICS = 8000 + +HomeMode = Literal["separate", "shared"] +PoolMode = Literal["separate", "shared", "unknown"] +InspectPool = Callable[[], "PoolObservation"] +ReconcilePool = Callable[[HomeMode, str], None] +Clock = Callable[[], str] + +_PHASES = { + "applying-shared", + "shared-ready", + "restoring-separate", + "unhealthy", +} +_HOME_MODES = {CANONICAL_HOME_MODE, SHARED_HOME_MODE} +_POOL_MODES = {*_HOME_MODES, UNKNOWN_HOME_MODE} + + +class SshHomeTransitionError(RuntimeError): + """An actionable SSH home-mode transition failure.""" + + +@dataclass(frozen=True) +class PoolObservation: + """Relevant live state of the SSH worker StatefulSet and its pods.""" + + home_mode: str + configuration_checksum: str + ready_nonterminating_pods: int + terminating_pods: int + diagnostics: str = "" + + def matches(self, mode: HomeMode, checksum: str) -> bool: + """Return whether the live pool is ready in the requested form.""" + return ( + self.home_mode == mode + and self.configuration_checksum == checksum + and self.ready_nonterminating_pods == EXPECTED_POD_COUNT + and self.terminating_pods == 0 + ) + + +@dataclass(frozen=True) +class TransitionState: + """Persistent identity and progress for one SSH pool transition.""" + + schema: int + run_id: str + scenario_id: str + prior_mode: PoolMode + requested_mode: HomeMode + expected_statefulset_checksum: str + phase: str + updated_at: str + failure_diagnostics: str = "" + + @classmethod + def from_document(cls, document: Mapping[str, Any]) -> "TransitionState": + """Validate and decode a persistent transition document.""" + required = { + "schema", + "run_id", + "scenario_id", + "prior_mode", + "requested_mode", + "expected_statefulset_checksum", + "phase", + "updated_at", + "failure_diagnostics", + } + if set(document) != required: + raise SshHomeTransitionError( + "invalid SSH home transition state fields: " + f"expected {sorted(required)}, found {sorted(document)}" + ) + state = cls(**document) + state.validate() + return state + + def validate(self) -> None: + """Reject a state document that cannot safely drive reconciliation.""" + if self.schema != STATE_SCHEMA: + raise SshHomeTransitionError( + f"unsupported SSH home transition schema: {self.schema!r}" + ) + if self.prior_mode not in _POOL_MODES or self.requested_mode not in _HOME_MODES: + raise SshHomeTransitionError("invalid SSH home mode in transition state") + if self.phase not in _PHASES: + raise SshHomeTransitionError( + f"invalid SSH home transition phase: {self.phase!r}" + ) + text_fields = { + "run_id": self.run_id, + "scenario_id": self.scenario_id, + "expected_statefulset_checksum": self.expected_statefulset_checksum, + "updated_at": self.updated_at, + } + empty = [name for name, value in text_fields.items() if not _valid_text(value)] + if empty: + raise SshHomeTransitionError( + "empty or invalid SSH home transition fields: " + ", ".join(empty) + ) + + +class TransitionStore: + """Atomically persist the SSH transition record in an existing state dir.""" + + def __init__(self, state_dir: Path): + self.state_dir = state_dir + self.path = state_dir / STATE_FILENAME + + def load(self) -> TransitionState | None: + """Return validated transition state, if present.""" + if not self.path.exists(): + return None + try: + document = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SshHomeTransitionError( + f"cannot read SSH home transition state {self.path}: {error}" + ) from error + if not isinstance(document, dict): + raise SshHomeTransitionError( + f"invalid SSH home transition state in {self.path}: expected object" + ) + try: + return TransitionState.from_document(document) + except (TypeError, SshHomeTransitionError) as error: + raise SshHomeTransitionError( + f"invalid SSH home transition state in {self.path}: {error}" + ) from error + + def save(self, state: TransitionState) -> None: + """Atomically replace the transition document.""" + state.validate() + self._require_state_dir() + payload = json.dumps(asdict(state), indent=2, sort_keys=True) + "\n" + descriptor, temporary_name = tempfile.mkstemp( + dir=self.state_dir, prefix=f".{STATE_FILENAME}." + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.path) + _sync_directory(self.state_dir) + finally: + temporary.unlink(missing_ok=True) + + def clear(self) -> None: + """Remove a completed transition record durably.""" + if not self.path.exists(): + return + self.path.unlink() + _sync_directory(self.state_dir) + + def _require_state_dir(self) -> None: + """Refuse to create or adopt lifecycle state implicitly.""" + if not self.state_dir.is_dir(): + raise SshHomeTransitionError( + f"integration state directory is absent: {self.state_dir}" + ) + + +class SshHomeTransitionManager: + """Drive and recover SSH home transitions through injected live hooks. + + ``reconcile_pool`` must apply the requested StatefulSet form, wait for two + ready nonterminating pods, regenerate known-host data, and validate SSH and + home semantics. Callers remain responsible for serializing fixture actions. + """ + + def __init__( + self, + state_dir: Path, + inspect_pool: InspectPool, + reconcile_pool: ReconcilePool, + *, + clock: Clock | None = None, + ): + self.store = TransitionStore(state_dir) + self.inspect_pool = inspect_pool + self.reconcile_pool = reconcile_pool + self.clock = clock or _utc_now + + def preflight( + self, + separate_checksum: str, + *, + run_id: str, + scenario_id: str, + ) -> PoolObservation: + """Ensure canonical separate-home state before an SSH setup or test.""" + retained = self.store.load() + try: + observed = self.inspect_pool() + except Exception as error: + self._save_unhealthy( + retained, + run_id=run_id, + scenario_id=scenario_id, + prior_mode=UNKNOWN_HOME_MODE, + requested_mode=CANONICAL_HOME_MODE, + checksum=separate_checksum, + error=error, + ) + raise self._blocked_error(error) from error + if retained is None and observed.matches( + CANONICAL_HOME_MODE, separate_checksum + ): + return observed + identity = retained or _new_state( + run_id, + scenario_id, + _known_mode(observed.home_mode), + CANONICAL_HOME_MODE, + separate_checksum, + "restoring-separate", + self.clock(), + ) + return self._reconcile_separate(identity, separate_checksum) + + def enter_shared( + self, + *, + run_id: str, + scenario_id: str, + separate_checksum: str, + shared_checksum: str, + ) -> PoolObservation: + """Enter shared-home mode after first recovering canonical state.""" + self.preflight(separate_checksum, run_id=run_id, scenario_id=scenario_id) + state = _new_state( + run_id, + scenario_id, + CANONICAL_HOME_MODE, + SHARED_HOME_MODE, + shared_checksum, + "applying-shared", + self.clock(), + ) + self.store.save(state) + try: + self.reconcile_pool(SHARED_HOME_MODE, shared_checksum) + observed = self.inspect_pool() + _require_observation(observed, SHARED_HOME_MODE, shared_checksum) + except Exception as error: + self._save_unhealthy_from_state(state, error) + raise self._blocked_error(error) from error + self.store.save(replace(state, phase="shared-ready", updated_at=self.clock())) + return observed + + def restore_separate( + self, + separate_checksum: str, + *, + run_id: str, + scenario_id: str, + ) -> PoolObservation: + """Restore canonical state after the complete shared-home batch.""" + retained = self.store.load() + identity = retained or _new_state( + run_id, + scenario_id, + SHARED_HOME_MODE, + CANONICAL_HOME_MODE, + separate_checksum, + "restoring-separate", + self.clock(), + ) + return self._reconcile_separate(identity, separate_checksum) + + def _reconcile_separate( + self, identity: TransitionState, checksum: str + ) -> PoolObservation: + state = replace( + identity, + requested_mode=CANONICAL_HOME_MODE, + expected_statefulset_checksum=checksum, + phase="restoring-separate", + updated_at=self.clock(), + failure_diagnostics="", + ) + self.store.save(state) + try: + self.reconcile_pool(CANONICAL_HOME_MODE, checksum) + observed = self.inspect_pool() + _require_observation(observed, CANONICAL_HOME_MODE, checksum) + except Exception as error: + self._save_unhealthy_from_state(state, error) + raise self._blocked_error(error) from error + self.store.clear() + return observed + + def _save_unhealthy_from_state( + self, state: TransitionState, error: Exception + ) -> None: + self.store.save( + replace( + state, + phase="unhealthy", + updated_at=self.clock(), + failure_diagnostics=_failure_text(error), + ) + ) + + def _save_unhealthy( + self, + retained: TransitionState | None, + *, + run_id: str, + scenario_id: str, + prior_mode: PoolMode, + requested_mode: HomeMode, + checksum: str, + error: Exception, + ) -> None: + state = ( + replace( + retained, + requested_mode=requested_mode, + expected_statefulset_checksum=checksum, + ) + if retained + else _new_state( + run_id, + scenario_id, + prior_mode, + requested_mode, + checksum, + "unhealthy", + self.clock(), + ) + ) + self._save_unhealthy_from_state(state, error) + + def _blocked_error(self, error: Exception) -> SshHomeTransitionError: + return SshHomeTransitionError( + "SSH worker pool reconciliation failed; further SSH scenarios are " + f"blocked. Retry setup to reconcile separate-home mode or teardown " + f"the fixture. State and diagnostics: {self.store.path}: {error}" + ) + + +def statefulset_checksum(manifest: str | bytes) -> str: + """Return a stable digest for one rendered StatefulSet configuration.""" + data = manifest.encode("utf-8") if isinstance(manifest, str) else manifest + return hashlib.sha256(data).hexdigest() + + +def _new_state( + run_id: str, + scenario_id: str, + prior_mode: PoolMode, + requested_mode: HomeMode, + checksum: str, + phase: str, + updated_at: str, +) -> TransitionState: + """Build and validate one transition state value.""" + state = TransitionState( + schema=STATE_SCHEMA, + run_id=run_id, + scenario_id=scenario_id, + prior_mode=prior_mode, + requested_mode=requested_mode, + expected_statefulset_checksum=checksum, + phase=phase, + updated_at=updated_at, + ) + state.validate() + return state + + +def _known_mode(value: str) -> PoolMode: + """Use a safe prior mode when live inspection finds a partial rollout.""" + if value == CANONICAL_HOME_MODE: + return CANONICAL_HOME_MODE + if value == SHARED_HOME_MODE: + return SHARED_HOME_MODE + return UNKNOWN_HOME_MODE + + +def _require_observation( + observed: PoolObservation, mode: HomeMode, checksum: str +) -> None: + """Require reconciliation to produce the exact healthy target form.""" + if observed.matches(mode, checksum): + return + detail = f"; {observed.diagnostics}" if observed.diagnostics else "" + raise SshHomeTransitionError( + "SSH worker pool did not reach the requested state: " + f"mode={observed.home_mode!r}, checksum=" + f"{observed.configuration_checksum!r}, ready=" + f"{observed.ready_nonterminating_pods}, terminating=" + f"{observed.terminating_pods}{detail}" + ) + + +def _failure_text(error: Exception) -> str: + """Return bounded persistent diagnostics for a reconciliation failure.""" + text = f"{type(error).__name__}: {error}".strip() + return text[-MAX_FAILURE_DIAGNOSTICS:] + + +def _valid_text(value: object) -> bool: + """Return whether a required document field is a nonempty string.""" + return isinstance(value, str) and bool(value.strip()) + + +def _utc_now() -> str: + """Return an ISO-8601 timestamp for persistent diagnostics.""" + return datetime.now(timezone.utc).isoformat() + + +def _sync_directory(directory: Path) -> None: + """Best-effort fsync of a directory after atomic state mutation.""" + try: + descriptor = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) diff --git a/integration-tests/manifests/kind-sbx-shared.yaml.tmpl b/integration-tests/manifests/kind-sbx-shared.yaml.tmpl new file mode 100644 index 0000000..4e0aade --- /dev/null +++ b/integration-tests/manifests/kind-sbx-shared.yaml.tmpl @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: @@CLUSTER_NAME@@ +nodes: + - role: control-plane + labels: + storage-scale-test/login: "true" + extraMounts: + - hostPath: @@SHARED_ROOT@@ + containerPath: /var/local/storage-scale-shared +@@KMSG_MOUNT@@ + - role: worker + labels: + storage-scale-test/target: "true" + extraMounts: + - hostPath: @@SHARED_ROOT@@ + containerPath: /var/local/storage-scale-shared +@@KMSG_MOUNT@@ + - role: worker + labels: + storage-scale-test/target: "true" + extraMounts: + - hostPath: @@SHARED_ROOT@@ + containerPath: /var/local/storage-scale-shared +@@KMSG_MOUNT@@ diff --git a/integration-tests/manifests/kind.yaml.tmpl b/integration-tests/manifests/kind.yaml.tmpl new file mode 100644 index 0000000..6f73886 --- /dev/null +++ b/integration-tests/manifests/kind.yaml.tmpl @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: @@CLUSTER_NAME@@ +nodes: + - role: control-plane + labels: + storage-scale-test/login: "true" + - role: worker + labels: + storage-scale-test/target: "true" + - role: worker + labels: + storage-scale-test/target: "true" diff --git a/integration-tests/manifests/mariadb-accounting.yaml.tmpl b/integration-tests/manifests/mariadb-accounting.yaml.tmpl new file mode 100644 index 0000000..98deed2 --- /dev/null +++ b/integration-tests/manifests/mariadb-accounting.yaml.tmpl @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-accounting-config + namespace: @@NAMESPACE@@ +data: + integration.cnf: | + [mariadb] + bind-address=0.0.0.0 + default_storage_engine=InnoDB + binlog_format=row + innodb_autoinc_lock_mode=2 + innodb_buffer_pool_size=128M + innodb_lock_wait_timeout=120 + max_allowed_packet=64M +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb + namespace: @@NAMESPACE@@ +spec: + clusterIP: None + selector: + app.kubernetes.io/name: mariadb-accounting + ports: + - name: mysql + port: 3306 + targetPort: mysql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mariadb-accounting + namespace: @@NAMESPACE@@ +spec: + serviceName: mariadb + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mariadb-accounting + template: + metadata: + labels: + app.kubernetes.io/name: mariadb-accounting + spec: + nodeSelector: + storage-scale-test/login: "true" + containers: + - name: mariadb + image: @@MARIADB_IMAGE@@ + imagePullPolicy: Never + ports: + - name: mysql + containerPort: 3306 + env: + - name: MARIADB_DATABASE + value: slurm_acct_db + - name: MARIADB_USER + value: slurm + - name: MARIADB_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-password + key: password + - name: MARIADB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-password + key: root-password + readinessProbe: + exec: + command: ["healthcheck.sh", "--connect", "--innodb_initialized"] + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: ["healthcheck.sh", "--connect", "--innodb_initialized"] + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: data + mountPath: /var/lib/mysql + - name: config + mountPath: /etc/mysql/conf.d/integration.cnf + subPath: integration.cnf + volumes: + - name: config + configMap: + name: mariadb-accounting-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: standard + resources: + requests: + storage: 1Gi diff --git a/integration-tests/manifests/nfs-csi-values.yaml b/integration-tests/manifests/nfs-csi-values.yaml new file mode 100644 index 0000000..cbed0e4 --- /dev/null +++ b/integration-tests/manifests/nfs-csi-values.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +controller: + replicas: 1 + runOnControlPlane: true + enableSnapshotter: false + +externalSnapshotter: + enabled: false + +feature: + enableFSGroupPolicy: true + +node: + maxUnavailable: 1 diff --git a/integration-tests/manifests/nfs-storage.yaml.tmpl b/integration-tests/manifests/nfs-storage.yaml.tmpl new file mode 100644 index 0000000..385052d --- /dev/null +++ b/integration-tests/manifests/nfs-storage.yaml.tmpl @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: storage-scale-nfs-csi +provisioner: nfs.csi.k8s.io +parameters: + server: @@NFS_SERVER@@ + share: / + subDir: ${pvc.metadata.namespace}/${pvc.metadata.name} + mountPermissions: "0770" + onDelete: retain +reclaimPolicy: Retain +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - nfsvers=4.1 + - hard + - timeo=600 + - retrans=2 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-test-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: storage-scale-nfs-csi + resources: + requests: + storage: @@STORAGE_TEST_CAPACITY@@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssh-home-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: storage-scale-nfs-csi + resources: + requests: + storage: @@SSH_HOME_CAPACITY@@ diff --git a/integration-tests/manifests/sbx-storage.yaml.tmpl b/integration-tests/manifests/sbx-storage.yaml.tmpl new file mode 100644 index 0000000..48cd3ba --- /dev/null +++ b/integration-tests/manifests/sbx-storage.yaml.tmpl @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: sbx-shared +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: Immediate +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage-scale-sbx-data +spec: + capacity: + storage: @@STORAGE_TEST_CAPACITY@@ + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: sbx-shared + claimRef: + namespace: @@NAMESPACE@@ + name: storage-test-rwx + hostPath: + path: /var/local/storage-scale-shared/storage-test + type: Directory +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-test-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: sbx-shared + volumeName: storage-scale-sbx-data + resources: + requests: + storage: @@STORAGE_TEST_CAPACITY@@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage-scale-sbx-home +spec: + capacity: + storage: @@SSH_HOME_CAPACITY@@ + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: sbx-shared + claimRef: + namespace: @@NAMESPACE@@ + name: ssh-home-rwx + hostPath: + path: /var/local/storage-scale-shared/ssh-home + type: Directory +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssh-home-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: sbx-shared + volumeName: storage-scale-sbx-home + resources: + requests: + storage: @@SSH_HOME_CAPACITY@@ diff --git a/integration-tests/manifests/slinky-operator-values.yaml b/integration-tests/manifests/slinky-operator-values.yaml new file mode 100644 index 0000000..843f55e --- /dev/null +++ b/integration-tests/manifests/slinky-operator-values.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +certManager: + enabled: false + +operator: + accountingWorkers: 1 + controllerWorkers: 1 + loginsetWorkers: 1 + nodesetWorkers: 1 + restapiWorkers: 1 + tokenWorkers: 1 + slurmclientWorkers: 1 + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + cpu: 300m + memory: 256Mi + +webhook: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi diff --git a/integration-tests/manifests/slinky-slurm-values.yaml.tmpl b/integration-tests/manifests/slinky-slurm-values.yaml.tmpl new file mode 100644 index 0000000..9e22190 --- /dev/null +++ b/integration-tests/manifests/slinky-slurm-values.yaml.tmpl @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +controller: + persistence: + enabled: false + slurmctld: + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + reconfigure: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + logfile: + image: + repository: @@SLINKY_HELPER_IMAGE_REPOSITORY@@ + tag: "@@SLINKY_HELPER_IMAGE_TAG@@" + digest: null + imagePullPolicy: Never + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + +restapi: + slurmrestd: + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + +accounting: + enabled: true + slurmdbd: + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + podSpec: + nodeSelector: + storage-scale-test/login: "true" + +loginsets: + test: + replicas: 1 + login: + image: + repository: storage-scale-integration-login + tag: slinky-26.05-user + digest: null + imagePullPolicy: Never + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: shared-storage + mountPath: /mnt/storage-test + initconf: + image: + repository: @@SLINKY_HELPER_IMAGE_REPOSITORY@@ + tag: "@@SLINKY_HELPER_IMAGE_TAG@@" + digest: null + imagePullPolicy: Never + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + podSpec: + nodeSelector: + storage-scale-test/login: "true" + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: storage-test-rwx + service: + spec: + type: ClusterIP + +nodesets: + storage: + scalingMode: DaemonSet + workloadDisruptionProtection: false + slurmd: + image: + repository: storage-scale-integration-slurmd + tag: slinky-26.05-user + digest: null + imagePullPolicy: Never + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 768Mi + volumeMounts: + - name: shared-storage + mountPath: /mnt/storage-test + logfile: + image: + repository: @@SLINKY_HELPER_IMAGE_REPOSITORY@@ + tag: "@@SLINKY_HELPER_IMAGE_TAG@@" + digest: null + imagePullPolicy: Never + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + podSpec: + nodeSelector: + storage-scale-test/target: "true" + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: storage-test-rwx + +partitions: + all: + enabled: true + nodesets: + - storage + configMap: + Default: "YES" + MaxTime: UNLIMITED diff --git a/integration-tests/manifests/ssh-workers.yaml.tmpl b/integration-tests/manifests/ssh-workers.yaml.tmpl new file mode 100644 index 0000000..17f0c4e --- /dev/null +++ b/integration-tests/manifests/ssh-workers.yaml.tmpl @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: ssh-workers + namespace: @@NAMESPACE@@ +spec: + clusterIP: None + selector: + app.kubernetes.io/name: storage-ssh-worker + ports: + - name: ssh + port: 22 + targetPort: ssh +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ssh-worker + namespace: @@NAMESPACE@@ + annotations: + storage-scale-test/ssh-home-mode: "@@SSH_HOME_MODE@@" + storage-scale-test/ssh-config-checksum: "@@SSH_CONFIG_CHECKSUM@@" +spec: + serviceName: ssh-workers + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: storage-ssh-worker + template: + metadata: + labels: + app.kubernetes.io/name: storage-ssh-worker + annotations: + storage-scale-test/ssh-home-mode: "@@SSH_HOME_MODE@@" + storage-scale-test/ssh-config-checksum: "@@SSH_CONFIG_CHECKSUM@@" + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + storage-scale-test/target: "true" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: storage-ssh-worker + topologyKey: kubernetes.io/hostname + initContainers: + - name: install-ssh-identity + image: storage-scale-integration-ssh:ubuntu-24.04 + imagePullPolicy: Never + command: + - sh + - -ec + - | + chown 2000:2000 /home/tester + chmod 0700 /home/tester + install -d -o 2000 -g 2000 -m 0700 /home/tester/.ssh + install -o 2000 -g 2000 -m 0600 /ssh-secret/id_ed25519 /home/tester/.ssh/id_ed25519 + install -o 2000 -g 2000 -m 0600 /ssh-secret/authorized_keys /home/tester/.ssh/authorized_keys + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: ssh-secret + mountPath: /ssh-secret + readOnly: true + - name: ssh-home + mountPath: /home/tester + containers: + - name: sshd + image: storage-scale-integration-ssh:ubuntu-24.04 + imagePullPolicy: Never + ports: + - name: ssh + containerPort: 22 + readinessProbe: + tcpSocket: + port: ssh + initialDelaySeconds: 1 + periodSeconds: 2 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: ssh-home + mountPath: /home/tester + - name: shared-storage + mountPath: /mnt/storage-test + volumes: + - name: ssh-secret + secret: + secretName: storage-ssh-identity + defaultMode: 0600 + - name: ssh-home + @@SSH_HOME_VOLUME@@ + - name: shared-storage + persistentVolumeClaim: + claimName: storage-test-rwx diff --git a/integration-tests/slinky-login-image.Dockerfile b/integration-tests/slinky-login-image.Dockerfile new file mode 100644 index 0000000..f4d4645 --- /dev/null +++ b/integration-tests/slinky-login-image.Dockerfile @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + file \ + && groupadd --gid 2000 storage-test \ + && useradd --uid 2000 --gid 2000 --create-home --shell /bin/bash tester \ + && rm -rf /var/lib/apt/lists/* diff --git a/integration-tests/slinky-slurmd-image.Dockerfile b/integration-tests/slinky-slurmd-image.Dockerfile new file mode 100644 index 0000000..d943bf8 --- /dev/null +++ b/integration-tests/slinky-slurmd-image.Dockerfile @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN groupadd --gid 2000 storage-test \ + && useradd --uid 2000 --gid 2000 --create-home --shell /bin/bash tester diff --git a/integration-tests/ssh-image.Dockerfile b/integration-tests/ssh-image.Dockerfile new file mode 100644 index 0000000..e48ae61 --- /dev/null +++ b/integration-tests/ssh-image.Dockerfile @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + coreutils \ + file \ + findutils \ + gawk \ + gzip \ + iproute2 \ + openssh-client \ + openssh-server \ + procps \ + psmisc \ + tar \ + util-linux \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 2000 storage-test \ + && useradd --uid 2000 --gid 2000 --create-home --shell /bin/bash tester \ + && install -d -m 0755 /run/sshd \ + && printf '%s\n' \ + 'PasswordAuthentication no' \ + 'PermitRootLogin no' \ + 'PubkeyAuthentication yes' \ + 'AllowUsers tester' \ + >> /etc/ssh/sshd_config \ + && rm -f /etc/ssh/ssh_host_* + +EXPOSE 22 +CMD ["bash", "-c", "ssh-keygen -A && exec /usr/sbin/sshd -D -e"] diff --git a/lib/_elbencho_functions.sh b/lib/_elbencho_functions.sh index 37e3e78..ba08d87 100644 --- a/lib/_elbencho_functions.sh +++ b/lib/_elbencho_functions.sh @@ -238,6 +238,13 @@ kill_elbencho_by_pid() { kill_all_elbencho_processes() { local port="${1:-1611}" + # A launcher or dynamic loader can give the service process a name other + # than "elbencho". Prefer the exact listening port when fuser is present, + # then retain the name-based cleanup for other processes and platforms. + if type -P fuser >/dev/null 2>&1; then + fuser -k "${port}/tcp" >/dev/null 2>&1 || true + fi + # Determine kill tool (prefer pkill over killall) local kill_tool="" if type -P pkill >/dev/null 2>&1; then @@ -1458,7 +1465,8 @@ _elbencho_workload_update_failure_cleanup_state() { _elbencho_workload_write } -# Parse elbencho newline-delimited JSON without a runtime JSON dependency. +# Parse elbencho streamed JSON without a runtime JSON dependency. Releases may +# delimit top-level phase objects with newlines or write them adjacently. # Prints canonical "entriesbytes-or-nullelapsed-ms" for one phase. # WRITE/READ require bytes; RMFILES must not contain a bytes counter. _elbencho_parse_phase_json() { @@ -1558,19 +1566,23 @@ _elbencho_parse_phase_json() { { if ($0 ~ /^[ \t\r]*$/) { fail(); next } s = $0; p = 1; n = length(s); records++ - phase_count = last_count = entries_count = bytes_count = elapsed_count = 0 - phase = entries = bytes = elapsed = "" - ws(); object("top"); ws() - if (bad || p <= n || phase_count != 1) { fail(); next } - if (phase == expected) { - matches++ - if (last_count != 1 || entries_count != 1 || elapsed_count != 1) fail() - if (expected == "RMFILES" && bytes_count != 0) fail() - if (expected != "RMFILES" && bytes_count != 1) fail() - found_entries = entries - found_bytes = expected == "RMFILES" ? "null" : bytes - found_elapsed = elapsed - } else if (phase != "SYNC") fail() + ws() + while (p <= n && !bad) { + phase_count = last_count = entries_count = bytes_count = elapsed_count = 0 + phase = entries = bytes = elapsed = "" + object("top"); ws() + if (bad || phase_count != 1) { fail(); break } + if (phase == expected) { + matches++ + if (last_count != 1 || entries_count != 1 || elapsed_count != 1) fail() + if (expected == "RMFILES" && bytes_count != 0) fail() + if (expected != "RMFILES" && bytes_count != 1) fail() + found_entries = entries + found_bytes = expected == "RMFILES" ? "null" : bytes + found_elapsed = elapsed + } else if (phase != "SYNC") fail() + if (p <= n) records++ + } } END { if (records == 0 || matches != 1 || bad) exit 1 @@ -3482,11 +3494,13 @@ reify_all_elbencho_executions() { local sweep_write_no_read="$7" local sweep_read_from="$8" - local -a node_counts - if ! mapfile -t node_counts < <(parse_range_specification "$nodes_spec"); then + local node_counts_output="" + if ! node_counts_output=$(parse_range_specification "$nodes_spec"); then echo "Error: Invalid node specification: $nodes_spec" >&2 return 1 fi + local -a node_counts + mapfile -t node_counts <<<"$node_counts_output" if [[ ${#node_counts[@]} -eq 0 ]]; then echo "Error: Node specification produced no node counts" >&2 return 1 diff --git a/lib/env_base.sh b/lib/env_base.sh index 8881916..208be84 100644 --- a/lib/env_base.sh +++ b/lib/env_base.sh @@ -58,9 +58,10 @@ export PATH # If TEST_DIR is defined and TEST_DIRS is empty, use TEST_DIR as the only test directory if [[ -n "${TEST_DIR:-}" ]]; then # Check if TEST_DIRS is not defined or is empty - if [[ -z "${TEST_DIRS+x}" ]] || [[ ${#TEST_DIRS[@]} -eq 0 ]]; then + if ! declare -p TEST_DIRS &>/dev/null || [[ ${#TEST_DIRS[@]} -eq 0 ]]; then # Initialize TEST_DIRS as an associative array if not already defined - declare -A TEST_DIRS + unset TEST_DIRS + declare -gA TEST_DIRS # Set TEST_DIR as the only test directory with weight 1 TEST_DIRS["$TEST_DIR"]=1 fi diff --git a/lib/env_functions.sh b/lib/env_functions.sh index 0f8b4df..ed20e4c 100644 --- a/lib/env_functions.sh +++ b/lib/env_functions.sh @@ -547,6 +547,89 @@ tail_until_complete() { done } +_cancel_slurm_job_and_wait() { + local job_id=$1 + local attempt + [[ -n "$job_id" ]] || return 0 + scancel "$job_id" 2>/dev/null || true + for ((attempt = 1; attempt <= 30; attempt++)); do + local active + if ! active=$(squeue -h -j "$job_id" -o '%i'); then + echo "Error: unable to verify cancellation of Slurm job $job_id" >&2 + return 1 + fi + [[ -z "$active" ]] && return 0 + sleep 2 + done + echo "Error: Slurm job $job_id remained active after cancellation" >&2 + return 1 +} + +_slurm_cancel_active_dispatch_once() { + if [[ "${ELBENCHO_SLURM_CANCEL_ARMED:-0}" != 1 ]]; then + return 0 + fi + ELBENCHO_SLURM_CANCEL_ARMED=0 + _cancel_slurm_job_and_wait "${ELBENCHO_SLURM_ACTIVE_JOB_ID:-}" +} + +_slurm_restore_dispatch_traps() { + trap - EXIT INT TERM + [[ -n "${ELBENCHO_SLURM_OLD_EXIT_TRAP:-}" ]] \ + && eval "$ELBENCHO_SLURM_OLD_EXIT_TRAP" + [[ -n "${ELBENCHO_SLURM_OLD_INT_TRAP:-}" ]] \ + && eval "$ELBENCHO_SLURM_OLD_INT_TRAP" + [[ -n "${ELBENCHO_SLURM_OLD_TERM_TRAP:-}" ]] \ + && eval "$ELBENCHO_SLURM_OLD_TERM_TRAP" + ELBENCHO_SLURM_CANCEL_ARMED=0 + ELBENCHO_SLURM_TRAPS_INSTALLED=0 +} + +# shellcheck disable=SC2317,SC2329 # Invoked by the active dispatch EXIT trap. +_slurm_active_dispatch_exit() { + local original_rc=$? + local old_exit_trap="${ELBENCHO_SLURM_OLD_EXIT_TRAP:-}" + trap - EXIT INT TERM + _slurm_cancel_active_dispatch_once || true + if [[ -n "$old_exit_trap" ]]; then + (eval "$old_exit_trap"; exit "$original_rc") || true + fi + return "$original_rc" +} + +# shellcheck disable=SC2317,SC2329 # Invoked by active dispatch signal traps. +_slurm_active_dispatch_signal() { + local signal_name=$1 + local signal_rc=1 + [[ "$signal_name" == INT ]] && signal_rc=130 + [[ "$signal_name" == TERM ]] && signal_rc=143 + trap - EXIT INT TERM + _slurm_cancel_active_dispatch_once || true + _slurm_restore_dispatch_traps + exit "$signal_rc" +} + +_slurm_arm_dispatch_cleanup() { + local job_id=$1 + ELBENCHO_SLURM_OLD_EXIT_TRAP=$(trap -p EXIT) + ELBENCHO_SLURM_OLD_INT_TRAP=$(trap -p INT) + ELBENCHO_SLURM_OLD_TERM_TRAP=$(trap -p TERM) + ELBENCHO_SLURM_ACTIVE_JOB_ID=$job_id + ELBENCHO_SLURM_CANCEL_ARMED=1 + ELBENCHO_SLURM_TRAPS_INSTALLED=1 + trap '_slurm_active_dispatch_exit' EXIT + trap '_slurm_active_dispatch_signal INT' INT + trap '_slurm_active_dispatch_signal TERM' TERM +} + +_slurm_disarm_dispatch_cleanup() { + if [[ "${ELBENCHO_SLURM_TRAPS_INSTALLED:-0}" != 1 ]]; then + return 0 + fi + ELBENCHO_SLURM_CANCEL_ARMED=0 + _slurm_restore_dispatch_traps +} + # Monitor array of Slurm array jobs until completion # Args: Array of job IDs to monitor # Returns: 0 on success, 1 on failure @@ -1825,7 +1908,7 @@ coordinator_run_one_execution() { test_dirs_csv=$(_compute_test_dirs_csv_for_execution) || exit 1 # shellcheck disable=SC2154 # nodes, io_size, thread_count, io_depth come from sourcing NNNN.sh - _echo_ts "[coordinator] starting execution ${id}: nodes=${nodes} io_size=${io_size} threads=${thread_count} iodepth=${io_depth}" + _echo_ts "[coordinator] starting execution ${id}: nodes=${nodes} hosts=${first_n_hosts_csv} io_size=${io_size} threads=${thread_count} iodepth=${io_depth}" run_elbencho_io_sweep_iteration ) 2>&1 | tee "$log_file" execution_pipe_status=("${PIPESTATUS[@]}") @@ -2125,13 +2208,13 @@ dispatch_slurm_executions() { fi _elbencho_verify_no_active_slurm_coordinator "$executions_dir" || return 1 local dispatch_lock_token="" - local dispatch_lock_handed_off=0 + local dispatch_lock_release=1 _elbencho_acquire_dispatch_lock \ "$executions_dir" slurm-submit dispatch_lock_token || return 1 local rc=0 _dispatch_slurm_executions_owned \ - "$output_dir" "$dispatch_lock_token" dispatch_lock_handed_off || rc=$? - if [[ "$dispatch_lock_handed_off" -eq 0 ]]; then + "$output_dir" "$dispatch_lock_token" dispatch_lock_release || rc=$? + if [[ "$dispatch_lock_release" -eq 1 ]]; then _elbencho_release_dispatch_lock \ "$executions_dir" "$dispatch_lock_token" || rc=1 fi @@ -2139,12 +2222,12 @@ dispatch_slurm_executions() { } # Run Slurm dispatch after the public entry point has acquired exclusive -# ownership. The output variable reports whether ownership transferred to -# the job. +# ownership. The output variable reports whether the submitter may release +# its dispatch lock; it stays false after handoff or unverified cancellation. _dispatch_slurm_executions_owned() { local output_dir="$1" local dispatch_lock_token="$2" - local dispatch_lock_handed_off_var="$3" + local dispatch_lock_release_var="$3" local executions_dir="${output_dir}/executions" _elbencho_sweep_running_to_pending "$executions_dir" || return 1 @@ -2183,15 +2266,27 @@ _dispatch_slurm_executions_owned() { echo "Error: sbatch submission failed" >&2 return 1 fi + _slurm_arm_dispatch_cleanup "$job_id" if ! _elbencho_adopt_slurm_dispatch_lock \ "$executions_dir" "$dispatch_lock_token" "$job_id"; then echo "Error: unable to transfer dispatch lock to coordinator job $job_id; cancelling it" >&2 - scancel "$job_id" || true + local cancel_rc=0 + _slurm_cancel_active_dispatch_once || cancel_rc=$? + _slurm_disarm_dispatch_cleanup + if [[ "$cancel_rc" -ne 0 ]]; then + printf -v "$dispatch_lock_release_var" '%s' 0 + echo "Error: cancellation of Slurm job $job_id was not verified; retaining dispatch lock" >&2 + fi return 1 fi - printf -v "$dispatch_lock_handed_off_var" '%s' 1 - tail_until_complete "$job_id" "${log_files[@]}" - return $? + printf -v "$dispatch_lock_release_var" '%s' 0 + local monitor_rc=0 + tail_until_complete "$job_id" "${log_files[@]}" || monitor_rc=$? + if [[ "$monitor_rc" -ne 0 ]]; then + _slurm_cancel_active_dispatch_once || monitor_rc=1 + fi + _slurm_disarm_dispatch_cleanup + return "$monitor_rc" } # SSH dispatcher: sequential per-execution dispatch with fresh host selection. @@ -2223,7 +2318,7 @@ dispatch_ssh_executions() { # The nameref lets the entry point stop services before releasing the lock. _dispatch_ssh_executions_owned() { local output_dir="$1" - local -n ssh_services_started="$2" + local -n ssh_services_started_ref="$2" local executions_dir="${output_dir}/executions" _elbencho_sweep_running_to_pending "$executions_dir" || return 1 @@ -2248,7 +2343,8 @@ _dispatch_ssh_executions_owned() { echo "Error: failed to start elbencho services on any reachable SSH host" >&2 return 1 fi - ssh_services_started=1 + # shellcheck disable=SC2034 # Nameref assignment updates the caller's flag. + ssh_services_started_ref=1 local max_nodes if ! max_nodes=$(max_nodes_remaining_executions "$executions_dir"); then @@ -2323,7 +2419,7 @@ _ssh_fan_out_to_each_host() { } local rc=0 - local -a successful_hosts=() + local -a completed_hosts=() local spawn_output if ! spawn_output=$(spawn_N_ssh "$status_dir" true "$scriptlet"); then rc=1 @@ -2341,7 +2437,7 @@ _ssh_fan_out_to_each_host() { echo "Warning: SSH command failed on $hostname (rc=$ssh_rc)" >&2 rc=1 else - successful_hosts+=("$hostname") + completed_hosts+=("$hostname") fi done fi @@ -2354,7 +2450,7 @@ _ssh_fan_out_to_each_host() { if [[ -n "$successful_hosts_array_name" ]]; then local -n successful_hosts_ref="$successful_hosts_array_name" # shellcheck disable=SC2034 # Assignment intentionally updates the caller's named array - successful_hosts_ref=("${successful_hosts[@]}") + successful_hosts_ref=("${completed_hosts[@]}") fi return "$rc" } diff --git a/lib/reporting_common.py b/lib/reporting_common.py index d4c3008..da584c1 100644 --- a/lib/reporting_common.py +++ b/lib/reporting_common.py @@ -216,23 +216,27 @@ def histogram_axis_ranges( for x_values_iter, y_values_iter in series: x_values = list(x_values_iter) y_values = list(y_values_iter) - if x_values: - min_latency = min(min_latency, *x_values) - max_latency = max(max_latency, *x_values) + positive_latencies = [value for value in x_values if value > 0] + if positive_latencies: + min_latency = min(min_latency, *positive_latencies) + max_latency = max(max_latency, *positive_latencies) if y_values: nonzero_counts = [value for value in y_values if value > 0] if nonzero_counts: min_count = min(min_count, *nonzero_counts) max_count = max(max_count, *y_values) - min_latency = 0.1 if min_latency == float("inf") else min_latency + latency_data_found = min_latency != float("inf") + min_latency = 0.1 if not latency_data_found else min_latency max_latency = 1000.0 if max_latency == 0 else max_latency min_count = 1.0 if min_count == float("inf") else min_count max_count = 100.0 if max_count == 0 else max_count padding_factor = 0.1 return { - "min_latency": max(0.1, min_latency * (1 - padding_factor)), + "min_latency": ( + min_latency * (1 - padding_factor) if latency_data_found else min_latency + ), "max_latency": max_latency * (1 + padding_factor), "min_count": max(1.0, min_count * (1 - padding_factor)), "max_count": max_count * (1 + padding_factor), diff --git a/storage-tests/fs/nv-elbencho-sweep.sh b/storage-tests/fs/nv-elbencho-sweep.sh index 8a0e618..64d46d2 100755 --- a/storage-tests/fs/nv-elbencho-sweep.sh +++ b/storage-tests/fs/nv-elbencho-sweep.sh @@ -445,10 +445,12 @@ if [[ "${ELBENCHO_SINGLE_BIG_FILE:-0}" == "1" && "$rand_option" == "1" ]]; then fi # Parse --nodes specification into array -if ! mapfile -t node_counts < <(parse_range_specification "$nodes_spec"); then +node_counts_output="" +if ! node_counts_output=$(parse_range_specification "$nodes_spec"); then echo "Error: Invalid node specification: $nodes_spec" >&2 exit 1 fi +mapfile -t node_counts <<<"$node_counts_output" if [[ ${#node_counts[@]} -eq 0 ]]; then echo "Error: Node specification produced no node counts" >&2 exit 1 diff --git a/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh b/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh index 6ee5a38..b8c1ef3 100755 --- a/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh +++ b/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh @@ -94,9 +94,31 @@ _elbencho_adopt_slurm_dispatch_lock \ "$EXECUTIONS_DIR" "$DISPATCH_LOCK_TOKEN" "$SLURM_JOB_ID" || exit 1 COORDINATOR_LOCK_OWNED=1 -# Build the allocation IPv4 nodelist (same approach as the previous sbatch -# driver: some clusters intermittently fail to resolve short hostnames in -# elbencho's resolver, so we resolve once here and propagate as IPs). +# Build the allocation IPv4 nodelist. Slurm may canonicalize SLURM_JOB_NODELIST +# instead of preserving the configured include-list order, so restore that +# order when ORDER_NODES is active before choosing per-execution prefixes. +mapfile -t allocation_nodes < <( + scontrol show hostname "${SLURM_JOB_NODELIST:-}" +) +if [[ -n "${ORDER_NODES_ENABLED:-}" && ${#SLURM_ORDERED_NODES[@]} -gt 0 ]]; then + ordered_allocation_nodes=() + for ordered_node in "${SLURM_ORDERED_NODES[@]}"; do + for allocated_node in "${allocation_nodes[@]}"; do + if [[ "$ordered_node" == "$allocated_node" ]]; then + ordered_allocation_nodes+=("$allocated_node") + break + fi + done + done + if [[ ${#ordered_allocation_nodes[@]} -ne ${#allocation_nodes[@]} ]]; then + echo "Error: configured node order does not match the Slurm allocation" >&2 + exit 1 + fi + allocation_nodes=("${ordered_allocation_nodes[@]}") +fi + +# Some clusters intermittently fail to resolve short hostnames in elbencho's +# resolver, so resolve once here and propagate IPs. nodelist_ips=() while read -r node; do ip=$(getent ahostsv4 "$node" | awk '{print $1; exit}') @@ -105,7 +127,7 @@ while read -r node; do ip="$node" fi nodelist_ips+=("$ip") -done < <(scontrol show hostname "${SLURM_JOB_NODELIST:-}") +done < <(printf '%s\n' "${allocation_nodes[@]}") ALLOC_HOSTS_CSV=$(IFS=,; printf '%s' "${nodelist_ips[*]}") # nodelist_expanded_comma_separated is read by run_an_elbencho (a bash # function in this same shell scope) as a fallback when no per-execution @@ -133,26 +155,44 @@ _elbencho_sweep_running_to_pending "$EXECUTIONS_DIR" || exit 1 # Start elbencho services on every allocation node, in background, ONCE. SRUN_ELBENCHO_PID="" SRUN_ELBENCHO_PID_FILE="" +SRUN_ELBENCHO_LOG="${OUTPUT_DIR}/elbencho-svc-j${SLURM_JOB_ID}-portdefault.log" ACTIVE_EXECUTION_ID="" if [[ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]]; then stop_elbencho_services_srun "" || true # initial cleanup of stale processes - SRUN_ELBENCHO_PID=$(start_elbencho_services_srun "") - if [[ ! "$SRUN_ELBENCHO_PID" =~ ^[0-9]+$ ]]; then - echo "Error: unable to start elbencho service owner" >&2 - exit 1 - fi SRUN_ELBENCHO_PID_FILE="${EXECUTIONS_DIR}/.service-srun.pid" - if ! _atomic_write_sentinel "$SRUN_ELBENCHO_PID_FILE" "$SRUN_ELBENCHO_PID"; then - echo "Error: unable to record elbencho service owner PID" >&2 - stop_elbencho_services_srun "$SRUN_ELBENCHO_PID" || true - exit 1 - fi - if ! check_elbencho_services_srun ""; then - echo "Error: elbencho services failed initial health check" >&2 + for attempt in 1 2; do + SRUN_ELBENCHO_PID=$(start_elbencho_services_srun "" "$OUTPUT_DIR") + if [[ ! "$SRUN_ELBENCHO_PID" =~ ^[0-9]+$ ]]; then + echo "Error: unable to start elbencho service owner" >&2 + exit 1 + fi + if ! _atomic_write_sentinel \ + "$SRUN_ELBENCHO_PID_FILE" "$SRUN_ELBENCHO_PID"; then + echo "Error: unable to record elbencho service owner PID" >&2 + stop_elbencho_services_srun "$SRUN_ELBENCHO_PID" || true + exit 1 + fi + if check_elbencho_services_srun ""; then + break + fi + echo "Warning: elbencho services failed initial health check "\ + "(attempt $attempt/2)" >&2 + if [[ -f "$SRUN_ELBENCHO_LOG" ]]; then + echo "Service log from failed attempt $attempt:" >&2 + tail -n 200 -- "$SRUN_ELBENCHO_LOG" >&2 || true + cp -f -- "$SRUN_ELBENCHO_LOG" \ + "${SRUN_ELBENCHO_LOG}.attempt-${attempt}" || true + else + echo "Service log is missing: $SRUN_ELBENCHO_LOG" >&2 + fi stop_elbencho_services_srun "$SRUN_ELBENCHO_PID" || true rm -f -- "$SRUN_ELBENCHO_PID_FILE" - exit 1 - fi + if [[ "$attempt" -eq 2 ]]; then + echo "Error: elbencho services failed after one restart" >&2 + exit 1 + fi + echo "Retrying initial elbencho service startup once" >&2 + done fi # Refresh the coordinator's copy after a phase-level check restarts services diff --git a/tests/test_check_license_headers.py b/tests/test_check_license_headers.py index 85b4380..3f5bb5b 100644 --- a/tests/test_check_license_headers.py +++ b/tests/test_check_license_headers.py @@ -76,6 +76,15 @@ def test_reports_all_text_failures_and_skips_binary(self) -> None: self.assertEqual([path for path, _error in violations], paths[:2]) + def test_skips_tracked_path_deleted_from_working_tree(self) -> None: + """A pending tracked-file deletion is not a header violation.""" + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + + violations = find_violations([Path("deleted.md")], root) + + self.assertEqual(violations, []) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ci_integration_wrapper.py b/tests/test_ci_integration_wrapper.py new file mode 100644 index 0000000..494823f --- /dev/null +++ b/tests/test_ci_integration_wrapper.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fault-path tests for the bounded integration lifecycle wrapper.""" + +import os +import platform +import subprocess +import sys +import time +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_WRAPPER = _REPO_ROOT / "integration-tests" / "bin" / "ci-integration.sh" + + +def _fixture(tmp_path: Path) -> tuple[dict[str, str], Path]: + driver = tmp_path / "fake-driver.py" + driver.write_text( + """#!/usr/bin/env python3 +import os +import sys +import time +from pathlib import Path + +root = Path(__file__).parent +action = sys.argv[-1] +with (root / "calls.log").open("a", encoding="utf-8") as handle: + handle.write(f"{action} {os.getpid()}\\n") +if os.environ.get("INTEGRATION_EXPECT_ROOT_REJECTION"): + raise SystemExit(1) +if action == "test" and (root / "block-test").exists(): + (root / "test.pid").write_text(str(os.getpid()), encoding="utf-8") + time.sleep(60) +if action == "teardown" and (root / "fail-teardown").exists(): + raise SystemExit(9) +""", + encoding="utf-8", + ) + driver.chmod(0o755) + privilege = tmp_path / "fake-sudo" + privilege.write_text('#!/bin/sh\nexec "$@"\n', encoding="utf-8") + privilege.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "INTEGRATION_DRIVER": str(driver), + "INTEGRATION_STATE_DIR": str(tmp_path / "state"), + "INTEGRATION_DIAGNOSTICS": str(tmp_path / "diagnostics"), + "INTEGRATION_PYTHON": sys.executable, + "INTEGRATION_PRIVILEGE_COMMAND": str(privilege), + } + ) + return environment, driver.parent / "calls.log" + + +def test_timeout_signals_child_and_runs_teardown(tmp_path): + """A process-group timeout cannot orphan the active lifecycle child.""" + environment, calls = _fixture(tmp_path) + (tmp_path / "block-test").touch() + architecture = {"x86_64": "amd64", "aarch64": "arm64"}[platform.machine()] + + result = subprocess.run( + [ + "timeout", + "--kill-after=5s", + "1s", + _WRAPPER, + architecture, + "sbx-shared", + ], + cwd=_REPO_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + assert result.returncode == 124, result.stderr + records = calls.read_text(encoding="utf-8").splitlines() + assert sum(line.startswith("teardown ") for line in records) == 2 + child_pid = int((tmp_path / "test.pid").read_text(encoding="utf-8")) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + try: + os.kill(child_pid, 0) + except ProcessLookupError: + break + time.sleep(0.05) + else: + raise AssertionError(f"timed-out lifecycle child {child_pid} survived") + + +def test_teardown_failure_makes_successful_lifecycle_fail(tmp_path): + """The EXIT handler propagates cleanup failure over lifecycle success.""" + environment, calls = _fixture(tmp_path) + environment.pop("INTEGRATION_PYTHON") + (tmp_path / "fail-teardown").touch() + architecture = {"x86_64": "amd64", "aarch64": "arm64"}[platform.machine()] + + result = subprocess.run( + [_WRAPPER, architecture, "sbx-shared"], + cwd=_REPO_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + assert result.returncode != 0 + assert ( + sum( + line.startswith("teardown ") + for line in calls.read_text(encoding="utf-8").splitlines() + ) + == 2 + ) + + +def test_unwritable_diagnostics_do_not_prevent_teardown(tmp_path): + """Cleanup runs twice even when no diagnostics log can be opened.""" + environment, calls = _fixture(tmp_path) + diagnostics = tmp_path / "diagnostics-file" + diagnostics.write_text("not a directory\n", encoding="utf-8") + environment["INTEGRATION_DIAGNOSTICS"] = str(diagnostics) + architecture = {"x86_64": "amd64", "aarch64": "arm64"}[platform.machine()] + + result = subprocess.run( + [_WRAPPER, architecture, "sbx-shared"], + cwd=_REPO_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + assert result.returncode == 0, result.stderr + records = calls.read_text(encoding="utf-8").splitlines() + assert sum(line.startswith("teardown ") for line in records) == 2 + assert "diagnostics path is unavailable" in result.stderr + + +def test_unwritable_teardown_logs_do_not_prevent_teardown(tmp_path): + """Log-open failures fall back to inherited output for both attempts.""" + environment, calls = _fixture(tmp_path) + diagnostics = tmp_path / "diagnostics" + diagnostics.mkdir() + (diagnostics / "teardown-1.log").mkdir() + (diagnostics / "teardown-2.log").mkdir() + environment["INTEGRATION_DIAGNOSTICS"] = str(diagnostics) + architecture = {"x86_64": "amd64", "aarch64": "arm64"}[platform.machine()] + + result = subprocess.run( + [_WRAPPER, architecture, "sbx-shared"], + cwd=_REPO_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + assert result.returncode == 0, result.stderr + records = calls.read_text(encoding="utf-8").splitlines() + assert sum(line.startswith("teardown ") for line in records) == 2 + assert result.stderr.count("cannot write teardown log") == 2 diff --git a/tests/test_elbencho_dispatch_shell.py b/tests/test_elbencho_dispatch_shell.py index 5d292e7..ba06791 100644 --- a/tests/test_elbencho_dispatch_shell.py +++ b/tests/test_elbencho_dispatch_shell.py @@ -47,6 +47,48 @@ def _run_bash(script: str) -> subprocess.CompletedProcess: class TestElbenchoDispatchShell(unittest.TestCase): """Dispatch helpers preserve per-execution state across retries.""" + def test_slurm_dispatch_signals_cancel_once_and_preserve_exit_trap(self) -> None: + """INT and TERM cancel once before the caller's EXIT cleanup runs.""" + for signal_name, expected_rc in (("INT", 130), ("TERM", 143)): + with self.subTest(signal=signal_name): + with tempfile.TemporaryDirectory() as temp_dir: + record = Path(temp_dir) / "record" + script = f""" + source "{_ENV_FUNCTIONS}" + RECORD={record!s} + trap 'printf "owner\\n" >> "$RECORD"' EXIT + scancel() {{ printf 'cancel\\n' >> "$RECORD"; }} + squeue() {{ return 0; }} + _slurm_arm_dispatch_cleanup 4242 + kill -{signal_name} $$ + """ + result = _run_bash(textwrap.dedent(script)) + self.assertEqual(result.returncode, expected_rc, result.stderr) + self.assertEqual( + record.read_text(encoding="utf-8").splitlines(), + ["cancel", "owner"], + ) + + def test_slurm_dispatch_exit_cancels_once_and_runs_inherited_exit(self) -> None: + """Unexpected shell exit retains both job and caller cleanup actions.""" + with tempfile.TemporaryDirectory() as temp_dir: + record = Path(temp_dir) / "record" + script = f""" + source "{_ENV_FUNCTIONS}" + RECORD={record!s} + trap 'printf "owner\\n" >> "$RECORD"' EXIT + scancel() {{ printf 'cancel\\n' >> "$RECORD"; }} + squeue() {{ return 0; }} + _slurm_arm_dispatch_cleanup 4242 + exit 9 + """ + result = _run_bash(textwrap.dedent(script)) + self.assertEqual(result.returncode, 9, result.stderr) + self.assertEqual( + record.read_text(encoding="utf-8").splitlines(), + ["cancel", "owner"], + ) + def test_reified_execution_records_stable_test_dirs(self) -> None: script = f""" set -e @@ -347,11 +389,15 @@ def test_phase_restart_persists_new_service_owner_pid(self) -> None: result = _run_bash(textwrap.dedent(script)) self.assertEqual(result.returncode, 0, result.stderr) - def test_coordinator_keeps_initial_check_without_post_execution_duplicate( + def test_coordinator_retries_initial_services_without_per_execution_probe( self, ) -> None: coordinator = _SLURM_COORDINATOR.read_text(encoding="utf-8") self.assertEqual(coordinator.count("check_elbencho_services_srun"), 1) + self.assertIn("for attempt in 1 2; do", coordinator) + self.assertIn('start_elbencho_services_srun "" "$OUTPUT_DIR"', coordinator) + self.assertIn('"${SRUN_ELBENCHO_LOG}.attempt-${attempt}"', coordinator) + self.assertIn("failed after one restart", coordinator) self.assertNotIn( 'maybe_restart_elbencho_services_slurm "execution-${ID}"', coordinator, @@ -481,6 +527,112 @@ def test_slurm_dispatch_preserves_lock_after_handoff(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("circular name reference", result.stderr) + def test_slurm_adoption_failure_cancels_armed_job_and_waits(self) -> None: + """A submitted job is cancellation-owned before lock adoption starts.""" + script = f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + mkdir -p "$tmp/executions" + printf 'export nodes=1\n' > "$tmp/executions/0001.sh" + printf 'PENDING\n' > "$tmp/executions/0001.status" + SCALE_TEST_BASE="{_REPO_ROOT}" + + build_sbatch_cmd() {{ local -n ref="$1"; ref=(sbatch); }} + run_sbatch_job() {{ JOBID=6161; log_files+=("$tmp/coordinator.log"); }} + _elbencho_adopt_slurm_dispatch_lock() {{ + [[ "${{ELBENCHO_SLURM_CANCEL_ARMED:-0}}" -eq 1 ]] + [[ "${{ELBENCHO_SLURM_ACTIVE_JOB_ID:-}}" == 6161 ]] + return 1 + }} + tail_until_complete() {{ touch "$tmp/unexpected-monitor"; }} + scancel() {{ [[ "$1" == 6161 ]]; touch "$tmp/cancelled"; }} + squeue() {{ [[ "$*" == *6161* ]]; return 0; }} + sleep() {{ :; }} + + set +e + dispatch_slurm_executions "$tmp" + rc=$? + set -e + [[ "$rc" -eq 1 ]] + [[ -e "$tmp/cancelled" ]] + [[ ! -e "$tmp/unexpected-monitor" ]] + [[ ! -e "$tmp/executions/.dispatch.lock" ]] + [[ "$(trap -p EXIT)" == *'rm -rf'* ]] + """ + result = _run_bash(textwrap.dedent(script)) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_slurm_adoption_failure_retains_lock_when_cancel_is_unverified( + self, + ) -> None: + """An allocation of unknown state keeps exclusive dispatch ownership.""" + script = f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + mkdir -p "$tmp/executions" + printf 'export nodes=1\n' > "$tmp/executions/0001.sh" + printf 'PENDING\n' > "$tmp/executions/0001.status" + SCALE_TEST_BASE="{_REPO_ROOT}" + + build_sbatch_cmd() {{ local -n ref="$1"; ref=(sbatch); }} + run_sbatch_job() {{ JOBID=6262; log_files+=("$tmp/coordinator.log"); }} + _elbencho_adopt_slurm_dispatch_lock() {{ return 1; }} + scancel() {{ [[ "$1" == 6262 ]]; }} + squeue() {{ return 1; }} + + set +e + dispatch_slurm_executions "$tmp" + rc=$? + set -e + [[ "$rc" -eq 1 ]] + [[ -d "$tmp/executions/.dispatch.lock" ]] + [[ "$(cat "$tmp/executions/.dispatch.lock/token")" != "" ]] + printf '999999\n' > "$tmp/executions/.dispatch.lock/pid" + retry_token="" + if _elbencho_acquire_dispatch_lock \ + "$tmp/executions" slurm-submit retry_token; then + exit 1 + fi + """ + result = _run_bash(textwrap.dedent(script)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("retaining dispatch lock", result.stderr) + + def test_slurm_monitor_timeout_cancels_exact_handed_off_job(self) -> None: + script = f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + mkdir -p "$tmp/executions" + printf 'export nodes=1\n' > "$tmp/executions/0001.sh" + printf 'PENDING\n' > "$tmp/executions/0001.status" + SCALE_TEST_BASE="{_REPO_ROOT}" + + build_sbatch_cmd() {{ local -n ref="$1"; ref=(sbatch); }} + run_sbatch_job() {{ JOBID=6262; log_files+=("$tmp/coordinator.log"); }} + tail_until_complete() {{ return 124; }} + scancel() {{ [[ "$1" == 6262 ]]; touch "$tmp/cancelled"; }} + squeue() {{ [[ "$*" == *6262* ]]; return 0; }} + sleep() {{ :; }} + + set +e + dispatch_slurm_executions "$tmp" + rc=$? + set -e + [[ "$rc" -eq 124 ]] + [[ -e "$tmp/cancelled" ]] + """ + result = _run_bash(textwrap.dedent(script)) + self.assertEqual(result.returncode, 0, result.stderr) + def test_active_lock_blocks_slurm_and_ssh_before_status_reset(self) -> None: script = f""" set -e diff --git a/tests/test_elbencho_shared_directory_shell.py b/tests/test_elbencho_shared_directory_shell.py index 15ea2b1..a40d611 100644 --- a/tests/test_elbencho_shared_directory_shell.py +++ b/tests/test_elbencho_shared_directory_shell.py @@ -124,6 +124,8 @@ def test_json_parser_accepts_sync_and_rejects_duplicate_counter(self) -> None: '{{"phase_type":"WRITE","last_done":{{"elapsed_time_ms":"7","entries":"2","bytes":"8192"}}}}' \ '{{"phase_type":"SYNC"}}' >"$tmp/good" [[ "$(_elbencho_parse_phase_json "$tmp/good" WRITE)" == $'2\t8192\t7' ]] + tr -d '\n' <"$tmp/good" >"$tmp/adjacent" + [[ "$(_elbencho_parse_phase_json "$tmp/adjacent" WRITE)" == $'2\t8192\t7' ]] printf '%s\n' \ '{{"phase_type":"WRITE","last_done":{{"elapsed_time_ms":"7","entries":"2","entries":"2","bytes":"8192"}}}}' \ >"$tmp/bad" diff --git a/tests/test_elbencho_sweep_contracts_shell.py b/tests/test_elbencho_sweep_contracts_shell.py new file mode 100644 index 0000000..5cd2d45 --- /dev/null +++ b/tests/test_elbencho_sweep_contracts_shell.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fast decision-partition tests for the filesystem sweep contract.""" + +import os +import shutil +import subprocess +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_ENV_BASE = _REPO_ROOT / "lib" / "env_base.sh" +_ENV_FUNCTIONS = _REPO_ROOT / "lib" / "env_functions.sh" +_ELBENCHO_FUNCTIONS = _REPO_ROOT / "lib" / "_elbencho_functions.sh" +_SWEEP = _REPO_ROOT / "storage-tests" / "fs" / "nv-elbencho-sweep.sh" +_BASH = shutil.which("bash") or "bash" + + +def _run_bash(script: str) -> subprocess.CompletedProcess[str]: + """Run a dedented Bash contract harness.""" + return subprocess.run( + [_BASH, "-c", textwrap.dedent(script)], + check=False, + cwd=_REPO_ROOT, + text=True, + capture_output=True, + ) + + +def test_node_specification_equivalence_classes_and_boundaries(): + """Lists and stepped ranges retain order while malformed forms fail.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + diff -u <(printf '3\n5\n7\n9\n10\n13\n16\n8\n8\n') \ + <(parse_range_specification '3-10+2,13,16,8,8') + for invalid in '' ',1' '1,' '1,,2' '0' '3-1' '1-3+0' '1-x'; do + ! parse_range_specification "$invalid" >/dev/null 2>&1 + done + """) + assert result.returncode == 0, result.stderr + + +def test_cartesian_reification_has_stable_order_and_rotation(tmp_path): + """Every coordinate is unique and numbered in documented loop order.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + declare -A TEST_DIRS=(["{tmp_path}/data"]=1) + generate_fs_test_directories() {{ printf '%s/target\n' "{tmp_path}"; }} + DS=20260920Z010203 + nodes_spec='2,1-3+2' + rand_option=0 + ELBENCHO_SCALE_IO_SIZES=(4K r8K) + ELBENCHO_SCALE_THREAD_LIST=(1 2) + ELBENCHO_IODEPTH_LIST=(1 4) + ELBENCHO_SCALE_READ_WRITE_DURATION=1 + ELBENCHO_LIVE_CSV_EXTENDED=0 + ELBENCHO_FILE_SIZE_MULTIPLIER=1 + ELBENCHO_FILE_LAYOUT=worker-directories + ELBENCHO_FILES_PER_NODE= + ELBENCHO_FILE_SIZE=1M + ELBENCHO_READ_AFTER_WRITE_PAUSE=0 + FS_MAX_AGG_THROUGHPUT=1 + FS_MAX_NODE_THROUGHPUT_GBPS=1 + FS_MAX_NODE_IOPS=1 + ELBENCHO_SINGLE_BIG_FILE=0 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + mkdir -p "{tmp_path}/result" + reify_all_elbencho_executions \ + "{tmp_path}/result" "$nodes_spec" dio 0 0 0 0 '' + [[ $(find "{tmp_path}/result/executions" -name '*.sh' | wc -l) -eq 24 ]] + [[ $(find "{tmp_path}/result/executions" -name '*.status' | wc -l) -eq 24 ]] + source "{tmp_path}/result/executions/0001.sh" + [[ "$nodes:$io_size:$thread_count:$io_depth:$ELBENCHO_READ_HOST_ROTATE_STEPS" == \ + '2:4K:1:1:0' ]] + source "{tmp_path}/result/executions/0008.sh" + [[ "$nodes:$io_size:$thread_count:$io_depth:$ELBENCHO_READ_HOST_ROTATE_STEPS" == \ + '2:r8K:2:4:7' ]] + source "{tmp_path}/result/executions/0024.sh" + [[ "$nodes:$io_size:$thread_count:$io_depth:$ELBENCHO_READ_HOST_ROTATE_STEPS" == \ + '3:r8K:2:4:23' ]] + [[ $(sort -u "{tmp_path}/result/executions/"*.status) == PENDING ]] + """) + assert result.returncode == 0, result.stderr + + +def test_env_precedence_selects_test_dirs_ssh_and_order_aliases(tmp_path): + """Populated TEST_DIRS wins and SSH disables otherwise valid Slurm state.""" + hosts = tmp_path / "hosts" + hosts.write_text("# comment\nhost-a, host-b\n\nhost-a\n", encoding="utf-8") + result = _run_bash(f""" + set -e + SCALE_TEST_BASE="{_REPO_ROOT}" + RESULTS_DIR="{tmp_path}/results" + LOGS_DIR="{tmp_path}/logs" + TEST_DIR="{tmp_path}/legacy" + declare -A TEST_DIRS=(["{tmp_path}/preferred"]=2) + OBJ_BUCKET= + OBJ_AUTH_FILE="{tmp_path}/missing-auth" + SSH_HOST_LIST="{hosts}" + SSH_USER=tester + ORDER_NODES=YeS + client_type=cpu + client_arch=x86_64 + source "{_ENV_BASE}" + [[ "${{#TEST_DIRS[@]}}" -eq 1 ]] + [[ "${{TEST_DIRS[{tmp_path}/preferred]}}" == 2 ]] + [[ -z "${{TEST_DIRS[{tmp_path}/legacy]+x}}" ]] + [[ "$SSH_ENABLED" == 1 && -z "$SLURM_ENABLED" ]] + [[ "$ORDER_NODES_ENABLED" == 1 ]] + [[ "${{SSH_ALL_HOSTS[*]}}" == 'host-a host-b host-a' ]] + """) + assert result.returncode == 0, result.stderr + + +def test_env_fallbacks_apply_only_when_primary_values_are_empty(tmp_path): + """Legacy directory and throughput names fill gaps without overriding values.""" + hosts = tmp_path / "hosts" + hosts.write_text("host-a\n", encoding="utf-8") + result = _run_bash(f""" + set -e + SCALE_TEST_BASE="{_REPO_ROOT}" + RESULTS_DIR="{tmp_path}/results" + LOGS_DIR="{tmp_path}/logs" + TEST_DIR="{tmp_path}/legacy" + declare -A TEST_DIRS=() + OBJ_BUCKET= + OBJ_AUTH_FILE="{tmp_path}/missing-auth" + SSH_HOST_LIST="{hosts}" + ORDER_NODES=off + client_type=cpu + client_arch=x86_64 + IOR_FS_MAX_AGG_THROUGHPUT=27 + FS_MAX_AGG_THROUGHPUT=19 + source "{_ENV_BASE}" + [[ "${{#TEST_DIRS[@]}}" -eq 1 ]] + [[ "${{TEST_DIRS[{tmp_path}/legacy]}}" == 1 ]] + [[ -z "$ORDER_NODES_ENABLED" ]] + [[ "$FS_MAX_AGG_THROUGHPUT" == 19 ]] + """) + assert result.returncode == 0, result.stderr + + +def test_ssh_selection_contract_covers_ordered_and_unordered_modes(): + """Ordered selection is a prefix; unordered selection is unique and bounded.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + SSH_ENABLED=1 + SSH_ALL_HOSTS=(host-a host-b host-c host-d) + ORDER_NODES_ENABLED=1 + choose_N_ssh_hosts 2 + [[ "$SSH_NODELIST" == host-a,host-b ]] + ORDER_NODES_ENABLED= + RANDOM=7 + choose_N_ssh_hosts 3 + IFS=, read -ra selected <<<"$SSH_NODELIST" + [[ "${{#selected[@]}}" -eq 3 ]] + [[ $(printf '%s\n' "${{selected[@]}}" | sort -u | wc -l) -eq 3 ]] + for host in "${{selected[@]}}"; do + [[ " ${{SSH_ALL_HOSTS[*]}} " == *" $host "* ]] + done + ! choose_N_ssh_hosts 0 >/dev/null 2>&1 + ! choose_N_ssh_hosts 5 >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_read_and_delete_path_guards_resolve_symlinks(tmp_path): + """Read permits the root, while deletion requires a real strict descendant.""" + root = tmp_path / "root" + child = root / "child" + outside = tmp_path / "outside" + child.mkdir(parents=True) + outside.mkdir() + (root / "escape").symlink_to(outside, target_is_directory=True) + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + declare -A TEST_DIRS=(["{root}"]=1) + validate_elbencho_sweep_read_from_path "{root}" + validate_elbencho_sweep_read_from_path "{child}" + validate_elbencho_sweep_delete_only_path "{child}" + ! validate_elbencho_sweep_delete_only_path "{root}" >/dev/null 2>&1 + ! validate_elbencho_sweep_delete_only_path "{outside}" >/dev/null 2>&1 + ! validate_elbencho_sweep_delete_only_path \ + "{root}/escape/victim" >/dev/null 2>&1 + ! validate_elbencho_sweep_read_from_path \ + "{root}/escape/input" >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_single_big_file_validation_partitions(tmp_path): + """Single-file mode enforces one root, sequential I/O, and extent rules.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + declare -A TEST_DIRS=(["{tmp_path}"]=1) + ELBENCHO_SINGLE_BIG_FILE=1 + ELBENCHO_SCALE_IO_SIZES=(4K '1M,4K') + ELBENCHO_SINGLE_BIG_FILE_SIZE=16M + validate_elbencho_single_big_file_env '' + ELBENCHO_SINGLE_BIG_FILE_SIZE= + validate_elbencho_single_big_file_env "{tmp_path}/existing-file" + ! validate_elbencho_single_big_file_env '' >/dev/null 2>&1 + ELBENCHO_SINGLE_BIG_FILE_SIZE=16M + ELBENCHO_SCALE_IO_SIZES=(r4K) + ! validate_elbencho_single_big_file_env '' >/dev/null 2>&1 + ELBENCHO_SCALE_IO_SIZES=(4K) + TEST_DIRS["{tmp_path}/second"]=1 + ! validate_elbencho_single_big_file_env '' >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_workload_precedence_and_incompatible_values(tmp_path): + """Only active branch inputs affect sizing and workload validation.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + declare -A TEST_DIRS=(["{tmp_path}"]=1) + ELBENCHO_SCALE_THREAD_LIST=(1) + ELBENCHO_SCALE_IO_SIZES=('6K,r4K') + ELBENCHO_FILE_LAYOUT=worker-directories + ELBENCHO_FILES_PER_NODE= + ELBENCHO_FILE_SIZE=6K + ELBENCHO_FILE_SIZE_MULTIPLIER=1024 + ELBENCHO_SINGLE_BIG_FILE=0 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + sweep_write_only=0 + sweep_write_no_read=0 + [[ $(_elbencho_resolve_generated_file_size 4K) == 6K ]] + ! validate_elbencho_sweep_workload_mode bio 0 '' >/dev/null 2>&1 + sweep_write_only=1 + validate_elbencho_sweep_workload_mode bio 0 '' + ELBENCHO_FILE_SIZE= + [[ $(_elbencho_resolve_generated_file_size 4K) == 4M ]] + sweep_write_only=0 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1 + ! validate_elbencho_sweep_workload_mode dio 0 '' >/dev/null 2>&1 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + ELBENCHO_FILE_LAYOUT=shared-directory + ELBENCHO_FILES_PER_NODE=3 + ELBENCHO_SCALE_THREAD_LIST=(2) + ELBENCHO_SCALE_IO_SIZES=(4K) + ELBENCHO_FILE_SIZE=4K + ! validate_elbencho_sweep_workload_mode dio 0 '' >/dev/null 2>&1 + ELBENCHO_FILES_PER_NODE=4 + validate_elbencho_sweep_workload_mode dio 0 '' + TEST_DIRS["{tmp_path}/second"]=1 + ! validate_elbencho_sweep_workload_mode dio 0 '' >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_weighted_sizing_uses_the_tightest_capacity_limit(): + """IOPS and aggregate bandwidth limits independently bound file counts.""" + result = _run_bash(f""" + set -e + source "{_ELBENCHO_FUNCTIONS}" + FS_MAX_NODE_THROUGHPUT_GBPS=8 + FS_MAX_AGG_THROUGHPUT=100 + FS_MAX_NODE_IOPS=512 + [[ $(compute_target_file_count_per_thread 4K 1M 2 1 1) -eq 2 ]] + FS_MAX_NODE_IOPS=999999999 + FS_MAX_AGG_THROUGHPUT=1 + [[ $(compute_target_file_count_per_thread 4K 1M 2 1 1) -eq 512 ]] + """) + assert result.returncode == 0, result.stderr + + +def test_slurm_command_boundaries_preserve_spaced_arguments(tmp_path): + """Scheduler arrays retain reservations, GPU requests, and one spaced value.""" + for name in ("sbatch", "srun"): + executable = tmp_path / name + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o755) + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + PATH="{tmp_path}:$PATH" + _SBATCH_OPTIONS_BASE='-A account-a -p batch --exclusive=user' + _SRUN_OPTIONS_BASE='-A account-a -p batch --exclusive=user' + SLURM_EXTRA_ARGS=(--qos=normal '--comment=integration scenario') + declare -a batch run + build_sbatch_cmd batch + build_srun_cmd run + [[ "${{batch[-1]}}" == '--comment=integration scenario' ]] + [[ "${{run[-1]}}" == '--comment=integration scenario' ]] + [[ "${{batch[-2]}}" == --qos=normal ]] + account=account-a + partition=gpu + run_time=00:05:00 + reservation=reserved-a + SLURM_GPUS_PER_NODE_OPT=--gpus-per-node=4 + build_srun_validate_scale_cmd run + [[ " ${{run[*]}} " == *' --reservation reserved-a '* ]] + [[ " ${{run[*]}} " == *' --gpus-per-node=4 '* ]] + [[ "${{run[-1]}}" == '--comment=integration scenario' ]] + SLURM_JOB_NAME_PREFIX='project-' + [[ $(make_sbatch_job_name elbencho 20260920Z010203 2-1) == \ + project-elbencho-20260920Z010203-2-1 ]] + """) + assert result.returncode == 0, result.stderr + + +def test_exclusive_user_cpu_discovery_uses_smallest_target(): + """Exclusive-user scheduling requests a capacity valid on every target.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + scontrol() {{ printf 'node-a\nnode-b\n'; }} + sinfo() {{ printf '16\n4\n8\n'; }} + SLURM_INCLUDE_COUNT=2 + SLURM_INCLUDES=--nodelist=node-a,node-b + [[ $(get_slurm_target_node_cpus) == 4 ]] + sinfo() {{ printf 'unknown\n'; }} + ! get_slurm_target_node_cpus >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def _make_sweep_fixture(tmp_path: Path) -> Path: + """Create a minimal deployment for parser-only sweep invocations.""" + script_dir = tmp_path / "storage-tests" / "fs" + script_dir.mkdir(parents=True) + sweep = script_dir / _SWEEP.name + shutil.copy2(_SWEEP, sweep) + root = tmp_path / "data" + root.mkdir() + env = tmp_path / "env.sh" + env.write_text( + textwrap.dedent(f""" + export SCALE_TEST_BASE={_REPO_ROOT!s} + export RESULTS_DIR={tmp_path / 'results'!s} + export LOGS_DIR={tmp_path / 'logs'!s} + declare -A TEST_DIRS=([{root!s}]=1) + ELBENCHO_SCALE_THREAD_LIST=(1) + ELBENCHO_SCALE_IO_SIZES=(4K) + ELBENCHO_IODEPTH_LIST=(1) + export ELBENCHO_SCALE_READ_WRITE_DURATION=1 + export ELBENCHO_FILE_LAYOUT=worker-directories + export ELBENCHO_FILES_PER_NODE= + export ELBENCHO_FILE_SIZE=1M + export ELBENCHO_FILE_SIZE_MULTIPLIER=1 + export ELBENCHO_LIVE_CSV_EXTENDED=0 + export ELBENCHO_SINGLE_BIG_FILE=0 + export ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + export FS_MAX_AGG_THROUGHPUT=1 + export FS_MAX_NODE_THROUGHPUT_GBPS=1 + export FS_MAX_NODE_IOPS=1 + export SSH_ENABLED= + export SLURM_ENABLED= + source "$SCALE_TEST_BASE/lib/env_functions.sh" + """), + encoding="utf-8", + ) + return sweep + + +@pytest.mark.parametrize( + "arguments, message", + [ + (("--nodes",), "requires an argument"), + (("--read-from",), "requires a path argument"), + (("--delete-only",), "requires a path argument"), + (("--resume",), "requires a path argument"), + (("--unknown",), "Unknown option"), + (("positional",), "Unexpected positional argument"), + ((), "--nodes is required"), + (("--nodes", "3-1"), "Invalid node specification"), + (("--nodes", "1,invalid"), "Invalid node specification"), + ( + ("--write-only", "--write-no-read", "--nodes", "1"), + "Use at most one", + ), + (("--resume", "/missing", "--bio"), "mutually exclusive"), + ], +) +def test_sweep_cli_rejects_invalid_equivalence_classes(tmp_path, arguments, message): + """Parser and mode errors fail before dispatching or creating a run.""" + sweep = _make_sweep_fixture(tmp_path) + result = subprocess.run( + [str(sweep), *arguments], + check=False, + cwd=tmp_path, + env={**os.environ, "SHELL": _BASH}, + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert message in result.stdout + result.stderr + assert not (tmp_path / "results").exists() + + +@pytest.mark.parametrize("flag", ["-h", "--help"]) +def test_sweep_help_is_environment_independent(tmp_path, flag): + """Both help aliases exit before workload validation or dispatch.""" + sweep = _make_sweep_fixture(tmp_path) + result = subprocess.run( + [str(sweep), flag], + check=False, + cwd=tmp_path, + env={**os.environ, "SHELL": _BASH}, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert "Usage:" in result.stdout + assert "--resume" in result.stdout diff --git a/tests/test_extract_elbencho_cli_contracts.py b/tests/test_extract_elbencho_cli_contracts.py new file mode 100644 index 0000000..2182b19 --- /dev/null +++ b/tests/test_extract_elbencho_cli_contracts.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fast CLI contract tests for the Elbencho report extractor.""" + +import sys +from dataclasses import replace + +import pytest + +from tests.extract_elbencho_test_support import load_extract_elbencho_module + +_EXTRACT = load_extract_elbencho_module("extract_elbencho_cli_contracts") + + +def _metric(): + """Return one valid metric suitable for a CSV-only report invocation.""" + return _EXTRACT.ElbenchoMetrics( + nodes=1, + io_size="4K", + threads=1, + io_depth=1, + operation="WRITE", + datestamp="20260101Z000000", + is_multi_node=False, + command="elbencho --size 4K", + file_size_bytes=4096, + direct_io=0, + random_io=0, + iops=1.0, + throughput_mib_s=1.0, + throughput_mb_s=1.048576, + throughput_gbps=0.008, + min_lat_sec=0.001, + avg_lat_sec=0.002, + max_lat_sec=0.003, + lat_pct_1=0.001, + lat_pct_50=0.002, + lat_pct_75=0.003, + lat_pct_99=0.004, + ) + + +def _run_main(monkeypatch, *arguments): + """Run the extractor entry point with a temporary argv.""" + monkeypatch.setattr(sys, "argv", ["extract-elbencho.py", *arguments]) + return _EXTRACT.main() + + +def test_from_csv_rejects_raw_result_directories(monkeypatch, tmp_path): + """CSV input is an alternative source, never a second source to merge.""" + with pytest.raises(SystemExit) as raised: + _run_main( + monkeypatch, + "--from-csv", + str(tmp_path / "metrics.csv"), + str(tmp_path / "raw-results"), + ) + assert raised.value.code == 2 + + +@pytest.mark.parametrize( + "option, value", + [ + ("--only-threads", "1,,2"), + ("--only-nodes", "2-1"), + ("--only-iodepths", "not-an-integer"), + ("--only-sizes", "4K;;1M"), + ("--only-sizes", "not-a-size"), + ], +) +def test_filter_parser_rejects_malformed_syntax(option, value): + """Malformed filters fail instead of silently broadening a report.""" + if option == "--only-sizes": + with pytest.raises(ValueError): + _EXTRACT.parse_only_sizes_filter([value]) + else: + with pytest.raises(ValueError): + _EXTRACT.parse_int_values_with_ranges(value) + + +def test_valid_filter_matching_no_metrics_is_an_error(monkeypatch, tmp_path): + """A successful process must not hide an accidentally empty report.""" + csv_path = tmp_path / "metrics.csv" + _EXTRACT.write_csv(str(csv_path), [_metric()]) + + with pytest.raises(SystemExit) as raised: + _run_main( + monkeypatch, + "--from-csv", + str(csv_path), + "--only-nodes", + "99", + "--markdown", + ) + assert raised.value.code == 1 + + +def _mixed_report_input(tmp_path, live_nodes): + """Create recognizable aggregate and live filenames for CLI tests.""" + stem = "elbencho-4K-c_001-s_001-d_001_20260101Z000000" + (tmp_path / f"{stem}.csv").write_text("aggregate\n", encoding="utf-8") + (tmp_path / f"{stem}.out").write_text("aggregate\n", encoding="utf-8") + live_stem = f"elbencho-4K-c_{live_nodes:03d}-s_001-d_001_20260101Z000000" + (tmp_path / f"{live_stem}.live.csv").write_text("live\n", encoding="utf-8") + + +def test_live_match_cannot_hide_empty_filtered_aggregate(monkeypatch, tmp_path): + """Each discovered report family must independently survive filtering.""" + _mixed_report_input(tmp_path, live_nodes=2) + monkeypatch.setattr(_EXTRACT, "parse_elbencho_files", lambda _base: [_metric()]) + + with pytest.raises(SystemExit) as raised: + _run_main( + monkeypatch, + str(tmp_path), + "--only-nodes", + "2", + "--markdown", + ) + + assert raised.value.code == 1 + + +def test_failed_live_reports_cannot_hide_behind_aggregate(monkeypatch, tmp_path): + """Aggregate output does not turn a failed live report into success.""" + _mixed_report_input(tmp_path, live_nodes=1) + monkeypatch.setattr(_EXTRACT, "parse_elbencho_files", lambda _base: [_metric()]) + monkeypatch.setattr(_EXTRACT, "print_markdown_table", lambda *_args: None) + monkeypatch.setattr(_EXTRACT, "plot_metrics", lambda *_args: None) + monkeypatch.setattr(_EXTRACT, "_report_one_live_file", lambda *_args: False) + + with pytest.raises(SystemExit) as raised: + _run_main(monkeypatch, str(tmp_path), "--markdown") + + assert raised.value.code == 1 + + +def test_csv_round_trip_preserves_report_dimensions(tmp_path): + """Persisted metrics retain the dimensions needed by later filtering.""" + metrics = [ + _metric(), + replace( + _metric(), + nodes=2, + io_size="1M,r4K", + threads=8, + io_depth=4, + operation="READ", + random_io=1, + ), + ] + path = tmp_path / "metrics.csv" + _EXTRACT.write_csv(str(path), metrics) + loaded = _EXTRACT.read_csv(str(path)) + assert [ + ( + item.nodes, + item.io_size, + item.threads, + item.io_depth, + item.operation, + item.random_io, + ) + for item in loaded + ] == [ + (1, "4K", 1, 1, "WRITE", 0), + (2, "1M,r4K", 8, 4, "READ", 1), + ] + + +def test_csv_cli_filters_compound_size_and_forwards_plot_mode(monkeypatch, tmp_path): + """CLI filters preserve compound sizes and the single-axis request.""" + path = tmp_path / "metrics.csv" + _EXTRACT.write_csv( + str(path), + [ + _metric(), + replace( + _metric(), + nodes=2, + io_size="1M,r4K", + threads=8, + io_depth=4, + operation="READ", + ), + ], + ) + reported = [] + plotted = [] + monkeypatch.setattr( + _EXTRACT, + "print_markdown_table", + lambda metrics, single_axis: reported.append((metrics, single_axis)), + ) + monkeypatch.setattr( + _EXTRACT, + "plot_metrics", + lambda metrics, output_dir, single_axis: plotted.append( + (metrics, output_dir, single_axis) + ), + ) + + _run_main( + monkeypatch, + "--from-csv", + str(path), + "--only-nodes", + "2", + "--only-threads", + "8", + "--only-iodepths", + "4", + "--only-sizes", + "1M,r4K", + "--no-dual-y-axis", + "--markdown", + ) + + assert len(reported) == len(plotted) == 1 + assert reported[0][1] is True + assert plotted[0][2] is True + assert [item.operation for item in reported[0][0]] == ["READ"] + + +def test_csv_cli_can_reemit_cache_and_default_terminal_report(monkeypatch, tmp_path): + """CSV input supports cache output and the normal mirrored text report.""" + source = tmp_path / "source.csv" + _EXTRACT.write_csv(str(source), [_metric()]) + mirrored = [] + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + _EXTRACT, + "mirror_stdout_to_file", + lambda path, printer, metrics: mirrored.append((path, printer, metrics)), + ) + monkeypatch.setattr(_EXTRACT, "plot_metrics", lambda *_arguments: None) + + _run_main(monkeypatch, "--from-csv", str(source), "--to-csv") + + cache = tmp_path / "elbencho-metrics.csv" + assert cache.is_file() + assert len(_EXTRACT.read_csv(str(cache))) == 1 + assert len(mirrored) == 1 + assert mirrored[0][0] == str(tmp_path / _EXTRACT.REPORT_TXT_FILENAME) + assert [item.operation for item in mirrored[0][2]] == ["WRITE"] + + +@pytest.mark.parametrize( + "option", + [ + "--client-outlier-threshold", + "--client-min-underperform-segments", + "--client-max-timeseries-lines", + "--client-max-heatmap-rows", + ], +) +def test_live_report_limits_must_be_positive(monkeypatch, option): + """Invalid live-report limits fail at the CLI boundary.""" + with pytest.raises(SystemExit) as raised: + _run_main(monkeypatch, "--from-csv", "unused.csv", option, "0") + assert raised.value.code == 2 diff --git a/tests/test_extract_elbencho_scale_efficiency.py b/tests/test_extract_elbencho_scale_efficiency.py new file mode 100644 index 0000000..0705a61 --- /dev/null +++ b/tests/test_extract_elbencho_scale_efficiency.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for Elbencho scaling-efficiency calculations.""" + +from tests.extract_elbencho_test_support import load_extract_elbencho_module + +_MODULE = load_extract_elbencho_module("extract_elbencho_scale_efficiency_under_test") +_scale_efficiency = getattr(_MODULE, "_scale_efficiency") + + +def test_scale_efficiency_preserves_relative_values(): + """The fastest per-unit result is the 100-percent reference.""" + assert _scale_efficiency([2.0, 1.0]) == [100.0, 50.0] + + +def test_scale_efficiency_accepts_all_zero_results(): + """Valid rounded-zero reports do not divide by zero.""" + assert _scale_efficiency([0.0, 0.0]) == [0.0, 0.0] diff --git a/tests/test_filesystem_scenario_specs.py b/tests/test_filesystem_scenario_specs.py new file mode 100644 index 0000000..879e0bb --- /dev/null +++ b/tests/test_filesystem_scenario_specs.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contract tests for declarative filesystem integration workloads.""" + +import importlib.util +from dataclasses import replace +from pathlib import Path +import sys + +import pytest + +_MODULE_PATH = ( + Path(__file__).resolve().parents[1] + / "integration-tests" + / "lib" + / "filesystem_scenario_specs.py" +) +_SPEC = importlib.util.spec_from_file_location( + "filesystem_scenario_specs", _MODULE_PATH +) +assert _SPEC is not None and _SPEC.loader is not None +_SCENARIOS = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _SCENARIOS +_SPEC.loader.exec_module(_SCENARIOS) + +_PLANNER_PATH = _MODULE_PATH.with_name("scenario_planner.py") +_PLANNER_SPEC = importlib.util.spec_from_file_location( + "filesystem_spec_scenario_planner", _PLANNER_PATH +) +assert _PLANNER_SPEC is not None and _PLANNER_SPEC.loader is not None +_PLANNER = importlib.util.module_from_spec(_PLANNER_SPEC) +sys.modules[_PLANNER_SPEC.name] = _PLANNER +_PLANNER_SPEC.loader.exec_module(_PLANNER) + +CommandKind = _SCENARIOS.CommandKind +DatasetExpectation = _SCENARIOS.DatasetExpectation +ExecutionStatus = _SCENARIOS.ExecutionStatus +FailureInjection = _SCENARIOS.FailureInjection +ScenarioSpecError = _SCENARIOS.ScenarioSpecError +WorkloadPhase = _SCENARIOS.WorkloadPhase +validate_scenario_specs = _SCENARIOS.validate_scenario_specs + + +EXPECTED_NAMES = { + "baseline", + "default-dio", + "failure-resume", + "retained-lifecycle", + "live-capture", + "slurm-cartesian", + "ssh-single-big-file", + "ssh-weighted-roots", + "ssh-shared-home", + "slurm-scheduling", +} + + +def _scenario(name): + return _SCENARIOS.SCENARIO_SPECS_BY_NAME[name] + + +def test_environment_overrides_use_variable_identity(): + """Overrides tolerate harmless shell quoting and formatting changes.""" + lines = ('export SAMPLE = "old"', "unset OTHER") + + assert _SCENARIOS._override_env( # pylint: disable=protected-access + lines, {"SAMPLE": 'export SAMPLE="new"'} + ) == ('export SAMPLE="new"', "unset OTHER") + + +@pytest.mark.parametrize( + "lines, name, matches", + [ + (("export PRESENT=1",), "MISSING", 0), + (("unset DUPLICATE", "export DUPLICATE=1"), "DUPLICATE", 2), + ], +) +def test_environment_overrides_reject_ambiguous_targets(lines, name, matches): + """Missing and duplicate scenario variables cannot silently drift.""" + with pytest.raises(ScenarioSpecError, match=rf"{name} \({matches} matches\)"): + _SCENARIOS._override_env( # pylint: disable=protected-access + lines, {name: f"export {name}=replacement"} + ) + + +def test_environment_overrides_reject_replacement_for_another_variable(): + """A valid target cannot silently be replaced with another variable.""" + with pytest.raises(ScenarioSpecError, match="FOO names BAR"): + _SCENARIOS._override_env( # pylint: disable=protected-access + ("export FOO=1",), {"FOO": "export BAR=2"} + ) + + +def test_environment_overrides_reject_multiline_replacements(): + """One override cannot smuggle in an additional shell statement.""" + with pytest.raises(ScenarioSpecError, match="require one statement: FOO"): + _SCENARIOS._override_env( # pylint: disable=protected-access + ("export FOO=1",), {"FOO": "export FOO=1\nexport BAR=2"} + ) + + +def test_catalog_defines_every_planned_real_scenario(): + """The execution catalog and workload catalog have matching stable names.""" + assert set(_SCENARIOS.SCENARIO_SPECS_BY_NAME) == EXPECTED_NAMES + assert {scenario.name for scenario in _PLANNER.SCENARIO_CATALOG} == EXPECTED_NAMES + planned_substrates = { + scenario.name: {substrate.value for substrate in scenario.substrates} + for scenario in _PLANNER.SCENARIO_CATALOG + } + specified_substrates = { + scenario.name: set(scenario.substrates) + for scenario in _SCENARIOS.SCENARIO_SPECS + } + assert planned_substrates == specified_substrates + validate_scenario_specs() + + +def test_each_step_is_bounded_and_has_semantic_expectations(): + """Real workloads remain bounded without freezing incidental artifacts.""" + for scenario in _SCENARIOS.SCENARIO_SPECS: + for step in scenario.steps: + assert 0 < step.timeout_seconds <= 600 + assert step.required_phases + assert step.dataset in DatasetExpectation + assert not hasattr(step, "expected_filenames") + assert not hasattr(step, "expected_native_arguments") + + +@pytest.mark.parametrize("name", ("baseline", "default-dio", "ssh-shared-home")) +def test_basic_real_sweeps_cover_one_and_two_nodes(name): + """Transport smokes exercise node subset changes on their real substrates.""" + step = _scenario(name).steps[0] + assert {execution.coordinate.nodes for execution in step.executions} == {1, 2} + assert all( + execution.status is ExecutionStatus.SUCCESS for execution in step.executions + ) + + +def test_default_dio_uses_worker_layout_and_derived_file_size(): + """The default-path scenario does not accidentally reuse the BIO fixture.""" + step = _scenario("default-dio").steps[0] + environment = "\n".join(step.env_lines) + + assert step.arguments == ("--nodes", "1,2") + assert 'ELBENCHO_FILE_LAYOUT="worker-directories"' in environment + assert "ELBENCHO_FILE_SIZE=" in environment + assert "ELBENCHO_FILE_SIZE_MULTIPLIER=4096" in environment + assert WorkloadPhase.TREE_SCAN in step.required_phases + + +def test_failure_resume_preserves_overlay_through_resume(): + """One atomic injected failure is not restaged between attempts.""" + initial, resume = _scenario("failure-resume").steps + statuses = [execution.status for execution in initial.executions] + + assert statuses == [ + ExecutionStatus.SUCCESS, + ExecutionStatus.FAILED, + ExecutionStatus.PENDING, + ExecutionStatus.PENDING, + ] + assert initial.failure_injection is FailureInjection.FAIL_AFTER_WRITE_ONCE + assert initial.preserve_failure_staging + assert resume.kind is CommandKind.RESUME + assert resume.requires == ("failed_results_dir",) + assert resume.preserve_failure_staging + assert all( + execution.status is ExecutionStatus.SUCCESS for execution in resume.executions + ) + + +def test_retained_lifecycle_expresses_cache_miss_hit_and_deletion(): + """The chained dataset contract survives reads and is deleted last.""" + steps = _scenario("retained-lifecycle").steps + + assert [step.name for step in steps] == [ + "write-only", + "read-cache-miss", + "read-cache-hit", + "delete-only", + ] + assert [step.dataset for step in steps] == [ + DatasetExpectation.PRESERVED, + DatasetExpectation.PRESERVED, + DatasetExpectation.PRESERVED, + DatasetExpectation.REMOVED, + ] + assert WorkloadPhase.TREE_SCAN in steps[1].required_phases + assert WorkloadPhase.TREE_SCAN not in steps[2].required_phases + assert steps[-1].kind is CommandKind.DELETE + assert not steps[-1].executions + + +def test_live_capture_has_stable_series_sampling_window(): + """The real live-data case requests several intervals without a huge run.""" + step = _scenario("live-capture").steps[0] + environment = "\n".join(step.env_lines) + + assert "ELBENCHO_SCALE_READ_WRITE_DURATION=3" in environment + assert "ELBENCHO_LIVE_CSV_EXTENDED=1" in environment + assert "ELBENCHO_LIVEINT=10" in environment + assert {execution.coordinate.nodes for execution in step.executions} == {2} + + +def test_representative_cartesian_sweep_has_all_sixteen_coordinates(): + """The real Slurm matrix is representative rather than exhaustive.""" + scenario = _scenario("slurm-cartesian") + coordinates = [execution.coordinate for execution in scenario.steps[0].executions] + + assert scenario.substrates == {"slurm"} + assert len(coordinates) == 16 + assert {coordinate.nodes for coordinate in coordinates} == {1, 2} + assert {coordinate.io_size for coordinate in coordinates} == {"4K", "4K,r8K"} + assert {coordinate.threads for coordinate in coordinates} == {1, 2} + assert {coordinate.io_depth for coordinate in coordinates} == {1, 2} + + +def test_single_file_sequence_uses_inferred_read_extent(): + """A retained cooperative file feeds the all-node staged read.""" + generated, read, delete = _scenario("ssh-single-big-file").steps + read_environment = "\n".join(read.env_lines) + + assert generated.dataset is DatasetExpectation.CLEANED + assert "{test_root}/staged-input/integration-bigfile" in read.arguments + assert 'ELBENCHO_SINGLE_BIG_FILE_SIZE="16M"' in read_environment + assert "export ELBENCHO_SINGLE_BIG_FILE_SIZE=" in read_environment + assert "ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1" in read_environment + assert read.dataset is DatasetExpectation.PRESERVED + assert delete.dataset is DatasetExpectation.REMOVED + + +def test_weighted_roots_activates_single_sizing_with_bounded_limits(): + """The SSH sizing case uses both weighted roots and deliberately low limits.""" + step = _scenario("ssh-weighted-roots").steps[0] + environment = "\n".join(step.env_lines) + + assert step.arguments == ("--bio", "--single", "--nodes", "1") + assert '["{test_root}"]=1' in environment + assert '["{test_root_secondary}"]=2' in environment + assert "FS_MAX_NODE_IOPS=100" in environment + + +def test_slurm_scheduling_renders_include_and_ignore_files(): + """The scheduling scenario constrains a real one-node allocation.""" + step = _scenario("slurm-scheduling").steps[0] + rendered = step.render_support_files( + { + "slurm_node_1": "compute-0", + "slurm_node_2": "compute-1", + "workspace": "/tmp/work", + "test_root": "/mnt/test", + } + ) + + assert step.arguments == ("--nodes", "1") + assert rendered[0].content == "compute-0\ncompute-1\n" + assert rendered[1].content == "compute-1\n" + assert "SLURM_EXCLUSIVE_USER=1" in step.env_lines + assert 'SLURM_JOB_NAME_PREFIX="itest-"' in step.env_lines + + +def test_template_rendering_resolves_driver_owned_paths(): + """The driver can render env and argument templates without shell discovery.""" + step = _scenario("retained-lifecycle").steps[1] + values = {"test_root": "/mnt/test", "retained_data_dir": "/mnt/test/data"} + + assert step.render_arguments(values) == ( + "--read-from", + "/mnt/test/data", + "--nodes", + "1", + ) + assert '["/mnt/test"]=1' in step.render_env(values) + + +def test_validation_rejects_missing_sequence_dependency(): + """A chained command cannot consume an artifact no prior step exported.""" + scenario = _scenario("retained-lifecycle") + broken_first = replace(scenario.steps[0], exports=()) + broken = replace(scenario, steps=(broken_first, *scenario.steps[1:])) + + with pytest.raises(ScenarioSpecError, match="unavailable values"): + validate_scenario_specs((broken,)) + + +def test_validation_rejects_duplicate_coordinates(): + """One command cannot claim two result records for the same coordinate.""" + scenario = _scenario("baseline") + step = scenario.steps[0] + broken_step = replace(step, executions=(step.executions[0],) * 2) + broken = replace(scenario, steps=(broken_step,)) + + with pytest.raises(ScenarioSpecError, match="duplicate execution"): + validate_scenario_specs((broken,)) + + +def test_validation_rejects_unbounded_timeout(): + """Every real command has an explicit upper wall-time bound.""" + scenario = _scenario("baseline") + broken_step = replace(scenario.steps[0], timeout_seconds=601) + broken = replace(scenario, steps=(broken_step,)) + + with pytest.raises(ScenarioSpecError, match="timeout"): + validate_scenario_specs((broken,)) diff --git a/tests/test_filesystem_scenarios.py b/tests/test_filesystem_scenarios.py new file mode 100644 index 0000000..bdf82af --- /dev/null +++ b/tests/test_filesystem_scenarios.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for filesystem integration scenario selection and scheduling.""" + +import importlib.util +import itertools +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_MODULE_PATH = _REPO_ROOT / "integration-tests" / "lib" / "scenario_planner.py" +_SPEC = importlib.util.spec_from_file_location( + "filesystem_scenarios_under_test", _MODULE_PATH +) +assert _SPEC and _SPEC.loader +_SCENARIOS = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _SCENARIOS +_SPEC.loader.exec_module(_SCENARIOS) + +Scenario = _SCENARIOS.Scenario +ScenarioPlanningError = _SCENARIOS.ScenarioPlanningError +SchedulePhase = _SCENARIOS.SchedulePhase +SshHomeMode = _SCENARIOS.SshHomeMode +SshHomeTransition = _SCENARIOS.SshHomeTransition +Substrate = _SCENARIOS.Substrate +WorkItem = _SCENARIOS.WorkItem +plan_scenarios = _SCENARIOS.plan_scenarios +format_scenario_listing = _SCENARIOS.format_scenario_listing +scenario_metadata = _SCENARIOS.scenario_metadata +select_scenarios = _SCENARIOS.select_scenarios + + +def _step_identity(step): + """Return a compact identity for a plan step.""" + if isinstance(step, WorkItem): + return ("work", step.scenario.name, step.substrate.value) + return ("transition", step.target.value) + + +def _plan_identity(**kwargs): + """Return compact identities for a scenario plan.""" + kwargs.setdefault("registry", _SCENARIOS.SCENARIO_CATALOG) + return tuple(_step_identity(step) for step in plan_scenarios(**kwargs)) + + +def test_catalog_exposes_stable_machine_metadata(): + """Listing metadata describes selection without freezing prose.""" + metadata = scenario_metadata(_SCENARIOS.SCENARIO_CATALOG) + + assert [item["name"] for item in metadata] == [ + "baseline", + "default-dio", + "failure-resume", + "retained-lifecycle", + "live-capture", + "slurm-cartesian", + "ssh-single-big-file", + "ssh-weighted-roots", + "ssh-shared-home", + "slurm-scheduling", + ] + shared = next(item for item in metadata if item["name"] == "ssh-shared-home") + assert shared["substrates"] == ["ssh"] + assert shared["ssh_home_mode"] == "shared" + assert shared["schedule_phase"] == "shared_home" + + +def test_human_listing_exposes_names_and_substrate_compatibility(): + """Human listing includes stable selection facts without freezing prose.""" + rows = { + fields[0]: fields[1] + for line in format_scenario_listing(_SCENARIOS.SCENARIO_CATALOG).splitlines() + if (fields := line.split("\t", 2)) + } + + assert rows["baseline"] == "slurm,ssh" + assert rows["ssh-shared-home"] == "ssh" + assert rows["slurm-scheduling"] == "slurm" + + +def test_default_plan_expands_substrates_and_batches_shared_home(): + """The full plan has one bounded shared-home SSH transition.""" + plan = _plan_identity() + + shared_start = plan.index(("transition", "shared")) + shared_stop = plan.index(("transition", "separate")) + assert plan[shared_start + 1 : shared_stop] == (("work", "ssh-shared-home", "ssh"),) + assert plan[shared_stop + 1 :] == (("work", "slurm-scheduling", "slurm"),) + assert all( + substrate != "ssh" + for kind, _name, substrate in plan[:shared_start] + if kind == "work" and _name == "ssh-shared-home" + ) + + +def test_ssh_selection_excludes_slurm_work(): + """An SSH plan contains only SSH work and still restores home mode.""" + plan = plan_scenarios(substrate="ssh", registry=_SCENARIOS.SCENARIO_CATALOG) + + work = [step for step in plan if isinstance(step, WorkItem)] + assert work + assert {item.substrate for item in work} == {Substrate.SSH} + assert isinstance(plan[-1], SshHomeTransition) + assert plan[-1].target is SshHomeMode.SEPARATE + + +def test_slurm_selection_needs_no_home_transition(): + """A Slurm-only plan cannot mutate the SSH worker pool.""" + plan = plan_scenarios( + substrate=Substrate.SLURM, registry=_SCENARIOS.SCENARIO_CATALOG + ) + + assert plan + assert all(isinstance(step, WorkItem) for step in plan) + assert {step.substrate for step in plan} == {Substrate.SLURM} + + +def test_requested_scenarios_are_order_independent(): + """Repeatable CLI selections do not determine execution order.""" + names = ("slurm-scheduling", "baseline", "ssh-shared-home") + expected = _plan_identity(requested=names) + + for permutation in itertools.permutations(names): + assert _plan_identity(requested=permutation) == expected + + +def test_registry_order_does_not_determine_execution_order(): + """Registry refactoring cannot silently change execution order.""" + expected = _plan_identity() + + assert _plan_identity(registry=reversed(_SCENARIOS.SCENARIO_CATALOG)) == expected + + +@pytest.mark.parametrize("substrate", ("ssh", "slurm")) +def test_explicit_incompatible_scenario_is_rejected(substrate): + """An explicit scenario/substrate mismatch is actionable.""" + scenario = "slurm-cartesian" if substrate == "ssh" else "ssh-shared-home" + + with pytest.raises(ScenarioPlanningError, match="incompatible"): + plan_scenarios( + substrate=substrate, + requested=(scenario,), + registry=_SCENARIOS.SCENARIO_CATALOG, + ) + + +def test_duplicate_and_unknown_requests_are_rejected(): + """Typos and duplicate repeatable options do not produce surprising work.""" + with pytest.raises(ScenarioPlanningError, match="duplicate requested"): + select_scenarios(requested=("baseline", "baseline")) + with pytest.raises(ScenarioPlanningError, match="unknown scenario"): + select_scenarios(requested=("not-a-scenario",)) + + +def test_duplicate_registry_names_are_rejected(): + """The registry has one authoritative definition per name.""" + duplicate = replace(_SCENARIOS.SCENARIO_CATALOG[0], description="Duplicate") + + with pytest.raises(ScenarioPlanningError, match="duplicate scenario in registry"): + scenario_metadata((*_SCENARIOS.SCENARIO_CATALOG, duplicate)) + + +def test_shared_home_metadata_constraints_are_enforced(): + """Shared mode cannot leak outside one SSH-only scheduling phase.""" + invalid = Scenario( + "invalid-shared", + "Invalid", + frozenset({Substrate.SSH, Substrate.SLURM}), + 1, + phase=SchedulePhase.SHARED_HOME, + ssh_home_mode=SshHomeMode.SHARED, + ) + + with pytest.raises(ScenarioPlanningError, match="must be SSH-only"): + scenario_metadata((invalid,)) + + +def test_selected_scenarios_follow_explicit_priority_and_tiebreaker(): + """Same-phase scenarios use order then stable name as tie-breakers.""" + later_name = Scenario( + "zeta", + "Zeta", + frozenset({Substrate.SLURM}), + 5, + ) + earlier_name = replace(later_name, name="alpha", description="Alpha") + + selected = select_scenarios(registry=(later_name, earlier_name)) + + assert [scenario.name for scenario in selected] == ["alpha", "zeta"] + + +def test_all_selector_filters_no_scenarios(): + """The all selector is accepted only as a request, not scenario metadata.""" + selected = select_scenarios(substrate=Substrate.ALL) + + assert len(selected) == len(_SCENARIOS.SCENARIOS) + invalid = Scenario( + "all-substrate", + "Invalid", + frozenset({Substrate.ALL}), + 1, + ) + with pytest.raises(ScenarioPlanningError, match="concrete substrates"): + scenario_metadata((invalid,)) + + +def test_unknown_substrate_is_actionable(): + """Planner wraps invalid substrate values in its public error type.""" + with pytest.raises(ScenarioPlanningError, match="unknown substrate"): + plan_scenarios(substrate="kubernetes") diff --git a/tests/test_integration_deployment_cache.py b/tests/test_integration_deployment_cache.py new file mode 100644 index 0000000..bc7f5bf --- /dev/null +++ b/tests/test_integration_deployment_cache.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for content-addressed integration deployments.""" + +import importlib.util +import json +import os +import subprocess +import sys +import tarfile +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_MODULE_PATH = _REPO_ROOT / "integration-tests" / "lib" / "deployment_cache.py" +_SPEC = importlib.util.spec_from_file_location( + "integration_deployment_cache_under_test", _MODULE_PATH +) +assert _SPEC and _SPEC.loader +_CACHE = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _CACHE +_SPEC.loader.exec_module(_CACHE) + + +class _Runner: + """Run local commands and record deployment-builder invocations.""" + + def __init__(self): + self.builder_calls = 0 + + def run(self, arguments, *, cwd=None, timeout=None): + """Run one command using the integration runner's result shape.""" + command = [str(argument) for argument in arguments] + if command[0].endswith("build_tarball.sh"): + self.builder_calls += 1 + result = subprocess.run( + command, + cwd=cwd, + timeout=timeout, + text=True, + capture_output=True, + check=False, + ) + if result.returncode: + raise RuntimeError( + f"command failed ({result.returncode}): {command}\n{result.stderr}" + ) + return SimpleNamespace(stdout=result.stdout, stderr=result.stderr) + + +def _write(path: Path, content: str, mode: int = 0o644) -> None: + """Write one fixture file with a controlled mode.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + path.chmod(mode) + + +def _repository(tmp_path: Path) -> tuple[Path, Path]: + """Create a minimal tracked tree with a compatible archive builder.""" + repository = tmp_path / "repository" + repository.mkdir() + _write(repository / "NOTICE", "notice\n") + _write(repository / "payload.txt", "first\n") + _write( + repository / "utils" / "build_tarball.sh", + """#!/usr/bin/env bash +set -eu +[[ $# -eq 0 ]] +printf 'generated\n' > utils/generated-tool +tar -czf ../storage-scale-test.tar.gz . +""", + 0o755, + ) + subprocess.run(["git", "init", "-q"], cwd=repository, check=True) + subprocess.run(["git", "add", "."], cwd=repository, check=True) + binary = tmp_path / "elbencho" + _write(binary, "#!/usr/bin/env bash\nexit 0\n", 0o755) + return repository, binary + + +def _request(tmp_path: Path, repository: Path, binary: Path): + """Return a standard request for one fixture deployment.""" + return _CACHE.DeploymentCacheRequest( + repo_root=repository, + cache_root=tmp_path / "cache", + architecture="x86_64", + binary=binary, + binary_name="elbencho", + ) + + +def _manifest(deployment) -> dict[str, object]: + """Read a cached deployment's published manifest.""" + return json.loads(deployment.manifest.read_text(encoding="utf-8")) + + +def test_reuses_verified_archive_for_identical_inputs(tmp_path): + """A second request reuses the archive built from the same snapshot.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + + first = _CACHE.get_or_build_deployment(runner, request) + second = _CACHE.get_or_build_deployment(runner, request) + + assert not first.cache_hit + assert second.cache_hit + assert first.archive == second.archive + assert runner.builder_calls == 1 + document = _manifest(first) + assert document["input_digest"] == first.key + assert document["archive"]["sha256"] + assert document["identity"]["binary"]["path"] == "utils/elbencho" + + +def test_snapshot_uses_current_tracked_content_and_excludes_untracked(tmp_path): + """The archive reflects tracked edits without admitting untracked files.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + _write(repository / "payload.txt", "modified\n") + _write(repository / "untracked.txt", "do not package\n") + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + with tarfile.open(second.archive, "r:gz") as archive: + names = archive.getnames() + payload = archive.extractfile("./payload.txt") + assert payload is not None + assert payload.read() == b"modified\n" + assert "./utils/generated-tool" in names + assert "./untracked.txt" not in names + + +def test_snapshot_honors_tracked_working_tree_deletions(tmp_path): + """A tracked file deleted locally is absent from the archive and identity.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + (repository / "payload.txt").unlink() + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + with tarfile.open(second.archive, "r:gz") as archive: + assert "./payload.txt" not in archive.getnames() + paths = [ + entry["path"] for entry in _manifest(second)["identity"]["source"]["entries"] + ] + assert "payload.txt" not in paths + + +def test_file_mode_participates_in_cache_identity(tmp_path): + """Changing a tracked mode invalidates an otherwise identical snapshot.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + (repository / "payload.txt").chmod(0o600) + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + entries = _manifest(second)["identity"]["source"]["entries"] + payload = next(entry for entry in entries if entry["path"] == "payload.txt") + assert payload["mode"] == 0o600 + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("architecture", "aarch64"), + ("recipe", 2), + ), +) +def test_build_identity_inputs_invalidate_cache(tmp_path, field, value): + """Architecture and builder recipe are cache-key inputs.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + + second = _CACHE.get_or_build_deployment(runner, replace(request, **{field: value})) + + assert first.key != second.key + assert runner.builder_calls == 2 + + +def test_binary_and_runtime_content_invalidate_cache(tmp_path): + """External executable and runtime identities participate in the key.""" + repository, binary = _repository(tmp_path) + runtime = tmp_path / "runtime" + _write(runtime / "lib" / "loader.so", "first runtime\n") + runner = _Runner() + request = replace(_request(tmp_path, repository, binary), runtime=runtime) + first = _CACHE.get_or_build_deployment(runner, request) + _write(binary, "#!/usr/bin/env bash\necho changed\n", 0o755) + + second = _CACHE.get_or_build_deployment(runner, request) + _write(runtime / "lib" / "loader.so", "second runtime\n") + third = _CACHE.get_or_build_deployment(runner, request) + + assert len({first.key, second.key, third.key}) == 3 + + +def test_runtime_directory_mode_participates_in_cache_identity(tmp_path): + """Runtime directory layout and modes cannot alias one cache entry.""" + repository, binary = _repository(tmp_path) + runtime = tmp_path / "runtime" + (runtime / "empty").mkdir(parents=True) + (runtime / "empty").chmod(0o755) + runner = _Runner() + request = replace(_request(tmp_path, repository, binary), runtime=runtime) + first = _CACHE.get_or_build_deployment(runner, request) + (runtime / "empty").chmod(0o700) + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + + +def test_unsafe_binary_name_is_rejected(tmp_path): + """A configured binary name cannot escape the snapshot's utils directory.""" + repository, binary = _repository(tmp_path) + request = replace( + _request(tmp_path, repository, binary), binary_name="../../outside" + ) + + with pytest.raises(_CACHE.DeploymentCacheError, match="not a basename"): + _CACHE.get_or_build_deployment(_Runner(), request) + + +def test_corrupt_archive_is_rebuilt_under_same_key(tmp_path): + """Checksum validation prevents reuse of a damaged cached archive.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + first.archive.write_bytes(b"corrupt") + + rebuilt = _CACHE.get_or_build_deployment(runner, request) + + assert rebuilt.key == first.key + assert not rebuilt.cache_hit + assert runner.builder_calls == 2 + assert rebuilt.archive.read_bytes() != b"corrupt" + assert not any( + path.name.startswith(".invalid-") for path in request.cache_root.iterdir() + ) + + +def test_builder_receives_snapshot_not_working_tree(tmp_path): + """The existing builder executes only from the manifested snapshot.""" + repository, binary = _repository(tmp_path) + + class SnapshotRunner(_Runner): + """Record the working directory used for the archive builder.""" + + def __init__(self): + super().__init__() + self.builder_cwd = None + self.builder_arguments = None + + def run(self, arguments, *, cwd=None, timeout=None): + if str(arguments[0]).endswith("build_tarball.sh"): + self.builder_cwd = Path(cwd) + self.builder_arguments = [str(argument) for argument in arguments] + assert self.builder_cwd != repository + assert self.builder_cwd.name == "source" + return super().run(arguments, cwd=cwd, timeout=timeout) + + runner = SnapshotRunner() + + _CACHE.get_or_build_deployment(runner, _request(tmp_path, repository, binary)) + + assert runner.builder_cwd is not None + assert runner.builder_arguments == [ + str(runner.builder_cwd / "utils" / "build_tarball.sh") + ] + + +def test_unsafe_tracked_symlink_is_rejected(tmp_path): + """A tracked symbolic link cannot escape the immutable snapshot.""" + repository, binary = _repository(tmp_path) + os.symlink("../outside", repository / "escape") + subprocess.run(["git", "add", "escape"], cwd=repository, check=True) + + with pytest.raises(_CACHE.DeploymentCacheError, match="unsafe tracked symbolic"): + _CACHE.get_or_build_deployment( + _Runner(), _request(tmp_path, repository, binary) + ) diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py new file mode 100644 index 0000000..9f89fe5 --- /dev/null +++ b/tests/test_integration_driver_safety.py @@ -0,0 +1,3095 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Safety and rollout regression tests for the integration driver.""" + +import importlib.util +import json +import os +import shutil +import sys +from dataclasses import replace +from pathlib import Path, PurePosixPath +from types import SimpleNamespace + +import pytest +import yaml + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_DRIVER_PATH = _REPO_ROOT / "integration-tests" / "bin" / "integration-test.py" +_SPEC = importlib.util.spec_from_file_location( + "integration_driver_under_test", _DRIVER_PATH +) +assert _SPEC and _SPEC.loader +_DRIVER = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _DRIVER +_SPEC.loader.exec_module(_DRIVER) +_FILESYSTEM = sys.modules["filesystem_integration"] +_bootstrap_state_dir = getattr(_DRIVER, "_bootstrap_state_dir") +_acquire_lock = getattr(_DRIVER, "_acquire_lock") +_check_docker_capacity = getattr(_DRIVER, "_check_docker_capacity") +_check_host_capacity = getattr(_DRIVER, "_check_host_capacity") +_collect_diagnostics = getattr(_DRIVER, "_collect_diagnostics") +_configure_nfs = getattr(_DRIVER, "_configure_nfs") +_create_cluster = getattr(_DRIVER, "_create_cluster") +_claim_nfs_host_owner = getattr(_DRIVER, "_claim_nfs_host_owner") +_configure_user_tool_path = getattr(_DRIVER, "_configure_user_tool_path") +_ensure_apt_packages = getattr(_DRIVER, "_ensure_apt_packages") +_ensure_export_marker = getattr(_DRIVER, "_ensure_export_marker") +_ensure_nfs_image_capacity = getattr(_DRIVER, "_ensure_nfs_image_capacity") +_observed_kubernetes_version = getattr(_DRIVER, "_observed_kubernetes_version") +_ensure_slurm_workload_account = getattr(_DRIVER, "_ensure_slurm_workload_account") +_export_paths = getattr(_DRIVER, "_export_paths") +_kind_clusters = getattr(_DRIVER, "_kind_clusters") +_login_pod = getattr(_DRIVER, "_login_pod") +_load_state = getattr(_FILESYSTEM, "_load_state") +_inspect_ssh_home_pool = getattr(_DRIVER, "_inspect_ssh_home_pool") +_migrate_export_data_to_image = getattr(_DRIVER, "_migrate_export_data_to_image") +_prepare_host_dependencies = getattr(_DRIVER, "_prepare_host_dependencies") +_prepare_sbx_shared = getattr(_DRIVER, "_prepare_sbx_shared") +_record_nfs_service_state = getattr(_DRIVER, "_record_nfs_service_state") +_nfs_host_lock = getattr(_DRIVER, "_nfs_host_lock") +_delete_nfs_firewall_rule = getattr(_DRIVER, "_delete_nfs_firewall_rule") +_nfs_host_owner_document = getattr(_DRIVER, "_nfs_host_owner_document") +_retained_cluster_matches_profile = getattr( + _DRIVER, "_retained_cluster_matches_profile" +) +_stop_owned_nfs = getattr(_DRIVER, "_stop_owned_nfs") +_driver_pods_with_container = getattr(_DRIVER, "_pods_with_container") +_remove_nfs_configuration = getattr(_DRIVER, "_remove_nfs_configuration") +_remove_sbx_shared = getattr(_DRIVER, "_remove_sbx_shared") +_restore_nfs_service_state = getattr(_DRIVER, "_restore_nfs_service_state") +_restore_preexisting_nfs_threads = getattr(_DRIVER, "_restore_preexisting_nfs_threads") +_select_storage_backend = getattr(_DRIVER, "_select_storage_backend") +_set_live_nfs_threads = getattr(_DRIVER, "_set_live_nfs_threads") +_storage_backend_document = getattr(_DRIVER, "_storage_backend_document") +_unmount_temporary_filesystem = getattr(_DRIVER, "_unmount_temporary_filesystem") +_validate_nfs_filesystem_capacity = getattr( + _DRIVER, "_validate_nfs_filesystem_capacity" +) +_validate_teardown_ownership = getattr(_DRIVER, "_validate_teardown_ownership") +_validate_lifecycle_paths = getattr(_DRIVER, "_validate_lifecycle_paths") +_validate_ssh_storage = getattr(_DRIVER, "_validate_ssh_storage") +_teardown_environment_locked = getattr(_DRIVER, "_teardown_environment_locked") +_begin_image_build = getattr(_DRIVER, "_begin_image_build") +_build_owned_image = getattr(_DRIVER, "_build_owned_image") +_acquire_pinned_image = getattr(_DRIVER, "_acquire_pinned_image") +_ensure_pinned_image = getattr(_DRIVER, "_ensure_pinned_image") +_install_slurm = getattr(_DRIVER, "_install_slurm") +_render_resource_text = getattr(_DRIVER, "_render_resource_text") +_transient_image_pull_failure = getattr(_DRIVER, "_transient_image_pull_failure") +_expected_kubernetes_version = getattr(_DRIVER, "_expected_kubernetes_version") +_image_ownership_state = getattr(_DRIVER, "_image_ownership_state") +_image_id = getattr(_DRIVER, "_image_id") +_inspect_pinned_image = getattr(_DRIVER, "_inspect_pinned_image") +_prepare_csi_images = getattr(_DRIVER, "_prepare_csi_images") +_load_image_into_nodes = getattr(_DRIVER, "_load_image_into_nodes") +_remove_harness_images = getattr(_DRIVER, "_remove_harness_images") +_write_image_ownership_state = getattr(_DRIVER, "_write_image_ownership_state") +_wait_for_kube_api = getattr(_DRIVER, "_wait_for_kube_api") +_wait_for_no_ssh_pods = getattr(_DRIVER, "_wait_for_no_ssh_pods") +_wait_for_slurm = getattr(_DRIVER, "_wait_for_slurm") +_wait_for_ssh = getattr(_DRIVER, "_wait_for_ssh") +_assert_execution_contract = getattr(_FILESYSTEM, "_assert_execution_contract") +_assert_ordered_workers = getattr(_FILESYSTEM, "_assert_ordered_workers") +_ensure_elbencho = getattr(_FILESYSTEM, "_ensure_elbencho") +_prepare_scenario_data = getattr(_FILESYSTEM, "_prepare_scenario_data") +_cleanup_scenario_storage = getattr(_FILESYSTEM, "_cleanup_scenario_storage") +_cleanup_ssh_remote_results = getattr(_FILESYSTEM, "_cleanup_ssh_remote_results") +_run_step = getattr(_FILESYSTEM, "_run_step") +_pod_command = getattr(_FILESYSTEM, "_pod_command") +_test_command = getattr(_FILESYSTEM, "_test_command") +_record_secondary_cleanup_failure = getattr( + _FILESYSTEM, "_record_secondary_cleanup_failure" +) +_slurm_run_time = getattr(_FILESYSTEM, "_slurm_run_time") +_preserve_scenario_failure_diagnostics = getattr( + _FILESYSTEM, "_preserve_scenario_failure_diagnostics" +) +_remote_staging_operations = getattr(_FILESYSTEM, "_remote_staging_operations") +_reset_result_base = getattr(_FILESYSTEM, "_reset_result_base") +_pods_with_container = getattr(_FILESYSTEM, "_pods_with_container") +_require_pods = getattr(_FILESYSTEM, "_require_pods") + + +def test_scenario_listing_short_circuits_before_privileged_state(monkeypatch, capsys): + """Listing scenarios needs no account, state, fixture, or root exception.""" + monkeypatch.setattr( + sys, + "argv", + [str(_DRIVER_PATH), "test", "--list-scenarios"], + ) + monkeypatch.setattr(os, "geteuid", lambda: 0) + monkeypatch.setattr( + _DRIVER, + "_config", + lambda _arguments: pytest.fail("scenario listing resolved an account"), + ) + + assert _DRIVER.main() == 0 + assert capsys.readouterr().out.startswith("baseline\t") + + +def test_integration_timeouts_control_command_process_groups(tmp_path): + """Remote and local bounds do not leave descendants outside timeout.""" + config = _config(tmp_path / "state", tmp_path / "export") + fixture = SimpleNamespace(login_pod="login", login_container="login") + + pod = _pod_command(config, "login", "login", "true") + ssh = _test_command(config, fixture, "ssh", "true", 30) + + assert "--foreground" not in pod + assert "--foreground" not in ssh + + +def test_root_lifecycle_is_rejected_before_state_resolution(monkeypatch): + """Every mutating or executing lifecycle action rejects root up front.""" + monkeypatch.setattr(sys, "argv", [str(_DRIVER_PATH), "setup"]) + monkeypatch.setattr(os, "geteuid", lambda: 0) + monkeypatch.setattr( + _DRIVER, + "_config", + lambda _arguments: pytest.fail("root lifecycle resolved setup state"), + ) + + assert _DRIVER.main() == 1 + + +def _config(state_dir: Path, export_dir: Path) -> object: + """Return a minimal real driver configuration.""" + return _DRIVER.Config( + cluster_name="test-cluster", + namespace="test-namespace", + state_dir=state_dir, + export_dir=export_dir, + storage_backend="sbx-shared", + sbx_shared_root=_REPO_ROOT / "tmp" / "test-shared", + test_user="tester", + test_uid=42424, + test_gid=43434, + verbose=False, + ) + + +def test_state_bootstrap_is_user_owned_and_idempotent(tmp_path, monkeypatch): + """State starts under the caller and repeated bootstrap preserves it.""" + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setenv("PATH", "/usr/bin") + + _bootstrap_state_dir(config) + _configure_user_tool_path(config) + first_marker = (config.state_dir / _DRIVER.STATE_MARKER).read_text(encoding="utf-8") + _bootstrap_state_dir(config) + + assert config.state_dir.stat().st_uid == os.getuid() + assert (config.state_dir.stat().st_mode & 0o777) == 0o750 + assert (config.state_dir / _DRIVER.STATE_MARKER).read_text( + encoding="utf-8" + ) == first_marker + assert os.environ["PATH"].split(os.pathsep)[0] == str(config.state_dir / "bin") + assert all( + path in os.environ["PATH"].split(os.pathsep) + for path in _DRIVER.SYSTEM_ADMIN_PATHS + ) + assert os.environ["HELM_CACHE_HOME"] == str( + config.state_dir / "tool-state" / "helm" / "cache" + ) + + +def test_lifecycle_lock_survives_state_tree_removal(tmp_path): + """Teardown cannot replace the inode locked by its active invocation.""" + config = _config(tmp_path / "state", tmp_path / "export") + config.state_dir.mkdir() + first = _acquire_lock(config) + try: + shutil.rmtree(config.state_dir) + config.state_dir.mkdir() + with pytest.raises(_DRIVER.ProvisionError, match="another integration"): + second = _acquire_lock(config) + second.close() + finally: + first.close() + + +def test_nfs_global_lock_rejects_unowned_directory_without_mutating_it(): + """A pre-created host-global lock path is never chmodded or adopted.""" + + class _UnsafeLockRunner: + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if "stat" in command: + return SimpleNamespace( + returncode=0, stdout="1001:1001:755\n", stderr="" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _UnsafeLockRunner() + with pytest.raises(_DRIVER.ProvisionError, match="unsafe NFS lock directory"): + with _nfs_host_lock(runner): + pytest.fail("unsafe lock directory was accepted") + + assert not any( + "chmod" in command or "touch" in command for command in runner.commands + ) + + +def test_standard_admin_path_resolves_nfs_tools(tmp_path, monkeypatch): + """Ordinary-user setup finds tools installed outside its initial PATH.""" + admin_path = tmp_path / "usr-sbin" + admin_path.mkdir() + losetup = admin_path / "losetup" + losetup.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + losetup.chmod(0o755) + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setenv("PATH", "/usr/bin") + monkeypatch.setattr(_DRIVER, "SYSTEM_ADMIN_PATHS", (str(admin_path),)) + + _configure_user_tool_path(config) + + assert _DRIVER.shutil.which("losetup") == str(losetup) + + +def test_capacity_check_uses_fixture_paths_nearest_existing_parent( + tmp_path, monkeypatch +): + """A configurable state path is checked on its actual backing filesystem.""" + existing = tmp_path / "existing" + existing.mkdir() + requested = existing / "not-yet-created" / "state" + observed = [] + monkeypatch.setattr(_DRIVER.os, "cpu_count", lambda: 4) + monkeypatch.setattr( + _DRIVER, + "_meminfo", + lambda: {"MemTotal": 16 * _DRIVER.GIB, "MemAvailable": 12 * _DRIVER.GIB}, + ) + monkeypatch.setattr( + _DRIVER.shutil, + "disk_usage", + lambda path: observed.append(Path(path)) + or SimpleNamespace(free=30 * _DRIVER.GIB), + ) + + _check_host_capacity(requested) + + assert observed == [existing] + + +def test_docker_capacity_checks_visible_data_root(tmp_path, monkeypatch): + """Docker storage is checked separately when its data root is host-visible.""" + observed = [] + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, stdout=f"{tmp_path}\n", stderr="" + ) + ) + monkeypatch.setattr( + _DRIVER.shutil, + "disk_usage", + lambda path: observed.append(Path(path)) + or SimpleNamespace(free=30 * _DRIVER.GIB), + ) + + _check_docker_capacity(runner) + + assert observed == [tmp_path] + + +class _TemporaryUnmountRunner: + """Model a temporary mount that disappears after a chosen attempt.""" + + def __init__(self, unmounted_after): + self.unmounted_after = unmounted_after + self.unmount_attempts = 0 + + def run(self, arguments, **_kwargs): + """Return deterministic umount and findmnt outcomes.""" + command = [str(item) for item in arguments] + if "umount" in command: + self.unmount_attempts += 1 + return SimpleNamespace( + returncode=0 if self.unmount_attempts >= self.unmounted_after else 1, + stdout="", + stderr=( + "" + if self.unmount_attempts >= self.unmounted_after + else "target is busy" + ), + ) + assert command[0] == "findmnt" + is_mounted = self.unmount_attempts < self.unmounted_after + return SimpleNamespace( + returncode=0 if is_mounted else 1, + stdout="ext4\n" if is_mounted else "", + stderr="", + ) + + +class _MigrationRunner(_TemporaryUnmountRunner): + """Model image creation plus the temporary migration mount.""" + + def run(self, arguments, **kwargs): + """Create the sparse placeholder and model privileged operations.""" + command = [str(item) for item in arguments] + if command[0] == "truncate": + Path(command[-1]).touch() + return SimpleNamespace(returncode=0, stdout="", stderr="") + if "umount" in command or command[0] == "findmnt": + return super().run(arguments, **kwargs) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_temporary_unmount_retries_until_mount_disappears(tmp_path, monkeypatch): + """A transient busy mount is retried and verified before cleanup continues.""" + runner = _TemporaryUnmountRunner(unmounted_after=3) + sleeps = [] + monkeypatch.setattr(_DRIVER.time, "sleep", sleeps.append) + + _unmount_temporary_filesystem(runner, tmp_path / "migration-mount") + + assert runner.unmount_attempts == 3 + assert sleeps == [ + _DRIVER.TEMPORARY_UNMOUNT_RETRY_SECONDS, + _DRIVER.TEMPORARY_UNMOUNT_RETRY_SECONDS, + ] + + +def test_temporary_unmount_rejects_persistent_mount(tmp_path, monkeypatch): + """Setup fails safely instead of deleting a still-mounted temporary path.""" + runner = _TemporaryUnmountRunner( + unmounted_after=_DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + 1 + ) + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + with pytest.raises(_DRIVER.ProvisionError, match="mount remains active"): + _unmount_temporary_filesystem(runner, tmp_path / "migration-mount") + + assert runner.unmount_attempts == _DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + + +def test_nfs_migration_retains_verified_active_mountpoint(tmp_path, monkeypatch): + """Failed unmount verification cannot trigger recursive mount cleanup.""" + state_dir = tmp_path / "state" + export_dir = tmp_path / "export" + state_dir.mkdir() + export_dir.mkdir() + mountpoint = state_dir / "known-migration-mount" + + def _make_mountpoint(**_kwargs): + mountpoint.mkdir() + return str(mountpoint) + + runner = _MigrationRunner(unmounted_after=_DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + 1) + monkeypatch.setattr(_DRIVER.tempfile, "mkdtemp", _make_mountpoint) + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + with pytest.raises(_DRIVER.ProvisionError, match="mount remains active"): + _migrate_export_data_to_image(runner, _config(state_dir, export_dir)) + + assert mountpoint.is_dir() + assert not (state_dir / "nfs-export.ext4").exists() + assert (state_dir / "nfs-export.ext4.new").exists() + assert runner.unmount_attempts == _DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + + +def test_nfs_migration_removes_mountpoint_only_after_verified_unmount( + tmp_path, monkeypatch +): + """Successful migration removes its now-unmounted temporary leaf.""" + state_dir = tmp_path / "state" + export_dir = tmp_path / "export" + state_dir.mkdir() + export_dir.mkdir() + mountpoint = state_dir / "known-migration-mount" + + def _make_mountpoint(**_kwargs): + mountpoint.mkdir() + return str(mountpoint) + + runner = _MigrationRunner(unmounted_after=1) + monkeypatch.setattr(_DRIVER.tempfile, "mkdtemp", _make_mountpoint) + + _migrate_export_data_to_image(runner, _config(state_dir, export_dir)) + + assert not mountpoint.exists() + assert (state_dir / "nfs-export.ext4").exists() + assert not (state_dir / "nfs-export.ext4.new").exists() + assert runner.unmount_attempts == 1 + + +class _CopyFailureMigrationRunner(_MigrationRunner): + """Fail while copying retained export data into the new image.""" + + def run(self, arguments, **kwargs): + command = [str(item) for item in arguments] + if "cp" in command: + raise _DRIVER.ProvisionError("copy failed") + return super().run(arguments, **kwargs) + + +def test_nfs_migration_does_not_publish_a_failed_image(tmp_path, monkeypatch): + """A migration failure cannot leave an incomplete image at the final path.""" + state_dir = tmp_path / "state" + export_dir = tmp_path / "export" + state_dir.mkdir() + export_dir.mkdir() + mountpoint = state_dir / "known-migration-mount" + + def _make_mountpoint(**_kwargs): + mountpoint.mkdir() + return str(mountpoint) + + monkeypatch.setattr(_DRIVER.tempfile, "mkdtemp", _make_mountpoint) + + with pytest.raises(_DRIVER.ProvisionError, match="copy failed"): + _migrate_export_data_to_image( + _CopyFailureMigrationRunner(unmounted_after=1), + _config(state_dir, export_dir), + ) + + assert not (state_dir / "nfs-export.ext4").exists() + assert not (state_dir / "nfs-export.ext4.new").exists() + assert not mountpoint.exists() + + +def test_nfs_server_configures_eight_workers(tmp_path, monkeypatch): + """Rendered NFS state requests enough workers for concurrent clients.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + runner = _RecordingRunner() + for name in ( + "_record_nfs_service_state", + "_ensure_export_filesystem", + "_ensure_export_marker", + "_ensure_nfs_firewall", + "_set_live_nfs_threads", + ): + monkeypatch.setattr(_DRIVER, name, lambda *_args: None) + monkeypatch.setattr(_DRIVER, "_read_system_file", lambda *_args: None) + + _configure_nfs(runner, config, "172.18.0.0/16", "172.18.0.1") + + daemon_config = (config.manifests_dir / "storage-scale-test-nfs.conf").read_text( + encoding="utf-8" + ) + assert f"threads = {_DRIVER.NFS_SERVER_THREADS}\n" in daemon_config + assert _DRIVER.NFS_SERVER_THREADS == 8 + + +def test_nfs_reconciliation_validates_previous_subnet_before_update( + tmp_path, monkeypatch +): + """A recreated kind subnet can replace its unchanged owned export file.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + old_subnet = "172.18.0.0/16" + new_subnet = "172.19.0.0/16" + old_export = ( + f"{config.export_dir} {old_subnet}(rw,sync,no_subtree_check,fsid=0," + f"all_squash,anonuid={_DRIVER.NFS_UID},anongid={_DRIVER.NFS_GID})\n" + ) + daemon = f"[nfsd]\nvers3 = n\nvers4 = y\nthreads = {_DRIVER.NFS_SERVER_THREADS}\n" + export_source = config.manifests_dir / "storage-scale-test.exports" + daemon_source = config.manifests_dir / "storage-scale-test-nfs.conf" + export_source.write_text(old_export, encoding="utf-8") + daemon_source.write_text(daemon, encoding="utf-8") + runner = _RecordingRunner() + for name in ( + "_record_nfs_service_state", + "_ensure_export_filesystem", + "_ensure_export_marker", + "_ensure_nfs_firewall", + "_set_live_nfs_threads", + ): + monkeypatch.setattr(_DRIVER, name, lambda *_args: None) + monkeypatch.setattr( + _DRIVER, + "_read_system_file", + lambda _runner, path: ( + old_export if path == _DRIVER.NFS_EXPORT_CONFIG else daemon + ), + ) + + _configure_nfs(runner, config, new_subnet, "172.19.0.1") + + assert new_subnet in export_source.read_text(encoding="utf-8") + + +def test_firewall_cleanup_removes_stored_rule_while_ufw_is_inactive(): + """Teardown consults UFW configuration instead of runtime status output.""" + + class _InactiveUfwRunner: + def __init__(self): + self.rule_exists = True + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[-2:] == ["show", "added"]: + output = "" + if self.rule_exists: + output = ( + "ufw allow from 172.18.0.0/16 to any port 2049 " + f"proto tcp comment '{_DRIVER.UFW_COMMENT}'\n" + ) + return SimpleNamespace(returncode=0, stdout=output, stderr="") + if "delete" in command: + self.rule_exists = False + return SimpleNamespace(returncode=0, stdout="Status: inactive\n", stderr="") + + runner = _InactiveUfwRunner() + _delete_nfs_firewall_rule(runner, "172.18.0.0/16") + + assert not runner.rule_exists + assert any("delete" in command for command in runner.commands) + + +class _ImageCapacityRunner: + """Apply sparse truncation while recording filesystem growth commands.""" + + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + """Record a command and enact only the harmless sparse resize.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if command[0] == "truncate": + os.truncate(command[-1], int(command[-2])) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_existing_nfs_image_grows_online_to_advertised_capacity(tmp_path): + """Repeated setup expands both a retained sparse image and loop device.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + config.nfs_image.touch() + os.truncate(config.nfs_image, 128 * 1024 * 1024) + runner = _ImageCapacityRunner() + + _ensure_nfs_image_capacity(runner, config, "/dev/loop9") + + assert config.nfs_image.stat().st_size == _DRIVER.NFS_IMAGE_BYTES + assert _DRIVER.NFS_IMAGE_BYTES == 4 * _DRIVER.GIB + rendered = [" ".join(command) for command in runner.commands] + assert any("losetup --set-capacity /dev/loop9" in line for line in rendered) + assert any("resize2fs /dev/loop9" in line for line in rendered) + + +def test_fixture_capacity_budget_drives_storage_declarations(): + """The backing image and manifests share one explicit capacity budget.""" + assert _DRIVER.NFS_IMAGE_BYTES >= _DRIVER.NFS_BUDGET_BYTES + assert _DRIVER.NFS_BUDGET_BYTES > 2 * _DRIVER.GIB + for name in ("nfs-storage.yaml.tmpl", "sbx-storage.yaml.tmpl"): + text = (_REPO_ROOT / "integration-tests" / "manifests" / name).read_text( + encoding="utf-8" + ) + assert "@@STORAGE_TEST_CAPACITY@@" in text + assert "@@SSH_HOME_CAPACITY@@" in text + + +def test_mounted_nfs_capacity_must_satisfy_the_runtime_budget(tmp_path): + """A retained image cannot pass based on sparse-file length alone.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, + stdout=f"{_DRIVER.NFS_BUDGET_BYTES // 4096} 4096 1\n", + stderr="", + ) + ) + + with pytest.raises(_DRIVER.ProvisionError, match="capacity budget"): + _validate_nfs_filesystem_capacity(runner, config) + + +class _NfsThreadRunner: + """Model a live pre-existing NFS service and its mutable worker count.""" + + def __init__(self, threads, *, active=True, enabled=False): + self.threads = threads + self.active = active + self.enabled = enabled + self.commands = [] + + def run(self, arguments, **_kwargs): + """Return and update the modeled live NFS worker count.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if "show" in command: + return SimpleNamespace(returncode=0, stdout="loaded\n", stderr="") + if "is-active" in command: + return SimpleNamespace( + returncode=0 if self.active else 3, + stdout="active\n" if self.active else "inactive\n", + stderr="", + ) + if "is-enabled" in command: + return SimpleNamespace( + returncode=0 if self.enabled else 1, + stdout="enabled\n" if self.enabled else "disabled\n", + stderr="", + ) + if str(_DRIVER.NFS_THREADS_PATH) in command: + return SimpleNamespace(returncode=0, stdout=f"{self.threads}\n", stderr="") + if any(Path(item).name == "rpc.nfsd" for item in command): + self.threads = int(command[-1]) + if "systemctl" in command and "stop" in command: + self.active = False + if "systemctl" in command and "enable" in command: + self.enabled = True + if "systemctl" in command and "disable" in command: + self.enabled = False + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_preexisting_nfs_worker_count_is_reconciled_and_restored(tmp_path, monkeypatch): + """Setup applies eight workers and teardown restores the host's prior count.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + runner = _NfsThreadRunner(threads=2) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _name: "/usr/sbin/rpc.nfsd") + + _record_nfs_service_state(runner, config) + _set_live_nfs_threads(runner, _DRIVER.NFS_SERVER_THREADS) + _record_nfs_service_state(runner, config) + + state = json.loads( + (config.state_dir / "nfs-service.json").read_text(encoding="utf-8") + ) + assert state == { + "previous_threads": 2, + "schema": _DRIVER.NFS_SERVICE_STATE_SCHEMA, + "service_existed": True, + "was_active": True, + "was_enabled": False, + } + assert runner.threads == 8 + + _restore_nfs_service_state(runner, config) + + assert runner.threads == 2 + assert any("disable" in command for command in runner.commands) + + +def test_nfs_service_records_active_and_enabled_states_independently( + tmp_path, monkeypatch +): + """Runtime ownership cannot be inferred from systemd boot policy.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + runner = _NfsThreadRunner(threads=3, enabled=True) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _name: "/usr/sbin/rpc.nfsd") + + _record_nfs_service_state(runner, config) + + state = json.loads( + (config.state_dir / "nfs-service.json").read_text(encoding="utf-8") + ) + assert state["service_existed"] is True + assert state["was_active"] is True + assert state["was_enabled"] is True + assert state["previous_threads"] == 3 + + +def test_enabled_inactive_nfs_service_is_stopped_but_remains_enabled( + tmp_path, monkeypatch +): + """Teardown restores runtime and boot states without conflating them.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + runner = _NfsThreadRunner(threads=0, active=False, enabled=True) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _name: "/usr/sbin/rpc.nfsd") + _record_nfs_service_state(runner, config) + runner.active = True + + _stop_owned_nfs(runner, config) + _restore_nfs_service_state(runner, config) + + assert runner.active is False + assert runner.enabled is True + + +def test_observed_kubernetes_version_requires_three_matching_nodes(tmp_path): + """Retained cluster identity comes from kubelets rather than configured pins.""" + config = _config(tmp_path / "state", tmp_path / "export") + + class _NodeRunner: + def __init__(self, versions): + self.versions = versions + + def run(self, _arguments, **_kwargs): + items = [ + {"status": {"nodeInfo": {"kubeletVersion": version}}} + for version in self.versions + ] + return SimpleNamespace( + returncode=0, stdout=json.dumps({"items": items}), stderr="" + ) + + assert ( + _observed_kubernetes_version( + _NodeRunner(["v1.34.0", "v1.34.0", "v1.34.0"]), config + ) + == "v1.34.0" + ) + with pytest.raises(_DRIVER.ProvisionError, match="one kubelet version"): + _observed_kubernetes_version( + _NodeRunner(["v1.34.0", "v1.34.0", "v1.33.0"]), config + ) + + +def test_expected_kubernetes_version_is_independent_of_kubectl(monkeypatch): + """A client-only kubectl upgrade does not invalidate a retained cluster.""" + monkeypatch.setattr(_DRIVER, "KUBECTL_VERSION", "v9.9.9") + monkeypatch.setattr(_DRIVER, "SBX_KUBECTL_VERSION", "v8.8.8") + + assert _expected_kubernetes_version("nfs") == _DRIVER.KUBERNETES_VERSION + assert _expected_kubernetes_version("sbx-shared") == _DRIVER.SBX_KUBERNETES_VERSION + + +class _DeadlineClock: + """Deterministic monotonic clock for overall-deadline tests.""" + + def __init__(self): + self.now = 0.0 + + def monotonic(self): + """Return the simulated time.""" + return self.now + + def sleep(self, seconds): + """Advance by one requested sleep.""" + self.now += seconds + + +class _DeadlineRunner: + """Consume each subprocess timeout without crossing the overall bound.""" + + def __init__(self, clock, deadline, *, pods=False): + self.clock = clock + self.deadline = deadline + self.pods = pods + self.calls = 0 + + def run(self, _arguments, *, timeout=None, **_kwargs): + """Model a subprocess that consumes its complete allowed timeout.""" + assert timeout is not None + assert self.clock.now + timeout <= self.deadline + self.clock.now += timeout + self.calls += 1 + items = [{"metadata": {"name": "ssh-worker-0"}}] if self.pods else [] + return SimpleNamespace( + returncode=1 if not self.pods else 0, + stdout=json.dumps({"items": items}), + stderr="unavailable", + ) + + +def test_kubernetes_api_poll_respects_overall_deadline(tmp_path, monkeypatch): + """One wedged kubectl call cannot exceed the readiness deadline.""" + clock = _DeadlineClock() + runner = _DeadlineRunner(clock, 90) + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setattr(_DRIVER.time, "monotonic", clock.monotonic) + monkeypatch.setattr(_DRIVER.time, "sleep", clock.sleep) + + with pytest.raises(_DRIVER.ProvisionError, match="within 90 seconds"): + _wait_for_kube_api(runner, config) + + assert runner.calls > 1 + assert clock.now == 90 + + +def test_ssh_disappearance_poll_respects_overall_deadline(tmp_path, monkeypatch): + """SSH transition timeout performs no extra post-deadline pod query.""" + clock = _DeadlineClock() + runner = _DeadlineRunner(clock, 180, pods=True) + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setattr(_DRIVER.time, "monotonic", clock.monotonic) + monkeypatch.setattr(_DRIVER.time, "sleep", clock.sleep) + + with pytest.raises(_DRIVER.ProvisionError, match="did not terminate"): + _wait_for_no_ssh_pods(runner, config) + + assert runner.calls > 1 + assert clock.now == 180 + + +def test_slurm_wait_reports_cleanly_when_deadline_precedes_first_poll( + tmp_path, monkeypatch +): + """Process suspension before the first query still yields a useful error.""" + times = iter((0.0, 301.0)) + monkeypatch.setattr(_DRIVER.time, "monotonic", lambda: next(times)) + config = _config(tmp_path / "state", tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="'slurmd': 0"): + _wait_for_slurm(None, config) + + +def test_retained_cluster_profile_uses_observed_kubelet_version(tmp_path, monkeypatch): + """A pin change marks an otherwise-running disposable cluster stale.""" + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setattr(_DRIVER, "_wait_for_kube_api", lambda *_args: None) + monkeypatch.setattr( + _DRIVER, "_observed_kubernetes_version", lambda *_args: "v1.33.0" + ) + + assert not _retained_cluster_matches_profile( + SimpleNamespace(), config, "sbx-shared" + ) + + +def test_local_scenario_failure_diagnostics_survive_workspace_cleanup(tmp_path): + """SSH logs and partial results are copied outside disposable workspace.""" + workspace = tmp_path / "workspace" + (workspace / "logs").mkdir(parents=True) + (workspace / "results").mkdir() + (workspace / "logs" / "service.log").write_text("service failed\n") + (workspace / "results" / "coordinator.log").write_text("probe failed\n") + scenario = SimpleNamespace(name="example") + runtime = _FILESYSTEM.ScenarioRuntime( + scenario=scenario, + selector="ssh", + workspace=str(workspace), + local_workspace=tmp_path, + artifact_root=tmp_path / "artifacts", + data_root="/mnt/storage-test/example", + values={}, + copied_results=[], + ) + log_dir = tmp_path / "retained" + log_dir.mkdir() + + _preserve_scenario_failure_diagnostics( + _RecordingRunner(), + SimpleNamespace(), + SimpleNamespace(), + runtime, + log_dir, + RuntimeError("scenario failed"), + ) + shutil.rmtree(workspace) + + diagnostics = log_dir / "failure-diagnostics" + assert (diagnostics / "logs" / "service.log").read_text() == "service failed\n" + assert (diagnostics / "results" / "coordinator.log").read_text() == "probe failed\n" + assert "scenario failed" in (diagnostics / "failure.txt").read_text() + + +def _cleanup_runtime(tmp_path): + """Return one marker-safe scenario runtime for cleanup tests.""" + return _FILESYSTEM.ScenarioRuntime( + scenario=SimpleNamespace(name="example"), + selector="ssh", + workspace=str(tmp_path / "workspace"), + local_workspace=tmp_path, + artifact_root=tmp_path / "artifacts", + data_root=f"{_FILESYSTEM.REMOTE_BASE}/test-data/example", + values={}, + copied_results=[], + ) + + +def test_scenario_storage_cleanup_failure_is_fatal(tmp_path): + """A successful workload cannot pass while isolated storage remains.""" + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=1, stdout="", stderr="permission denied" + ) + ) + fixture = SimpleNamespace(login_pod="login", login_container="login") + + with pytest.raises(_FILESYSTEM.IntegrationTestError, match="permission denied"): + _cleanup_scenario_storage( + runner, + _config(tmp_path / "state", tmp_path / "export"), + fixture, + _cleanup_runtime(tmp_path), + ) + + +def test_expected_failure_step_rejects_zero_exit(tmp_path, monkeypatch): + """Failure injection must propagate failure through the sweep entry point.""" + monkeypatch.setattr(_FILESYSTEM, "_test_command", lambda *_args: ["sweep"]) + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, stdout="recorded FAILED state\n", stderr="" + ) + ) + + with pytest.raises( + _FILESYSTEM.IntegrationTestError, match="unexpectedly succeeded" + ): + _run_step( + runner, + object(), + object(), + "slurm", + "injected-attempt", + "true", + tmp_path, + 30, + expected_failure=True, + ) + + +def test_ssh_home_cleanup_failure_is_fatal(tmp_path): + """Failed worker-home cleanup cannot silently contaminate later scenarios.""" + pod = { + "metadata": {"name": "ssh-worker-0"}, + "spec": {"containers": [{"name": "sshd"}]}, + "status": { + "phase": "Running", + "containerStatuses": [{"name": "sshd", "ready": True}], + }, + } + + class _SshCleanupRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if "get" in command and "pods" in command: + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"items": [pod]}), + stderr="", + ) + return SimpleNamespace( + returncode=1, stdout="", stderr="remote cleanup failed" + ) + + with pytest.raises(_FILESYSTEM.IntegrationTestError, match="remote cleanup"): + _cleanup_ssh_remote_results( + _SshCleanupRunner(), + _config(tmp_path / "state", tmp_path / "export"), + ) + + +def test_secondary_cleanup_failure_is_retained_without_replacing_primary(tmp_path): + """Cleanup diagnostics survive beside an already-recorded workload error.""" + _record_secondary_cleanup_failure(tmp_path, RuntimeError("cleanup failed")) + + error = tmp_path / "failure-diagnostics" / "cleanup-error.txt" + assert error.read_text(encoding="utf-8") == "RuntimeError: cleanup failed\n" + + +def test_slurm_allocation_outlives_slowest_scenario(): + """The scheduler cannot expire before a declared harness deadline.""" + assert _slurm_run_time(600) == "00:12:00" + + +def test_default_state_bootstrap_creates_missing_tmp_parent(tmp_path, monkeypatch): + """A clean checkout need not contain the ignored default tmp directory.""" + checkout = tmp_path / "checkout" + checkout.mkdir() + state_dir = checkout / "tmp" / "integration-state" + monkeypatch.setattr(_DRIVER, "DEFAULT_STATE_DIR", state_dir) + config = _config(state_dir, tmp_path / "export") + + _validate_lifecycle_paths(config) + _bootstrap_state_dir(config) + + assert state_dir.is_dir() + assert (state_dir / _DRIVER.STATE_MARKER).is_file() + + +def test_custom_state_still_requires_an_existing_parent(tmp_path): + """Parent creation is limited to the repository's known default path.""" + state_dir = tmp_path / "missing-parent" / "state" + config = _config(state_dir, tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="leaf below an existing"): + _validate_lifecycle_paths(config) + + +class _RecordingRunner: + """Record commands and return an empty successful process.""" + + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + """Record one command without executing it.""" + command = [str(item) for item in arguments] + self.commands.append(command) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_nfs_service_ownership_precedes_package_install(tmp_path, monkeypatch): + """Package activation cannot obscure who started the NFS service.""" + events = [] + config = _config(tmp_path / "state", tmp_path / "export") + runner = _RecordingRunner() + monkeypatch.setattr( + _DRIVER, + "_record_nfs_service_state", + lambda *_args: events.append("record"), + ) + monkeypatch.setattr( + _DRIVER, + "_ensure_apt_packages", + lambda *_args: events.append("packages"), + ) + + _prepare_host_dependencies(runner, config, "nfs") + + assert events == ["record", "packages"] + + +def test_missing_kind_is_an_empty_cluster_listing(monkeypatch): + """Repeated teardown does not require a deleted private kind client.""" + + class _UnexpectedRunner: + def run(self, *_args, **_kwargs): + pytest.fail("kind was invoked after its private client was removed") + + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: None) + + assert _kind_clusters(_UnexpectedRunner()) == set() + + +def test_missing_exportfs_is_an_empty_export_listing(monkeypatch): + """Cleanup after an interrupted package install needs no NFS client tool.""" + + class _UnexpectedRunner: + def run(self, *_args, **_kwargs): + pytest.fail("exportfs was invoked when it is unavailable") + + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: None) + + assert _export_paths(_UnexpectedRunner()) == [] + + +def test_sbx_shared_directories_allow_replacement_pod_cleanup(tmp_path, monkeypatch): + """SBX UID remapping cannot make prior pod files sticky and undeletable.""" + repository = tmp_path / "repository" + (repository / "tmp").mkdir(parents=True) + config = replace( + _config(tmp_path / "state", tmp_path / "export"), + sbx_shared_root=repository / "tmp" / "shared", + ) + + events = [] + + class _SbxProbeRunner: + def run(self, arguments, **_kwargs): + events.append(("run", str(arguments[0]))) + token = str(arguments[-1]) + (config.sbx_shared_root / "engine-probe").write_text( + token + "\n", encoding="utf-8" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(_DRIVER, "_repository_root", lambda: repository) + monkeypatch.setattr( + _DRIVER, + "_ensure_pinned_image", + lambda _runner, image: events.append(("acquire", image)), + ) + + _prepare_sbx_shared(_SbxProbeRunner(), config) + + assert events[:2] == [ + ("acquire", _DRIVER.SBX_KIND_NODE_IMAGE), + ("run", "docker"), + ] + for name in ("storage-test", "ssh-home"): + mode = (config.sbx_shared_root / name).stat().st_mode & 0o7777 + assert mode == _DRIVER.SBX_SHARED_DIRECTORY_MODE == 0o777 + + +def test_sbx_cleanup_preacquires_its_container_image(tmp_path, monkeypatch): + """SBX teardown cannot bypass verified image acquisition either.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), + sbx_shared_root=tmp_path / "shared", + ) + config.sbx_shared_root.mkdir() + (config.sbx_shared_root / _DRIVER.EXPORT_MARKER).touch() + events = [] + + class _CleanupRunner: + def run(self, arguments, **_kwargs): + events.append(("run", str(arguments[0]))) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr( + _DRIVER, + "_ensure_pinned_image", + lambda _runner, image: events.append(("acquire", image)), + ) + + _remove_sbx_shared(_CleanupRunner(), config) + + assert events == [ + ("acquire", _DRIVER.SBX_KIND_NODE_IMAGE), + ("run", "docker"), + ] + + +def test_sbx_diagnostics_never_invoke_privileged_nfs_tools(tmp_path, monkeypatch): + """SBX failure reporting remains entirely within its unprivileged profile.""" + config = _config(tmp_path / "state", tmp_path / "export") + runner = _RecordingRunner() + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: "/bin/tool") + + _collect_diagnostics(runner, config) + + rendered = "\n".join(" ".join(command) for command in runner.commands) + assert "sudo" not in rendered + assert "systemctl" not in rendered + assert "exportfs" not in rendered + + +def test_sbx_missing_packages_fail_without_sudo(): + """The SBX profile reports host prerequisites instead of escalating.""" + runner = _RecordingRunner() + + with pytest.raises(_DRIVER.ProvisionError, match="never invokes sudo"): + _ensure_apt_packages(runner, "sbx-shared") + + assert runner.commands + assert all(command[0] == "dpkg-query" for command in runner.commands) + + +def test_existing_state_requires_ownership_marker(tmp_path): + """Bootstrap cannot adopt an arbitrary existing directory.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + config = _config(state_dir, tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="unowned setup state"): + _validate_lifecycle_paths(config) + + +def test_existing_state_accepts_matching_ownership_marker(tmp_path): + """A correctly marked state directory remains reusable.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + marker = state_dir / _DRIVER.STATE_MARKER + marker.write_text( + json.dumps({"schema": _DRIVER.STATE_SCHEMA, "cluster_name": "test-cluster"}), + encoding="utf-8", + ) + config = _config(state_dir, tmp_path / "export") + + _validate_lifecycle_paths(config) + + +def _write_backend_state(config, backend): + """Write the retained backend identity used by repeated setup.""" + path = config.state_dir / "storage-backend.json" + path.write_text( + json.dumps(_storage_backend_document(config, backend)), encoding="utf-8" + ) + + +def _write_completed_state(config, backend): + """Write the immutable subset of a successful setup summary.""" + state = { + "schema": _DRIVER.STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + "storage_backend": backend, + } + (config.state_dir / "state.json").write_text(json.dumps(state), encoding="utf-8") + + +def test_retained_setup_rejects_changed_namespace(tmp_path): + """Repeated setup cannot create a second namespace workload stack.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + original = replace(_config(state_dir, tmp_path / "export"), storage_backend="nfs") + _write_completed_state(original, "nfs") + _write_backend_state(original, "nfs") + changed = replace(original, namespace="other-namespace") + + with pytest.raises(_DRIVER.ProvisionError, match="teardown first"): + _select_storage_backend(changed) + + +def test_retained_nfs_setup_rejects_changed_export(tmp_path): + """Repeated setup cannot mount retained NFS data at a second path.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + original = replace(_config(state_dir, tmp_path / "export"), storage_backend="nfs") + _write_completed_state(original, "nfs") + _write_backend_state(original, "nfs") + changed = replace(original, export_dir=tmp_path / "other-export") + + with pytest.raises(_DRIVER.ProvisionError, match="teardown first"): + _select_storage_backend(changed) + + +def test_test_state_rejects_explicit_backend_mismatch(tmp_path): + """A test action cannot silently ignore its explicit backend selection.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + retained = replace(_config(state_dir, tmp_path / "export"), storage_backend="nfs") + _write_completed_state(retained, "nfs") + requested = replace(retained, storage_backend="sbx-shared") + + with pytest.raises(_FILESYSTEM.IntegrationTestError, match="does not match"): + _load_state(requested) + + +def test_system_directory_cannot_be_used_as_state(tmp_path): + """An existing broad system directory cannot be marked during setup.""" + config = _config(Path("/usr"), tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="not owned by the current user"): + _validate_lifecycle_paths(config) + + +class _UnownedExportRunner: + """Record commands while presenting an existing unmarked export.""" + + def __init__(self, export_dir): + self.commands = [] + self.export_dir = str(export_dir) + + def run(self, arguments, **_kwargs): + """Answer ownership probes without executing privileged commands.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if "test" in command and "-e" in command: + exists = command[-1] == self.export_dir + return SimpleNamespace(returncode=0 if exists else 1, stdout="", stderr="") + if "cat" in command: + return SimpleNamespace(returncode=1, stdout="", stderr="missing") + raise AssertionError(f"unexpected mutation command: {command}") + + +def test_existing_export_requires_marker_before_mutation(tmp_path): + """An unmarked export is rejected before install or chown runs.""" + config = _config(tmp_path / "state", tmp_path / "export") + runner = _UnownedExportRunner(config.export_dir) + + with pytest.raises(_DRIVER.ProvisionError, match="unowned export directory"): + _ensure_export_marker(runner, config) + + assert not any( + "install" in command or "chown" in command for command in runner.commands + ) + + +def test_host_nfs_claim_rejects_unowned_fixed_configuration(tmp_path, monkeypatch): + """Setup cannot replace administrator-owned files at fixed NFS paths.""" + config = _config(tmp_path / "state", tmp_path / "export") + config.manifests_dir.mkdir(parents=True) + monkeypatch.setattr( + _DRIVER, + "_read_system_file", + lambda _runner, path: ( + "admin configuration\n" if path == _DRIVER.NFS_EXPORT_CONFIG else None + ), + ) + + with pytest.raises(_DRIVER.ProvisionError, match="unowned host NFS"): + _claim_nfs_host_owner(object(), config) + + +class _PodRunner: + """Return one terminating and one active ready login pod.""" + + def run(self, _arguments, **_kwargs): + """Return a fixed rollout-overlap pod list.""" + container = {"name": "login"} + ready = {"phase": "Running", "containerStatuses": [{"ready": True}]} + items = [ + { + "metadata": { + "name": "old-login", + "deletionTimestamp": "2026-09-19T00:00:00Z", + }, + "spec": {"containers": [container]}, + "status": ready, + }, + { + "metadata": {"name": "new-login"}, + "spec": {"containers": [container]}, + "status": ready, + }, + ] + return SimpleNamespace( + returncode=0, stdout=json.dumps({"items": items}), stderr="" + ) + + +def test_login_selection_ignores_terminating_rollout_pod(tmp_path): + """A terminating predecessor does not look like a second login node.""" + config = _config(tmp_path / "state", tmp_path / "export") + + assert _login_pod(_PodRunner(), config) == "new-login" + + +def test_fixture_discovery_ignores_terminating_ready_pod(): + """Test preflight observes only ready pods that are not terminating.""" + container = {"name": "login"} + ready = {"phase": "Running", "containerStatuses": [{"ready": True}]} + pods = [ + { + "metadata": {"name": "old", "deletionTimestamp": "now"}, + "spec": {"containers": [container]}, + "status": ready, + }, + { + "metadata": {"name": "new"}, + "spec": {"containers": [container]}, + "status": ready, + }, + ] + + selected = _pods_with_container(pods, "login") + + assert [pod["metadata"]["name"] for pod in selected] == ["new"] + + +def test_driver_worker_discovery_ignores_terminating_ready_pod(): + """Slinky worker rollout waits cannot count terminating pods as ready.""" + pods = [ + { + "metadata": {"name": "old", "deletionTimestamp": "now"}, + "spec": {"containers": [{"name": "slurmd"}]}, + }, + { + "metadata": {"name": "new"}, + "spec": {"containers": [{"name": "slurmd"}]}, + }, + ] + + selected = _driver_pods_with_container(pods, "slurmd") + + assert [pod["metadata"]["name"] for pod in selected] == ["new"] + + +def test_slurm_workload_account_reconciliation_is_idempotent(tmp_path, monkeypatch): + """Repeated setup does not add an existing Slurm account or association.""" + + class _AccountingRunner: + def __init__(self): + self.accounts = set() + self.associations = set() + self.users = {} + self.mutations = [] + + def run(self, command, **_kwargs): + arguments = [str(item) for item in command] + index = arguments.index("sacctmgr") + operation = arguments[index + 1 :] + if "show" in operation: + entity = operation[operation.index("show") + 1] + rows = { + "account": ((account,) for account in self.accounts), + "association": iter(self.associations), + "user": ((user, account) for user, account in self.users.items()), + }[entity] + output = "".join("|".join(row) + "|\n" for row in rows) + return SimpleNamespace(stdout=output, returncode=0) + self.mutations.append(operation) + action = operation[1] + if action == "add" and operation[2] == "account": + self.accounts.add(operation[3]) + elif action == "add" and operation[2] == "user": + user = operation[3] + account = operation[4].removeprefix("Account=") + self.associations.add((user, account)) + self.users[user] = operation[5].removeprefix("DefaultAccount=") + elif action == "modify": + user = operation[operation.index("where") + 1].removeprefix("Name=") + self.users[user] = operation[-1].removeprefix("DefaultAccount=") + return SimpleNamespace(stdout="", returncode=0) + + runner = _AccountingRunner() + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setattr(_DRIVER, "_login_pod", lambda *_args: "login") + + _ensure_slurm_workload_account(runner, config) + first_mutations = list(runner.mutations) + _ensure_slurm_workload_account(runner, config) + + assert len(first_mutations) == 2 + assert runner.mutations == first_mutations + + +def _ready_pod(name, container): + """Return one minimal ready, nonterminating fixture pod.""" + return { + "metadata": {"name": name}, + "spec": {"containers": [{"name": container}]}, + "status": {"phase": "Running", "containerStatuses": [{"ready": True}]}, + } + + +def test_ssh_rollout_waits_for_statefulset_revision(monkeypatch, tmp_path): + """Pod readiness is checked only after Kubernetes finishes its rollout.""" + runner = _RecordingRunner() + config = _config(tmp_path / "state", tmp_path / "export") + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + monkeypatch.setattr(_DRIVER, "_ssh_pods", lambda *_args, **_kwargs: pods) + + _wait_for_ssh(runner, config) + + assert runner.commands[0][-4:] == [ + "rollout", + "status", + "statefulset/ssh-worker", + "--timeout=180s", + ] + + +def test_ssh_pool_inspection_rejects_unobserved_revision(tmp_path): + """Ready old pods cannot satisfy a newly annotated StatefulSet form.""" + config = _config(tmp_path / "state", tmp_path / "export") + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + statefulset = { + "metadata": { + "generation": 2, + "annotations": { + _DRIVER.SSH_HOME_ANNOTATION: "separate", + _DRIVER.SSH_CONFIG_ANNOTATION: "new-checksum", + }, + }, + "status": { + "observedGeneration": 1, + "currentRevision": "old-revision", + "updateRevision": "new-revision", + "updatedReplicas": 0, + "readyReplicas": 2, + }, + } + + class _PoolRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + document = ( + statefulset if "statefulset/ssh-worker" in command else {"items": pods} + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps(document), + stderr="", + ) + + observed = _inspect_ssh_home_pool(_PoolRunner(), config) + + assert not observed.matches("separate", "new-checksum") + assert observed.ready_nonterminating_pods == 0 + assert "old-revision" in observed.diagnostics + + +def test_ssh_storage_visibility_retries_unique_probes(tmp_path, monkeypatch): + """Transient NFS visibility cannot fail a healthy home-mode transition.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + commands = [] + storage_attempts = 0 + + def pod_exec(_runner, _config, pod, script, *, check=True, timeout=60): + nonlocal storage_attempts + commands.append((pod, script, check, timeout)) + returncode = 0 + if script.startswith("test -e /home/tester/"): + returncode = 1 + elif script.startswith('test "$(cat /mnt/storage-test/'): + storage_attempts += 1 + returncode = 1 if storage_attempts == 1 else 0 + return SimpleNamespace(returncode=returncode, stdout="", stderr="") + + monkeypatch.setattr(_DRIVER, "_pod_exec", pod_exec) + monkeypatch.setattr(_DRIVER, "_select_storage_backend", lambda _config: "nfs") + monkeypatch.setattr(_DRIVER.secrets, "token_hex", lambda _length: "unique-token") + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + _validate_ssh_storage(runner=None, config=config, pods=pods, home_mode="separate") + + assert storage_attempts == 2 + assert all("unique-token" in script for _pod, script, _check, _timeout in commands) + cleanup = [ + script + for _pod, script, _check, _timeout in commands + if script.startswith("rm -f") + ] + assert len(cleanup) == 2 + + +def test_ssh_storage_visibility_bounds_fractional_remaining_time(tmp_path, monkeypatch): + """The second probe cannot begin after a fractional overall deadline.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + clock = _DeadlineClock() + probe_timeouts = [] + + def pod_exec(_runner, _config, _pod, script, *, check=True, timeout=60): + del check + if script.startswith("test -e ") or script.startswith('test "$(cat '): + assert clock.now + timeout <= 0.5 + probe_timeouts.append(timeout) + clock.now += timeout + return SimpleNamespace(returncode=1, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(_DRIVER, "_pod_exec", pod_exec) + monkeypatch.setattr(_DRIVER, "_select_storage_backend", lambda _config: "nfs") + monkeypatch.setattr(_DRIVER.time, "monotonic", clock.monotonic) + monkeypatch.setattr(_DRIVER, "SSH_STORAGE_VISIBILITY_TIMEOUT_SECONDS", 0.5) + + with pytest.raises(_DRIVER.ProvisionError, match="did not converge"): + _validate_ssh_storage(None, config, pods, "separate") + + assert probe_timeouts == [0.5] + assert clock.now == 0.5 + + +def test_slurm_only_fixture_does_not_require_ssh_workers(): + """Slurm diagnosis remains available while the SSH pool is unhealthy.""" + pods = [ + _ready_pod("login", "login"), + _ready_pod("slurmd-1", "slurmd"), + _ready_pod("slurmd-2", "slurmd"), + ] + + login, ssh = _require_pods(pods, {"slurm"}) + + assert login["metadata"]["name"] == "login" + assert ssh == [] + with pytest.raises(_FILESYSTEM.IntegrationTestError, match=r"required=\['ssh'\]"): + _require_pods(pods, {"ssh"}) + + +def test_pod_storage_commands_use_only_the_workload_identity(tmp_path): + """Conspicuous host IDs never leak into pod-side storage operations.""" + + class _CaptureRunner: + def __init__(self): + self.commands = [] + + def run(self, command, **_kwargs): + self.commands.append([str(item) for item in command]) + return SimpleNamespace(stdout="", returncode=0) + + runner = _CaptureRunner() + config = _config(tmp_path / "state", tmp_path / "export") + fixture = SimpleNamespace(login_pod="login", login_container="login") + runtime = SimpleNamespace(selector="slurm") + _prepare_scenario_data( + runner, + config, + fixture, + "/mnt/storage-test/integration-regression/test-data/sentinel", + ) + _reset_result_base( + runner, + config, + fixture, + runtime, + "/mnt/storage-test/integration-regression/results/sentinel", + ) + operations = _remote_staging_operations( + runner, + config, + fixture, + "slurm", + None, + None, + ) + operations.make_directory( + "login", + PurePosixPath("/mnt/storage-test/integration-regression/failure/sentinel"), + ) + + rendered = "\n".join(" ".join(command) for command in runner.commands) + assert "42424" not in rendered + assert "43434" not in rendered + assert "chown" not in rendered + assert "runuser -u tester --" in rendered + assert "umask 0007" in rendered + + +def test_workload_totals_require_resume_metadata(tmp_path, monkeypatch): + """Expected dataset totals cannot silently pass without workload metadata.""" + execution_root = tmp_path / "result" / "executions" + execution_root.mkdir(parents=True) + (execution_root / "0001.sh").write_text("coordinates\n", encoding="utf-8") + (execution_root / "0001.status").write_text("SUCCESS\n", encoding="utf-8") + (execution_root / "0001.exitcode").write_text("0\n", encoding="utf-8") + coordinate = SimpleNamespace(nodes=1, io_size="4K", threads=1, io_depth=1) + expected = SimpleNamespace( + coordinate=coordinate, + status=_FILESYSTEM.ExecutionStatus.SUCCESS, + ) + step = SimpleNamespace(name="bounded", executions=(expected,), required_phases=()) + scenario = SimpleNamespace(name="baseline") + monkeypatch.setattr( + _FILESYSTEM, + "_coordinate_from_execution", + lambda _path: (1, "4K", 1, 1), + ) + + with pytest.raises( + _FILESYSTEM.IntegrationTestError, match="missing required workload metadata" + ): + _assert_execution_contract(scenario, step, tmp_path / "result") + + +def test_slurm_ordering_uses_copied_execution_logs(tmp_path): + """Asynchronous Slurm ordering comes from durable result evidence.""" + result = tmp_path / "result" + executions = result / "executions" + executions.mkdir(parents=True) + (executions / "0001.log").write_text( + "[coordinator] starting execution 1: nodes=1 hosts=10.0.0.1 io_size=4K\n", + encoding="utf-8", + ) + (executions / "0002.log").write_text( + "[coordinator] starting execution 2: " + "nodes=2 hosts=10.0.0.1,10.0.0.2 io_size=4K\n", + encoding="utf-8", + ) + fixture = SimpleNamespace(slurm_addresses=("10.0.0.1", "10.0.0.2")) + + _assert_ordered_workers( + fixture, + "slurm", + "submission output contains no coordinator execution lines", + result, + ) + + for path in executions.iterdir(): + path.unlink() + with pytest.raises(_FILESYSTEM.IntegrationTestError, match=r"was \{\}"): + _assert_ordered_workers(fixture, "slurm", "", result) + + +def test_teardown_validation_allows_unrelated_nfs_exports(tmp_path, monkeypatch): + """Owned fixture cleanup is valid while an unrelated export remains active.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + config.manifests_dir.mkdir() + owner = json.dumps( + {"schema": _DRIVER.STATE_SCHEMA, "cluster_name": config.cluster_name} + ) + (config.state_dir / _DRIVER.STATE_MARKER).write_text(owner, encoding="utf-8") + (config.state_dir / "cluster-owner.json").write_text(owner, encoding="utf-8") + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": False}), encoding="utf-8" + ) + export_config = "owned export\n" + daemon_config = "owned daemon config\n" + (config.manifests_dir / "storage-scale-test.exports").write_text( + export_config, encoding="utf-8" + ) + (config.manifests_dir / "storage-scale-test-nfs.conf").write_text( + daemon_config, encoding="utf-8" + ) + installed = { + config.export_dir / _DRIVER.EXPORT_MARKER: owner, + _DRIVER.NFS_EXPORT_CONFIG: export_config, + _DRIVER.NFS_DAEMON_CONFIG: daemon_config, + _DRIVER.NFS_HOST_OWNER: json.dumps( + _nfs_host_owner_document(config), sort_keys=True + ) + + "\n", + } + monkeypatch.setattr( + _DRIVER, "_read_system_file", lambda _runner, path: installed.get(path) + ) + monkeypatch.setattr(_DRIVER, "_export_mount_type", lambda *_args: "") + monkeypatch.setattr(_DRIVER, "_validate_loop_associations", lambda *_args: None) + monkeypatch.setattr(_DRIVER, "_validate_image_ownership", lambda *_args: None) + monkeypatch.setattr( + _DRIVER, "_export_paths", lambda _runner: ["/srv/unrelated-export"] + ) + + ownership = _validate_teardown_ownership(object(), config) + + assert ownership == (True, True, True, True) + + +def test_teardown_accepts_service_only_partial_nfs_bootstrap(tmp_path, monkeypatch): + """An early setup failure remains recoverable before cluster ownership.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + _bootstrap_state_dir(config) + (config.state_dir / "nfs-service.json").write_text( + json.dumps( + { + "schema": _DRIVER.NFS_SERVICE_STATE_SCHEMA, + "service_existed": True, + "was_active": True, + "was_enabled": False, + "previous_threads": 2, + } + ), + encoding="utf-8", + ) + host_owner = json.dumps(_nfs_host_owner_document(config)) + monkeypatch.setattr( + _DRIVER, + "_read_system_file", + lambda _runner, path: host_owner if path == _DRIVER.NFS_HOST_OWNER else None, + ) + monkeypatch.setattr(_DRIVER, "_export_mount_type", lambda *_args: "") + monkeypatch.setattr(_DRIVER, "_validate_loop_associations", lambda *_args: None) + + class _PartialRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if "test" in command and "-d" in command: + return SimpleNamespace(returncode=1, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + assert _validate_teardown_ownership(_PartialRunner(), config) == ( + True, + False, + False, + True, + ) + + +def test_teardown_ignores_preexisting_unowned_fixed_nfs_files(tmp_path, monkeypatch): + """A rejected setup can remove local state without touching admin NFS files.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + _bootstrap_state_dir(config) + monkeypatch.setattr( + _DRIVER, + "_read_system_file", + lambda _runner, path: ( + "admin configuration\n" if path == _DRIVER.NFS_EXPORT_CONFIG else None + ), + ) + monkeypatch.setattr(_DRIVER, "_export_mount_type", lambda *_args: "") + monkeypatch.setattr(_DRIVER, "_validate_loop_associations", lambda *_args: None) + + class _ExternalConfigRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if "test" in command and "-d" in command: + return SimpleNamespace(returncode=1, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + assert _validate_teardown_ownership(_ExternalConfigRunner(), config) == ( + True, + False, + False, + False, + ) + + +def test_service_only_partial_bootstrap_tears_down_twice_and_restores_host( + tmp_path, monkeypatch +): + """Early NFS mutation is fully reversible without a cluster owner marker.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + _bootstrap_state_dir(config) + (config.state_dir / "nfs-service.json").write_text( + json.dumps( + { + "schema": _DRIVER.NFS_SERVICE_STATE_SCHEMA, + "service_existed": True, + "was_active": True, + "was_enabled": False, + "previous_threads": 2, + } + ), + encoding="utf-8", + ) + + class _EarlyFailureRunner: + def __init__(self): + self.host_owner = True + self.active = True + self.enabled = False + self.threads = 8 + self.exports = ["/srv/unrelated-export"] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if "test" in command and "-d" in command: + return SimpleNamespace(returncode=1, stdout="", stderr="") + if "is-active" in command: + return SimpleNamespace( + returncode=0 if self.active else 3, stdout="", stderr="" + ) + if "is-enabled" in command: + return SimpleNamespace( + returncode=0 if self.enabled else 1, stdout="", stderr="" + ) + if "--property=LoadState" in command: + return SimpleNamespace(returncode=0, stdout="loaded\n", stderr="") + if str(_DRIVER.NFS_THREADS_PATH) in command: + return SimpleNamespace( + returncode=0, stdout=f"{self.threads}\n", stderr="" + ) + if any(Path(item).name == "rpc.nfsd" for item in command): + self.threads = int(command[-1]) + if "systemctl" in command and "enable" in command: + self.enabled = True + if "systemctl" in command and "disable" in command: + self.enabled = False + if command[-1] == str(_DRIVER.NFS_HOST_OWNER) and "rm" in command: + self.host_owner = False + stdout = "" + if ( + any(Path(item).name == "exportfs" for item in command) + and "-v" in command + ): + stdout = "/srv/unrelated-export 10.0.0.0/24(options)\n" + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + runner = _EarlyFailureRunner() + owner = json.dumps(_nfs_host_owner_document(config)) + "\n" + + def read_system(_runner, path): + if path == _DRIVER.NFS_HOST_OWNER and runner.host_owner: + return owner + return None + + def remove_owned(_runner, _config_value, *, remove_state, remove_export): + assert not remove_export + if remove_state and config.state_dir.exists(): + shutil.rmtree(config.state_dir) + + monkeypatch.setattr(_DRIVER, "_read_system_file", read_system) + monkeypatch.setattr(_DRIVER, "_export_mount_type", lambda *_args: "") + monkeypatch.setattr(_DRIVER, "_export_paths", lambda _runner: runner.exports) + monkeypatch.setattr(_DRIVER, "_remove_owned_directories", remove_owned) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda command: f"/usr/sbin/{command}") + + _teardown_environment_locked(runner, config, "nfs") + _teardown_environment_locked(runner, config, "nfs") + + assert runner.active + assert not runner.enabled + assert runner.threads == 2 + assert runner.exports == ["/srv/unrelated-export"] + assert not runner.host_owner + assert not config.state_dir.exists() + + +class _NfsRemovalRunner: + """Record NFS cleanup while reporting one unrelated active export.""" + + def __init__(self, threads=None): + self.commands = [] + self.threads = threads + + def run(self, arguments, **_kwargs): + """Record one cleanup command and return stable export state.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if "is-active" in command: + return SimpleNamespace(returncode=0, stdout="active\n", stderr="") + if str(_DRIVER.NFS_THREADS_PATH) in command: + return SimpleNamespace(returncode=0, stdout=f"{self.threads}\n", stderr="") + if any(Path(item).name == "rpc.nfsd" for item in command): + self.threads = int(command[-1]) + stdout = ( + "/srv/unrelated-export 10.0.0.0/24(options)\n" if "-v" in command else "" + ) + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + +def test_nfs_cleanup_preserves_preexisting_service(tmp_path, monkeypatch): + """Removing owned NFS configuration never disables a pre-existing service.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + (config.manifests_dir / "storage-scale-test.exports").write_text( + f"{config.export_dir} 10.0.0.0/24(options)\n", encoding="utf-8" + ) + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": False}), encoding="utf-8" + ) + runner = _NfsRemovalRunner() + monkeypatch.setattr( + _DRIVER.shutil, + "which", + lambda command: f"/usr/sbin/{command}", + ) + + _remove_nfs_configuration(runner, config) + + assert not any( + "systemctl" in command + and any(action in command for action in ("enable", "disable", "stop")) + for command in runner.commands + ) + assert any( + any(Path(item).name == "exportfs" for item in command) and "-u" in command + for command in runner.commands + ) + + +def test_nfs_cleanup_restores_preexisting_worker_count(tmp_path, monkeypatch): + """Teardown restores the live count after removing fixture configuration.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": False, "previous_threads": 2}), + encoding="utf-8", + ) + runner = _NfsRemovalRunner(threads=8) + monkeypatch.setattr( + _DRIVER.shutil, + "which", + lambda command: f"/usr/sbin/{command}", + ) + + _remove_nfs_configuration(runner, config) + + assert runner.threads == 2 + assert any( + any(Path(item).name == "rpc.nfsd" for item in command) and command[-1] == "2" + for command in runner.commands + ) + + +def test_nfs_cleanup_survives_missing_exportfs(tmp_path, monkeypatch): + """Interrupted package bootstrap leaves teardown able to remove state.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": True}), encoding="utf-8" + ) + runner = _NfsRemovalRunner() + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: None) + + _remove_nfs_configuration(runner, config) + + rendered = [" ".join(command) for command in runner.commands] + assert not any("exportfs" in command for command in rendered) + assert sum("rm --force" in command for command in rendered) == 2 + + +class _DockerTagRunner: + """Model only the Docker tag operations used by the ownership journal.""" + + def __init__(self, tags=None): + self.tags = dict(tags or {}) + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{.Id}}", + ]: + image = command[-1] + image_id = self.tags.get(image) + return SimpleNamespace( + returncode=0 if image_id else 1, + stdout=f"{image_id}\n" if image_id else "", + stderr="" if image_id else f"Error: No such image: {image}\n", + ) + if command[:3] == ["docker", "image", "tag"]: + self.tags[command[-1]] = self.tags[command[-2]] + elif command[:3] == ["docker", "image", "rm"]: + self.tags.pop(command[-1], None) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_docker_image_publish_recovers_interrupted_fixed_tag_update(tmp_path): + """A crash after publishing remains owned and recoverable on retry.""" + config = _config(tmp_path / "state", tmp_path / "export") + config.state_dir.mkdir() + image = "fixture:test" + runner = _DockerTagRunner({image: "sha256:original"}) + temporary = _begin_image_build(runner, config, image) + runner.tags[temporary] = "sha256:built" + state = _image_ownership_state(config) + state[image]["pending_id"] = "sha256:built" + _write_image_ownership_state(config, state) + runner.tags[image] = "sha256:built" + + replacement = _begin_image_build(runner, config, image) + + ownership = _image_ownership_state(config)[image] + assert ownership["built_id"] == "sha256:built" + assert ownership["pending_tag"] == replacement + assert temporary not in runner.tags + + +def test_docker_image_build_rejects_external_fixed_tag_change(tmp_path): + """Repeated setup never silently overwrites a fixture tag retagged by a user.""" + config = _config(tmp_path / "state", tmp_path / "export") + config.state_dir.mkdir() + image = "fixture:test" + runner = _DockerTagRunner({image: "sha256:original"}) + temporary = _begin_image_build(runner, config, image) + runner.tags.pop(temporary, None) + state = _image_ownership_state(config) + state[image]["pending_tag"] = None + _write_image_ownership_state(config, state) + runner.tags[image] = "sha256:external" + + with pytest.raises(_DRIVER.ProvisionError, match="outside the fixture"): + _begin_image_build(runner, config, image) + + +def test_docker_image_absence_is_distinct_from_daemon_failure(): + """Ownership decisions fail closed on Docker transport errors.""" + absent = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=1, stdout="", stderr="Error: No such image: fixture:test" + ) + ) + broken = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=1, + stdout="", + stderr="Cannot connect to the Docker daemon", + ) + ) + + assert _image_id(absent, "fixture:test") is None + with pytest.raises(_DRIVER.ProvisionError, match="could not inspect"): + _image_id(broken, "fixture:test") + with pytest.raises(_DRIVER.ProvisionError, match="could not inspect"): + _inspect_pinned_image(broken, "fixture@test", "sha256:test", "amd64") + + +def test_pinned_image_requires_matching_platform_and_index_digest(): + """A cached image for another architecture is not accepted offline.""" + metadata = { + "Architecture": "arm64", + "RepoDigests": ["registry.example/fixture@sha256:index"], + } + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, stdout=json.dumps(metadata), stderr="" + ) + ) + + assert not _inspect_pinned_image( + runner, "registry.example/fixture@sha256:index", "sha256:index", "amd64" + ) + assert not _inspect_pinned_image( + runner, "registry.example/fixture@sha256:other", "sha256:other", "arm64" + ) + + +def test_pinned_fixture_base_uses_cache_before_network(monkeypatch): + """A valid digest-qualified base is pulled only when genuinely absent.""" + reference = "registry.example/fixture:1@sha256:index" + + class _PinnedRunner: + def __init__(self, present): + self.present = present + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + if not self.present: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": ["registry.example/fixture@sha256:index"], + } + ), + stderr="", + ) + if command[:2] == ["docker", "pull"]: + self.present = True + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + cached = _PinnedRunner(True) + absent = _PinnedRunner(False) + + _ensure_pinned_image(cached, reference) + _ensure_pinned_image(absent, reference) + + assert not any(command[:2] == ["docker", "pull"] for command in cached.commands) + assert sum(command[:2] == ["docker", "pull"] for command in absent.commands) == 1 + + +@pytest.mark.parametrize( + "detail", + ( + "429 Too Many Requests", + "toomanyrequests: unauthenticated pull rate limit", + "request timed out", + "context deadline exceeded", + "connection reset by peer", + "unexpected status from registry: 503 Service Unavailable", + ), +) +def test_transient_image_pull_failures_are_retryable(detail): + """Only temporary registry and transport failures enter the retry loop.""" + assert _transient_image_pull_failure(detail) + + +def test_pinned_image_pull_retries_rate_limit_then_verifies(monkeypatch): + """Host Docker retries a temporary quota response before accepting an image.""" + reference = "docker.io/library/fixture:1@sha256:index" + + class _RetryRunner: + def __init__(self): + self.present = False + self.pull_attempts = 0 + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + if not self.present: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": ["fixture@sha256:index"], + } + ), + stderr="", + ) + if command[:2] == ["docker", "pull"]: + self.pull_attempts += 1 + if self.pull_attempts < 3: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="429 Too Many Requests", + ) + self.present = True + return SimpleNamespace(returncode=0, stdout="", stderr="") + + sleeps = [] + runner = _RetryRunner() + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr(_DRIVER.secrets, "randbelow", lambda _limit: 0) + monkeypatch.setattr(_DRIVER.time, "sleep", sleeps.append) + + assert _ensure_pinned_image(runner, reference) == reference + assert runner.pull_attempts == 3 + assert sleeps == [10.0, 20.0] + + +def test_pinned_image_pull_retries_command_timeout(monkeypatch): + """A bounded host-Docker timeout receives the same transient retry.""" + reference = "docker.io/library/fixture:1@sha256:index" + + class _TimeoutRunner: + def __init__(self): + self.present = False + self.pull_attempts = 0 + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + if not self.present: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": ["fixture@sha256:index"], + } + ), + stderr="", + ) + self.pull_attempts += 1 + if self.pull_attempts == 1: + raise _DRIVER.subprocess.TimeoutExpired( + command, _DRIVER.IMAGE_PULL_TIMEOUT_SECONDS + ) + self.present = True + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _TimeoutRunner() + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr(_DRIVER.secrets, "randbelow", lambda _limit: 0) + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + assert _ensure_pinned_image(runner, reference) == reference + assert runner.pull_attempts == 2 + + +def test_pinned_image_pull_reports_final_registry_failure(monkeypatch): + """Exhausted retries retain the final actionable registry diagnostic.""" + reference = "docker.io/library/fixture:1@sha256:index" + + class _LimitedRunner: + def __init__(self): + self.pull_attempts = 0 + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + self.pull_attempts += 1 + return SimpleNamespace( + returncode=1, + stdout="", + stderr="429 Too Many Requests: quota exhausted", + ) + + runner = _LimitedRunner() + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr(_DRIVER.secrets, "randbelow", lambda _limit: 0) + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + with pytest.raises( + _DRIVER.ProvisionError, match="429 Too Many Requests: quota exhausted" + ): + _ensure_pinned_image(runner, reference) + + assert runner.pull_attempts == _DRIVER.IMAGE_PULL_ATTEMPTS + + +def test_pinned_image_pull_respects_overall_deadline(monkeypatch): + """Slow retries cannot exceed the shared image-acquisition deadline.""" + reference = "docker.io/library/fixture:1@sha256:index" + + class _Clock: + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + class _ImageDeadlineRunner: + def __init__(self, clock): + self.clock = clock + self.pull_attempts = 0 + + def run(self, arguments, *, timeout=None, **_kwargs): + command = [str(item) for item in arguments] + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + self.pull_attempts += 1 + assert timeout is not None + assert self.clock.now + timeout <= _DRIVER.IMAGE_PULL_DEADLINE_SECONDS + self.clock.now += timeout + return SimpleNamespace( + returncode=1, + stdout="", + stderr="429 Too Many Requests", + ) + + clock = _Clock() + runner = _ImageDeadlineRunner(clock) + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr(_DRIVER.secrets, "randbelow", lambda _limit: 0) + monkeypatch.setattr(_DRIVER.time, "monotonic", clock.monotonic) + monkeypatch.setattr(_DRIVER.time, "sleep", clock.sleep) + + with pytest.raises(_DRIVER.ProvisionError, match="acquisition deadline"): + _ensure_pinned_image(runner, reference) + + assert runner.pull_attempts == 3 + assert clock.now < _DRIVER.IMAGE_PULL_DEADLINE_SECONDS + + +def test_pinned_image_pull_reserves_fallback_attempt(monkeypatch): + """A hanging primary leaves one full pull and inspect window per fallback.""" + digest = "sha256:index" + primary = f"registry.example/fixture@{digest}" + fallback = f"mirror.example/fixture@{digest}" + + class _Clock: + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + class _FallbackRunner: + def __init__(self, clock): + self.clock = clock + self.present = set() + self.fallback_timeouts = [] + + def run(self, arguments, *, timeout=None, **_kwargs): + command = [str(item) for item in arguments] + reference = command[-1] + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + if reference not in self.present: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": [f"fixture@{digest}"], + } + ), + stderr="", + ) + assert command[:2] == ["docker", "pull"] + assert timeout is not None + if reference == primary: + self.clock.now += timeout + raise _DRIVER.subprocess.TimeoutExpired(command, timeout) + self.fallback_timeouts.append(timeout) + self.present.add(reference) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + clock = _Clock() + runner = _FallbackRunner(clock) + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr(_DRIVER.secrets, "randbelow", lambda _limit: 0) + monkeypatch.setattr(_DRIVER.time, "monotonic", clock.monotonic) + monkeypatch.setattr(_DRIVER.time, "sleep", clock.sleep) + + selected = _acquire_pinned_image(runner, (primary, fallback), digest) + + assert selected == fallback + assert runner.fallback_timeouts == [float(_DRIVER.IMAGE_PULL_TIMEOUT_SECONDS)] + assert clock.now < _DRIVER.IMAGE_PULL_DEADLINE_SECONDS + + +@pytest.mark.parametrize( + ("backend", "expected"), + ( + ("nfs", _DRIVER.KIND_NODE_IMAGE), + ("sbx-shared", _DRIVER.SBX_KIND_NODE_IMAGE), + ), +) +def test_cluster_creation_preacquires_pinned_node_image( + tmp_path, monkeypatch, backend, expected +): + """kind cannot bypass verified host-Docker image acquisition.""" + config = _config(tmp_path / "state", tmp_path / "export") + events = [] + + class _KindRunner: + def run(self, arguments, **_kwargs): + events.append(("run", [str(item) for item in arguments])) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr( + _DRIVER, + "_render_kind_config", + lambda _config, _backend: tmp_path / "kind.yaml", + ) + monkeypatch.setattr( + _DRIVER, + "_ensure_pinned_image", + lambda _runner, image: events.append(("acquire", image)), + ) + + _create_cluster(_KindRunner(), config, backend) + + assert events[0] == ("acquire", expected) + assert events[1][0] == "run" + assert events[1][1][:3] == ["kind", "create", "cluster"] + assert events[1][1][events[1][1].index("--image") + 1] == expected + + +def test_fixture_build_does_not_force_registry_refresh(tmp_path, monkeypatch): + """Digest-qualified cached bases are not defeated by docker build --pull.""" + runner = _RecordingRunner() + config = _config(tmp_path / "state", tmp_path / "export") + base = "registry.example/base:1@sha256:index" + checked = [] + monkeypatch.setattr( + _DRIVER, "_ensure_pinned_image", lambda _runner, image: checked.append(image) + ) + monkeypatch.setattr(_DRIVER, "_begin_image_build", lambda *_args: "fixture:pending") + monkeypatch.setattr(_DRIVER, "_publish_image_build", lambda *_args: None) + + _build_owned_image( + runner, + config, + "fixture:latest", + tmp_path / "Dockerfile", + base, + ) + + build = next( + command for command in runner.commands if command[:2] == ["docker", "build"] + ) + assert checked == [base] + assert "--pull" not in build + + +def test_slurm_install_preloads_verified_mariadb_and_helper_images( + tmp_path, monkeypatch +): + """Slurm workload images use host Docker and the deployed private tag.""" + config = _config(tmp_path / "state", tmp_path / "export") + _bootstrap_state_dir(config) + + class _WorkloadImageRunner: + def __init__(self): + self.commands = [] + self.present = set() + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + reference = command[-1] + if reference not in self.present: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + digest = reference.rpartition("@")[2] + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": [f"fixture@{digest}"], + } + ), + stderr="", + ) + if command[:2] == ["docker", "pull"]: + self.present.add(command[-1]) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _WorkloadImageRunner() + published = [] + loaded = [] + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr( + _DRIVER, + "_begin_image_build", + lambda _runner, _config, image: f"{image}-pending", + ) + monkeypatch.setattr( + _DRIVER, + "_publish_image_build", + lambda _runner, _config, image, temporary: published.append((image, temporary)), + ) + monkeypatch.setattr( + _DRIVER, + "_load_image_into_nodes", + lambda _runner, image, nodes: loaded.append((image, nodes)), + ) + monkeypatch.setattr(_DRIVER, "_build_slinky_image", lambda *_args, **_kwargs: None) + for name in ( + "_helm_slinky", + "_wait_for_slurm", + "_restart_slinky_login", + "_ensure_slurm_workload_account", + "_validate_slurm", + ): + monkeypatch.setattr(_DRIVER, name, lambda *_args, **_kwargs: None) + + _install_slurm(runner, config) + + pulls = { + command[-1] for command in runner.commands if command[:2] == ["docker", "pull"] + } + assert pulls == { + _DRIVER.MARIADB_BASE_IMAGE, + _DRIVER.SLINKY_HELPER_BASE_IMAGE, + } + control_plane = f"{config.cluster_name}-control-plane" + assert (_DRIVER.MARIADB_IMAGE, [control_plane]) in loaded + assert ( + _DRIVER.SLINKY_HELPER_IMAGE, + [ + control_plane, + f"{config.cluster_name}-worker", + f"{config.cluster_name}-worker2", + ], + ) in loaded + assert {image for image, _temporary in published} == { + _DRIVER.MARIADB_IMAGE, + _DRIVER.SLINKY_HELPER_IMAGE, + } + apply = next(command for command in runner.commands if "apply" in command) + manifest = Path(apply[-1]).read_text(encoding="utf-8") + assert f"image: {_DRIVER.MARIADB_IMAGE}" in manifest + assert "imagePullPolicy: Never" in manifest + assert _DRIVER.MARIADB_BASE_IMAGE not in manifest + + +def _yaml_image_references(value): + """Yield scalar or structured image references from nested fixture YAML.""" + if isinstance(value, dict): + for key, item in value.items(): + if key == "image" and isinstance(item, str): + yield item + elif key == "image" and isinstance(item, dict) and item.get("repository"): + reference = str(item["repository"]) + if item.get("tag"): + reference += f":{item['tag']}" + if item.get("digest"): + reference += f"@{item['digest']}" + yield reference + yield from _yaml_image_references(item) + elif isinstance(value, list): + for item in value: + yield from _yaml_image_references(item) + + +def _is_docker_hub_reference(reference): + """Return whether Docker resolves an image reference through Docker Hub.""" + first_component = reference.split("/", maxsplit=1)[0] + if "/" not in reference: + return True + if first_component in {"docker.io", "index.docker.io", "registry-1.docker.io"}: + return True + return ( + "." not in first_component + and ":" not in first_component + and first_component != "localhost" + ) + + +def test_workload_image_contract_has_no_mutable_or_unstaged_dockerhub_images(): + """Checked-in workload values cannot restore direct mutable Hub pulls.""" + mariadb = _render_resource_text( + "manifests/mariadb-accounting.yaml.tmpl", + { + "MARIADB_IMAGE": _DRIVER.MARIADB_IMAGE, + "NAMESPACE": "fixture", + }, + ) + slinky = _render_resource_text( + "manifests/slinky-slurm-values.yaml.tmpl", + { + "SLINKY_HELPER_IMAGE_REPOSITORY": (_DRIVER.SLINKY_HELPER_IMAGE_REPOSITORY), + "SLINKY_HELPER_IMAGE_TAG": _DRIVER.SLINKY_HELPER_IMAGE_TAG, + }, + ) + ssh = _render_resource_text( + "manifests/ssh-workers.yaml.tmpl", + { + "NAMESPACE": "fixture", + "SSH_CONFIG_CHECKSUM": "checksum", + "SSH_HOME_MODE": "separate", + "SSH_HOME_VOLUME": "emptyDir: {}", + }, + ) + slinky_values = yaml.safe_load(slinky) + documents = [ + *yaml.safe_load_all(mariadb), + slinky_values, + *yaml.safe_load_all(ssh), + ] + references = { + reference + for document in documents + for reference in _yaml_image_references(document) + } + preloaded = { + _DRIVER.MARIADB_IMAGE, + _DRIVER.SLINKY_HELPER_IMAGE, + _DRIVER.SLINKY_LOGIN_IMAGE, + _DRIVER.SLINKY_SLURMD_IMAGE, + _DRIVER.SSH_IMAGE, + } + dockerhub = { + reference for reference in references if _is_docker_hub_reference(reference) + } + + assert _is_docker_hub_reference("bitnami/example:1") + assert not _is_docker_hub_reference("registry.example.com/example:1") + assert ":latest" not in "\n".join(references) + assert dockerhub <= preloaded + helper_containers = ( + slinky_values["controller"]["logfile"], + slinky_values["loginsets"]["test"]["initconf"], + slinky_values["nodesets"]["storage"]["logfile"], + ) + assert all( + container["image"]["repository"] == _DRIVER.SLINKY_HELPER_IMAGE_REPOSITORY + and container["image"]["tag"] == _DRIVER.SLINKY_HELPER_IMAGE_TAG + and container["image"]["digest"] is None + and container["imagePullPolicy"] == "Never" + for container in helper_containers + ) + assert slinky_values["loginsets"]["test"]["login"]["imagePullPolicy"] == "Never" + assert slinky_values["nodesets"]["storage"]["slurmd"]["imagePullPolicy"] == "Never" + + +def test_cached_csi_images_avoid_network_and_shared_host_tags(tmp_path, monkeypatch): + """Verified cached indexes load through temporary fixture-only tags.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + + class _CachedCsiRunner: + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + reference = command[-1] + image = next( + entry for entry in _DRIVER.CSI_IMAGES if entry[0] in reference + ) + repository = reference.split(":", maxsplit=1)[0] + document = { + "Architecture": "amd64", + "RepoDigests": [f"{repository}@{image[2]}"], + } + return SimpleNamespace( + returncode=0, stdout=json.dumps(document), stderr="" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _CachedCsiRunner() + loaded = [] + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr( + _DRIVER, + "_kind_containers", + lambda *_args, **_kwargs: ["control", "worker", "worker2"], + ) + monkeypatch.setattr( + _DRIVER, + "_load_image_into_nodes", + lambda _runner, image, nodes, *, destination=None: loaded.append( + (image, nodes, destination) + ), + ) + + _prepare_csi_images(runner, config) + + assert len(loaded) == len(_DRIVER.CSI_IMAGES) + assert not any(command[:2] == ["docker", "pull"] for command in runner.commands) + tag_commands = [ + command + for command in runner.commands + if command[:3] == ["docker", "image", "tag"] + ] + assert all( + "storage-scale-integration-csi/" in command[-1] for command in tag_commands + ) + + +def test_csi_pull_uses_verified_staging_fallback(tmp_path, monkeypatch): + """A primary-registry outage falls back to the established staging mirror.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + image = ("nfsplugin", "v-test", "sha256:index") + + class _FallbackRunner: + def __init__(self): + self.commands = [] + self.fallback_present = False + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + reference = command[-1] + if self.fallback_present and "k8s-staging-sig-storage" in reference: + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": [ + "gcr.io/k8s-staging-sig-storage/" + "nfsplugin@sha256:index" + ], + } + ), + stderr="", + ) + return SimpleNamespace( + returncode=1, + stdout="", + stderr="Error: No such image", + ) + if command[:2] == ["docker", "pull"]: + if "registry.k8s.io" in command[-1]: + return SimpleNamespace( + returncode=1, + stdout="", + stderr="registry unavailable", + ) + self.fallback_present = True + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _FallbackRunner() + monkeypatch.setattr(_DRIVER, "CSI_IMAGES", (image,)) + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr( + _DRIVER, + "_kind_containers", + lambda *_args, **_kwargs: ["control", "worker", "worker2"], + ) + monkeypatch.setattr( + _DRIVER, "_load_image_into_nodes", lambda *_args, **_kwargs: None + ) + + _prepare_csi_images(runner, config) + + pulls = [ + command[-1] for command in runner.commands if command[:2] == ["docker", "pull"] + ] + assert pulls == [ + "registry.k8s.io/sig-storage/nfsplugin@sha256:index", + "gcr.io/k8s-staging-sig-storage/nfsplugin@sha256:index", + ] + assert all("k8s-artifacts-prod" not in item for item in pulls) + + +def test_csi_setup_reconciles_interrupted_temporary_aliases(tmp_path, monkeypatch): + """Repeated setup removes fixture-private host and node aliases after SIGKILL.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + image = ("nfsplugin", "v-test", "sha256:index") + host_alias = "storage-scale-integration-csi/nfsplugin:v-old-stale" + node_alias = f"docker.io/{host_alias}" + + class _StaleAliasRunner: + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": [ + "registry.k8s.io/sig-storage/nfsplugin@sha256:index" + ], + } + ), + stderr="", + ) + if command[:4] == ["docker", "image", "ls", "--format"]: + return SimpleNamespace( + returncode=0, stdout=f"{host_alias}\n", stderr="" + ) + if command[-3:] == ["images", "list", "--quiet"]: + return SimpleNamespace( + returncode=0, stdout=f"{node_alias}\n", stderr="" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _StaleAliasRunner() + monkeypatch.setattr(_DRIVER, "CSI_IMAGES", (image,)) + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr( + _DRIVER, + "_kind_containers", + lambda *_args, **_kwargs: ["control", "worker", "worker2"], + ) + monkeypatch.setattr( + _DRIVER, "_load_image_into_nodes", lambda *_args, **_kwargs: None + ) + + _prepare_csi_images(runner, config) + + assert ["docker", "image", "rm", host_alias] in runner.commands + for node in ("control", "worker", "worker2"): + assert [ + "docker", + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "remove", + node_alias, + ] in runner.commands + + +def test_teardown_removes_interrupted_csi_host_alias(tmp_path, monkeypatch): + """Teardown cleans a private CSI tag even when setup never reruns.""" + config = _config(tmp_path / "state", tmp_path / "export") + alias = "storage-scale-integration-csi/nfsplugin:v-old-interrupted" + + class _InterruptedAliasRunner: + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:4] == ["docker", "image", "ls", "--format"]: + return SimpleNamespace(returncode=0, stdout=f"{alias}\n", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _InterruptedAliasRunner() + monkeypatch.setattr( + _DRIVER, "CSI_IMAGES", (("nfsplugin", "v-test", "sha256:index"),) + ) + + _remove_harness_images(runner, config) + + assert ["docker", "image", "rm", alias] in runner.commands + + +def test_csi_alias_cleanup_does_not_mask_staging_failure(tmp_path, monkeypatch): + """A secondary temporary-tag failure preserves the node-import error.""" + config = _config(tmp_path / "state", tmp_path / "export") + + class _CleanupFailureRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if command[:5] == [ + "docker", + "image", + "inspect", + "--format", + "{{json .}}", + ]: + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "Architecture": "amd64", + "RepoDigests": [ + "registry.k8s.io/sig-storage/nfsplugin@sha256:index" + ], + } + ), + stderr="", + ) + if command[:4] == ["docker", "image", "ls", "--format"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + if command[-3:] == ["images", "list", "--quiet"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + if command[:3] == ["docker", "image", "rm"]: + raise _DRIVER.ProvisionError("alias cleanup failed") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr( + _DRIVER, "CSI_IMAGES", (("nfsplugin", "v-test", "sha256:index"),) + ) + monkeypatch.setattr(_DRIVER, "_check_platform", lambda: "amd64") + monkeypatch.setattr( + _DRIVER, + "_kind_containers", + lambda *_args, **_kwargs: ["control", "worker", "worker2"], + ) + + def fail_node_import(*_args, **_kwargs): + raise _DRIVER.ProvisionError("node import failed") + + monkeypatch.setattr(_DRIVER, "_load_image_into_nodes", fail_node_import) + + with pytest.raises(_DRIVER.ProvisionError, match="node import failed"): + _prepare_csi_images(_CleanupFailureRunner(), config) + + +def test_csi_chart_tag_is_published_only_inside_kind_nodes(): + """The imported private tag is aliased in containerd, not host Docker.""" + + class _ImageLoadRunner: + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + self.commands.append(command) + if command[:3] == ["docker", "save", "--output"]: + Path(command[3]).write_bytes(b"archive") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runner = _ImageLoadRunner() + private = "storage-scale-integration-csi/nfsplugin:v4.13.4-42" + destination = "registry.k8s.io/sig-storage/nfsplugin:v4.13.4" + + _load_image_into_nodes(runner, private, ["kind-worker"], destination=destination) + + tag = next(command for command in runner.commands if "tag" in command) + assert tag[-2:] == [f"docker.io/{private}", destination] + assert [ + "docker", + "exec", + "kind-worker", + "ctr", + "-n", + "k8s.io", + "images", + "remove", + f"docker.io/{private}", + ] in runner.commands + assert not any( + command[:3] == ["docker", "image", "tag"] for command in runner.commands + ) + + +def test_markerless_sbx_elbencho_bundle_is_rebuilt(tmp_path, monkeypatch): + """A pre-recipe cached wrapper cannot survive a repository update.""" + cache = tmp_path / "test-cache" + cache.mkdir() + binary_name = "elbencho.aarch64" + binary = cache / f"v3.1-11-{binary_name}" + runtime = cache / f"v3.1-11-{binary_name}.runtime" + binary.write_text("stale wrapper\n", encoding="utf-8") + runtime.mkdir() + calls = [] + + def fake_extract(_runner, _cache, target_binary, target_runtime, architecture): + calls.append(architecture) + target_binary.write_text("current wrapper\n", encoding="utf-8") + target_runtime.mkdir(exist_ok=True) + + monkeypatch.setattr(_FILESYSTEM, "_extract_container_elbencho", fake_extract) + config = SimpleNamespace(state_dir=tmp_path) + + _ensure_elbencho(object(), config, "aarch64", "sbx-shared") + _ensure_elbencho(object(), config, "aarch64", "sbx-shared") + + assert calls == ["aarch64"] + marker = cache / f"v3.1-11-{binary_name}.bundle.json" + document = json.loads(marker.read_text(encoding="utf-8")) + assert document["container"] == _FILESYSTEM.ELBENCHO_CONTAINER + assert document["recipe"] == _FILESYSTEM.SBX_ELBENCHO_BUNDLE_RECIPE diff --git a/tests/test_integration_failure_injection.py b/tests/test_integration_failure_injection.py new file mode 100644 index 0000000..0ab5cfd --- /dev/null +++ b/tests/test_integration_failure_injection.py @@ -0,0 +1,500 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for scenario-owned Elbencho failure injection.""" + +import os +import shutil +import signal +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_REPOSITORY_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPOSITORY_ROOT / "integration-tests" / "lib")) + +from failure_injection import ( # pylint: disable=wrong-import-position + FailureInjectionError, + FailureInjectionOperations, + build_slurm_failure_injection_plan, + build_ssh_failure_injection_plan, + cleanup_failure_injection, + stage_failure_injection, + staged_failure_injection, +) +import filesystem_integration as _INTEGRATION # pylint: disable=wrong-import-position + +TARGET_ARGUMENT = "/mnt/storage-test/results-e0002" + + +def _write_delegate(path, body): + """Create one executable delegate used by wrapper process tests.""" + path.write_text(f"#!/usr/bin/env bash\nset -u\n{body}\n", encoding="utf-8") + path.chmod(0o755) + + +def _local_plan(tmp_path, *, runtime=False, exit_code=97): + """Return a staged local Slurm-shaped plan.""" + source = tmp_path / "source-ussegl" + _write_delegate(source, 'printf "stdout:%s\\n" "$*"; printf "stderr\\n" >&2') + source_runtime = None + if runtime: + source_runtime = tmp_path / "source-runtime" + source_runtime.mkdir() + (source_runtime / "library.so").write_text("runtime", encoding="utf-8") + plan = build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path.as_posix(), + source_binary=source, + source_runtime=source_runtime, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + injected_exit_code=exit_code, + ) + operations = _local_operations(tmp_path) + stage_failure_injection(plan, operations) + return plan, operations + + +def _local_operations(base): + """Return staging operations that require the expected local endpoint.""" + + def _check(endpoint, path): + assert endpoint == "local" + assert Path(path).is_relative_to(base) + return Path(path) + + def _make(endpoint, path): + _check(endpoint, path).mkdir(parents=True, exist_ok=True) + + def _copy_file(source, endpoint, path, mode): + destination = _check(endpoint, path) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + destination.chmod(mode) + + def _copy_tree(source, endpoint, path): + shutil.copytree(source, _check(endpoint, path)) + + def _write(endpoint, path, content, mode): + destination = _check(endpoint, path) + destination.write_text(content, encoding="utf-8") + destination.chmod(mode) + + def _remove(endpoint, path): + shutil.rmtree(_check(endpoint, path), ignore_errors=True) + + return FailureInjectionOperations(_make, _copy_file, _copy_tree, _write, _remove) + + +def test_target_succeeds_then_fails_once_and_marker_survives_resume(tmp_path): + """A successful target fails once, while resume delegates successfully.""" + plan, _ = _local_plan(tmp_path) + command = [str(plan.layout.wrapper), "--write", TARGET_ARGUMENT] + + first = subprocess.run(command, text=True, capture_output=True, check=False) + second = subprocess.run(command, text=True, capture_output=True, check=False) + + assert first.returncode == 97 + assert first.stdout == f"stdout:--write {TARGET_ARGUMENT}\n" + assert first.stderr == "stderr\n" + assert (Path(plan.layout.marker) / "state").read_text( + encoding="utf-8" + ) == "injected\n" + assert second.returncode == 0 + assert second.stdout == first.stdout + assert second.stderr == first.stderr + assert Path(plan.layout.marker).is_dir() + + +def test_service_mode_execs_delegate_with_same_process_identity(tmp_path): + """Service mode replaces the wrapper instead of leaving a parent process.""" + plan, _ = _local_plan(tmp_path) + _write_delegate(Path(plan.layout.delegate), 'printf "%s\\n" "$BASHPID"') + + process = subprocess.Popen( + [str(plan.layout.wrapper), "--service", TARGET_ARGUMENT], + text=True, + stdout=subprocess.PIPE, + ) + output = process.communicate(timeout=5)[0] + + assert process.returncode == 0 + assert int(output.strip()) == process.pid + assert not Path(plan.layout.marker).exists() + + +def test_nontarget_and_real_failure_are_not_replaced(tmp_path): + """Only a successful target is eligible for the injected exit status.""" + plan, _ = _local_plan(tmp_path) + + nontarget = subprocess.run( + [str(plan.layout.wrapper), "--write", "/another/path"], check=False + ) + assert nontarget.returncode == 0 + assert not Path(plan.layout.marker).exists() + + _write_delegate(Path(plan.layout.delegate), "exit 23") + failed = subprocess.run([str(plan.layout.wrapper), TARGET_ARGUMENT], check=False) + assert failed.returncode == 23 + assert not Path(plan.layout.marker).exists() + + +def test_coordinator_forwards_signal_and_preserves_child_status(tmp_path): + """Signals reach the real coordinator and its failure remains authoritative.""" + plan, _ = _local_plan(tmp_path) + _write_delegate( + Path(plan.layout.delegate), + "trap 'exit 42' TERM\nprintf '%s\\n' \"$BASHPID\"\nwhile true; do sleep 1; done", + ) + process = subprocess.Popen( + [str(plan.layout.wrapper), TARGET_ARGUMENT], + text=True, + stdout=subprocess.PIPE, + ) + assert process.stdout is not None + child_pid = int(process.stdout.readline().strip()) + + process.send_signal(signal.SIGTERM) + process.wait(timeout=5) + + assert process.returncode == 42 + with pytest.raises(ProcessLookupError): + os.kill(child_pid, 0) + assert not Path(plan.layout.marker).exists() + + +class _Recorder: + """Record substrate-neutral staging operations without touching hosts.""" + + def __init__(self): + self.calls = [] + + def operations(self): + """Return callbacks backed by this recorder.""" + return FailureInjectionOperations( + lambda endpoint, path: self.calls.append(("mkdir", endpoint, path)), + lambda source, endpoint, path, mode: self.calls.append( + ("copy-file", source, endpoint, path, mode) + ), + lambda source, endpoint, path: self.calls.append( + ("copy-tree", source, endpoint, path) + ), + lambda endpoint, path, content, mode: self.calls.append( + ("write", endpoint, path, content, mode) + ), + lambda endpoint, path: self.calls.append(("remove", endpoint, path)), + ) + + +def test_ssh_staging_places_delegate_runtime_on_every_worker(tmp_path): + """SSH workers receive delegate trees but production copies their wrapper.""" + source = tmp_path / "elbencho" + _write_delegate(source, "exit 0") + runtime = tmp_path / "elbencho-runtime" + runtime.mkdir() + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root="/var/tmp/storage-scale-test", + source_binary=source, + source_runtime=runtime, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="coordinator", + worker_endpoints=("worker-1", "worker-2"), + ) + recorder = _Recorder() + + stage_failure_injection(plan, recorder.operations()) + + file_endpoints = {call[2] for call in recorder.calls if call[0] == "copy-file"} + tree_endpoints = {call[2] for call in recorder.calls if call[0] == "copy-tree"} + writes = [call for call in recorder.calls if call[0] == "write"] + assert file_endpoints == {"coordinator", "worker-1", "worker-2"} + assert tree_endpoints == file_endpoints + assert len(writes) == 1 + assert writes[0][1] == "coordinator" + assert f"readonly delegate={plan.layout.delegate}" in writes[0][3] + + +def test_ssh_coordinator_adapter_stages_local_delegate_and_runtime(tmp_path): + """The concrete SSH adapter supplies every path referenced by its wrapper.""" + source = tmp_path / "source-elbencho" + _write_delegate(source, 'printf "delegate:%s\\n" "$*"') + runtime = tmp_path / "source-runtime" + runtime.mkdir() + (runtime / "library.so").write_text("runtime\n", encoding="utf-8") + packaged = tmp_path / "workspace" / "utils" / "elbencho" + packaged.parent.mkdir(parents=True) + shutil.copy2(source, packaged) + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path / "scenario-staging", + source_binary=source, + source_runtime=runtime, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="local", + worker_endpoints=(), + ) + operations = ( + _INTEGRATION._remote_staging_operations( # pylint: disable=protected-access + object(), + SimpleNamespace(namespace="unused"), + SimpleNamespace(login_container="login"), + "ssh", + packaged, + source, + ) + ) + + stage_failure_injection(plan, operations) + + assert Path(plan.layout.delegate).is_file() + assert (Path(plan.layout.runtime) / "library.so").is_file() + injected = subprocess.run( + [str(packaged), TARGET_ARGUMENT], text=True, capture_output=True, check=False + ) + assert injected.returncode == 97 + assert injected.stdout == f"delegate:{TARGET_ARGUMENT}\n" + + cleanup_failure_injection(plan, operations) + + assert packaged.read_bytes() == source.read_bytes() + assert not Path(plan.layout.root).exists() + + +def test_outer_context_retains_artifacts_through_resume_then_cleans(tmp_path): + """No per-attempt cleanup removes the marker needed by resume.""" + source = tmp_path / "source-elbencho" + _write_delegate(source, "exit 0") + plan = build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path.as_posix(), + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + ) + operations = _local_operations(tmp_path) + + with staged_failure_injection(plan, operations): + command = [str(plan.layout.wrapper), TARGET_ARGUMENT] + assert subprocess.run(command, check=False).returncode == 97 + assert Path(plan.layout.marker).is_dir() + assert subprocess.run(command, check=False).returncode == 0 + assert Path(plan.layout.marker).is_dir() + + assert not Path(plan.layout.root).exists() + + +def test_staging_rejects_missing_or_unsafe_inputs(tmp_path): + """Plans reject unsafe roots and staging rejects missing delegates.""" + source = tmp_path / "missing" + with pytest.raises(FailureInjectionError, match="dedicated absolute"): + build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root="relative", + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + ) + + plan = build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path.as_posix(), + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + ) + with pytest.raises(FailureInjectionError, match="not a regular file"): + stage_failure_injection(plan, _local_operations(tmp_path)) + + +def test_cleanup_runs_in_reverse_target_order(tmp_path): + """Cleanup removes workers before the coordinator marker owner.""" + source = tmp_path / "elbencho" + _write_delegate(source, "exit 0") + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root="/var/tmp/storage-scale-test", + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="coordinator", + worker_endpoints=("worker-1", "worker-2"), + ) + recorder = _Recorder() + + cleanup_failure_injection(plan, recorder.operations()) + + assert [call[1] for call in recorder.calls] == [ + "worker-2", + "worker-1", + "coordinator", + ] + + +@pytest.mark.parametrize("fail_at", range(1, 11)) +def test_partial_staging_failure_cleans_every_endpoint(tmp_path, fail_at): + """Every partial staging boundary remains inside the cleanup scope.""" + source = tmp_path / "elbencho" + _write_delegate(source, "exit 0") + runtime = tmp_path / "runtime" + runtime.mkdir() + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root="/var/tmp/storage-scale-test", + source_binary=source, + source_runtime=runtime, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="coordinator", + worker_endpoints=("worker-1", "worker-2"), + ) + recorder = _Recorder() + base = recorder.operations() + calls = 0 + + def _failing(operation): + def _wrapped(*arguments): + nonlocal calls + calls += 1 + if calls == fail_at: + raise RuntimeError(f"staging failure {fail_at}") + return operation(*arguments) + + return _wrapped + + operations = FailureInjectionOperations( + _failing(base.make_directory), + _failing(base.copy_file), + _failing(base.copy_tree), + _failing(base.write_text), + base.remove_tree, + ) + + with pytest.raises(RuntimeError, match="staging failure"): + with staged_failure_injection(plan, operations): + pytest.fail("staging unexpectedly completed") + + assert [call[1] for call in recorder.calls if call[0] == "remove"] == [ + "worker-2", + "worker-1", + "coordinator", + ] + + +def test_cleanup_attempts_every_endpoint_and_aggregates_errors(tmp_path): + """One removal failure cannot prevent cleanup of later endpoints.""" + source = tmp_path / "elbencho" + _write_delegate(source, "exit 0") + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root="/var/tmp/storage-scale-test", + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="coordinator", + worker_endpoints=("worker-1", "worker-2"), + ) + attempted = [] + + def _remove(endpoint, _path): + attempted.append(endpoint) + if endpoint != "worker-1": + raise OSError(f"cannot remove {endpoint}") + + operations = FailureInjectionOperations( + lambda *_args: None, + lambda *_args: None, + lambda *_args: None, + lambda *_args: None, + _remove, + ) + + with pytest.raises(FailureInjectionError, match="worker-2.*coordinator"): + cleanup_failure_injection(plan, operations) + + assert attempted == ["worker-2", "worker-1", "coordinator"] + + +def test_real_binary_validation_precedes_wrapper_staging(tmp_path, monkeypatch): + """The validator observes Elbencho before the failure wrapper replaces it.""" + events = [] + first = SimpleNamespace(name="inject-one-failure") + resume = SimpleNamespace(name="resume") + runtime = SimpleNamespace( + selector="ssh", + scenario=SimpleNamespace(steps=(first, resume)), + workspace=tmp_path.as_posix(), + ) + + class _StopAfterStaging: + def __enter__(self): + events.append("stage-wrapper") + raise RuntimeError("stop after staging") + + def __exit__(self, *_args): + return False + + monkeypatch.setattr( + _INTEGRATION, + "_failure_plan", + lambda *_args: (object(), object()), + ) + monkeypatch.setattr( + _INTEGRATION, + "_reset_result_base", + lambda *_args: events.append("reset"), + ) + monkeypatch.setattr( + _INTEGRATION, + "_sync_step_runtime", + lambda *_args: events.append("render-real-env"), + ) + monkeypatch.setattr( + _INTEGRATION, + "_validate_step_environment", + lambda *_args: events.append("validate-real-binary"), + ) + monkeypatch.setattr( + _INTEGRATION, + "staged_failure_injection", + lambda *_args: _StopAfterStaging(), + ) + + with pytest.raises(RuntimeError, match="stop after staging"): + _INTEGRATION._run_failure_resume( # pylint: disable=protected-access + object(), + object(), + object(), + runtime, + tmp_path / "env.sh.template", + tmp_path, + tmp_path, + "elbencho", + tmp_path / "elbencho", + None, + ) + + assert events == [ + "reset", + "render-real-env", + "validate-real-binary", + "stage-wrapper", + ] diff --git a/tests/test_reporting_common.py b/tests/test_reporting_common.py index 389c80f..745333f 100644 --- a/tests/test_reporting_common.py +++ b/tests/test_reporting_common.py @@ -92,3 +92,9 @@ def test_scale_filter_and_histogram_ranges(): "min_count": 1.0, "max_count": 110.00000000000001, } + assert histogram_axis_ranges([([0.0012, 0.091], [4080.0, 1.0])]) == { + "min_latency": 0.00108, + "max_latency": 0.10010000000000001, + "min_count": 1.0, + "max_count": 4488.0, + } diff --git a/tests/test_ssh_home_transition.py b/tests/test_ssh_home_transition.py new file mode 100644 index 0000000..5355f39 --- /dev/null +++ b/tests/test_ssh_home_transition.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for crash-recoverable SSH home transitions.""" + +import json +import sys +from pathlib import Path + +import pytest + +_REPOSITORY_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPOSITORY_ROOT / "integration-tests" / "lib")) + +from ssh_home_transition import ( # pylint: disable=wrong-import-position + STATE_FILENAME, + PoolObservation, + SshHomeTransitionError, + SshHomeTransitionManager, + TransitionState, + TransitionStore, + statefulset_checksum, +) + +SEPARATE_CHECKSUM = "separate-checksum" +SHARED_CHECKSUM = "shared-checksum" +FIXED_TIME = "2026-09-19T00:00:00+00:00" + + +def _observation( + mode="separate", + checksum=SEPARATE_CHECKSUM, + ready=2, + terminating=0, + diagnostics="", +): + """Return one compact live-pool observation.""" + return PoolObservation(mode, checksum, ready, terminating, diagnostics) + + +class _Controller: + """Injectable live pool with observable reconciliation calls.""" + + def __init__(self, state_dir, observed=None): + self.state_dir = state_dir + self.observed = observed or _observation() + self.calls = [] + self.fail = None + self.state_during_reconcile = None + + def inspect(self): + """Return the current simulated StatefulSet and pod state.""" + return self.observed + + def reconcile(self, mode, checksum): + """Record and apply one simulated StatefulSet reconciliation.""" + self.calls.append((mode, checksum)) + state_path = self.state_dir / STATE_FILENAME + self.state_during_reconcile = json.loads(state_path.read_text(encoding="utf-8")) + if self.fail: + raise self.fail + self.observed = _observation(mode, checksum) + + +def _manager(tmp_path, controller): + """Return a deterministic transition manager.""" + return SshHomeTransitionManager( + tmp_path, + controller.inspect, + controller.reconcile, + clock=lambda: FIXED_TIME, + ) + + +def _state( + *, + phase="applying-shared", + requested_mode="shared", + checksum=SHARED_CHECKSUM, +): + """Return one valid retained state document.""" + return TransitionState( + schema=1, + run_id="run-1", + scenario_id="ssh-shared-home", + prior_mode="separate", + requested_mode=requested_mode, + expected_statefulset_checksum=checksum, + phase=phase, + updated_at=FIXED_TIME, + ) + + +def test_clean_canonical_preflight_does_not_roll_out_workers(tmp_path): + """A healthy canonical pool is accepted without replacing SSH pods.""" + controller = _Controller(tmp_path) + + observed = _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="setup", scenario_id="preflight" + ) + + assert observed.matches("separate", SEPARATE_CHECKSUM) + assert controller.calls == [] + assert not (tmp_path / STATE_FILENAME).exists() + + +@pytest.mark.parametrize( + "observed", + [ + _observation("shared", SHARED_CHECKSUM), + _observation("separate", "stale-separate-checksum"), + _observation("separate", SEPARATE_CHECKSUM, terminating=1), + _observation("unknown", "partial", ready=1), + ], +) +def test_preflight_reconciles_noncanonical_live_state(tmp_path, observed): + """Setup and SSH preflight restore any partial or shared pool.""" + controller = _Controller(tmp_path, observed) + + result = _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-2", scenario_id="preflight" + ) + + assert result.matches("separate", SEPARATE_CHECKSUM) + assert controller.calls == [("separate", SEPARATE_CHECKSUM)] + assert controller.state_during_reconcile["phase"] == "restoring-separate" + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_interrupted_transition_forces_validation_rollout(tmp_path): + """Retained intent forces reconciliation even if pods look canonical.""" + TransitionStore(tmp_path).save(_state()) + controller = _Controller(tmp_path) + + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="new-run", scenario_id="preflight" + ) + + assert controller.calls == [("separate", SEPARATE_CHECKSUM)] + assert controller.state_during_reconcile["run_id"] == "run-1" + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_enter_shared_persists_intent_before_rollout(tmp_path): + """A crash during rollout leaves enough state for the next preflight.""" + controller = _Controller(tmp_path) + + observed = _manager(tmp_path, controller).enter_shared( + run_id="run-3", + scenario_id="ssh-shared-home", + separate_checksum=SEPARATE_CHECKSUM, + shared_checksum=SHARED_CHECKSUM, + ) + + assert observed.matches("shared", SHARED_CHECKSUM) + assert controller.calls == [("shared", SHARED_CHECKSUM)] + assert controller.state_during_reconcile["phase"] == "applying-shared" + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "shared-ready" + assert retained.run_id == "run-3" + + +def test_restore_clears_state_only_after_canonical_validation(tmp_path): + """The transition record remains present while restoration runs.""" + TransitionStore(tmp_path).save(_state(phase="shared-ready")) + controller = _Controller(tmp_path, _observation("shared", SHARED_CHECKSUM)) + + observed = _manager(tmp_path, controller).restore_separate( + SEPARATE_CHECKSUM, run_id="run-3", scenario_id="ssh-shared-home" + ) + + assert observed.matches("separate", SEPARATE_CHECKSUM) + assert controller.state_during_reconcile["phase"] == "restoring-separate" + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_failed_reconciliation_persists_actionable_unhealthy_state(tmp_path): + """A rollout failure blocks SSH work and survives another invocation.""" + controller = _Controller(tmp_path, _observation("shared", SHARED_CHECKSUM)) + controller.fail = RuntimeError("rollout timed out") + + with pytest.raises(SshHomeTransitionError, match="further SSH scenarios"): + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-4", scenario_id="preflight" + ) + + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "unhealthy" + assert retained.prior_mode == "shared" + assert retained.requested_mode == "separate" + assert "rollout timed out" in retained.failure_diagnostics + + +def test_later_preflight_retries_unhealthy_reconciliation(tmp_path): + """A later setup can recover an SSH fixture marked unhealthy.""" + TransitionStore(tmp_path).save( + _state( + phase="unhealthy", + requested_mode="separate", + checksum=SEPARATE_CHECKSUM, + ) + ) + controller = _Controller(tmp_path) + + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-5", scenario_id="preflight" + ) + + assert controller.calls == [("separate", SEPARATE_CHECKSUM)] + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_inspection_failure_retains_canonical_recovery_target(tmp_path): + """A failed preflight still records that separate mode must be restored.""" + + def failed_inspection(): + raise RuntimeError("Kubernetes API unavailable") + + manager = SshHomeTransitionManager( + tmp_path, + failed_inspection, + lambda _mode, _checksum: None, + clock=lambda: FIXED_TIME, + ) + + with pytest.raises(SshHomeTransitionError, match="blocked"): + manager.preflight(SEPARATE_CHECKSUM, run_id="run-6", scenario_id="preflight") + + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "unhealthy" + assert retained.prior_mode == "unknown" + assert retained.requested_mode == "separate" + assert retained.expected_statefulset_checksum == SEPARATE_CHECKSUM + assert "Kubernetes API unavailable" in retained.failure_diagnostics + + +def test_post_reconcile_observation_must_be_ready_and_nonterminating(tmp_path): + """A callback cannot claim success while rollout overlap remains.""" + controller = _Controller(tmp_path, _observation("shared", SHARED_CHECKSUM)) + + def incomplete_reconcile(_mode, _checksum): + controller.observed = _observation( + "separate", SEPARATE_CHECKSUM, ready=2, terminating=1 + ) + + manager = SshHomeTransitionManager( + tmp_path, + controller.inspect, + incomplete_reconcile, + clock=lambda: FIXED_TIME, + ) + + with pytest.raises(SshHomeTransitionError, match="blocked"): + manager.preflight(SEPARATE_CHECKSUM, run_id="run-6", scenario_id="preflight") + + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "unhealthy" + assert "terminating=1" in retained.failure_diagnostics + + +def test_invalid_state_is_not_silently_replaced(tmp_path): + """Malformed ownership state requires diagnosis instead of blind cleanup.""" + (tmp_path / STATE_FILENAME).write_text('{"schema": 99}\n', encoding="utf-8") + controller = _Controller(tmp_path) + + with pytest.raises(SshHomeTransitionError, match="invalid SSH home transition"): + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-7", scenario_id="preflight" + ) + + assert controller.calls == [] + + +def test_state_store_requires_existing_state_directory(tmp_path): + """Transition code never creates or adopts lifecycle state directories.""" + absent = tmp_path / "absent" + + with pytest.raises(SshHomeTransitionError, match="state directory is absent"): + TransitionStore(absent).save(_state()) + + +def test_statefulset_checksum_uses_rendered_bytes(): + """Configuration identities change with the rendered StatefulSet form.""" + assert statefulset_checksum("emptyDir: {}") == statefulset_checksum(b"emptyDir: {}") + assert statefulset_checksum("emptyDir: {}") != statefulset_checksum( + "persistentVolumeClaim: {}" + ) diff --git a/tests/test_validate_env_filesystem_shell.py b/tests/test_validate_env_filesystem_shell.py new file mode 100644 index 0000000..f16ea61 --- /dev/null +++ b/tests/test_validate_env_filesystem_shell.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shell-level regressions for validate_env filesystem identity checks.""" + +import shlex +import shutil +import subprocess +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_VALIDATE_ENV = _REPO_ROOT / "validate_env.sh" +_BASH = shutil.which("bash") or "bash" + + +def _filesystem_functions() -> str: + """Extract the production filesystem-check functions without running main.""" + source = _VALIDATE_ENV.read_text(encoding="utf-8") + start = source.index("check_fs() {") + end = source.index("check_obj() {", start) + return source[start:end] + + +def _write_fake_stat(directory: Path) -> None: + """Install a stat whose statfs value would collide but st_dev does not.""" + command = directory / "stat" + command.write_text( + textwrap.dedent("""\ + #!/bin/sh + if [ "$1" != -c ] || [ "$2" != %d ]; then + printf 'unexpected stat arguments: %s\\n' "$*" >&2 + exit 90 + fi + shift 2 + [ "${1-}" != -- ] || shift + if [ "${SAME_DEVICE-}" = 1 ] || [ "$1" = / ]; then + printf '11\\n' + else + printf '22\\n' + fi + """), + encoding="utf-8", + ) + command.chmod(0o755) + + +def _run_check( + tmp_path: Path, substrate: str, *, same_device: bool +) -> subprocess.CompletedProcess[str]: + """Run both validation paths for one substrate against the fake stat.""" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + _write_fake_stat(fake_bin) + target = tmp_path / "test directory" + target.mkdir() + trace = tmp_path / "trace" + script = _filesystem_functions() + textwrap.dedent(f""" + declare -a captured_errors=() + FAKE_BIN={shlex.quote(str(fake_bin))} + TRACE={shlex.quote(str(trace))} + TARGET={shlex.quote(str(target))} + export FAKE_BIN TRACE + {"export SAME_DEVICE=1" if same_device else "unset SAME_DEVICE"} + + register_error() {{ + captured_errors+=("$1") + }} + run_scriptlet() {{ + local scriptlet="$1" + shift + PATH="$FAKE_BIN:$PATH" bash -s -- "$@" <<< "$scriptlet" + }} + check_sbatch_with() {{ + local scriptlet="$1" + shift + printf 'sbatch\\n' >> "$TRACE" + run_scriptlet "$scriptlet" "$@" + }} + check_srun_with() {{ + local scriptlet="$1" + shift + printf 'srun\\n' >> "$TRACE" + PATH="$FAKE_BIN:$PATH" bash -c "$scriptlet" "$@" + }} + check_ssh_with() {{ + local scriptlet="$1" + shift + printf 'ssh\\n' >> "$TRACE" + if [[ -n "$scriptlet" ]]; then + run_scriptlet "$scriptlet" "$@" + else + PATH="$FAKE_BIN:$PATH" bash -c "$1" + fi + }} + + if [[ {shlex.quote(substrate)} == slurm ]]; then + check_fs 0 "$TARGET" + [[ $(grep -c '^sbatch$' "$TRACE") -eq 1 ]] + [[ $(grep -c '^srun$' "$TRACE") -eq 1 ]] + else + SSH_ENABLED=1 + check_fs 1 "$TARGET" + [[ $(grep -c '^ssh$' "$TRACE") -eq 2 ]] + fi + printf '%s\\n' "${{captured_errors[@]}}" + """) + return subprocess.run( + [_BASH, "-c", script], + check=False, + cwd=_REPO_ROOT, + text=True, + capture_output=True, + ) + + +@pytest.mark.parametrize("substrate", ("ssh", "slurm")) +def test_distinct_device_passes_every_remote_validation_path(tmp_path, substrate): + """SSH, sbatch, and srun compare st_dev rather than statfs free inodes.""" + result = _run_check(tmp_path, substrate, same_device=False) + + assert result.returncode == 0, result.stderr + assert result.stdout == "\n" + + +@pytest.mark.parametrize("substrate", ("ssh", "slurm")) +def test_root_device_is_rejected_by_every_substrate(tmp_path, substrate): + """A test directory sharing root's device remains a validation error.""" + result = _run_check(tmp_path, substrate, same_device=True) + + assert result.returncode == 0, result.stderr + assert "not on a filesystem distinct from /" in result.stdout diff --git a/utils/check_license_headers.py b/utils/check_license_headers.py index 83c06ba..b3c0d3c 100755 --- a/utils/check_license_headers.py +++ b/utils/check_license_headers.py @@ -113,7 +113,7 @@ def find_violations( violations: list[tuple[Path, str]] = [] for path in paths: absolute_path = repo_root / path - if absolute_path.is_symlink(): + if not absolute_path.exists() or absolute_path.is_symlink(): continue data = absolute_path.read_bytes() if b"\0" in data: diff --git a/utils/extract-elbencho.py b/utils/extract-elbencho.py index a18e156..88c0192 100644 --- a/utils/extract-elbencho.py +++ b/utils/extract-elbencho.py @@ -53,6 +53,7 @@ Tuple, TypedDict, cast, + get_type_hints, ) import matplotlib.pyplot as plt @@ -81,7 +82,6 @@ write_client_summaries, ) from lib.join_datestamps import join_datestamps, join_datestamps_for_filename -from lib.parse_only_sizes import parse_only_sizes_arg from lib.reporting_common import histogram_axis_ranges from lib.stdout_report_file import ( # pylint: disable=wrong-import-position REPORT_TXT_FILENAME, @@ -267,6 +267,9 @@ class ElbenchoMetrics: histogram: Dict[float, int] = field(default_factory=dict) +_ELBENCHO_METRIC_FIELD_TYPES = get_type_hints(ElbenchoMetrics) + + @dataclass(frozen=True) class ExecutionCoordinates: """Coordinates and configured layout captured in one reified execution.""" @@ -2330,6 +2333,14 @@ def plot_performance_metrics( ) +def _scale_efficiency(throughput_per_unit: List[float]) -> List[float]: + """Return relative efficiency, including a defined all-zero result.""" + maximum = max(throughput_per_unit) + if maximum <= 0: + return [0.0] * len(throughput_per_unit) + return [value / maximum * 100 for value in throughput_per_unit] + + def plot_throughput_scale_efficiency( annotated_size_group: AnnotatedSizeGroup, output_dir: str, @@ -2434,15 +2445,9 @@ def plot_throughput_scale_efficiency( # Calculate throughput per unit (node/thread) for each data point bw_per_unit = [bw_val / x_val for bw_val, x_val in zip(bw, x_data)] - # Find the maximum throughput per unit to use as the reference (100%) - max_bw_per_unit = max(bw_per_unit) - # Calculate efficiency values relative to the maximum throughput per unit # Formula: (bw_per_unit / max_bw_per_unit) * 100 - efficiency = [ - (bw_val / x_val) / max_bw_per_unit * 100 - for bw_val, x_val in zip(bw, x_data) - ] + efficiency = _scale_efficiency(bw_per_unit) # Select color and marker using the pre-computed mapping color_index = color_key_to_index[color_key] @@ -2681,13 +2686,14 @@ def _elbencho_csv_coerce_field_types(row: Dict[str, Any]) -> None: for a_field in fields(ElbenchoMetrics): field_name = a_field.name field_value = row[field_name] + field_type = _ELBENCHO_METRIC_FIELD_TYPES[field_name] - if a_field.type == bool: + if field_type is bool: if isinstance(field_value, str): row[field_name] = field_value.lower() == "true" - elif a_field.type == int: + elif field_type is int: row[field_name] = int(float(field_value)) if field_value else 0 - elif a_field.type == float: + elif field_type is float: row[field_name] = float(field_value) if field_value else 0.0 @@ -3531,40 +3537,64 @@ def test_parse(base_filename: str) -> None: # Helper function to parse value lists with ranges -def parse_int_values_with_ranges(value_str): - """Parse a comma-separated list of integers or ranges. +def parse_int_values_with_ranges(value_str: str) -> Set[int]: + """Parse a strict comma-separated list of integers or inclusive ranges. - Ranges are specified as 'start-end'. Returns a set of integer values. - Example: '1,2,5-10,15' would return {1, 2, 5, 6, 7, 8, 9, 10, 15} + A report filter is an operator request, not a best-effort hint. Silently + dropping a malformed token can turn a narrow report into an unfiltered + report, so invalid syntax raises ``ValueError`` for the CLI to report. """ - result = set() - if not value_str: - return result + if not value_str or not value_str.strip(): + raise ValueError("filter must not be empty") + + result: Set[int] = set() + for raw_part in value_str.split(","): + part = raw_part.strip() + if not part: + raise ValueError("empty filter item") + pieces = part.split("-") + if len(pieces) > 2 or not all(piece.isdigit() for piece in pieces): + raise ValueError(f"invalid integer or range: {part}") + start = int(pieces[0]) + end = int(pieces[-1]) + if len(pieces) == 2 and start > end: + raise ValueError(f"range start exceeds end: {part}") + result.update(range(start, end + 1)) + return result + + +_REPORT_SIZE_FILTER_RE = re.compile(r"^r?[0-9]+[KMG](,r?[0-9]+[KMG])?$") - for part in value_str.split(","): - part = part.strip() - if "-" in part: - # Handle range (e.g., "1-10") - try: - start, end = map(int, part.split("-", 1)) - # Add all integers in the range (inclusive) - result.update(range(start, end + 1)) - except ValueError: - eprint( - f"Warning: Invalid range format: {part}. Using individual values only." - ) - continue - else: - # Handle individual value - try: - result.add(int(part)) - except ValueError: - eprint(f"Warning: Invalid integer value: {part}. Skipping.") - continue +def parse_only_sizes_filter(values: Optional[List[str]]) -> Optional[Set[str]]: + """Parse and validate repeated ``--only-sizes`` filter arguments.""" + if not values: + return None + + result: Set[str] = set() + for raw_value in values: + parts = raw_value.split(";") + if any(not part.strip() for part in parts): + raise ValueError("size filter contains an empty item") + for part in parts: + size = part.strip() + if not _REPORT_SIZE_FILTER_RE.fullmatch(size): + raise ValueError(f"invalid IO size: {size}") + result.add(size) return result +def _parse_report_integer_filter( + parser: argparse.ArgumentParser, option_name: str, value: str +) -> Set[int]: + """Parse one integer filter and convert syntax errors to CLI errors.""" + try: + return parse_int_values_with_ranges(value) + except ValueError as exc: + parser.error(f"invalid {option_name} filter: {exc}") + return set() # ``parser.error`` raises; keeps static type checkers happy. + + def parse_benchmark_filename(filename: str) -> Optional[Dict[str, Any]]: """Parse an elbencho benchmark filename to extract parameters. @@ -4507,6 +4537,9 @@ def main() -> None: ) args = parser.parse_args() + if args.from_csv and args.input_dirs: + parser.error("--from-csv cannot be combined with input directories") + positive_live_options = ( ("--client-outlier-threshold", args.client_outlier_threshold), ( @@ -4593,6 +4626,7 @@ def main() -> None: ) had_aggregate_metrics = bool(metrics) + had_live_files = bool(live_files) # Verify that all metrics have a datestamp if metrics: @@ -4610,20 +4644,32 @@ def main() -> None: node_filter = None size_filter = None iodepth_filter = None - if args.only_threads or args.only_sizes or args.only_nodes or args.only_iodepths: + filter_requested = bool( + args.only_threads or args.only_sizes or args.only_nodes or args.only_iodepths + ) + if filter_requested: thread_filter = ( - parse_int_values_with_ranges(args.only_threads) - if args.only_threads - else None + None + if not args.only_threads + else _parse_report_integer_filter( + parser, "--only-threads", args.only_threads + ) ) node_filter = ( - parse_int_values_with_ranges(args.only_nodes) if args.only_nodes else None + None + if not args.only_nodes + else _parse_report_integer_filter(parser, "--only-nodes", args.only_nodes) ) - size_filter = parse_only_sizes_arg(args.only_sizes) + try: + size_filter = parse_only_sizes_filter(args.only_sizes) + except ValueError as exc: + parser.error(f"invalid --only-sizes filter: {exc}") iodepth_filter = ( - parse_int_values_with_ranges(args.only_iodepths) - if args.only_iodepths - else None + None + if not args.only_iodepths + else _parse_report_integer_filter( + parser, "--only-iodepths", args.only_iodepths + ) ) metrics = filter_metrics( @@ -4643,6 +4689,13 @@ def main() -> None: ] eprint(f"After filtering, {len(live_files)} live CSV files remain") + if had_aggregate_metrics and not metrics: + eprint("ERROR: Filters matched no aggregate metrics") + sys.exit(1) + if had_live_files and not live_files: + eprint("ERROR: Filters matched no live CSV files") + sys.exit(1) + # Write to CSV if requested if args.to_csv: # Use the first input directory or current directory if none provided @@ -4662,8 +4715,7 @@ def main() -> None: elif args.from_csv: output_dir = os.path.dirname(os.path.abspath(args.from_csv)) - # Preserve the pre-live behavior when filters remove all aggregate metrics. - if had_aggregate_metrics: + if metrics: if args.markdown: print_markdown_table(metrics, args.no_dual_y_axis) else: @@ -4698,7 +4750,7 @@ def main() -> None: except Exception as exc: # pylint: disable=broad-exception-caught eprint(f"Error reporting live CSV {metadata.path}: {exc}") traceback.print_exc() - if live_files and not successful_live_reports and not had_aggregate_metrics: + if live_files and not successful_live_reports: eprint("ERROR: No live CSV reports were generated") sys.exit(1) diff --git a/validate_env.sh b/validate_env.sh index 69653c7..9a20661 100755 --- a/validate_env.sh +++ b/validate_env.sh @@ -578,14 +578,17 @@ check_fs() { local slurm_rc="$1" # 0 if we can run slurm local fs_path="$2" # filesystem path to test - # This scriptlet is used to test if the filesystem is a mountpoint. - # It is used for both SLURM and SSH-based testing. + # This scriptlet verifies that the test path is a directory on a different + # filesystem device than /. It does not prove that the path itself is the + # exact mountpoint. It is used for both Slurm and SSH-based testing. local scriptlet scriptlet=$(cat << 'EOF' TEST_DIR="$1" -test_dir_inode=$(stat -f -c %d "$TEST_DIR") -root_inode=$(stat -f -c %d "/") -test -d "$TEST_DIR" && [ "$test_dir_inode" != "$root_inode" ] || (echo "failed_$(hostname -s)" && exit 1) +fs_device_failure() { echo "failed_$(hostname -s)"; exit 1; } +test -d "$TEST_DIR" || fs_device_failure +test_dir_device=$(stat -c %d -- "$TEST_DIR") || fs_device_failure +root_device=$(stat -c %d -- "/") || fs_device_failure +[ "$test_dir_device" != "$root_device" ] || fs_device_failure test -w "$TEST_DIR" || (echo "nowrite_$(hostname -s)" && exit 1) EOF ) @@ -614,23 +617,25 @@ check_fs_ssh() { ssh_rc=$? if [[ "$ssh_output" =~ failed_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" - register_error "ssh: ${fs_path} is not a mountpoint on compute node ${nodename}" + register_error "ssh: ${fs_path} is not on a filesystem distinct from / on compute node ${nodename}" elif [[ "$ssh_output" =~ nowrite_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" register_error "ssh: ${fs_path} is not writable on compute node ${nodename}" elif [[ $ssh_rc != 0 ]]; then - register_error "ssh: $fs_path is not a mountpoint on some node" + register_error "ssh: $fs_path is not on a filesystem distinct from / on some node" register_error "ssh: output: $ssh_output" fi + local fs_path_quoted + printf -v fs_path_quoted '%q' "$fs_path" # shellcheck disable=SC2016 - local fs_check_cmd='test -d '"$fs_path"' && [ $(stat -f -c %d '"$fs_path"') != $(stat -f -c %d /) ] || echo "failed_$(hostname -s)"' + local fs_check_cmd='TEST_DIR='"$fs_path_quoted"'; test -d "$TEST_DIR" && test_dir_device=$(stat -c %d -- "$TEST_DIR") && root_device=$(stat -c %d -- /) && [ "$test_dir_device" != "$root_device" ] || echo "failed_$(hostname -s)"' ssh_output=$(check_ssh_with "" "$fs_check_cmd") - # Check for mountpoint failure regardless of ssh exit status + # Check for filesystem-device failure regardless of ssh exit status if [[ "$ssh_output" =~ failed_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" - register_error "ssh: ${fs_path} is not a mountpoint on compute node ${nodename}" + register_error "ssh: ${fs_path} is not on a filesystem distinct from / on compute node ${nodename}" fi } @@ -641,17 +646,17 @@ check_fs_slurm() { sbatch_output=$(check_sbatch_with "$scriptlet" "$fs_path") sbatch_rc=$? if [[ $sbatch_rc != 0 ]]; then - register_error "sbatch: $fs_path is not a mountpoint on some node" + register_error "sbatch: $fs_path is not on a filesystem distinct from / on some node" register_error "sbatch: output: $sbatch_output" fi # shellcheck disable=SC2016 - srun_output=$(check_srun_with 'TEST_DIR="$1"; h=$(hostname -s); test -d ${TEST_DIR} && [ $(stat -f -c %d "$TEST_DIR") != $(stat -f -c %d /) ] || echo "failed_${h}"' _ "$fs_path") + srun_output=$(check_srun_with 'TEST_DIR="$1"; h=$(hostname -s); test -d "$TEST_DIR" && test_dir_device=$(stat -c %d -- "$TEST_DIR") && root_device=$(stat -c %d -- /) && [ "$test_dir_device" != "$root_device" ] || echo "failed_${h}"' _ "$fs_path") - # Check for mountpoint failure regardless of srun exit status + # Check for filesystem-device failure regardless of srun exit status if [[ "$srun_output" =~ failed_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" - register_error "srun: ${fs_path} is not a mountpoint on compute node ${nodename}" + register_error "srun: ${fs_path} is not on a filesystem distinct from / on compute node ${nodename}" fi }