From 193e7e1ce4c55da44a81c40e0c0427d2cedb54ec Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 14:48:37 -0700
Subject: [PATCH 1/8] Run e2e tests from the runner via Holodeck remoteAccess

Enable kubernetes.remoteAccess in tests/holodeck.yaml so Holodeck hands the
GitHub Actions runner a kubeconfig for the test cluster, and rework the e2e
workflow to use it. The case scripts now run on the runner instead of being
rsynced to the EC2 instance and driven over SSH, which removes the scp of the
values override file, the ci-run-e2e.sh/local.sh/push.sh/pull.sh chain and the
key.pem written into the workspace.

SSH is still needed for the two host-mutating operations, so both jobs write
the key under RUNNER_TEMP, export NODE_SSH_HOST/NODE_SSH_KEY/
NODE_SSH_KNOWN_HOSTS for tests/scripts/node-exec.sh, and delete the key
directory at the end of the job.

helm, kubectl and jq are now installed on the runner at pinned versions. helm
was previously installed on the instance by tests/scripts/prerequisites.sh
from the get-helm-3 master script, so it was whatever release happened to be
current; pinning it is a deliberate change. kubectl is pinned to the
Kubernetes version in tests/holodeck.yaml.

Also add a preflight step that fails the job early if the kubeconfig is
unusable, and an always() step that dumps nodes, pods, events and helm
releases into the log directory so the existing artifacts are more useful.

tests/local.sh and friends are unchanged and remain the documented developer
path.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 168 ++++++++++++++++++++++++-------
 tests/README.md                  |  17 ++++
 tests/holodeck.yaml              |   1 +
 3 files changed, 152 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f0f1fb1237..ee18b84e70 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -55,6 +55,16 @@ on:
         type: boolean
         default: false
 
+env:
+  # Pinned versions of the tooling the test scripts drive from the runner.
+  # helm used to be installed on the test node by tests/scripts/prerequisites.sh
+  # straight from the helm get-helm-3 master script, i.e. whatever release was
+  # current at the time the job ran. Pinning it here is a deliberate change.
+  HELM_VERSION: v3.19.0
+  # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
+  KUBECTL_VERSION: v1.35.4
+  JQ_VERSION: "1.7.1"
+
 jobs:
   variables:
     uses: ./.github/workflows/variables.yaml
@@ -79,6 +89,11 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
+      # kubeconfig pointing at the node's public API server endpoint here.
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@v7
         name: Check out code
@@ -90,6 +105,25 @@ jobs:
         with:
           name: values-overrides
           path: ${{ github.workspace }}
+      - name: Install helm
+        uses: azure/setup-helm@v4
+        with:
+          version: ${{ env.HELM_VERSION }}
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@v0.3.7
         with:
@@ -102,36 +136,54 @@ jobs:
         uses: mikefarah/yq@v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
           OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/defaults.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          ./tests/cases/defaults.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v7
@@ -139,6 +191,9 @@ jobs:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove node SSH credentials
+        if: always()
+        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -147,6 +202,11 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
+      # kubeconfig pointing at the node's public API server endpoint here.
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@v7
         name: Check out code
@@ -158,6 +218,25 @@ jobs:
         with:
           name: values-overrides
           path: ${{ github.workspace }}
+      - name: Install helm
+        uses: azure/setup-helm@v4
+        with:
+          version: ${{ env.HELM_VERSION }}
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@v0.3.7
         with:
@@ -170,36 +249,54 @@ jobs:
         uses: mikefarah/yq@v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
           OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/nvidia-driver.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          ./tests/cases/nvidia-driver.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v7
@@ -207,3 +304,6 @@ jobs:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove node SSH credentials
+        if: always()
+        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
diff --git a/tests/README.md b/tests/README.md
index 9b14de2462..3a4096ce5c 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,5 +1,22 @@
 # GPU operator test utilities
 
+## Testing in CI
+CI no longer uses `local.sh` or `ci-run-e2e.sh`, and no longer syncs the project
+folder to the test instance. Those remain the developer path described below.
+
+Instead, the e2e workflow provisions a Holodeck environment with
+`kubernetes.remoteAccess` enabled, which gives the GitHub Actions runner a
+kubeconfig for the cluster. The case scripts (`cases/defaults.sh`,
+`cases/nvidia-driver.sh`) then run on the runner itself, and everything they do
+-- helm, kubectl, log collection -- goes over that kubeconfig.
+
+Only two operations still need a shell on the instance, and both go through
+`scripts/node-exec.sh`, which dispatches `scripts/node-operations.sh` over SSH:
+loading the `i2c_core` and `ipmi_msghandler` kernel modules, and killing the
+gpu-operator container for the operator restart test. With `NODE_SSH_HOST`
+unset, `node-exec.sh` runs the operation locally, so the developer path below is
+unaffected.
+
 ## Testing locally
 The `local.sh` script allows for triggering basic end-to-end testing of the GPU
 operator from a local machine.
diff --git a/tests/holodeck.yaml b/tests/holodeck.yaml
index 3050eaf680..880aa417a6 100644
--- a/tests/holodeck.yaml
+++ b/tests/holodeck.yaml
@@ -24,3 +24,4 @@ spec:
     version: v1.35.4
     crictlVersion: v1.35.0
     calicoVersion: v3.31.5
+    remoteAccess: true

From f4ef16484ebf47aac55d4d46f777a0defc966e41 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 14:49:06 -0700
Subject: [PATCH 2/8] feat(ci): add node-operation dispatch for e2e tests

The e2e tests are moving to run against the cluster from the GitHub Actions
runner instead of over SSH on the node. Two operations still have to run on
the node itself: loading the i2c_core and ipmi_msghandler kernel modules, and
killing the gpu-operator container in the restart test.

Add node-operations.sh, which implements both operations and is self-contained
so that it can be streamed to the node over SSH stdin, and node-exec.sh, which
dispatches an operation over SSH when NODE_SSH_HOST is set and runs it locally
otherwise so that the existing developer workflow keeps working.

test_restart_operator now calls node-exec.sh instead of running crictl or
docker inline. The container selection logic is unchanged, except that an
empty container ID is now reported as an error instead of being handed to the
removal command.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh          | 13 ++---
 tests/scripts/node-exec.sh       | 91 +++++++++++++++++++++++++++++
 tests/scripts/node-operations.sh | 99 ++++++++++++++++++++++++++++++++
 3 files changed, 195 insertions(+), 8 deletions(-)
 create mode 100755 tests/scripts/node-exec.sh
 create mode 100755 tests/scripts/node-operations.sh

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 5056d48b2f..1577a4ae18 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -85,14 +85,11 @@ test_restart_operator() {
 	local ns=${1}
 	local runtime=${2}
 
-	if [[ x"${runtime}" == x"containerd" ]]; then
-		# The operator is the only container that has the string '"gpu-operator"'
-		# TODO: This requires permissions on containerd.sock
-		sudo crictl rm --force "$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')"
-	else
-		# The operator is the only container that has the string '"gpu-operator"'
-	    docker kill "$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')"
-	fi
+	# Killing the operator container mutates the node, so it is dispatched to the
+	# node itself. node-exec.sh runs it either over SSH or locally.
+	local checks_script_dir
+	checks_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+	"${checks_script_dir}"/node-exec.sh restart-operator-container "${runtime}"
 
 	for i in $(seq 1 10); do
 		# Sleep a reasonable amount of time for k8s to update the container status to crashing
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
new file mode 100755
index 0000000000..9f4b351e4d
--- /dev/null
+++ b/tests/scripts/node-exec.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# This script dispatches a host-mutating operation to the node hosting the
+# cluster. When NODE_SSH_HOST is set the operation is streamed to the node over
+# SSH; otherwise it is executed locally, which is the developer path where the
+# tests already run on the node itself.
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
+
+usage() {
+	cat <<'EOF'
+Usage: node-exec.sh <operation> [args...]
+
+Runs one of the node-operations.sh operations on the node hosting the cluster.
+
+Environment:
+  NODE_SSH_HOST         user@host of the node, e.g. ubuntu@ec2-1-2-3-4.compute.amazonaws.com.
+                        If unset or empty the operation is executed locally.
+  NODE_SSH_KEY          Path to the private key. Required when NODE_SSH_HOST is set.
+  NODE_SSH_KNOWN_HOSTS  Path to the known_hosts file. Required when NODE_SSH_HOST is set.
+
+Operations:
+  load-modules
+  restart-operator-container <runtime>
+EOF
+}
+
+require_readable_file() {
+	local name="${1}"
+	local value="${2}"
+
+	if [[ -z "${value}" ]]; then
+		echo "Error: ${name} must be set when NODE_SSH_HOST is set" >&2
+		exit 1
+	fi
+	if [[ ! -r "${value}" ]]; then
+		echo "Error: ${name} '${value}' does not exist or is not readable" >&2
+		exit 1
+	fi
+}
+
+if [[ $# -lt 1 ]]; then
+	usage >&2
+	exit 2
+fi
+
+if [[ ! -r "${NODE_OPERATIONS}" ]]; then
+	echo "Error: ${NODE_OPERATIONS} does not exist or is not readable" >&2
+	exit 1
+fi
+
+if [[ -z "${NODE_SSH_HOST:-}" ]]; then
+	echo "Running '$*' locally"
+	bash "${NODE_OPERATIONS}" "$@"
+	exit $?
+fi
+
+require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
+require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
+
+# Quote each argument so that it survives the remote shell.
+REMOTE_COMMAND="bash -s --"
+for arg in "$@"; do
+	REMOTE_COMMAND+=" $(printf '%q' "${arg}")"
+done
+
+echo "Running '$*' on ${NODE_SSH_HOST}"
+ssh -i "${NODE_SSH_KEY}" \
+	-o BatchMode=yes \
+	-o ConnectTimeout=30 \
+	-o StrictHostKeyChecking=accept-new \
+	-o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
+	"${NODE_SSH_HOST}" \
+	"${REMOTE_COMMAND}" < "${NODE_OPERATIONS}"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
new file mode 100755
index 0000000000..fd46b1e80b
--- /dev/null
+++ b/tests/scripts/node-operations.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# This script runs ON the node hosting the cluster. It is either executed
+# locally or streamed over SSH stdin via `bash -s --`, which means it MUST stay
+# fully self-contained: it must not source sibling scripts such as
+# .definitions.sh and must not refer to any path relative to the repository,
+# since the repository is not guaranteed to exist on the node.
+
+usage() {
+	cat <<'EOF'
+Usage: node-operations.sh <operation> [args...]
+
+Operations:
+  load-modules
+      Load the kernel modules required by the GPU Operator.
+
+  restart-operator-container <runtime>
+      Kill the running gpu-operator container so that kubernetes restarts it.
+      Supported runtimes: containerd, docker.
+EOF
+}
+
+load_modules() {
+	echo "Load kernel modules i2c_core and ipmi_msghandler"
+	sudo modprobe -a i2c_core ipmi_msghandler
+}
+
+# The x-prefixed comparisons and the container selection pipelines below are
+# kept as they were in tests/scripts/checks.sh so that the behaviour of the
+# restart test does not change.
+# shellcheck disable=SC2268
+restart_operator_container() {
+	local runtime="${1:-}"
+	local container_id=""
+
+	if [[ x"${runtime}" == x"containerd" ]]; then
+		# The operator is the only container that has the string '"gpu-operator"'
+		# TODO: This requires permissions on containerd.sock
+		container_id="$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via crictl" >&2
+			return 1
+		fi
+		sudo crictl rm --force "${container_id}"
+	elif [[ x"${runtime}" == x"docker" ]]; then
+		# The operator is the only container that has the string '"gpu-operator"'
+		container_id="$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via docker" >&2
+			return 1
+		fi
+		docker kill "${container_id}"
+	else
+		echo "Error: unknown runtime '${runtime}'. Supported runtimes: containerd, docker" >&2
+		return 1
+	fi
+}
+
+main() {
+	if [[ $# -lt 1 ]]; then
+		usage >&2
+		exit 2
+	fi
+
+	local operation="${1}"
+	shift
+
+	case "${operation}" in
+		load-modules)
+			load_modules "$@"
+			;;
+		restart-operator-container)
+			restart_operator_container "$@"
+			;;
+		*)
+			echo "Error: unknown operation '${operation}'" >&2
+			usage >&2
+			exit 2
+			;;
+	esac
+}
+
+main "$@"

From 232e572e5cbd83b9e9d7f9a100d753f6cb1abbda Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 15:07:36 -0700
Subject: [PATCH 3/8] Harden the e2e workflow after review

node-exec.sh deliberately falls back to running the operation locally when
NODE_SSH_HOST is empty, which is what the developer path relies on. In CI that
fallback would run modprobe and crictl against the shared self-hosted runner
instead of the test instance, so guard both call sites with a non-empty check
on NODE_SSH_HOST and fail the job loudly.

Also from review:

- verify the downloaded jq binary against the sha256 published in the jq 1.7.1
  release checksum file before putting it on PATH
- fail early if the public-dns-name lookup came back empty, rather than
  building NODE_SSH_HOST=ubuntu@ and getting an opaque ssh error later
- add || true to the mkdir in the diagnostics step so a green job cannot be
  turned red by diagnostics
- remove the kubeconfig alongside the SSH key in the always() cleanup, since
  it holds cluster-admin credentials
- use GITHUB_WORKSPACE instead of interpolating github.workspace into a run
  block, matching how the rest of the job passes values through env

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 44 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ee18b84e70..f9ff74786d 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -64,6 +64,8 @@ env:
   # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
   KUBECTL_VERSION: v1.35.4
   JQ_VERSION: "1.7.1"
+  # From https://github.com/jqlang/jq/releases/download/jq-1.7.1/sha256sum.txt
+  JQ_SHA256: "5942c9b0934e510ee61eb3e30273f1b3fe2590df93933a93d7c58b81d19c8ff5"
 
 jobs:
   variables:
@@ -119,6 +121,7 @@ jobs:
           mkdir -p "${RUNNER_TEMP}/bin"
           curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
             "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
           chmod +x "${RUNNER_TEMP}/bin/jq"
           echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
           helm version
@@ -142,6 +145,7 @@ jobs:
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
           set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
           SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
           mkdir -p "${SSH_DIR}"
           chmod 700 "${SSH_DIR}"
@@ -161,10 +165,15 @@ jobs:
         if: ${{ inputs.use_values_override }}
         run: |
           set -euo pipefail
-          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
+          # node-exec.sh falls back to running the operation locally when
+          # NODE_SSH_HOST is empty, which is the developer path. In CI that
+          # would run modprobe against the shared runner, so fail loudly
+          # instead.
+          test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
@@ -174,12 +183,16 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
+          # The restart-operator test shells out to node-exec.sh, which runs
+          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
+          # let crictl run against the shared runner.
+          test -n "${NODE_SSH_HOST}"
           mkdir -p "${GITHUB_WORKSPACE}/logs"
           ./tests/cases/defaults.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}"
+          mkdir -p "${LOG_DIR}" || true
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -191,9 +204,11 @@ jobs:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
-      - name: Remove node SSH credentials
+      - name: Remove credentials from the runner
         if: always()
-        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
+          rm -f "${KUBECONFIG}" || true
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -232,6 +247,7 @@ jobs:
           mkdir -p "${RUNNER_TEMP}/bin"
           curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
             "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
           chmod +x "${RUNNER_TEMP}/bin/jq"
           echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
           helm version
@@ -255,6 +271,7 @@ jobs:
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
           set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
           SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
           mkdir -p "${SSH_DIR}"
           chmod 700 "${SSH_DIR}"
@@ -274,10 +291,15 @@ jobs:
         if: ${{ inputs.use_values_override }}
         run: |
           set -euo pipefail
-          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
+          # node-exec.sh falls back to running the operation locally when
+          # NODE_SSH_HOST is empty, which is the developer path. In CI that
+          # would run modprobe against the shared runner, so fail loudly
+          # instead.
+          test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
@@ -287,12 +309,16 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
+          # The restart-operator test shells out to node-exec.sh, which runs
+          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
+          # let crictl run against the shared runner.
+          test -n "${NODE_SSH_HOST}"
           mkdir -p "${GITHUB_WORKSPACE}/logs"
           ./tests/cases/nvidia-driver.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}"
+          mkdir -p "${LOG_DIR}" || true
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -304,6 +330,8 @@ jobs:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
-      - name: Remove node SSH credentials
+      - name: Remove credentials from the runner
         if: always()
-        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
+          rm -f "${KUBECONFIG}" || true

From 3fa5cda2e329ef744d383db771b6b9ee8a866a74 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 16:24:31 -0700
Subject: [PATCH 4/8] fix(ci): bound the e2e polling loops by wall clock

The polling loops in checks.sh bounded themselves with a counter that was
incremented by 5 on every iteration, on the assumption that an iteration costs
only the 5 second sleep. That has been close enough while the tests ran on the
node itself, but each iteration also issues a number of kubectl calls, and once
those calls cross a network the iteration takes considerably longer than 5
seconds. The counter then runs slower than the clock and the nominal 45 minute
bound stretches to several hours, which is long enough for the job timeout to
cancel the run before any of the loops give up on their own.

Measure elapsed time with the SECONDS builtin against a baseline taken when the
loop starts, so the bound means what it says regardless of how long an
iteration takes. The 45 minute budget itself is unchanged.

wait_for_driver_upgrade_done printed its debug dump when the counter was
divisible by 30. Elapsed time no longer advances in fixed steps, so that test
can step over every multiple and the dump would never be printed. Track the
time at which the next dump is due instead.

Also pass --tail to the per-pod log fetch in check_gpu_pod_ready. It runs
inside the poll loop for every pod in every namespace and refetches each
complete log every five seconds, which is a lot of traffic to repeat for up to
45 minutes. The log collection on failure is left untouched.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh | 42 ++++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 1577a4ae18..20449ef6d3 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -2,7 +2,8 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
 		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
@@ -21,7 +22,7 @@ check_pod_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -30,14 +31,14 @@ check_pod_ready() {
 		kubectl get pods -n ${TEST_NAMESPACE}
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
 
 check_pod_deleted() {
 	local pod_label=$1
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
 		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
@@ -53,7 +54,7 @@ check_pod_deleted() {
 			echo "Pod $pod_label has not been deleted"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -62,7 +63,6 @@ check_pod_deleted() {
 		kubectl get pods -n ${TEST_NAMESPACE}
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
@@ -111,7 +111,8 @@ test_restart_operator() {
 
 check_gpu_pod_ready() {
 	local log_dir=$1
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
@@ -125,7 +126,7 @@ check_gpu_pod_ready() {
 			break;
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1
 		fi
@@ -138,7 +139,8 @@ check_gpu_pod_ready() {
 			echo "Generating logs for pod: ${pod} ns: ${ns}"
 			echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
 			kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
+			# This runs on every poll iteration, so bound the volume we re-fetch each time.
+			kubectl -n "${ns}" logs "${pod}" --all-containers=true --tail=200 > "${log_dir}/${pod}.logs" || true
 		done
 
 		echo "Generating cluster logs"
@@ -146,14 +148,14 @@ check_gpu_pod_ready() {
 		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs"
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5;
 	done
 }
 
 # TODO: deduplicate the logic found in this file by moving the duplicate to a common method and parameterizing the labels to select on
 check_nvidia_driver_pods_ready() {
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 	while :; do
 		echo "Checking nvidia driver pod"
 		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE}
@@ -172,7 +174,7 @@ check_nvidia_driver_pods_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -181,7 +183,6 @@ check_nvidia_driver_pods_ready() {
 		kubectl get pods -n ${TEST_NAMESPACE}
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
@@ -239,7 +240,13 @@ print_driver_upgrade_debug() {
 
 wait_for_driver_upgrade_done() {
 	gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present --no-headers | wc -l)
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
+	local elapsed=0
+	# Next elapsed time at which the full debug dump is due. Iterations can take
+	# much longer than the nominal sleep, so track a due time instead of testing
+	# the elapsed time for divisibility, which would skip dumps entirely.
+	local next_debug=0
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
 		local upgraded_count=0
@@ -256,21 +263,22 @@ wait_for_driver_upgrade_done() {
 			echo "gpu driver still in progress. $upgraded_count/$gpu_node_count node(s) upgraded"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		elapsed=$((SECONDS - start_time))
+		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			print_driver_upgrade_debug
 			exit 1;
 		fi
 
-		if [[ $((current_time % 30)) -eq 0 ]]; then
+		if [[ "${elapsed}" -ge "${next_debug}" ]]; then
 			print_driver_upgrade_debug
+			next_debug=$((elapsed + 30))
 		else
 			kubectl get node -l nvidia.com/gpu.present \
 				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
 		fi
 		
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }

From ab591ed0967b088edb33bc7ea1d71ec2e81017b7 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 16:50:03 -0700
Subject: [PATCH 5/8] fix(ci): collect e2e pod logs on a slower cadence instead
 of truncating them

check_gpu_pod_ready regenerates a describe and a log file for every pod in
every namespace on each pass of its five second poll loop. Passing --tail to
bound that traffic was the wrong call: the log file is overwritten rather than
appended, so the file left behind when the loop gives up is the one that gets
uploaded as the failure artifact, and truncating it drops exactly the output
that explains a failed driver build.

Fetch the whole log again and instead regenerate on a thirty second cadence,
which cuts the traffic by the same order without shortening anything. The
readiness check keeps running every five seconds so success is still noticed
promptly. Both the timeout and the success path collect once more on the way
out so the files on disk are current rather than up to thirty seconds old.

update-nvidiadriver.sh has seven loops with the same counter-based timeout that
checks.sh had, guarding the nvidiadriver test that runs in the same job under
the same job timeout. Convert them the same way. wait_for_nvidiadriver_owner is
the worst of them: its counter needs 181 iterations to reach a fifteen minute
bound, which is half an hour of wall clock once each iteration waits on a
round trip.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh              | 42 ++++++++++++++++++++++------
 tests/scripts/update-nvidiadriver.sh | 42 ++++++++++++++--------------
 2 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 20449ef6d3..437b6a299d 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -109,10 +109,33 @@ test_restart_operator() {
 	exit 1
 }
 
+# Regenerate the describe and log files for every pod passed in. The log files
+# are rewritten in full rather than tailed, so that the artifact left behind
+# holds the complete output of every container.
+collect_pod_logs() {
+	local log_dir=$1
+	local pods=$2
+
+	for pod in $(echo "$pods" | jq -r .[].name); do
+		ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
+		echo "Generating logs for pod: ${pod} ns: ${ns}"
+		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
+		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
+		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
+	done
+}
+
 check_gpu_pod_ready() {
 	local log_dir=$1
 	# SECONDS counts from shell start, so record a baseline and measure against it.
 	local start_time=${SECONDS}
+	local elapsed=0
+	# Regenerating every pod's describe and log files is the expensive part of
+	# this loop, so it runs on its own slower cadence while the readiness check
+	# below keeps polling every 5 seconds. Track the time at which the next
+	# regeneration is due rather than testing the elapsed time for divisibility,
+	# which would skip regenerations when an iteration runs long.
+	local next_collection=0
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
@@ -123,25 +146,26 @@ check_gpu_pod_ready() {
 		if [ "${status}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
 			rc=0
+			collect_pod_logs "${log_dir}" "${pods}"
 			break;
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		elapsed=$((SECONDS - start_time))
+		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
+			# Collect once more so that the artifact reflects the state at the
+			# timeout rather than the state at the last scheduled collection.
+			collect_pod_logs "${log_dir}" "${pods}"
 			exit 1
 		fi
 
 		# Echo useful information on stdout
 		kubectl get pods --all-namespaces
 
-		for pod in $(echo "$pods" | jq -r .[].name); do
-			ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
-			echo "Generating logs for pod: ${pod} ns: ${ns}"
-			echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
-			# This runs on every poll iteration, so bound the volume we re-fetch each time.
-			kubectl -n "${ns}" logs "${pod}" --all-containers=true --tail=200 > "${log_dir}/${pod}.logs" || true
-		done
+		if [[ "${elapsed}" -ge "${next_collection}" ]]; then
+			collect_pod_logs "${log_dir}" "${pods}"
+			next_collection=$((elapsed + 30))
+		fi
 
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index d104673654..efab216afe 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -56,7 +56,8 @@ set_default_driver() {
 
 wait_for_default_nvidiadriver() {
     local expected_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for NVIDIADriver/${expected_name} to be the only default"
     while :; do
@@ -67,14 +68,13 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
             kubectl get nvidiadriver
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -120,7 +120,8 @@ create_nvidiadriver() {
 
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
     local gpu_node_count
 
     gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
@@ -133,7 +134,7 @@ wait_for_nvidiadriver_owner() {
             break
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
             kubectl get nodes -l nvidia.com/gpu.present=true -o json |
                 jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
@@ -142,7 +143,6 @@ wait_for_nvidiadriver_owner() {
 
         echo "NVIDIADriver/${driver_name} owns ${owned_count}/${gpu_node_count} GPU node(s)"
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -154,7 +154,8 @@ get_nvidiadriver_daemonsets() {
 
 wait_for_nvidiadriver_daemonsets() {
     local driver_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for daemonsets owned by NVIDIADriver/${driver_name}"
     while :; do
@@ -164,14 +165,13 @@ wait_for_nvidiadriver_daemonsets() {
             break
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
             kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -184,20 +184,20 @@ test_driver_image_updates() {
     fi
 
     # Verify update is applied to Driver Daemonset
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
     while :; do
         if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e --arg version "${TARGET_DRIVER_VERSION}" 'length > 0 and all(.[]; .spec.template.spec.containers[0].image | contains($version))' >/dev/null; then
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "Image update failed for driver daemonset to version $TARGET_DRIVER_VERSION"
             get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
     echo "driver daemonset image updated successfully to version $TARGET_DRIVER_VERSION"
 
@@ -223,20 +223,20 @@ test_custom_labels_override() {
 
   # Wait for the operator to update the pod template with new labels
   echo "Waiting for DaemonSet pod template to be updated with new labels..."
-  local current_time=0
+  # SECONDS counts from shell start, so record a baseline and measure against it.
+  local start_time=${SECONDS}
   while :; do
     if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e 'length > 0 and all(.[]; .spec.template.metadata.labels.cloudprovider == "aws" and .spec.template.metadata.labels.platform == "kubernetes")' >/dev/null; then
       break
     fi
 
-    if [[ "${current_time}" -gt 120 ]]; then
+    if [[ $((SECONDS - start_time)) -gt 120 ]]; then
       echo "timeout reached waiting for DaemonSet pod template labels"
       get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
       exit 1
     fi
 
     sleep 5
-    current_time=$((${current_time} + 5))
   done
 
   # Delete driver pod to force recreation with updated labels. Existing pods are not automatically restarted due to the DaemonSet's 'OnDelete` updateStrategy.
@@ -276,7 +276,8 @@ assert_nvidiadriver_owner_count() {
 wait_for_nvidiadriver_condition_message() {
     local driver_name=$1
     local message=$2
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
@@ -287,20 +288,20 @@ wait_for_nvidiadriver_condition_message() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
             kubectl get nvidiadriver/"${driver_name}" -o yaml
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_ready() {
     local driver_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
@@ -312,14 +313,13 @@ wait_for_nvidiadriver_ready() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
             kubectl get nvidiadriver/"${driver_name}" -o yaml
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 

From 949db5bbe0e36c76d9cfe43c97b1202f668fb72b Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 17:19:06 -0700
Subject: [PATCH 6/8] fix(ci): upload e2e logs on cancelled runs too

The archive step was gated on failure(), but a job stopped by
timeout-minutes is cancelled rather than failed, so failure() evaluates
false and the upload is skipped. That loses the logs on exactly the runs
that are hardest to diagnose.

Now that the e2e tests run from the runner, the polling loops in
tests/scripts talk to the API server over the internet rather than over
loopback, so a job is more likely to reach the 90 minute cap than it was
when everything ran on the node. Switch both jobs to always() so a
cancelled run still produces artifacts.

The step still runs before the credential cleanup and still uploads only
./logs/, so neither the kubeconfig nor the SSH key can end up in the
artifact.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f9ff74786d..6217b84849 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -198,7 +198,10 @@ jobs:
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
           helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        # always() rather than failure(): a job stopped by timeout-minutes is
+        # cancelled, not failed, so failure() would skip the upload on exactly
+        # the runs whose logs we most need.
+        if: always()
         uses: actions/upload-artifact@v7
         with:
           name: containerd-e2e-test-logs
@@ -324,7 +327,10 @@ jobs:
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
           helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        # always() rather than failure(): a job stopped by timeout-minutes is
+        # cancelled, not failed, so failure() would skip the upload on exactly
+        # the runs whose logs we most need.
+        if: always()
         uses: actions/upload-artifact@v7
         with:
           name: nvidiadriver-e2e-test-logs

From e323085fd62b15edcd7f3f8a5f239e2630ec53e2 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 18:58:47 -0700
Subject: [PATCH 7/8] fix(ci): survive a transient API outage during the driver
 upgrade

Both e2e jobs died in wait_for_driver_upgrade_done. The driver upgrade
restarts the container runtime on the node, and now that the tests drive the
cluster from the runner rather than from the node itself, the API server is
briefly unreachable across the public address instead of on localhost. The
first casualty was the opening kubectl in print_driver_upgrade_debug, which
unlike its three siblings had no guard, so a debug dump ended the run.

Guard it, and make the wait itself tolerate the outage: the node count and the
per-node upgrade state are now read through checks that treat a failure as
"not upgraded yet" and retry until the existing wall-clock deadline expires.
A count that could not be read stays empty rather than defaulting to zero, so
an unreachable API can never be mistaken for a finished upgrade. Failures are
announced with a timestamp so the next run shows how long such an outage
lasts, which this one died too quickly to reveal.

The calls in these two functions also carry an explicit request timeout, since
the default behaviour was to spend thirty seconds per call discovering that
the address was black-holed.

The same shape exists elsewhere: a kubectl whose output only exists for a
human to read, usually just before exit 1, sitting unguarded next to siblings
that already end in || true. Guard those too, in the readiness and log
collection loops in checks.sh and in the timeout dumps in
update-clusterpolicy.sh, migrate-clusterpolicy-to-nvidiadriver.sh and
update-nvidiadriver.sh. Assertions are left alone: a check that cannot reach
the API still fails the test.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh                       | 100 +++++++++++++-----
 .../migrate-clusterpolicy-to-nvidiadriver.sh  |  10 +-
 tests/scripts/update-clusterpolicy.sh         |   2 +-
 tests/scripts/update-nvidiadriver.sh          |  14 +--
 4 files changed, 85 insertions(+), 41 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 437b6a299d..073e0c5426 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -6,7 +6,7 @@ check_pod_ready() {
 	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
 
 		echo "Checking $pod_label pod readiness"
 		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -28,7 +28,7 @@ check_pod_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n ${TEST_NAMESPACE} || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -41,7 +41,7 @@ check_pod_deleted() {
 	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
 
 		echo "Checking if $pod_label pod has been deleted"
 		# note: $(kubectl get pods <options> -o jsonpath='.items' | jq length) does not work for older kubectl clients
@@ -60,7 +60,7 @@ check_pod_deleted() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n ${TEST_NAMESPACE} || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -72,7 +72,7 @@ check_no_restarts() {
 	restartCount=$(kubectl get pod -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "$pod_label restarted multiple times: $restartCount"
-		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE}
+		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE} || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for pod $pod_label"
@@ -120,7 +120,7 @@ collect_pod_logs() {
 		ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
 		echo "Generating logs for pod: ${pod} ns: ${ns}"
 		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
+		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe" || true
 		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
 	done
 }
@@ -160,7 +160,7 @@ check_gpu_pod_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods --all-namespaces
+		kubectl get pods --all-namespaces || true
 
 		if [[ "${elapsed}" -ge "${next_collection}" ]]; then
 			collect_pod_logs "${log_dir}" "${pods}"
@@ -169,7 +169,7 @@ check_gpu_pod_ready() {
 
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs"
+		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5;
@@ -182,7 +182,7 @@ check_nvidia_driver_pods_ready() {
 	local start_time=${SECONDS}
 	while :; do
 		echo "Checking nvidia driver pod"
-		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE}
+		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} || true
 
 		echo "Checking nvidia driver pod readiness"
 		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -204,7 +204,7 @@ check_nvidia_driver_pods_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n ${TEST_NAMESPACE} || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -215,34 +215,46 @@ check_no_driver_pod_restarts() {
 	restartCount=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "nvidia driver pod restarted multiple times: $restartCount"
-		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE}
+		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE} || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for the nvidia driver pod"
 	return 0
 }
 
+# Report that the cluster API could not be reached. The tests now run from a
+# runner outside the cluster, so any call can fail while the node restarts its
+# container runtime during a driver upgrade. Callers treat this as "not ready
+# yet" and keep retrying until their own deadline expires. The timestamp is
+# here so that a later run can show whether such an outage is brief or
+# permanent.
+api_unreachable() {
+	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
+}
+
+# Purely diagnostic. Every call here is guarded so that a debug dump can never
+# be the thing that ends the run.
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
-	kubectl get node -l nvidia.com/gpu.present \
-		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 
 	echo ""
 	echo "driver pods"
-	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
 
 	echo ""
 	echo "gpu operator operands"
-	kubectl get pods -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
 
 	echo ""
 	echo "driver daemonsets"
-	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
 
 	echo ""
 	echo "NVIDIADriver status"
 	local nvidiadriver_status
-	if nvidiadriver_status=$(kubectl get nvidiadriver -o json 2>/dev/null); then
+	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" 2>/dev/null); then
 		echo "${nvidiadriver_status}" | jq -r '
 			(["NAME", "DEFAULT", "STATE", "REASON", "MESSAGE"] | @tsv),
 			(
@@ -263,7 +275,6 @@ print_driver_upgrade_debug() {
 }
 
 wait_for_driver_upgrade_done() {
-	gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present --no-headers | wc -l)
 	# SECONDS counts from shell start, so record a baseline and measure against it.
 	local start_time=${SECONDS}
 	local elapsed=0
@@ -271,20 +282,53 @@ wait_for_driver_upgrade_done() {
 	# much longer than the nominal sleep, so track a due time instead of testing
 	# the elapsed time for divisibility, which would skip dumps entirely.
 	local next_debug=0
+	local node_list=""
+	local upgraded_count=0
+	local upgrade_state=""
+
+	# The driver upgrade restarts the container runtime on the node, so from a
+	# runner outside the cluster every query below can fail for a while. A failed
+	# query means "not upgraded yet" and is retried until the deadline; it must
+	# never be read as the upgrade having finished.
+	gpu_node_count=""
+
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
-		local upgraded_count=0
-		for node in $(kubectl get nodes -o NAME); do
-			upgrade_state=$(kubectl get $node -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}')
-			if [ "${upgrade_state}" = "upgrade-done" ]; then
-				upgraded_count=$((${upgraded_count} + 1))
+		upgraded_count=0
+
+		# Resolve the expected node count once and keep it. Re-reading it every
+		# iteration would risk sampling a transient count while the upgrade churns
+		# node labels, which could make the comparison below succeed early.
+		if [[ -z "${gpu_node_count}" ]]; then
+			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+				gpu_node_count=$(echo "${node_list}" | grep -c . || true)
+			else
+				api_unreachable "counting the GPU nodes"
 			fi
-		done
-		if [[ $upgraded_count -eq $gpu_node_count ]]; then
+		fi
+
+		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+			for node in ${node_list}; do
+				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+					if [ "${upgrade_state}" = "upgrade-done" ]; then
+						upgraded_count=$((upgraded_count + 1))
+					fi
+				else
+					api_unreachable "reading the upgrade state of ${node}"
+				fi
+			done
+		else
+			api_unreachable "listing the nodes"
+		fi
+
+		# The node count guard keeps an unreachable API from being mistaken for a
+		# finished upgrade, which is what comparing 0 against an empty count would
+		# otherwise do.
+		if [[ -n "${gpu_node_count}" ]] && [[ $upgraded_count -eq $gpu_node_count ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
 		else
-			echo "gpu driver still in progress. $upgraded_count/$gpu_node_count node(s) upgraded"
+			echo "gpu driver still in progress. $upgraded_count/${gpu_node_count:-unknown} node(s) upgraded"
 		fi
 
 		elapsed=$((SECONDS - start_time))
@@ -298,8 +342,8 @@ wait_for_driver_upgrade_done() {
 			print_driver_upgrade_debug
 			next_debug=$((elapsed + 30))
 		else
-			kubectl get node -l nvidia.com/gpu.present \
-				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 		fi
 		
 		echo "Sleeping 5 seconds"
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 8cbaae72c5..02bbab6778 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -34,7 +34,7 @@ wait_for_legacy_driver_daemonset_deleted() {
 
         if [[ "${elapsed_time}" -gt 300 ]]; then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide || true
             exit 1
         fi
 
@@ -57,7 +57,7 @@ wait_for_orphaned_legacy_driver_pod() {
 
         if [[ "${elapsed_time}" -gt 300 ]]; then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
@@ -127,7 +127,7 @@ wait_for_nvidiadriver_daemonset() {
 
         if [[ "${elapsed_time}" -gt 300 ]]; then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
@@ -162,14 +162,14 @@ wait_for_legacy_driver_pod_deleted() {
 legacy_driver_pod=$(kubectl get pod -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}')
 if [[ -z "${legacy_driver_pod}" ]]; then
     echo "legacy ClusterPolicy driver pod not found"
-    kubectl get pods -n "${TEST_NAMESPACE}" -o wide
+    kubectl get pods -n "${TEST_NAMESPACE}" -o wide || true
     exit 1
 fi
 
 operator_name=$(get_helm_release_name)
 if [[ -z "${operator_name}" ]]; then
     echo "GPU Operator Helm release not found in namespace ${TEST_NAMESPACE}"
-    ${HELM} list -n "${TEST_NAMESPACE}"
+    ${HELM} list -n "${TEST_NAMESPACE}" || true
     exit 1
 fi
 
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 7a53901c74..6aacbdb107 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -150,7 +150,7 @@ test_gpu_sharing() {
     kubectl wait --for=condition=available --timeout=300s deployment/nvidia-plugin-test -n $TEST_NAMESPACE
     if [ $? -ne 0 ]; then
         echo "cannot run parallel pods with GPU sharing enabled"
-        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE
+        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE || true
         exit 1
     fi
 
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index efab216afe..4682db5835 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -70,7 +70,7 @@ wait_for_default_nvidiadriver() {
 
         if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
-            kubectl get nvidiadriver
+            kubectl get nvidiadriver || true
             exit 1
         fi
 
@@ -83,7 +83,7 @@ test_arbitrary_name_default_nvidiadriver() {
     current_default=$(get_default_nvidiadriver_name)
     if [[ -z "${current_default}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver || true
         exit 1
     fi
 
@@ -110,7 +110,7 @@ create_nvidiadriver() {
     default_name=$(get_default_nvidiadriver_name)
     if [[ -z "${default_name}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver || true
         exit 1
     fi
 
@@ -167,7 +167,7 @@ wait_for_nvidiadriver_daemonsets() {
 
         if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
-            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml
+            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml || true
             exit 1
         fi
 
@@ -253,7 +253,7 @@ test_custom_labels_override() {
   gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
   if [[ "${labeled_pod_count}" -ne "${gpu_node_count}" ]]; then
     echo "Custom labels are missing from one or more NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
-    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels
+    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels || true
     exit 1
   fi
 }
@@ -290,7 +290,7 @@ wait_for_nvidiadriver_condition_message() {
 
         if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1
         fi
 
@@ -315,7 +315,7 @@ wait_for_nvidiadriver_ready() {
 
         if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1
         fi
 

From 8213db15c25189697a447091e8fa643a6ec63ead Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 21:56:05 -0700
Subject: [PATCH 8/8] refactor(ci): tidy the e2e test scripts

Store a deadline rather than a start time in the polling loops. Comparing
SECONDS against a deadline computed once says what the loop means without the
subtraction, and it removes the comment each loop was carrying to explain that
SECONDS counts from shell start rather than from the loop.

migrate-clusterpolicy-to-nvidiadriver.sh had six loops still counting sleeps
instead of measuring time, the same bug already fixed in checks.sh and
update-nvidiadriver.sh. It runs on the containerd path, so convert those too.

check_gpu_pod_ready listed every pod in the cluster as json on every pass of a
five second loop, but only used the result when it regenerated the log files
every thirty seconds. Fetch it where it is used, ask for two custom columns
instead of the whole object, and read it with the shell rather than a jq
invocation per pod. The same loop printed the pod table once for the console
and fetched it again for the log file, which tee does in one call.

The request timeout default now lives in .definitions.sh with the other
defaults instead of being repeated at each use.

Also guard five more diagnostic calls that the earlier pass missed, replace
the runtime comparison in node-operations.sh with a case statement, and drop
the argument and readability checks in node-exec.sh that only repeat what
node-operations.sh and the redirect already report.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml              |  16 +--
 tests/scripts/.definitions.sh                 |   1 +
 tests/scripts/checks.sh                       | 113 +++++++++---------
 .../migrate-clusterpolicy-to-nvidiadriver.sh  |  32 ++---
 tests/scripts/node-exec.sh                    |  60 ++++------
 tests/scripts/node-operations.sh              |  19 +--
 tests/scripts/update-nvidiadriver.sh          |  43 +++----
 7 files changed, 127 insertions(+), 157 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 6217b84849..36712d1890 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -187,12 +187,12 @@ jobs:
           # locally when NODE_SSH_HOST is empty. Refuse to start rather than
           # let crictl run against the shared runner.
           test -n "${NODE_SSH_HOST}"
-          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          mkdir -p "${LOG_DIR}"
           ./tests/cases/defaults.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}" || true
+          mkdir -p "${LOG_DIR}"
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -210,8 +210,8 @@ jobs:
       - name: Remove credentials from the runner
         if: always()
         run: |
-          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
-          rm -f "${KUBECONFIG}" || true
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -316,12 +316,12 @@ jobs:
           # locally when NODE_SSH_HOST is empty. Refuse to start rather than
           # let crictl run against the shared runner.
           test -n "${NODE_SSH_HOST}"
-          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          mkdir -p "${LOG_DIR}"
           ./tests/cases/nvidia-driver.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}" || true
+          mkdir -p "${LOG_DIR}"
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -339,5 +339,5 @@ jobs:
       - name: Remove credentials from the runner
         if: always()
         run: |
-          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
-          rm -f "${KUBECONFIG}" || true
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh
index f422c31adf..fa9af4c2cc 100644
--- a/tests/scripts/.definitions.sh
+++ b/tests/scripts/.definitions.sh
@@ -14,6 +14,7 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}"
 
 # Set default values if not defined
 : ${HELM:="helm"}
+: ${KUBECTL_REQUEST_TIMEOUT:="15s"}
 : ${LOG_DIR:="/tmp/logs"}
 : ${PROJECT:="$(basename "${PROJECT_DIR}")"}
 : ${TEST_NAMESPACE:="test-operator"}
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 073e0c5426..492a90b792 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -2,8 +2,7 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking $pod_label pod"
 		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
@@ -22,7 +21,7 @@ check_pod_ready() {
 			fi
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -37,15 +36,20 @@ check_pod_ready() {
 
 check_pod_deleted() {
 	local pod_label=$1
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
+	local deadline=$((SECONDS + 60 * 45))
+	local pod_list
 	while :; do
 		echo "Checking $pod_label pod"
 		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
 
 		echo "Checking if $pod_label pod has been deleted"
-		# note: $(kubectl get pods <options> -o jsonpath='.items' | jq length) does not work for older kubectl clients
-		num_pods=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o json | jq '.items' | jq length)
+		# Leave the count empty when the query itself fails, so that an
+		# unreachable API is never read as the pod having been deleted.
+		if pod_list=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} --no-headers); then
+			num_pods=$(echo "${pod_list}" | grep -c . || true)
+		else
+			num_pods=""
+		fi
 
 		if [ "${num_pods}" = 0 ]; then
 			echo "Pod $pod_label has been deleted"
@@ -54,7 +58,7 @@ check_pod_deleted() {
 			echo "Pod $pod_label has not been deleted"
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -109,67 +113,66 @@ test_restart_operator() {
 	exit 1
 }
 
-# Regenerate the describe and log files for every pod passed in. The log files
-# are rewritten in full rather than tailed, so that the artifact left behind
-# holds the complete output of every container.
+# Every pod in the cluster as "<namespace> <name>" lines. custom-columns keeps
+# this to a few hundred bytes; the equivalent -o json is hundreds of kilobytes.
+list_all_pods() {
+	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers || true
+}
+
+# Regenerate the describe and log files for the pods listed by list_all_pods.
+# The log files are rewritten in full rather than tailed, so that the artifact
+# left behind holds the complete output of every container.
 collect_pod_logs() {
 	local log_dir=$1
 	local pods=$2
+	local ns pod
 
-	for pod in $(echo "$pods" | jq -r .[].name); do
-		ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
+	while read -r ns pod; do
+		[[ -n "${pod}" ]] || continue
 		echo "Generating logs for pod: ${pod} ns: ${ns}"
 		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
 		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe" || true
 		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
-	done
+	done <<< "${pods}"
 }
 
 check_gpu_pod_ready() {
 	local log_dir=$1
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
-	local elapsed=0
+	local deadline=$((SECONDS + 60 * 45))
 	# Regenerating every pod's describe and log files is the expensive part of
 	# this loop, so it runs on its own slower cadence while the readiness check
-	# below keeps polling every 5 seconds. Track the time at which the next
-	# regeneration is due rather than testing the elapsed time for divisibility,
-	# which would skip regenerations when an iteration runs long.
+	# below keeps polling every 5 seconds.
 	local next_collection=0
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
 
 	while :; do
-		pods="$(kubectl get --all-namespaces pods -o json | jq '.items[] | {name: .metadata.name, ns: .metadata.namespace}' | jq -s -c .)"
-		status=$(kubectl get pods gpu-operator-test -o json | jq -r .status.phase)
+		status=$(kubectl get pods gpu-operator-test -o jsonpath='{.status.phase}' || true)
 		if [ "${status}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
 			rc=0
-			collect_pod_logs "${log_dir}" "${pods}"
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			break;
 		fi
 
-		elapsed=$((SECONDS - start_time))
-		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			# Collect once more so that the artifact reflects the state at the
 			# timeout rather than the state at the last scheduled collection.
-			collect_pod_logs "${log_dir}" "${pods}"
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			exit 1
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods --all-namespaces || true
-
-		if [[ "${elapsed}" -ge "${next_collection}" ]]; then
-			collect_pod_logs "${log_dir}" "${pods}"
-			next_collection=$((elapsed + 30))
-		fi
-
+		# Echo useful information on stdout and record it at the same time
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs" || true
+		kubectl get pods --all-namespaces | tee -a "${log_dir}/cluster.logs" || true
+
+		if (( SECONDS >= next_collection )); then
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
+			next_collection=$((SECONDS + 30))
+		fi
 
 		echo "Sleeping 5 seconds"
 		sleep 5;
@@ -178,8 +181,7 @@ check_gpu_pod_ready() {
 
 # TODO: deduplicate the logic found in this file by moving the duplicate to a common method and parameterizing the labels to select on
 check_nvidia_driver_pods_ready() {
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking nvidia driver pod"
 		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} || true
@@ -198,7 +200,7 @@ check_nvidia_driver_pods_ready() {
 			fi
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -236,25 +238,25 @@ api_unreachable() {
 # be the thing that ends the run.
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
-	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
 		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 
 	echo ""
 	echo "driver pods"
-	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
+	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "gpu operator operands"
-	kubectl get pods -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
+	kubectl get pods -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "driver daemonsets"
-	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
+	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "NVIDIADriver status"
 	local nvidiadriver_status
-	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" 2>/dev/null); then
+	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" 2>/dev/null); then
 		echo "${nvidiadriver_status}" | jq -r '
 			(["NAME", "DEFAULT", "STATE", "REASON", "MESSAGE"] | @tsv),
 			(
@@ -275,12 +277,10 @@ print_driver_upgrade_debug() {
 }
 
 wait_for_driver_upgrade_done() {
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
-	local elapsed=0
-	# Next elapsed time at which the full debug dump is due. Iterations can take
-	# much longer than the nominal sleep, so track a due time instead of testing
-	# the elapsed time for divisibility, which would skip dumps entirely.
+	local deadline=$((SECONDS + 60 * 45))
+	# Time at which the next full debug dump is due. Iterations can take much
+	# longer than the nominal sleep, so track a due time rather than testing the
+	# elapsed time for divisibility, which would skip dumps entirely.
 	local next_debug=0
 	local node_list=""
 	local upgraded_count=0
@@ -300,16 +300,16 @@ wait_for_driver_upgrade_done() {
 		# iteration would risk sampling a transient count while the upgrade churns
 		# node labels, which could make the comparison below succeed early.
 		if [[ -z "${gpu_node_count}" ]]; then
-			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 				gpu_node_count=$(echo "${node_list}" | grep -c . || true)
 			else
 				api_unreachable "counting the GPU nodes"
 			fi
 		fi
 
-		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 			for node in ${node_list}; do
-				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 					if [ "${upgrade_state}" = "upgrade-done" ]; then
 						upgraded_count=$((upgraded_count + 1))
 					fi
@@ -331,18 +331,17 @@ wait_for_driver_upgrade_done() {
 			echo "gpu driver still in progress. $upgraded_count/${gpu_node_count:-unknown} node(s) upgraded"
 		fi
 
-		elapsed=$((SECONDS - start_time))
-		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			print_driver_upgrade_debug
 			exit 1;
 		fi
 
-		if [[ "${elapsed}" -ge "${next_debug}" ]]; then
+		if (( SECONDS >= next_debug )); then
 			print_driver_upgrade_debug
-			next_debug=$((elapsed + 30))
+			next_debug=$((SECONDS + 30))
 		else
-			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
 				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 		fi
 		
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 02bbab6778..14be3937bc 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -23,7 +23,7 @@ get_helm_release_name() {
 }
 
 wait_for_legacy_driver_daemonset_deleted() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for ClusterPolicy-owned driver DaemonSet to be deleted"
     while :; do
@@ -32,20 +32,19 @@ wait_for_legacy_driver_daemonset_deleted() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
             kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_orphaned_legacy_driver_pod() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for legacy driver pod/${pod_name} to become orphaned"
     while :; do
@@ -55,19 +54,18 @@ wait_for_orphaned_legacy_driver_pod() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
             kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_default_nvidiadriver() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for default NVIDIADriver to be rendered"
     while :; do
@@ -76,20 +74,19 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for default NVIDIADriver"
             kubectl get nvidiadriver || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_owner_labels() {
     local driver_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
     local gpu_node_count
 
     gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
@@ -101,21 +98,20 @@ wait_for_nvidiadriver_owner_labels() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver owner labels"
             kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_daemonset() {
     local driver_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for NVIDIADriver-owned driver DaemonSet"
     while :; do
@@ -125,20 +121,19 @@ wait_for_nvidiadriver_daemonset() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
             kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_legacy_driver_pod_deleted() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for orphaned legacy driver pod/${pod_name} to be deleted by the upgrade flow"
     while :; do
@@ -146,7 +141,7 @@ wait_for_legacy_driver_pod_deleted() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for orphaned legacy driver pod deletion"
             print_driver_upgrade_debug
             kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
@@ -155,7 +150,6 @@ wait_for_legacy_driver_pod_deleted() {
 
         print_driver_upgrade_debug
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
index 9f4b351e4d..27b7a5cf1d 100755
--- a/tests/scripts/node-exec.sh
+++ b/tests/scripts/node-exec.sh
@@ -16,32 +16,23 @@
 
 set -euo pipefail
 
-# This script dispatches a host-mutating operation to the node hosting the
-# cluster. When NODE_SSH_HOST is set the operation is streamed to the node over
-# SSH; otherwise it is executed locally, which is the developer path where the
-# tests already run on the node itself.
+# Usage: node-exec.sh <operation> [args...]
+#
+# Runs one of the node-operations.sh operations on the node hosting the cluster.
+# When NODE_SSH_HOST is set the operation is streamed to the node over SSH;
+# otherwise it is executed locally, which is the developer path where the tests
+# already run on the node itself. The operation names and their arguments are
+# documented by node-operations.sh, which also reports them on bad input.
+#
+# Environment:
+#   NODE_SSH_HOST         user@host of the node, e.g. ubuntu@ec2-1-2-3-4.compute.amazonaws.com.
+#                         If unset or empty the operation is executed locally.
+#   NODE_SSH_KEY          Path to the private key. Required when NODE_SSH_HOST is set.
+#   NODE_SSH_KNOWN_HOSTS  Path to the known_hosts file. Required when NODE_SSH_HOST is set.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
 
-usage() {
-	cat <<'EOF'
-Usage: node-exec.sh <operation> [args...]
-
-Runs one of the node-operations.sh operations on the node hosting the cluster.
-
-Environment:
-  NODE_SSH_HOST         user@host of the node, e.g. ubuntu@ec2-1-2-3-4.compute.amazonaws.com.
-                        If unset or empty the operation is executed locally.
-  NODE_SSH_KEY          Path to the private key. Required when NODE_SSH_HOST is set.
-  NODE_SSH_KNOWN_HOSTS  Path to the known_hosts file. Required when NODE_SSH_HOST is set.
-
-Operations:
-  load-modules
-  restart-operator-container <runtime>
-EOF
-}
-
 require_readable_file() {
 	local name="${1}"
 	local value="${2}"
@@ -56,30 +47,19 @@ require_readable_file() {
 	fi
 }
 
-if [[ $# -lt 1 ]]; then
-	usage >&2
-	exit 2
-fi
-
-if [[ ! -r "${NODE_OPERATIONS}" ]]; then
-	echo "Error: ${NODE_OPERATIONS} does not exist or is not readable" >&2
-	exit 1
-fi
-
 if [[ -z "${NODE_SSH_HOST:-}" ]]; then
 	echo "Running '$*' locally"
-	bash "${NODE_OPERATIONS}" "$@"
-	exit $?
+	exec bash "${NODE_OPERATIONS}" "$@"
 fi
 
 require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
 require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
 
-# Quote each argument so that it survives the remote shell.
-REMOTE_COMMAND="bash -s --"
-for arg in "$@"; do
-	REMOTE_COMMAND+=" $(printf '%q' "${arg}")"
-done
+# Quote the arguments so that they survive the remote shell.
+REMOTE_ARGS=""
+if (( $# )); then
+	printf -v REMOTE_ARGS ' %q' "$@"
+fi
 
 echo "Running '$*' on ${NODE_SSH_HOST}"
 ssh -i "${NODE_SSH_KEY}" \
@@ -88,4 +68,4 @@ ssh -i "${NODE_SSH_KEY}" \
 	-o StrictHostKeyChecking=accept-new \
 	-o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
 	"${NODE_SSH_HOST}" \
-	"${REMOTE_COMMAND}" < "${NODE_OPERATIONS}"
+	"bash -s --${REMOTE_ARGS}" < "${NODE_OPERATIONS}"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
index fd46b1e80b..58c4e17c97 100755
--- a/tests/scripts/node-operations.sh
+++ b/tests/scripts/node-operations.sh
@@ -41,15 +41,15 @@ load_modules() {
 	sudo modprobe -a i2c_core ipmi_msghandler
 }
 
-# The x-prefixed comparisons and the container selection pipelines below are
-# kept as they were in tests/scripts/checks.sh so that the behaviour of the
-# restart test does not change.
-# shellcheck disable=SC2268
+# The container selection pipelines below are kept as they were in
+# tests/scripts/checks.sh so that the behaviour of the restart test does not
+# change.
 restart_operator_container() {
 	local runtime="${1:-}"
 	local container_id=""
 
-	if [[ x"${runtime}" == x"containerd" ]]; then
+	case "${runtime}" in
+	containerd)
 		# The operator is the only container that has the string '"gpu-operator"'
 		# TODO: This requires permissions on containerd.sock
 		container_id="$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')" || true
@@ -58,7 +58,8 @@ restart_operator_container() {
 			return 1
 		fi
 		sudo crictl rm --force "${container_id}"
-	elif [[ x"${runtime}" == x"docker" ]]; then
+		;;
+	docker)
 		# The operator is the only container that has the string '"gpu-operator"'
 		container_id="$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')" || true
 		if [[ -z "${container_id}" ]]; then
@@ -66,10 +67,12 @@ restart_operator_container() {
 			return 1
 		fi
 		docker kill "${container_id}"
-	else
+		;;
+	*)
 		echo "Error: unknown runtime '${runtime}'. Supported runtimes: containerd, docker" >&2
 		return 1
-	fi
+		;;
+	esac
 }
 
 main() {
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index 4682db5835..3816b09312 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -56,8 +56,7 @@ set_default_driver() {
 
 wait_for_default_nvidiadriver() {
     local expected_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${expected_name} to be the only default"
     while :; do
@@ -68,7 +67,7 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
             kubectl get nvidiadriver || true
             exit 1
@@ -120,8 +119,7 @@ create_nvidiadriver() {
 
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + $((60 * 15))))
     local gpu_node_count
 
     gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
@@ -134,10 +132,10 @@ wait_for_nvidiadriver_owner() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
             kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
@@ -154,8 +152,7 @@ get_nvidiadriver_daemonsets() {
 
 wait_for_nvidiadriver_daemonsets() {
     local driver_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + $((60 * 15))))
 
     echo "Waiting for daemonsets owned by NVIDIADriver/${driver_name}"
     while :; do
@@ -165,7 +162,7 @@ wait_for_nvidiadriver_daemonsets() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
             kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml || true
             exit 1
@@ -184,16 +181,15 @@ test_driver_image_updates() {
     fi
 
     # Verify update is applied to Driver Daemonset
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
     while :; do
         if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e --arg version "${TARGET_DRIVER_VERSION}" 'length > 0 and all(.[]; .spec.template.spec.containers[0].image | contains($version))' >/dev/null; then
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "Image update failed for driver daemonset to version $TARGET_DRIVER_VERSION"
-            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
             exit 1
         fi
 
@@ -223,16 +219,15 @@ test_custom_labels_override() {
 
   # Wait for the operator to update the pod template with new labels
   echo "Waiting for DaemonSet pod template to be updated with new labels..."
-  # SECONDS counts from shell start, so record a baseline and measure against it.
-  local start_time=${SECONDS}
+  local deadline=$((SECONDS + 120))
   while :; do
     if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e 'length > 0 and all(.[]; .spec.template.metadata.labels.cloudprovider == "aws" and .spec.template.metadata.labels.platform == "kubernetes")' >/dev/null; then
       break
     fi
 
-    if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+    if (( SECONDS > deadline )); then
       echo "timeout reached waiting for DaemonSet pod template labels"
-      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
       exit 1
     fi
 
@@ -268,7 +263,7 @@ assert_nvidiadriver_owner_count() {
     if [[ "${owned_count}" -ne "${gpu_node_count}" ]]; then
         echo "Expected ${gpu_node_count} GPU node(s) to remain owned by NVIDIADriver/${driver_name}, found ${owned_count}"
         kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
         exit 1
     fi
 }
@@ -276,8 +271,7 @@ assert_nvidiadriver_owner_count() {
 wait_for_nvidiadriver_condition_message() {
     local driver_name=$1
     local message=$2
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
@@ -288,7 +282,7 @@ wait_for_nvidiadriver_condition_message() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
             kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1
@@ -300,8 +294,7 @@ wait_for_nvidiadriver_condition_message() {
 
 wait_for_nvidiadriver_ready() {
     local driver_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
@@ -313,7 +306,7 @@ wait_for_nvidiadriver_ready() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
             kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1