From 7ec12f18b33f3ab2db3c29511bf82bb799720723 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Wed, 16 Sep 2026 14:06:32 +0800
Subject: [PATCH 01/19] [TRTLLMINF-381][fix] classify CI failures from
 originating step logs

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/Build.groovy   |   4 +-
 jenkins/L0_Test.groovy | 215 ++++++++++++++++++-----------------------
 2 files changed, 96 insertions(+), 123 deletions(-)

diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy
index cccabe235ccd..d9c32e7671a3 100644
--- a/jenkins/Build.groovy
+++ b/jenkins/Build.groovy
@@ -18,6 +18,7 @@
 
 import java.lang.InterruptedException
 import groovy.transform.Field
+import com.nvidia.bloom.FailureEvidenceCollector
 import trtllm.FailureClassifier
 import trtllm.exceptions.InfraFailure
 
@@ -739,7 +740,8 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars)
         // ENABLE_INFRA_SCOPED_FAILFAST; off = plain failFast + parallel, as before.
         trtllm_utils.runBranchesWithInfraDefer(pipeline, parallelJobs, enableFailFast,
                 ENABLE_INFRA_SCOPED_FAILFAST) { e, stageName ->
-            FailureClassifier.isDeferrableInfra(e, InfraFailure.K8S)
+            def evidence = FailureEvidenceCollector.collectOriginatingStepEvidence(pipeline, e, FailureClassifier.failureEvidenceQueries(InfraFailure.K8S))
+            FailureClassifier.isDeferrableInfra(e, InfraFailure.K8S, evidence)
         }
     } // Build stage
 }
diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index cd2571fcbee8..fb520ef84f8a 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -27,6 +27,7 @@ import com.nvidia.bloom.SlurmCluster
 import com.nvidia.bloom.SlurmPartition
 import com.nvidia.bloom.Utils
 import com.nvidia.bloom.ContainerRuntime
+import com.nvidia.bloom.FailureEvidenceCollector
 import com.nvidia.bloom.SshAuthMethod
 import org.jenkinsci.plugins.workflow.cps.CpsThread
 import org.jsoup.Jsoup
@@ -357,43 +358,6 @@ def scrapeSlurmLogForDeviceFault(def pipeline, Map remote, String remoteLogPath)
     }
 }
 
-// Read back the sbatch output the submit script captured to sbatch_output.txt
-// in the job workspace. A failed sh step surfaces only the exit code ("script
-// returned exit code 1"), so sbatch's stderr -- e.g. "Slurm backup controller
-// in standby mode" during a slurmctld failover -- never reaches
-// FailureClassifier.classify(), which matches the exception chain only. Like
-// scrapeSlurmLogForDeviceFault above, this is a GATE only: the caller folds
-// the returned text into a fresh exception and the shared-lib PATTERN_CATALOG
-// (the authoritative list) makes the real retry/severity decision. Returns ""
-// when the file is missing or unreadable, so the caller can rethrow the
-// original exception unchanged.
-def readSlurmSubmitOutput(def pipeline, Map remote, String jobWorkspace, String stageName) {
-    def outputPath = "${jobWorkspace}/sbatch_output.txt"
-    try {
-        return Utils.exec(
-            pipeline,
-            script: Utils.sshUserCmd(remote,
-                "\"bash -c 'if [ -f \\\"${outputPath}\\\" ]; then head -c 1000 -- \\\"${outputPath}\\\"; fi'\""),
-            returnStdout: true,
-            numRetries: 1,
-        )?.trim()
-    } catch (InterruptedException e) {
-        throw e
-    } catch (Exception readEx) {
-        // A dead frontend must propagate so the enclosing withSlurmFrontendFailover
-        // fails over to another remote; swallowing it as "" would return an empty
-        // read to the caller, which then rethrows the original (generic) submit
-        // exception and strands the stage on the unreachable frontend. Any other
-        // read failure (missing file, transient) is non-fatal -- the caller
-        // rethrows the original exception unchanged.
-        if (CloudManager.isSlurmFrontendConnectionFailure(readEx)) {
-            throw readEx
-        }
-        pipeline.echo("Ignorable warning: could not read ${outputPath} on ${remote.host}: ${readEx.message}")
-        return ""
-    }
-}
-
 // `postTag` uniquifies the uploaded tar filename, the Artifactory guard key and
 // the locally-staged result XMLs when the same stageName is uploaded more than
 // once in a build (e.g. SLURM infra-failure retries). First attempt passes "".
@@ -1104,6 +1068,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
 
     def slurmJobID = null
     def dockerArgs = null
+    Throwable primaryError = null
 
     // The in-flight stage failure (test/bring-up/interrupt), if any. Recorded so the
     // cleanup finally below can tell "cleanup is the only failure" (throw a retryable
@@ -1522,7 +1487,14 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
             pendingStageFailure = classified
             throw classified
         }
+    } catch (InterruptedException e) {
+        primaryError = e
+        throw e
+    } catch (Exception e) {
+        primaryError = e
+        throw e
     } finally {
+        preservePrimaryFailure(primaryError) {
         // Resource cleanup must run even if SLURM metadata capture is interrupted.
         try {
             captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, slurmJobID, placementContext, stageName)
@@ -1569,6 +1541,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
                 deregisterSlurmResource(stageName)
             }
         }
+        }
     }
 }
 
@@ -1810,6 +1783,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
     // a retryable infra failure (a retry follows) -- otherwise a stage that fails
     // an intermediate attempt and passes on retry leaves the build UNSTABLE.
     def caughtStageError = null
+    Throwable primaryError = null
 
     try {
         // Run ssh command to start node in desired cluster via SLURM
@@ -2336,14 +2310,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     find "${jobWorkspace}" -maxdepth 1 -mindepth 1 ${findKeepWhenRetryArgs} -exec rm -rf {} +
 
                     touch ${slurmJobLogPath}
-                    # Capture sbatch's combined output and persist it before acting on
-                    # the exit code: a failed sh step carries only the exit code back
-                    # to Jenkins, so a submission rejection's stderr (e.g. "Slurm
-                    # backup controller in standby mode") would otherwise never reach
-                    # the failure classifier. The pipeline reads sbatch_output.txt
-                    # back on failure (readSlurmSubmitOutput) and folds it into the
-                    # exception it throws.
-                    #
                     # Control-plane failover backoff: during a slurmctld failover the
                     # backup controller rejects submissions with "Slurm backup
                     # controller in standby mode" until it promotes, which takes up to
@@ -2364,7 +2330,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                         sbatch_rc=0
                         sbatch_output=\$(sbatch ${scriptLaunchPathNode} 2>&1) || sbatch_rc=\$?
                         printf '%s\\n' "\${sbatch_output}"
-                        printf '%s\\n' "\${sbatch_output}" > "${jobWorkspace}/sbatch_output.txt"
                         if [ "\${sbatch_rc}" -eq 0 ]; then
                             break
                         fi
@@ -2404,39 +2369,12 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 // share the job workspace (slurm_job_id.txt, scripts) on that login
                 // node; a frontend disconnect fails the whole closure over to a fresh
                 // frontend as a unit (the submit script reuses an active job).
-                try {
-                    Utils.exec(
-                        pipeline,
-                        timeout: false,
-                        script: Utils.sshUserCmd(remote, scriptSubmitPathNode),
-                        numRetries: 3
-                    )
-                } catch (InterruptedException e) {
-                    throw e
-                } catch (Exception submitEx) {
-                    // Only enrich failures the classifier cannot already act on. An
-                    // interrupt (user abort / pipeline timeout) or an exception that
-                    // already classifies as infra must propagate unchanged --
-                    // wrapping them in a fresh cause-less Exception would erase the
-                    // FlowInterruptedException / typed-infra signal from the chain.
-                    def preClassified = FailureClassifier.classify(submitEx, InfraFailure.SLURM)
-                    if (preClassified instanceof PipelineInterruption || preClassified instanceof InfraFailure) {
-                        throw submitEx
-                    }
-                    // The sh step's exception carries only the exit code, so fold the
-                    // sbatch output captured by the submit script (stderr included)
-                    // into a fresh exception for FailureClassifier.classify() at the
-                    // runLLMTestlistOnSlurm caller; the shared-lib PATTERN_CATALOG
-                    // (e.g. its "Slurm backup controller in standby mode" row) then
-                    // drives the retry/severity decision. An empty read (file
-                    // missing or unreadable) preserves today's behavior by
-                    // rethrowing the original exception unchanged.
-                    def sbatchOutput = readSlurmSubmitOutput(pipeline, remote, jobWorkspace, stageName)
-                    if (sbatchOutput) {
-                        throw new Exception("SLURM job submission failed for ${stageName}: ${sbatchOutput}")
-                    }
-                    throw submitEx
-                }
+                Utils.exec(
+                    pipeline,
+                    timeout: false,
+                    script: Utils.sshUserCmd(remote, scriptSubmitPathNode),
+                    numRetries: 3
+                )
 
                 def slurmMetadata = captureSlurmWorkspaceMetadata(pipeline, remote, jobWorkspace, placementContext, stageName)
                 def slurmJobId = slurmMetadata.slurmJobId
@@ -2743,12 +2681,15 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             }  // end CloudManager.withSlurmFrontendFailover
         }  // end withCredentials
     } catch (InterruptedException e) {
+        primaryError = e
         stageIsInterrupted = true
         throw e
     } catch (Exception e) {
+        primaryError = e
         caughtStageError = e
         throw e
     } finally {
+        preservePrimaryFailure(primaryError) {
         // Resource cleanup must run even if metadata capture or result upload is interrupted.
         try {
             captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, placementContext?.slurmJobId ?: null, placementContext, stageName, jobWorkspace)
@@ -2757,7 +2698,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             // the build UNSTABLE from an intermediate attempt's results. A genuine test
             // failure classifies as UserFailure -> not suppressed -> reported.
             boolean suppressTestReporting = (caughtStageError != null && retryContext != null) &&
-                retryContextAllowsRetry(null, retryContext, caughtStageError, false)
+                retryContextAllowsRetry(pipeline, retryContext, caughtStageError, false)
             uploadResults(pipeline, cluster, partition.clusterName, jobUID, stageName, postTag, suppressTestReporting)
             deleteProgressArtifact(stageName, postTag)
         } finally {
@@ -2806,6 +2747,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 deregisterSlurmResource(stageName)
             }
         }
+        }
     }
 }
 
@@ -2926,6 +2868,16 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
     Map attemptPlacementContext = [
       excludedSlurmNodeListsByCluster: avoidedSlurmNodeListsByCluster.collectEntries { c, ns -> [(c): ns.collect()] }
     ]
+    // Describes this attempt so the stage body can suppress its junit when a
+    // retryable infra failure means another attempt will follow (mirrors the K8s
+    // path's retryContext). scope=SLURM so classification/budget match this loop.
+    // Shared by result suppression and the retry decision for this attempt only.
+    Map slurmRetryContext = [
+      scope: InfraFailure.SLURM,
+      stageName: stageName,
+      attempt: attempt,
+      backoffMs: 60L * 1000L,
+    ]
     try {
       if (attempt > 1) {
         echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${slurmInfraRetryMax + 1}"
@@ -2949,16 +2901,6 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
       def innerSuffix = (attempt == 1) ? "" : "-attempt-${attempt}"
       def postTag = "${outerAttemptTag}${innerSuffix}"
 
-      // Describes this attempt so the stage body can suppress its junit when a
-      // retryable infra failure means another attempt will follow (mirrors the K8s
-      // path's retryContext). scope=SLURM so classification/budget match this loop.
-      def slurmRetryContext = [
-        scope: InfraFailure.SLURM,
-        stageName: stageName,
-        attempt: attempt,
-        backoffMs: 60L * 1000L,
-      ]
-
       if (nodeCount > 1 || runWithSbatch) {
         runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext)
       } else {
@@ -2984,11 +2926,11 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
         echo "[INFRA-RETRY] ${stageName}: dispatcher pod died mid-run; not retrying on the dead pod (${e.toString()}). Failing closed for off-pod reconciliation."
         throw e
       }
-      // classify() handles FlowInterruptedException + exit-code-143 +
-      // typed throws + cause-chain pattern matching, returning one of
+      // classifyFailure() handles FlowInterruptedException + exit-code-143 +
+      // typed throws + originating-step evidence and cause-chain fallback, returning one of
       // PipelineInterruption / InfraFailure / UserFailure. Scope=SLURM
       // ensures we only match catalog rows tagged SLURM or BOTH.
-      def c = FailureClassifier.classify(e, InfraFailure.SLURM)
+      def c = classifyFailure(pipeline, e, InfraFailure.SLURM, slurmRetryContext)
       if (c instanceof PipelineInterruption) throw e
       if (!(c instanceof InfraFailure)) {
         // UserFailure -> don't retry, but leave a trace: a failure whose
@@ -3005,12 +2947,12 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
       if (attempt > effectiveMax) {
         echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) " +
              "but max retries (${effectiveMax}) exhausted after ${attempt} attempts. Failing."
-        throw e
+        throw c
       }
       if (!hasBudgetForInfraRetry(pipeline, stageName, InfraFailure.SLURM, c, attempt, effectiveMax, 60L * 1000L, true)) {
         echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) is retryable, " +
              "but remaining CI timeout budget is too small for another SLURM attempt. Failing without retry."
-        throw e
+        throw c
       }
 
       echo "[INFRA-RETRY] ${stageName}: Infrastructure failure detected on attempt ${attempt}: " +
@@ -3252,13 +3194,38 @@ boolean hasBudgetForInfraRetry(def pipeline, String stageName, String scope, Inf
     ])
 }
 
+def classifyFailure(def pipeline, Throwable error, String scope, Map retryContext=null)
+{
+    if (error instanceof TrtllmCiException) return FailureClassifier.classify(error, scope)
+
+    Map evidence = retryContext?.failureEvidence
+    if (evidence == null) {
+        evidence = FailureEvidenceCollector.collectOriginatingStepEvidence(pipeline, error, FailureClassifier.failureEvidenceQueries(scope))
+        if (retryContext != null) retryContext.failureEvidence = evidence
+    }
+    return FailureClassifier.classify(error, scope, evidence)
+}
+
+def preservePrimaryFailure(Throwable primaryError, Closure actions)
+{
+    try {
+        actions()
+    } catch (InterruptedException e) {
+        throw e
+    } catch (Exception e) {
+        // Post-processing failures must not replace the failure that triggered them.
+        if (primaryError == null || FailureClassifier.classify(e, InfraFailure.BOTH) instanceof PipelineInterruption) throw e
+        echo "[INFRA-RETRY] Secondary failure while handling ${primaryError}: ${e}"
+    }
+}
+
 boolean retryContextAllowsRetry(def pipeline, Map retryContext, Throwable error, boolean logDecision=false)
 {
     if (retryContext == null || error == null) {
         return false
     }
     String scope = retryContext.scope ?: InfraFailure.K8S
-    def classified = FailureClassifier.classify(error, scope)
+    def classified = classifyFailure(pipeline, error, scope, retryContext)
     if (!(classified instanceof InfraFailure)) {
         return false
     }
@@ -3328,7 +3295,7 @@ def readSlurmWorkspaceFile(def pipeline, Map remote, String path, String stageNa
         // fails over to another remote; swallowing it as "" would strand the stage on
         // the unreachable frontend. Any other read failure (missing file, transient)
         // is non-fatal -- the metadata is best-effort, so return "" and carry on.
-        if (CloudManager.isSlurmFrontendConnectionFailure(e)) {
+        if (CloudManager.isSlurmFrontendConnectionFailure(pipeline, e)) {
             throw e
         }
         echo "[INFRA-RETRY] ${stageName}: unable to read SLURM metadata file ${path}: ${e.toString()}"
@@ -3499,12 +3466,14 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
         taskRunner()
         stageIsFailed = false
     } catch (InterruptedException e) {
+        caughtError = e
         stageIsInterrupted = true
         throw e
     } catch (Exception e) {
         caughtError = e
         throw e
     } finally {
+        preservePrimaryFailure(caughtError) {
         ensureStageResultNotUploaded(stageName + postTag)
         if (stageIsInterrupted) {
             echo "Stage is interrupted, skip to upload test result."
@@ -3532,9 +3501,9 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
             boolean suppressTestReporting = false
             if (stageIsFailed && caughtError != null) {
                 if (retryContext != null) {
-                    suppressTestReporting = retryContextAllowsRetry(null, retryContext, caughtError, false)
+                    suppressTestReporting = retryContextAllowsRetry(pipeline, retryContext, caughtError, false)
                 } else if (!isFinalAttempt) {
-                    def c = FailureClassifier.classify(caughtError, InfraFailure.K8S)
+                    def c = classifyFailure(pipeline, caughtError, InfraFailure.K8S)
                     suppressTestReporting = c instanceof InfraFailure
                 }
                 if (suppressTestReporting) {
@@ -3606,6 +3575,7 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
         """
 
         echo "Finished test stage execution."
+        }
     }
 }
 
@@ -6076,7 +6046,7 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN
                     }
                     throw e
                 }
-                def c = FailureClassifier.classify(e, InfraFailure.K8S)
+                def c = classifyFailure(pipeline, e, InfraFailure.K8S)
                 if (c instanceof PipelineInterruption) throw e
                 if (!(c instanceof InfraFailure)) {
                     // UserFailure -> don't retry, but leave a trace: a pod/agent
@@ -6092,12 +6062,12 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN
                 if (launchAttempt > K8S_INFRA_RETRY_MAX) {
                     echo "[INFRA-RETRY] ${stageName}: pod launch failed before execution (${c.detectedPattern}), " +
                          "but max launch retries (${K8S_INFRA_RETRY_MAX}) exhausted after ${launchAttempt} attempts. Failing."
-                    throw e
+                    throw c
                 }
                 if (!hasBudgetForInfraRetry(pipeline, stageName, InfraFailure.K8S, c, launchAttempt, K8S_INFRA_RETRY_MAX, 60L * 1000L, true)) {
                     echo "[INFRA-RETRY] ${stageName}: pod launch failed (${c.detectedPattern}) is retryable, " +
                          "but remaining CI timeout budget is too small for a relaunch. Failing without retry."
-                    throw e
+                    throw c
                 }
 
                 echo "[INFRA-RETRY] ${stageName}: pod/agent launch failed before execution on attempt ${launchAttempt}: ${c.detectedPattern}"
@@ -6118,6 +6088,15 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN
     while (true) {
         attempt++
         Map attemptPlacementContext = [:]
+        // Shared by result suppression and the retry decision for this attempt only.
+        Map retryContext = [
+            scope: InfraFailure.K8S,
+            stageName: stageName,
+            attempt: attempt,
+            backoffMs: 60L * 1000L,
+            infraRetryMax: opts.infraRetryMax,
+            excludedKubernetesHostNodes: avoidedKubernetesHostNodes.collect(),
+        ]
         try {
             if (attempt > 1) {
                 echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${k8sInfraRetryMax + 1}"
@@ -6143,14 +6122,6 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN
             // another intermediate attempt.
             def effectiveMaxThisAttempt = (lastSeverity == InfraFailure.PERSISTENT) ? Math.min(1, k8sInfraRetryMax) : k8sInfraRetryMax
             boolean isFinalAttempt = (attempt > effectiveMaxThisAttempt)
-            def retryContext = [
-                scope: InfraFailure.K8S,
-                stageName: stageName,
-                attempt: attempt,
-                backoffMs: 60L * 1000L,
-                infraRetryMax: opts.infraRetryMax,
-                excludedKubernetesHostNodes: avoidedKubernetesHostNodes.collect(),
-            ]
             def attemptPodSpec = trtllm_utils.withKubernetesHostNodeExclusion(podSpec, avoidedKubernetesHostNodes)
             trtllm_utils.launchKubernetesPodWithPlacement(pipeline, attemptPodSpec, containerName, attemptPlacementContext, { runner(attemptTag, isFinalAttempt, retryContext) })
             if (attempt > 1) {
@@ -6161,14 +6132,14 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN
             // User abort / pipeline timeout -- never retry
             throw e
         } catch (Exception e) {
-            // classify() handles FlowInterruptedException + exit-code-143 +
-            // typed throws + cause-chain pattern matching, returning one of
+            // classifyFailure() handles FlowInterruptedException + exit-code-143 +
+            // typed throws + originating-step evidence and cause-chain fallback, returning one of
             // PipelineInterruption / InfraFailure / UserFailure. Scope=K8S
             // ensures we only match catalog rows tagged K8S or BOTH; the new
             // scope-isolation guard inside classify() also prevents a typed
             // SLURM-scoped InfraFailure (e.g. from an inner SLURM retry that
             // exhausted its own budget) from being treated as K8s infra here.
-            def c = FailureClassifier.classify(e, InfraFailure.K8S)
+            def c = classifyFailure(pipeline, e, InfraFailure.K8S, retryContext)
             if (c instanceof PipelineInterruption) throw e
             if (!(c instanceof InfraFailure)) {
                 // UserFailure -> don't retry, but leave a trace so the decline
@@ -6184,12 +6155,12 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN
             if (attempt > effectiveMax) {
                 echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) " +
                      "but max retries (${effectiveMax}) exhausted after ${attempt} attempts. Failing."
-                throw e
+                throw c
             }
             if (!hasBudgetForInfraRetry(pipeline, stageName, InfraFailure.K8S, c, attempt, effectiveMax, 60L * 1000L, true)) {
                 echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) is retryable, " +
                      "but remaining CI timeout budget is too small for another K8s attempt. Failing without retry."
-                throw e
+                throw c
             }
 
             echo "[INFRA-RETRY] ${stageName}: Infrastructure failure detected on attempt ${attempt}: " +
@@ -6223,11 +6194,11 @@ def buildStageConfigs(stageName, platform, testlist, testCount, gpuCount, nodeCo
 // via failFast. The inner SLURM retry (runLLMTestlistOnSlurm) is already
 // exhausted by the time the branch body returns, so this stays post-retry,
 // mirroring the K8s path. Stages absent from stageScopes are K8S-only.
-def infraDeferPredicate(Map stageScopes) {
+def infraDeferPredicate(def pipeline, Map stageScopes) {
     return { e, stageName ->
         boolean slurmScoped = (stageScopes[stageName] == InfraFailure.SLURM)
-        FailureClassifier.isDeferrableInfra(e, InfraFailure.K8S) ||
-            (slurmScoped && FailureClassifier.isDeferrableInfra(e, InfraFailure.SLURM))
+        classifyFailure(pipeline, e, InfraFailure.K8S) instanceof InfraFailure ||
+            (slurmScoped && classifyFailure(pipeline, e, InfraFailure.SLURM) instanceof InfraFailure)
     }
 }
 
@@ -7428,7 +7399,7 @@ pipeline {
                             }
                             if (singleGpuJobs.size() > 0) {
                                 trtllm_utils.runBranchesWithInfraDefer(this, singleGpuJobs, params.enableFailFast,
-                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(stageInfraScope))
+                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(this, stageInfraScope))
                             } else if (isInfraDryRun()) {
                                 error "Skip single-GPU testing. No test to run for infrastructure dry run."
                             } else {
@@ -7438,14 +7409,14 @@ pipeline {
                             echo "Only run multi-GPU tests."
                             if (dgxJobs.size() > 0) {
                                 trtllm_utils.runBranchesWithInfraDefer(this, dgxJobs, params.enableFailFast,
-                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(stageInfraScope))
+                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(this, stageInfraScope))
                             } else {
                                 error "Skip multi-GPU testing. No test to run."
                             }
                         } else {
                             if (singleGpuJobs.size() > 0) {
                                 trtllm_utils.runBranchesWithInfraDefer(this, singleGpuJobs, params.enableFailFast,
-                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(stageInfraScope))
+                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(this, stageInfraScope))
                             } else {
                                 echo "Skip single-GPU testing. No test to run."
                             }
@@ -7453,7 +7424,7 @@ pipeline {
                             if (dgxJobs.size() > 0) {
                                 stage(testPhase2StageName) {
                                     trtllm_utils.runBranchesWithInfraDefer(this, dgxJobs, params.enableFailFast,
-                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(stageInfraScope))
+                                    ENABLE_INFRA_SCOPED_FAILFAST, infraDeferPredicate(this, stageInfraScope))
                                 }
                             }
                         }

From 3f43f660f834fcf739fa4faa12d391cd566dc080 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Wed, 16 Sep 2026 17:15:15 +0800
Subject: [PATCH 02/19] fix: preserve SLURM retry and cleanup semantics

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 104 ++++++++++-------------------------------
 1 file changed, 25 insertions(+), 79 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index fb520ef84f8a..2fd4e991cb26 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1070,12 +1070,6 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
     def dockerArgs = null
     Throwable primaryError = null
 
-    // The in-flight stage failure (test/bring-up/interrupt), if any. Recorded so the
-    // cleanup finally below can tell "cleanup is the only failure" (throw a retryable
-    // typed InfraFailure) from "a stage failure is already pending" (preserve it, since
-    // a throw from the finally would supersede the real failure per JVM semantics).
-    Throwable pendingStageFailure = null
-
     try {
         // Run ssh command to start node in desired cluster via SLURM
         CloudManager.withSlurmFrontendFailover(pipeline, partition.clusterName, cluster) { remote ->
@@ -1475,7 +1469,6 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
         try {
             executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, slurmRunner, postTag, useClusterDurations, retryContext, classifySlurmFailure)
         } catch (InterruptedException e) {
-            pendingStageFailure = e
             throw e
         } catch (Exception e) {
             // A test-execution failure was already labeled inside the task runner, so
@@ -1484,7 +1477,6 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
             // (image pull, node/agent bring-up, etc.) were never labeled, so classify
             // them here on the agent.
             Throwable classified = (slurmFailureClassified || e instanceof TrtllmCiException ? e : classifySlurmFailure(e))
-            pendingStageFailure = classified
             throw classified
         }
     } catch (InterruptedException e) {
@@ -1499,47 +1491,26 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
         try {
             captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, slurmJobID, placementContext, stageName)
         } finally {
-            boolean cleanupFailed = false
             stage("Clean Up Slurm Resource") {
-                // Workaround to handle the interruption during clean up SLURM resources
+                // Retry transient cleanup failures before leaving recovery to the sweep.
                 retry(3) {
                     try {
                         cleanUpNodeResources(pipeline, cluster, partition.clusterName, nodeName, slurmJobID)
+                    } catch (InterruptedException e) {
+                        throw e
                     } catch (Exception e) {
-                        if (pendingStageFailure != null) {
-                            // A stage failure (test/bring-up/interrupt) is already in flight.
-                            // Throwing the cleanup error from this finally would supersede it
-                            // (JVM finally semantics), dropping the real failure and letting
-                            // runLLMTestlistOnSlurm reclassify the stage as retryable SLURM
-                            // infra -- masking a genuine test failure as an infra retry. Swallow
-                            // the cleanup error (keep it in the console for diagnostics) so the
-                            // pending failure propagates unchanged.
-                            cleanupFailed = true
-                            echo "Clean Up SLURM resources failed while a stage failure was already " +
-                                 "pending; preserving the pending failure and leaving the resource " +
-                                 "ledger entry for the post-build sweep. Cleanup error: ${e.toString()}"
-                        } else {
-                            // Cleanup is the only failure. Throw it typed (chaining the cause)
-                            // instead of error(...): error(...) flattens the exception to a bare
-                            // string, dropping the cause chain and the ssh transport signature,
-                            // which is how a cleanup ssh drop (exit 255) reaches FailureClassifier
-                            // as an unmatched UserFailure and fail-fast-kills sibling stages. A
-                            // typed throw still de-interrupts into a retryable throw for the
-                            // retry(3) workaround, and keeps any pod-death signal in the cause
-                            // chain for isDispatcherPodFailure() upstream.
-                            throw new InfraFailure(
-                                "Error during clean up SLURM resources: ${e.getMessage()} and retrying.",
-                                e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "<typed:slurm-cleanup-failure>")
-                        }
+                        // retry(3) retries any exception. Keep the cleanup failure typed
+                        // so its cause and scope survive if all attempts fail; the outer
+                        // preservePrimaryFailure then retains any original stage failure.
+                        throw new InfraFailure(
+                            "Error during clean up SLURM resources: ${e.getMessage()} and retrying.",
+                            e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "<typed:slurm-cleanup-failure>")
                     }
                 }
             }
-            // Cleanup ran on the live pod; drop the registry entry so the off-pod
-            // finalizer/sweep does not reconcile already-freed resources. If cleanup was
-            // skipped under a pending stage failure, leave the entry live for the sweep.
-            if (!cleanupFailed) {
-                deregisterSlurmResource(stageName)
-            }
+            // Reaching here means cleanup succeeded within its retry budget. If all
+            // attempts fail, the throw skips deregistration and leaves the entry for sweep.
+            deregisterSlurmResource(stageName)
         }
         }
     }
@@ -1778,7 +1749,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
 
     Utils.exec(pipeline, script: "env | sort && pwd && ls -alh")
 
-    def stageIsInterrupted = false
     // Captured so the finally can suppress this attempt's junit when the failure is
     // a retryable infra failure (a retry follows) -- otherwise a stage that fails
     // an intermediate attempt and passes on retry leaves the build UNSTABLE.
@@ -2682,7 +2652,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
         }  // end withCredentials
     } catch (InterruptedException e) {
         primaryError = e
-        stageIsInterrupted = true
         throw e
     } catch (Exception e) {
         primaryError = e
@@ -2702,50 +2671,26 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             uploadResults(pipeline, cluster, partition.clusterName, jobUID, stageName, postTag, suppressTestReporting)
             deleteProgressArtifact(stageName, postTag)
         } finally {
-            // A stage failure (test/bring-up) or interruption already in flight; recorded
-            // above as caughtStageError / stageIsInterrupted. If either is set, a throw from
-            // this cleanup finally would supersede it.
-            boolean stageFailurePending = (caughtStageError != null || stageIsInterrupted)
-            boolean cleanupFailed = false
             stage("Clean Up Slurm Resource") {
-                // Workaround to handle the interruption during clean up SLURM resources
+                // Retry transient cleanup failures before leaving recovery to the sweep.
                 retry(3) {
                     try {
                         cleanUpSlurmResources(pipeline, cluster, partition.clusterName, jobUID)
+                    } catch (InterruptedException e) {
+                        throw e
                     } catch (Exception e) {
-                        if (stageFailurePending) {
-                            // Throwing the cleanup error from this finally would supersede the
-                            // pending failure (JVM finally semantics), dropping the real failure
-                            // and letting runLLMTestlistOnSlurm reclassify the stage as retryable
-                            // SLURM infra -- masking a genuine test failure (or an interrupt) as
-                            // an infra retry. Swallow the cleanup error (keep it in the console
-                            // for diagnostics) so the pending failure propagates unchanged.
-                            cleanupFailed = true
-                            echo "Clean Up SLURM resources failed while a stage failure was already " +
-                                 "pending; preserving the pending failure and leaving the resource " +
-                                 "ledger entry for the post-build sweep. Cleanup error: ${e.toString()}"
-                        } else {
-                            // Cleanup is the only failure. Throw it typed (chaining the cause)
-                            // instead of error(...): error(...) flattens the exception to a bare
-                            // string, dropping the cause chain and the ssh transport signature,
-                            // which is how a cleanup ssh drop (exit 255) reaches FailureClassifier
-                            // as an unmatched UserFailure and fail-fast-kills sibling stages. A
-                            // typed throw still de-interrupts into a retryable throw for the
-                            // retry(3) workaround, and keeps any pod-death signal in the cause
-                            // chain for isDispatcherPodFailure() upstream.
-                            throw new InfraFailure(
-                                "Error during clean up SLURM resources: ${e.getMessage()} and retrying.",
-                                e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "<typed:slurm-cleanup-failure>")
-                        }
+                        // retry(3) retries any exception. Keep the cleanup failure typed
+                        // so its cause and scope survive if all attempts fail; the outer
+                        // preservePrimaryFailure then retains any original stage failure.
+                        throw new InfraFailure(
+                            "Error during clean up SLURM resources: ${e.getMessage()} and retrying.",
+                            e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "<typed:slurm-cleanup-failure>")
                     }
                 }
             }
-            // Cleanup ran on the live pod; drop the registry entry so the off-pod
-            // finalizer/sweep does not reconcile already-freed resources. If cleanup was
-            // skipped under a pending stage failure, leave the entry live for the sweep.
-            if (!cleanupFailed) {
-                deregisterSlurmResource(stageName)
-            }
+            // Reaching here means cleanup succeeded within its retry budget. If all
+            // attempts fail, the throw skips deregistration and leaves the entry for sweep.
+            deregisterSlurmResource(stageName)
         }
         }
     }
@@ -2877,6 +2822,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
       stageName: stageName,
       attempt: attempt,
       backoffMs: 60L * 1000L,
+      infraRetryMax: slurmInfraRetryMax,
     ]
     try {
       if (attempt > 1) {

From 075196d0ad6a0e34ea7c0848e97eab52011aa3fd Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Wed, 16 Sep 2026 18:04:43 +0800
Subject: [PATCH 03/19] [TRTLLMINF-381] fix failure evidence follow-ups

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                        | 75 ++++++++++++-------
 .../tools/test_infra_dry_run_pipeline.py      |  2 +-
 2 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 2fd4e991cb26..d50f44c79319 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1520,7 +1520,7 @@ def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
 {
     runner {
         // TODO: refactor the finallyRunner to reuse within slurm or nonslurm job.
-        cacheErrorAndUploadResult(stageName, {
+        cacheErrorAndUploadResult(pipeline, stageName, {
             try {
                 runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations)
             } catch (InterruptedException e) {
@@ -1785,6 +1785,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             def scriptLaunchPathNode = "${jobWorkspace}/${jobUID}-slurm_launch.sh"
             def scriptSubmitPathLocal = Utils.createTempLocation(pipeline, "./slurm_submit.sh")
             def scriptSubmitPathNode = "${jobWorkspace}/${jobUID}-slurm_submit.sh"
+            def sbatchFailureMarker = "[TRTLLM_SBATCH_FINAL_FAILURE]"
             def scriptTrackPathLocal = Utils.createTempLocation(pipeline, "./slurm_track.sh")
             def scriptTrackPathNode = "${jobWorkspace}/${jobUID}-slurm_track.sh"
             def s3SecretKeyPathLocal = Utils.createTempLocation(pipeline, "./s3_secret_key")
@@ -2294,6 +2295,16 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     # "standby mode" PERSISTENT so the stage does not keep re-running
                     # against a down controller. Any other sbatch failure still fails
                     # immediately.
+                    report_sbatch_failure() {
+                        local reason="\$1"
+                        local summary
+                        summary=\$(printf '%s' "\${sbatch_output}" | tr '\\r\\n' '  ')
+                        summary="\${summary:0:3000}"
+                        set +x
+                        printf '${sbatchFailureMarker} %s: %s\\n' "\${reason}" "\${summary}"
+                        set -x
+                    }
+
                     sbatch_max_attempts=6
                     sbatch_attempt=1
                     while true; do
@@ -2309,11 +2320,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                             sleep 30
                             continue
                         fi
+                        report_sbatch_failure "exit code \${sbatch_rc}"
                         echo "Error: Slurm job submission failed with exit code \${sbatch_rc}."
                         exit "\${sbatch_rc}"
                     done
                     jobId=\$(printf '%s\\n' "\${sbatch_output}" | awk '/Submitted batch job/ {print \$4; exit}')
                     if [ -z "\$jobId" ]; then
+                        report_sbatch_failure "no job ID returned"
                         echo "Error: Slurm job submission failed, no job ID returned."
                         exit 1
                     fi
@@ -2339,12 +2352,33 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 // share the job workspace (slurm_job_id.txt, scripts) on that login
                 // node; a frontend disconnect fails the whole closure over to a fresh
                 // frontend as a unit (the submit script reuses an active job).
-                Utils.exec(
-                    pipeline,
-                    timeout: false,
-                    script: Utils.sshUserCmd(remote, scriptSubmitPathNode),
-                    numRetries: 3
-                )
+                try {
+                    Utils.exec(
+                        pipeline,
+                        timeout: false,
+                        script: Utils.sshUserCmd(remote, scriptSubmitPathNode),
+                        numRetries: 3
+                    )
+                } catch (InterruptedException e) {
+                    throw e
+                } catch (Exception submitEx) {
+                    def classified = classifyFailure(pipeline, submitEx, InfraFailure.SLURM, retryContext)
+                    if (classified instanceof PipelineInterruption || classified instanceof InfraFailure) {
+                        throw submitEx
+                    }
+
+                    // If the error msg didn't match any predefined pattern. Extract
+                    // only the bounded final-failure line emitted by our submit script.
+                    def diagnosticEvidence = FailureEvidenceCollector.collectOriginatingStepEvidence(
+                        pipeline,
+                        submitEx,
+                        [[id: "slurm-sbatch-final-failure", anyOf: [sbatchFailureMarker]]]
+                    )
+                    if (diagnosticEvidence.collectionStatus == "MATCHED") {
+                        throw new UserFailure("SLURM job submission failed for ${stageName}: ${diagnosticEvidence.matchedMessage}", submitEx)
+                    }
+                    throw submitEx
+                }
 
                 def slurmMetadata = captureSlurmWorkspaceMetadata(pipeline, remote, jobWorkspace, placementContext, stageName)
                 def slurmJobId = slurmMetadata.slurmJobId
@@ -3161,7 +3195,7 @@ def preservePrimaryFailure(Throwable primaryError, Closure actions)
     } catch (Exception e) {
         // Post-processing failures must not replace the failure that triggered them.
         if (primaryError == null || FailureClassifier.classify(e, InfraFailure.BOTH) instanceof PipelineInterruption) throw e
-        echo "[INFRA-RETRY] Secondary failure while handling ${primaryError}: ${e}"
+        echo "[INFRA-RETRY] Secondary failure while handling ${primaryError}:\n${Utils.getStackTrace(e)}"
     }
 }
 
@@ -3402,7 +3436,7 @@ def getHostNodeName() {
     ''', returnStdout: true).trim()
 }
 
-def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSuccess=false, postTag="", boolean isFinalAttempt=true, Map retryContext=null)
+def cacheErrorAndUploadResult(pipeline, stageName, taskRunner, finallyRunner, noResultIfSuccess=false, postTag="", boolean isFinalAttempt=true, Map retryContext=null)
 {
     checkStageName([stageName])
     def Boolean stageIsInterrupted = false
@@ -3426,18 +3460,8 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
         } else {
             // Temporarily disable to reduce the log size
             // sh 'if [ "$(id -u)" -eq 0 ]; then dmesg || true; fi'
-            if (noResultIfSuccess && !stageIsFailed) {
-                // Clean up the workspace
-                sh """
-                    env | sort
-                    pwd && ls -alh
-                    rm -rf ./*
-                """
-
-                echo "Finished test stage execution."
-                return
-            }
-
+            boolean shouldProcessResults = !noResultIfSuccess || stageIsFailed
+            if (shouldProcessResults) {
             // Suppress synthetic stage-fail XML and junit() when this attempt is
             // an intermediate retry that classified as a retryable infra failure.
             // Without this, every transient pod-level failure poisons the per-build
@@ -3511,6 +3535,7 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
             if (!suppressTestReporting) {
                 junit(testResults: "${stageName}/results*.xml")
             }
+            }
         }
 
         // Clean up the workspace
@@ -5460,7 +5485,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
 // and junit() for intermediate retryable failures).
 def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", postTag="", boolean isFinalAttempt=true, Map retryContext=null, boolean useClusterDurations=false)
 {
-    cacheErrorAndUploadResult(stageName, {
+    cacheErrorAndUploadResult(pipeline, stageName, {
         // PMIx builds a singleton ID, singleton.{hostname}.{pid}, that must fit a
         // 50-byte buffer; our 63-character pod names overflow it and every
         // MpiPoolSession spawn fails. PMIX_HOSTNAME replaces only the name PMIx
@@ -6692,7 +6717,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
         // pod-launch attempt; isFinalAttempt suppresses synthetic stage-fail XML
         // and junit() on intermediate retryable infra failures.
         stage("[${key}] Run") {
-            cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, isFinalAttempt, retryContext)
+            cacheErrorAndUploadResult(pipeline, "${key}", values[1], {}, true, attemptTag, isFinalAttempt, retryContext)
         }
     }]]}
 
@@ -6725,7 +6750,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
 
     agentFlowTestJobs = agentFlowTestConfigs.collectEntries{key, values -> [key, [values[0], { attemptTag, isFinalAttempt, retryContext = null ->
         stage("[${key}] Run") {
-            cacheErrorAndUploadResult("${key}", values[1], {}, false, attemptTag, isFinalAttempt, retryContext)
+            cacheErrorAndUploadResult(pipeline, "${key}", values[1], {}, false, attemptTag, isFinalAttempt, retryContext)
         }
     }]]}
 
@@ -6797,7 +6822,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
     }
 
     sanityCheckJobs = sanityCheckConfigs.collectEntries {key, values -> [toStageName(values[1], key), {
-        cacheErrorAndUploadResult(toStageName(values[1], key), {
+        cacheErrorAndUploadResult(pipeline, toStageName(values[1], key), {
             def cpu_arch = values[2]
             def gpu_type = values[1].toLowerCase()
             if (values[1] == "B200_PCIe") {
diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index 5687293e6520..cfaa99976fa6 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -90,7 +90,7 @@ def test_dry_run_skips_merge_request_diff_lookups(self) -> None:
 
     def test_docs_skip_junit_after_a_successful_build(self) -> None:
         self.assertIn(
-            'cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, '
+            'cacheErrorAndUploadResult(pipeline, "${key}", values[1], {}, true, attemptTag, '
             "isFinalAttempt, retryContext)",
             L0_TEST,
         )

From 023549dc8dd580836cf98471f4471d35279011cd Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Thu, 17 Sep 2026 15:49:08 +0800
Subject: [PATCH 04/19] [TRTLLMINF-381][ci] scope failure evidence cache to
 error

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index d50f44c79319..131adc752348 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -3178,10 +3178,17 @@ def classifyFailure(def pipeline, Throwable error, String scope, Map retryContex
 {
     if (error instanceof TrtllmCiException) return FailureClassifier.classify(error, scope)
 
-    Map evidence = retryContext?.failureEvidence
+    // Reuse evidence only when it was collected for this exact error and classification scope.
+    boolean cachedForFailure = retryContext?.failureEvidence != null &&
+        retryContext.failureEvidenceError?.is(error) && retryContext.failureEvidenceScope == scope
+    Map evidence = cachedForFailure ? retryContext.failureEvidence : null
     if (evidence == null) {
         evidence = FailureEvidenceCollector.collectOriginatingStepEvidence(pipeline, error, FailureClassifier.failureEvidenceQueries(scope))
-        if (retryContext != null) retryContext.failureEvidence = evidence
+        if (retryContext != null) {
+            retryContext.failureEvidence = evidence
+            retryContext.failureEvidenceError = error
+            retryContext.failureEvidenceScope = scope
+        }
     }
     return FailureClassifier.classify(error, scope, evidence)
 }

From 48b0aa78fee5926124de54d693d282ba2cc4cb2a Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Thu, 17 Sep 2026 16:06:59 +0800
Subject: [PATCH 05/19] [TRTLLMINF-381][test] cover docs result suppression

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 tests/unittest/tools/test_infra_dry_run_pipeline.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index cfaa99976fa6..6987511c7172 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -88,12 +88,22 @@ def test_dry_run_skips_merge_request_diff_lookups(self) -> None:
                 self.assertLess(body.index(infra_dry_run_check), body.index("def githubPrApiUrl"))
                 self.assertLess(body.index(empty_result), body.index("def githubPrApiUrl"))
 
-    def test_docs_skip_junit_after_a_successful_build(self) -> None:
+    def test_docs_skip_results_only_after_a_successful_build(self) -> None:
+        result_handler = _function_body(
+            L0_TEST, "cacheErrorAndUploadResult", "createKubernetesPodConfig"
+        )
         self.assertIn(
             'cacheErrorAndUploadResult(pipeline, "${key}", values[1], {}, true, attemptTag, '
             "isFinalAttempt, retryContext)",
             L0_TEST,
         )
+        process_results = "boolean shouldProcessResults = !noResultIfSuccess || stageIsFailed"
+        process_results_branch = "if (shouldProcessResults) {"
+        self.assertIn(f"{process_results}\n            {process_results_branch}", result_handler)
+        self.assertLess(
+            result_handler.index(process_results_branch),
+            result_handler.index('junit(testResults: "${stageName}/results*.xml")'),
+        )
 
 
 if __name__ == "__main__":

From 4e037fef06749fea8e15991134960cfca7653693 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Mon, 21 Sep 2026 12:50:50 +0800
Subject: [PATCH 06/19] [TRTLLMINF-381][fix] avoid Jenkins sandbox rejection

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 131adc752348..e70925394819 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -3180,7 +3180,7 @@ def classifyFailure(def pipeline, Throwable error, String scope, Map retryContex
 
     // Reuse evidence only when it was collected for this exact error and classification scope.
     boolean cachedForFailure = retryContext?.failureEvidence != null &&
-        retryContext.failureEvidenceError?.is(error) && retryContext.failureEvidenceScope == scope
+        retryContext.failureEvidenceError == error && retryContext.failureEvidenceScope == scope
     Map evidence = cachedForFailure ? retryContext.failureEvidence : null
     if (evidence == null) {
         evidence = FailureEvidenceCollector.collectOriginatingStepEvidence(pipeline, error, FailureClassifier.failureEvidenceQueries(scope))

From 9309eb8f104d91ac13b4a2c7fcb874ae169db073 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Mon, 21 Sep 2026 17:33:47 +0800
Subject: [PATCH 07/19] [TRTLLMINF-381][ci] log matched failure evidence

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                              |  3 +++
 tests/unittest/tools/test_infra_dry_run_pipeline.py | 10 ++++++++++
 2 files changed, 13 insertions(+)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index e70925394819..32de6cef05de 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -3184,6 +3184,9 @@ def classifyFailure(def pipeline, Throwable error, String scope, Map retryContex
     Map evidence = cachedForFailure ? retryContext.failureEvidence : null
     if (evidence == null) {
         evidence = FailureEvidenceCollector.collectOriginatingStepEvidence(pipeline, error, FailureClassifier.failureEvidenceQueries(scope))
+        if (evidence.collectionStatus == "MATCHED") {
+            echo "[FAILURE-EVIDENCE] ${scope}: originating step matched ${evidence.matchedQueryId} [${evidence.matchedTerms.join(', ')}]"
+        }
         if (retryContext != null) {
             retryContext.failureEvidence = evidence
             retryContext.failureEvidenceError = error
diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index 6987511c7172..1bb0076ded89 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -105,6 +105,16 @@ def test_docs_skip_results_only_after_a_successful_build(self) -> None:
             result_handler.index('junit(testResults: "${stageName}/results*.xml")'),
         )
 
+    def test_failure_evidence_match_is_logged_before_caching(self) -> None:
+        classifier = _function_body(L0_TEST, "classifyFailure", "preservePrimaryFailure")
+
+        self.assertIn("[FAILURE-EVIDENCE]", classifier)
+        self.assertIn("evidence.matchedQueryId", classifier)
+        self.assertLess(
+            classifier.index("[FAILURE-EVIDENCE]"),
+            classifier.index("retryContext.failureEvidence = evidence"),
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

From de5236bd5bea48482e0b4a2e21a9ad2a0081f039 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Mon, 21 Sep 2026 18:50:15 +0800
Subject: [PATCH 08/19] [TRTLLMINF-381][fix] preserve primary failure during
 upload

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                        |  4 ++++
 .../tools/test_infra_dry_run_pipeline.py      | 24 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 32de6cef05de..d4f46bead8ed 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -364,6 +364,10 @@ def scrapeSlurmLogForDeviceFault(def pipeline, Map remote, String remoteLogPath)
 def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String nodeName, String stageName, String postTag="", boolean suppressTestReporting=false) {
     pipeline.stage('Submit Test Result') {
         sh "ls -al ${stageName}/ || true"
+        if (!fileExists(stageName)) {
+            echo "No local test-result directory for ${stageName}; skipping upload."
+            return
+        }
 
         if (suppressTestReporting) {
             // This attempt is superseded by a planned retry. Rename its XMLs so
diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index 1bb0076ded89..35256aa717c8 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -115,6 +115,30 @@ def test_failure_evidence_match_is_logged_before_caching(self) -> None:
             classifier.index("retryContext.failureEvidence = evidence"),
         )
 
+    def test_post_actions_keep_interruptions_higher_priority_than_primary_failures(self) -> None:
+        result_handler = _function_body(
+            L0_TEST, "cacheErrorAndUploadResult", "createKubernetesPodConfig"
+        )
+        preserve_primary = _function_body(
+            L0_TEST, "preservePrimaryFailure", "rememberAvoidedKubernetesHostNodes"
+        )
+
+        self.assertIn("caughtError = e\n        stageIsInterrupted = true", result_handler)
+        self.assertIn("catch (InterruptedException e) {\n        throw e", preserve_primary)
+        self.assertIn(
+            "FailureClassifier.classify(e, InfraFailure.BOTH) instanceof PipelineInterruption",
+            preserve_primary,
+        )
+
+    def test_slurm_upload_skips_a_missing_result_directory(self) -> None:
+        upload_results = _function_body(L0_TEST, "uploadResults", "runIsolatedTests")
+
+        self.assertIn("if (!fileExists(stageName))", upload_results)
+        self.assertLess(
+            upload_results.index("if (!fileExists(stageName))"),
+            upload_results.index('sh "tar -czvf results-${stageName}${postTag}.tar.gz'),
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

From 1f4fe6e6f53822f305779a8f029107b0b99d2740 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Tue, 22 Sep 2026 13:48:41 +0800
Subject: [PATCH 09/19] [TRTLLMINF-384][fix] classify failures from Slurm job
 logs

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                        | 144 +++++++++++-------
 .../extract_failure_evidence_candidates.py    |  67 ++++++++
 2 files changed, 154 insertions(+), 57 deletions(-)
 create mode 100644 jenkins/scripts/extract_failure_evidence_candidates.py

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index d4f46bead8ed..ac55e9aa363e 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -159,6 +159,10 @@ REQUIRED_NO_DRIVER_TYPES = ["dgx-h100", "dgx-h200", "gh200", "gb10x"]
 // K8s pod retry, so 1 pod × 2 SLURM attempts = 2 total.
 SLURM_INFRA_RETRY_MAX = 1
 
+// Only this bounded tail is copied from a failed SLURM job for local evidence
+// matching. The temporary copy is deleted after matching and is never uploaded.
+SLURM_FAILURE_EVIDENCE_LOG_BYTES = 20 * 1024 * 1024
+
 // Maximum K8s infra-failure retries (total attempts = K8S_INFRA_RETRY_MAX + 1).
 // Kept distinct from SLURM_INFRA_RETRY_MAX so the two paths can be tuned
 // independently as production telemetry comes in. Patterns tagged scope=K8S
@@ -311,50 +315,69 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2
     }
 }
 
-// Scrape the SLURM job output log for a device / driver / interconnect fault
-// signature and return the matched signature itself, or "" for no match.
-//
-// Device faults (CUDA/NVLink/ECC/driver) print into job-output.log but never
-// reach the stage exception chain -- the tracker squashes a failed job to
-// `exit 1` -- so classify() otherwise sees only a generic failure and cannot
-// steer the retry off the bad node. This is a GATE only: the returned signature
-// is folded into a fresh exception so FailureClassifier.PATTERN_CATALOG (the
-// authoritative list) makes the real retry/severity decision. A signature the
-// catalog does not recognize simply falls through to a normal rethrow.
-// App-induced CUDA errors (illegal memory access, unspecified launch failure,
-// OOM) are deliberately excluded -- the OpenSearch stage data shows those are
-// overwhelmingly code regressions, not node faults, and must not trigger a
-// node-avoiding retry.
-//
-// grep -o returns only the matched signature (not the whole line), so a long
-// log line cannot truncate the signature out of the result before it reaches
-// classify(). Each alternative must therefore be catalog-exact: it must match
-// (via `.` wildcards for shell-hostile chars) the full catalog substring, so
-// grep -o emits text that still contains the catalog pattern.
-def scrapeSlurmLogForDeviceFault(def pipeline, Map remote, String remoteLogPath) {
-    def deviceFaultRegex = "cudaErrorMapBufferObjectFailed|mapping of buffer object failed|" +
-        "uncorrectable NVLink error|cudaErrorNvlinkUncorrectable|CUDA_ERROR_SYSTEM_NOT_READY|" +
-        "uncorrectable ECC error|CUDA_ERROR_ECC_UNCORRECTABLE|has fallen off the bus|GPU is lost|" +
-        "Unable to determine the device handle for GPU|RmInitAdapter failed|Failed to initialize NVML|" +
-        "could... communicate with the NVIDIA driver|CUDA_ERROR_DEVICE_UNAVAILABLE|" +
-        "no CUDA-capable device is detected|CUDA_ERROR_UNKNOWN: 999|CUDA unknown error|" +
-        "CUDA-capable device.s. is/are busy or unavailable"
+// Copy a bounded failed-job log tail to the Jenkins agent, reduce it to a small
+// set of catalog candidate lines, and delete both temporary copies. Matching
+// policy remains in the shared libraries: this helper only collects evidence.
+def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath,
+                               String evidenceExtractorPath) {
+    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM)
+    def evidenceId = UUID.randomUUID().toString()
+    def localLogPath = Utils.createTempLocation(pipeline, "./${evidenceId}-slurm-job-output.log")
+    def localQueriesPath = Utils.createTempLocation(pipeline, "./${evidenceId}-slurm-failure-queries.json")
+    def remoteLogTailPath = "${remoteLogPath}.${evidenceId}.failure-evidence"
     try {
-        // Wrap the body in `bash -c` so it is shell-agnostic: cluster login shells
-        // are often csh/tcsh, which can't parse this bash test/pipe/redirection
-        // syntax. The login shell only has to run `bash -c '<single-quoted body>'`.
-        return Utils.exec(
+        pipeline.writeFile(file: localQueriesPath, text: JsonOutput.toJson(queries))
+        def copyLogTail = "test -f '${remoteLogPath}' && " +
+            "tail -c ${SLURM_FAILURE_EVIDENCE_LOG_BYTES} -- '${remoteLogPath}' > '${remoteLogTailPath}'"
+        Utils.exec(
             pipeline,
-            script: Utils.sshUserCmd(remote,
-                "\"bash -c 'if [ -f \\\"${remoteLogPath}\\\" ]; then grep -aioE \\\"${deviceFaultRegex}\\\" \\\"${remoteLogPath}\\\" 2>/dev/null | tail -n 1 | cut -c1-500; fi'\""),
+            script: Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd(copyLogTail)),
+            retryOnFail: false,
+            noNVDFEvent: true,
+        )
+        Utils.exec(
+            pipeline,
+            script: scpFromRemoteCmd(remote, remoteLogTailPath, localLogPath),
+            retryOnFail: false,
+            noNVDFEvent: true,
+        )
+        def candidateText = pipeline.sh(
             returnStdout: true,
-            numRetries: 1,
+            script: """#!/bin/bash
+                set +x
+                python3 '${evidenceExtractorPath}' --log '${localLogPath}' --queries '${localQueriesPath}'
+            """,
         )?.trim()
-    } catch (InterruptedException e) {
-        throw e
-    } catch (Exception scrapeEx) {
-        pipeline.echo("Ignorable warning: could not scrape ${remoteLogPath} for device faults on ${remote.host}: ${scrapeEx.message}")
-        return ""
+        def match = FailureEvidenceCollector.matchQueries(candidateText, queries)
+        return [
+            source: "SLURM_JOB_LOG",
+            collectionStatus: match.matchedQueryId == null ? "NO_MATCH" : "MATCHED",
+            matchedQueryId: match.matchedQueryId,
+            matchedTerms: match.matchedTerms,
+            matchedMessage: match.matchedMessage,
+        ]
+    } catch (InterruptedException interruptedError) {
+        throw interruptedError
+    } catch (Exception collectionError) {
+        pipeline.echo("Ignorable warning: could not collect failure evidence from ${remoteLogPath} on " +
+            "${remote.host}: ${collectionError.message}")
+        return [source: "SLURM_JOB_LOG", collectionStatus: "COLLECTION_ERROR"]
+    } finally {
+        try {
+            pipeline.sh(returnStatus: true, script: "rm -f '${localLogPath}' '${localQueriesPath}'")
+            Utils.exec(
+                pipeline,
+                script: Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd("rm -f '${remoteLogTailPath}'")),
+                returnStatus: true,
+                timeout: 60,
+                noNVDFEvent: true,
+            )
+        } catch (InterruptedException interruptedError) {
+            throw interruptedError
+        } catch (Exception cleanupError) {
+            pipeline.echo("Ignorable warning: could not delete temporary SLURM failure evidence: " +
+                cleanupError.message)
+        }
     }
 }
 
@@ -2419,7 +2442,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     trap 'rc=\$?; echo "Error in file \${BASH_SOURCE[0]} on line \$LINENO: \$BASH_COMMAND (exit \$rc)"; exit \$rc' ERR
 
                     jobId=${slurmJobId}
-                    tail -f ${slurmJobLogPath} &
+                    tail -n +1 -F -- "${slurmJobLogPath}" &
                     tailPid=\$!
 
                     noLogTimeoutSecs=7200
@@ -2462,6 +2485,12 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                         fi
                     done
 
+                    # A failed job can reach a terminal accounting state before
+                    # its final output is visible on the login node's shared FS.
+                    if [[ \$STATUS != "COMPLETED" ]]; then
+                        sleep 15
+                    fi
+
                     # Stop and reap the log follower. It may have already exited
                     # when the remote log stream closes; that is not a test failure.
                     kill \$tailPid 2>/dev/null || true
@@ -2665,24 +2694,25 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                             "test failure.",
                             null, InfraFailure.TRANSIENT, InfraFailure.SLURM, "<typed:slurm-job-still-running>")
                     }
-                    // A terminal FAILED state may be a node/device fault whose signature
-                    // (CUDA/NVLink/ECC/driver) printed only into the SLURM job output log,
-                    // never into this verdict. Scrape the log and, on a hit, surface the
-                    // matched line into a fresh exception so the authoritative catalog
-                    // (FailureClassifier.classify at the runLLMTestlistWithSbatch caller)
-                    // can match it and steer the retry off the bad node. A miss falls
-                    // through to the plain "Pytest failed" rethrow below.
-                    if (slurmState == "FAILED") {
-                        def deviceHit = scrapeSlurmLogForDeviceFault(pipeline, remote, slurmJobLogPath)
-                        if (deviceHit) {
-                            echo "[INFRA-RETRY] ${stageName}: device-fault signature in SLURM job ${slurmJobId} log; " +
-                                 "surfacing to classifier: ${deviceHit}"
-                            throw new Exception(
-                                "Device/interconnect fault on SLURM node during job ${slurmJobId} for ${stageName}: ${deviceHit}")
-                        }
+                    def jobFailure = new Exception(
+                        "SLURM job ${slurmJobId} for ${stageName} failed with state=" +
+                        "${slurmState ?: 'unknown'}, exit=${jobExit ?: 'unknown'}")
+                    def evidence = collectSlurmJobLogEvidence(
+                        pipeline,
+                        remote,
+                        slurmJobLogPath,
+                        "${llmSrcLocal}/jenkins/scripts/extract_failure_evidence_candidates.py",
+                    )
+                    def classified = FailureClassifier.classify(jobFailure, InfraFailure.SLURM, evidence)
+                    if (classified instanceof InfraFailure) {
+                        def classificationSource = evidence.collectionStatus == "MATCHED" ? "job log" : "job state"
+                        echo "[INFRA-RETRY] ${stageName}: SLURM ${classificationSource} matched " +
+                            "${classified.detectedPattern}: " +
+                            classified.message
+                        throw classified
                     }
                     echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobId} state=${slurmState ?: 'unknown'}, exit=${jobExit ?: 'unknown'}; deferring to classifier."
-                    throw new Exception("Pytest failed in SLURM job ${slurmJobId} for ${stageName}")
+                    throw jobFailure
                 }
             }
             echo "Finished test stage execution."
diff --git a/jenkins/scripts/extract_failure_evidence_candidates.py b/jenkins/scripts/extract_failure_evidence_candidates.py
new file mode 100644
index 000000000000..838ac6baf303
--- /dev/null
+++ b/jenkins/scripts/extract_failure_evidence_candidates.py
@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Extract bounded candidate lines for shared-library failure matching."""
+
+import argparse
+import json
+from pathlib import Path
+
+MAX_CANDIDATE_CHARS = 4096
+
+
+def _query_terms(queries: list[dict]) -> list[str]:
+    terms = []
+    seen = set()
+    for query in queries:
+        query_terms = query.get("anyOf") or query.get("allOf") or []
+        for term in query_terms:
+            folded = term.casefold()
+            if folded not in seen:
+                seen.add(folded)
+                terms.append(term)
+    return terms
+
+
+def _match_context(line: str, term: str) -> str:
+    match_index = line.casefold().find(term.casefold())
+    start = max(0, match_index - MAX_CANDIDATE_CHARS // 2)
+    end = min(len(line), start + MAX_CANDIDATE_CHARS)
+    if end - start < MAX_CANDIDATE_CHARS:
+        start = max(0, end - MAX_CANDIDATE_CHARS)
+    return line[start:end].strip().replace("\x00", "\ufffd")
+
+
+def extract_candidates(log_path: Path, queries: list[dict]) -> list[str]:
+    terms = _query_terms(queries)
+    folded_terms = [(term, term.casefold()) for term in terms]
+    matches = {}
+    with log_path.open(encoding="utf-8", errors="replace") as log_file:
+        for line in log_file:
+            folded_line = line.casefold()
+            for term, folded_term in folded_terms:
+                if folded_term in folded_line:
+                    matches[folded_term] = _match_context(line, term)
+
+    candidates = []
+    seen = set()
+    for term in terms:
+        candidate = matches.get(term.casefold())
+        if candidate and candidate not in seen:
+            seen.add(candidate)
+            candidates.append(candidate)
+    return candidates
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--log", required=True, type=Path)
+    parser.add_argument("--queries", required=True, type=Path)
+    args = parser.parse_args()
+
+    queries = json.loads(args.queries.read_text(encoding="utf-8"))
+    print("\n".join(extract_candidates(args.log, queries)))
+
+
+if __name__ == "__main__":
+    main()

From 0976b580f960545da33f4dbd79ca6ea3f1ac7a7e Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Wed, 23 Sep 2026 10:09:05 +0800
Subject: [PATCH 10/19] [TRTLLMINF-384][fix] restrict Slurm job log evidence

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                        | 75 +++++++------------
 .../extract_failure_evidence_candidates.py    | 67 -----------------
 2 files changed, 27 insertions(+), 115 deletions(-)
 delete mode 100644 jenkins/scripts/extract_failure_evidence_candidates.py

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index ac55e9aa363e..3cd5d2bd37a0 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -159,9 +159,17 @@ REQUIRED_NO_DRIVER_TYPES = ["dgx-h100", "dgx-h200", "gh200", "gb10x"]
 // K8s pod retry, so 1 pod × 2 SLURM attempts = 2 total.
 SLURM_INFRA_RETRY_MAX = 1
 
-// Only this bounded tail is copied from a failed SLURM job for local evidence
-// matching. The temporary copy is deleted after matching and is never uploaded.
-SLURM_FAILURE_EVIDENCE_LOG_BYTES = 20 * 1024 * 1024
+// Only machine-specific device / driver / interconnect signatures are safe to
+// match against arbitrary test stdout/stderr in the SLURM job log. Keep the
+// concrete terms and their retry policy owned by FailureClassifier.
+SLURM_JOB_LOG_EVIDENCE_QUERY_IDS = [
+    "cuda-device-unavailable",
+    "cuda-buffer-mapping",
+    "nvidia-driver-unavailable",
+    "gpu-lost",
+    "gpu-ecc-uncorrectable",
+    "nvlink-fabric-failure",
+]
 
 // Maximum K8s infra-failure retries (total attempts = K8S_INFRA_RETRY_MAX + 1).
 // Kept distinct from SLURM_INFRA_RETRY_MAX so the two paths can be tuned
@@ -315,38 +323,26 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2
     }
 }
 
-// Copy a bounded failed-job log tail to the Jenkins agent, reduce it to a small
-// set of catalog candidate lines, and delete both temporary copies. Matching
-// policy remains in the shared libraries: this helper only collects evidence.
-def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath,
-                               String evidenceExtractorPath) {
-    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM)
-    def evidenceId = UUID.randomUUID().toString()
-    def localLogPath = Utils.createTempLocation(pipeline, "./${evidenceId}-slurm-job-output.log")
-    def localQueriesPath = Utils.createTempLocation(pipeline, "./${evidenceId}-slurm-failure-queries.json")
-    def remoteLogTailPath = "${remoteLogPath}.${evidenceId}.failure-evidence"
+// Stream-scan the complete remote job log for the source-safe catalog subset,
+// returning at most one occurrence of each signature. Matching policy and
+// query precedence remain in the shared libraries.
+def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
+    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM).findAll {
+        SLURM_JOB_LOG_EVIDENCE_QUERY_IDS.contains(it.id)
+    }
+    def grepPatterns = queries.collectMany { it.anyOf }.collect {
+        "-e '" + it.replace("'", "'\"'\"'") + "'"
+    }.join(" ")
     try {
-        pipeline.writeFile(file: localQueriesPath, text: JsonOutput.toJson(queries))
-        def copyLogTail = "test -f '${remoteLogPath}' && " +
-            "tail -c ${SLURM_FAILURE_EVIDENCE_LOG_BYTES} -- '${remoteLogPath}' > '${remoteLogTailPath}'"
-        Utils.exec(
+        def scanLog = "if [ -f '${remoteLogPath}' ]; then " +
+            "grep -aFio ${grepPatterns} -- '${remoteLogPath}' 2>/dev/null " +
+            '''| awk '{ key=tolower($0); if (!seen[key]++) print }'; fi'''
+        def candidateText = Utils.exec(
             pipeline,
-            script: Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd(copyLogTail)),
-            retryOnFail: false,
-            noNVDFEvent: true,
-        )
-        Utils.exec(
-            pipeline,
-            script: scpFromRemoteCmd(remote, remoteLogTailPath, localLogPath),
+            script: Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd(scanLog)),
+            returnStdout: true,
             retryOnFail: false,
             noNVDFEvent: true,
-        )
-        def candidateText = pipeline.sh(
-            returnStdout: true,
-            script: """#!/bin/bash
-                set +x
-                python3 '${evidenceExtractorPath}' --log '${localLogPath}' --queries '${localQueriesPath}'
-            """,
         )?.trim()
         def match = FailureEvidenceCollector.matchQueries(candidateText, queries)
         return [
@@ -362,22 +358,6 @@ def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath,
         pipeline.echo("Ignorable warning: could not collect failure evidence from ${remoteLogPath} on " +
             "${remote.host}: ${collectionError.message}")
         return [source: "SLURM_JOB_LOG", collectionStatus: "COLLECTION_ERROR"]
-    } finally {
-        try {
-            pipeline.sh(returnStatus: true, script: "rm -f '${localLogPath}' '${localQueriesPath}'")
-            Utils.exec(
-                pipeline,
-                script: Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd("rm -f '${remoteLogTailPath}'")),
-                returnStatus: true,
-                timeout: 60,
-                noNVDFEvent: true,
-            )
-        } catch (InterruptedException interruptedError) {
-            throw interruptedError
-        } catch (Exception cleanupError) {
-            pipeline.echo("Ignorable warning: could not delete temporary SLURM failure evidence: " +
-                cleanupError.message)
-        }
     }
 }
 
@@ -2701,7 +2681,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                         pipeline,
                         remote,
                         slurmJobLogPath,
-                        "${llmSrcLocal}/jenkins/scripts/extract_failure_evidence_candidates.py",
                     )
                     def classified = FailureClassifier.classify(jobFailure, InfraFailure.SLURM, evidence)
                     if (classified instanceof InfraFailure) {
diff --git a/jenkins/scripts/extract_failure_evidence_candidates.py b/jenkins/scripts/extract_failure_evidence_candidates.py
deleted file mode 100644
index 838ac6baf303..000000000000
--- a/jenkins/scripts/extract_failure_evidence_candidates.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Extract bounded candidate lines for shared-library failure matching."""
-
-import argparse
-import json
-from pathlib import Path
-
-MAX_CANDIDATE_CHARS = 4096
-
-
-def _query_terms(queries: list[dict]) -> list[str]:
-    terms = []
-    seen = set()
-    for query in queries:
-        query_terms = query.get("anyOf") or query.get("allOf") or []
-        for term in query_terms:
-            folded = term.casefold()
-            if folded not in seen:
-                seen.add(folded)
-                terms.append(term)
-    return terms
-
-
-def _match_context(line: str, term: str) -> str:
-    match_index = line.casefold().find(term.casefold())
-    start = max(0, match_index - MAX_CANDIDATE_CHARS // 2)
-    end = min(len(line), start + MAX_CANDIDATE_CHARS)
-    if end - start < MAX_CANDIDATE_CHARS:
-        start = max(0, end - MAX_CANDIDATE_CHARS)
-    return line[start:end].strip().replace("\x00", "\ufffd")
-
-
-def extract_candidates(log_path: Path, queries: list[dict]) -> list[str]:
-    terms = _query_terms(queries)
-    folded_terms = [(term, term.casefold()) for term in terms]
-    matches = {}
-    with log_path.open(encoding="utf-8", errors="replace") as log_file:
-        for line in log_file:
-            folded_line = line.casefold()
-            for term, folded_term in folded_terms:
-                if folded_term in folded_line:
-                    matches[folded_term] = _match_context(line, term)
-
-    candidates = []
-    seen = set()
-    for term in terms:
-        candidate = matches.get(term.casefold())
-        if candidate and candidate not in seen:
-            seen.add(candidate)
-            candidates.append(candidate)
-    return candidates
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--log", required=True, type=Path)
-    parser.add_argument("--queries", required=True, type=Path)
-    args = parser.parse_args()
-
-    queries = json.loads(args.queries.read_text(encoding="utf-8"))
-    print("\n".join(extract_candidates(args.log, queries)))
-
-
-if __name__ == "__main__":
-    main()

From 6e95b879bf85651d1caa28f03a95d22c5fcc1eaf Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Wed, 14 Oct 2026 16:28:23 +0800
Subject: [PATCH 11/19] [TRTLLMINF-384][fix] bound Slurm job log scan

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index e352578ae5b9..7530d34e7025 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -30,6 +30,7 @@ import com.nvidia.bloom.ContainerRuntime
 import com.nvidia.bloom.FailureEvidenceCollector
 import com.nvidia.bloom.SshAuthMethod
 import org.jenkinsci.plugins.workflow.cps.CpsThread
+import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException
 import org.jsoup.Jsoup
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils as jUtils
 import trtllm.FailureClassifier
@@ -347,6 +348,7 @@ def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
             returnStdout: true,
             retryOnFail: false,
             noNVDFEvent: true,
+            timeout: 300,
         )?.trim()
         def match = FailureEvidenceCollector.matchQueries(candidateText, queries)
         return [
@@ -356,6 +358,15 @@ def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
             matchedTerms: match.matchedTerms,
             matchedMessage: match.matchedMessage,
         ]
+    } catch (FlowInterruptedException interruptedError) {
+        // A bounded evidence scan is best-effort. Utils.exec marks its timeout
+        // as a non-actual interruption; user aborts must keep propagating.
+        if (interruptedError.isActualInterruption()) {
+            throw interruptedError
+        }
+        pipeline.echo("Ignorable warning: timed out collecting failure evidence from ${remoteLogPath} on " +
+            "${remote.host} after 5 minutes")
+        return [source: "SLURM_JOB_LOG", collectionStatus: "COLLECTION_ERROR"]
     } catch (InterruptedException interruptedError) {
         throw interruptedError
     } catch (Exception collectionError) {

From b4a2921c5ea06801a713f4a2fcc9d6db48633555 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Wed, 14 Oct 2026 16:31:14 +0800
Subject: [PATCH 12/19] [TRTLLMINF-384][fix] guard empty Slurm evidence queries

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 7530d34e7025..377b235b075a 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -335,7 +335,9 @@ def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
     def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM).findAll {
         SLURM_JOB_LOG_EVIDENCE_QUERY_IDS.contains(it.id)
     }
-    def grepPatterns = queries.collectMany { it.anyOf }.collect {
+    def queryTerms = queries.collectMany { it.anyOf ?: [] }
+    if (!queryTerms) return [source: "SLURM_JOB_LOG", collectionStatus: "NO_MATCH"]
+    def grepPatterns = queryTerms.collect {
         "-e '" + it.replace("'", "'\"'\"'") + "'"
     }.join(" ")
     try {

From 7d1efa4299fb84bf53df71a90d6b55b3a8b5ce55 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Thu, 15 Oct 2026 12:06:12 +0800
Subject: [PATCH 13/19] [TRTLLMINF-384][fix] scan all SLURM log signatures

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 377b235b075a..c09f7834ffd5 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -160,18 +160,6 @@ REQUIRED_NO_DRIVER_TYPES = ["dgx-h100", "dgx-h200", "gh200", "gb10x"]
 // K8s pod retry, so 1 pod × 2 SLURM attempts = 2 total.
 SLURM_INFRA_RETRY_MAX = 1
 
-// Only machine-specific device / driver / interconnect signatures are safe to
-// match against arbitrary test stdout/stderr in the SLURM job log. Keep the
-// concrete terms and their retry policy owned by FailureClassifier.
-SLURM_JOB_LOG_EVIDENCE_QUERY_IDS = [
-    "cuda-device-unavailable",
-    "cuda-buffer-mapping",
-    "nvidia-driver-unavailable",
-    "gpu-lost",
-    "gpu-ecc-uncorrectable",
-    "nvlink-fabric-failure",
-]
-
 // Maximum K8s infra-failure retries (total attempts = K8S_INFRA_RETRY_MAX + 1).
 // Kept distinct from SLURM_INFRA_RETRY_MAX so the two paths can be tuned
 // independently as production telemetry comes in. Patterns tagged scope=K8S
@@ -328,14 +316,12 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2
     }
 }
 
-// Stream-scan the complete remote job log for the source-safe catalog subset,
-// returning at most one occurrence of each signature. Matching policy and
-// query precedence remain in the shared libraries.
+// Stream-scan the complete remote job log for SLURM failure signatures,
+// returning at most one occurrence of each term. Matching policy and query
+// precedence remain in the shared libraries.
 def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
-    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM).findAll {
-        SLURM_JOB_LOG_EVIDENCE_QUERY_IDS.contains(it.id)
-    }
-    def queryTerms = queries.collectMany { it.anyOf ?: [] }
+    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM)
+    def queryTerms = queries.collectMany { it.anyOf ?: (it.allOf ?: []) }.unique()
     if (!queryTerms) return [source: "SLURM_JOB_LOG", collectionStatus: "NO_MATCH"]
     def grepPatterns = queryTerms.collect {
         "-e '" + it.replace("'", "'\"'\"'") + "'"

From 338ad95f4e99a660de39897de229d3dbd147f6f7 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Thu, 15 Oct 2026 13:02:01 +0800
Subject: [PATCH 14/19] Restrict Slurm job log evidence queries

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index c09f7834ffd5..cc87a40790d2 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -160,6 +160,19 @@ REQUIRED_NO_DRIVER_TYPES = ["dgx-h100", "dgx-h200", "gh200", "gb10x"]
 // K8s pod retry, so 1 pod × 2 SLURM attempts = 2 total.
 SLURM_INFRA_RETRY_MAX = 1
 
+// Only source-specific signatures are safe to match against arbitrary test
+// stdout/stderr in the SLURM job log. Keep the concrete terms and their retry
+// policy owned by FailureClassifier.
+SLURM_JOB_LOG_EVIDENCE_QUERY_IDS = [
+    "slurm-ucx-worker-address",
+    "cuda-device-unavailable",
+    "cuda-buffer-mapping",
+    "nvidia-driver-unavailable",
+    "gpu-lost",
+    "gpu-ecc-uncorrectable",
+    "nvlink-fabric-failure",
+]
+
 // Maximum K8s infra-failure retries (total attempts = K8S_INFRA_RETRY_MAX + 1).
 // Kept distinct from SLURM_INFRA_RETRY_MAX so the two paths can be tuned
 // independently as production telemetry comes in. Patterns tagged scope=K8S
@@ -316,11 +329,13 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2
     }
 }
 
-// Stream-scan the complete remote job log for SLURM failure signatures,
-// returning at most one occurrence of each term. Matching policy and query
-// precedence remain in the shared libraries.
+// Stream-scan the complete remote job log for the source-safe catalog subset,
+// returning at most one occurrence of each signature. Matching policy and
+// query precedence remain in the shared libraries.
 def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
-    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM)
+    def queries = FailureClassifier.failureEvidenceQueries(InfraFailure.SLURM).findAll {
+        SLURM_JOB_LOG_EVIDENCE_QUERY_IDS.contains(it.id)
+    }
     def queryTerms = queries.collectMany { it.anyOf ?: (it.allOf ?: []) }.unique()
     if (!queryTerms) return [source: "SLURM_JOB_LOG", collectionStatus: "NO_MATCH"]
     def grepPatterns = queryTerms.collect {

From 390611feb5dd8bef37a66f3f20f2f90a8a79d100 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Tue, 20 Oct 2026 16:34:13 +0800
Subject: [PATCH 15/19] Fix SLURM job log evidence scanning

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                        | 20 ++++++-------------
 .../tools/test_infra_dry_run_pipeline.py      | 17 ++++++++++++++++
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index cc87a40790d2..7254c1c411ac 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -30,7 +30,6 @@ import com.nvidia.bloom.ContainerRuntime
 import com.nvidia.bloom.FailureEvidenceCollector
 import com.nvidia.bloom.SshAuthMethod
 import org.jenkinsci.plugins.workflow.cps.CpsThread
-import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException
 import org.jsoup.Jsoup
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils as jUtils
 import trtllm.FailureClassifier
@@ -343,15 +342,17 @@ def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
     }.join(" ")
     try {
         def scanLog = "if [ -f '${remoteLogPath}' ]; then " +
-            "grep -aFio ${grepPatterns} -- '${remoteLogPath}' 2>/dev/null " +
-            '''| awk '{ key=tolower($0); if (!seen[key]++) print }'; fi'''
+            "grep -aFi ${grepPatterns} -- '${remoteLogPath}' 2>/dev/null " +
+            '''| awk 'index($0, ".py::") == 0' ''' +
+            "| grep -aFio ${grepPatterns} " +
+            '''| awk '{ key=tolower($0); if (seen[key]++ == 0) print }'; fi'''
         def candidateText = Utils.exec(
             pipeline,
-            script: Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd(scanLog)),
+            script: "timeout 300s " + Utils.sshUserCmd(remote, Utils.bashWrappedRemoteCmd(scanLog)),
             returnStdout: true,
             retryOnFail: false,
             noNVDFEvent: true,
-            timeout: 300,
+            timeout: false,
         )?.trim()
         def match = FailureEvidenceCollector.matchQueries(candidateText, queries)
         return [
@@ -361,15 +362,6 @@ def collectSlurmJobLogEvidence(def pipeline, Map remote, String remoteLogPath) {
             matchedTerms: match.matchedTerms,
             matchedMessage: match.matchedMessage,
         ]
-    } catch (FlowInterruptedException interruptedError) {
-        // A bounded evidence scan is best-effort. Utils.exec marks its timeout
-        // as a non-actual interruption; user aborts must keep propagating.
-        if (interruptedError.isActualInterruption()) {
-            throw interruptedError
-        }
-        pipeline.echo("Ignorable warning: timed out collecting failure evidence from ${remoteLogPath} on " +
-            "${remote.host} after 5 minutes")
-        return [source: "SLURM_JOB_LOG", collectionStatus: "COLLECTION_ERROR"]
     } catch (InterruptedException interruptedError) {
         throw interruptedError
     } catch (Exception collectionError) {
diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index 6b261d642496..78cc3114970f 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -151,6 +151,23 @@ def test_sbatch_final_failure_is_classified_without_scanning_the_whole_step_log(
             "classifyFailure(pipeline, submitEx, InfraFailure.SLURM, retryContext)", sbatch
         )
 
+    def test_slurm_job_log_evidence_scan_is_bounded_without_script_approval(self) -> None:
+        scanner = _function_body(L0_TEST, "collectSlurmJobLogEvidence", "uploadResults")
+
+        self.assertIn(
+            'script: "timeout 300s " + Utils.sshUserCmd',
+            scanner,
+        )
+        self.assertIn("timeout: false,", scanner)
+        self.assertNotIn("isActualInterruption()", scanner)
+
+    def test_slurm_job_log_evidence_ignores_pytest_nodeids(self) -> None:
+        scanner = _function_body(L0_TEST, "collectSlurmJobLogEvidence", "uploadResults")
+
+        self.assertIn("""awk 'index($0, \".py::\") == 0'""", scanner)
+        self.assertIn("seen[key]++ == 0", scanner)
+        self.assertNotIn("!seen[key]++", scanner)
+
     def test_cbts_parser_preserves_coverage_preflight_fields(self) -> None:
         parser = _function_body(
             L0_PARENT, "_cbtsParseSelectionResult", "getOssComplianceFileChanged"

From 63c3cf6bd0bfe962af5f1b3c78e8355ea689bc54 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Tue, 20 Oct 2026 16:37:46 +0800
Subject: [PATCH 16/19] Remove SLURM evidence scanner tests

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 .../tools/test_infra_dry_run_pipeline.py        | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index 78cc3114970f..6b261d642496 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -151,23 +151,6 @@ def test_sbatch_final_failure_is_classified_without_scanning_the_whole_step_log(
             "classifyFailure(pipeline, submitEx, InfraFailure.SLURM, retryContext)", sbatch
         )
 
-    def test_slurm_job_log_evidence_scan_is_bounded_without_script_approval(self) -> None:
-        scanner = _function_body(L0_TEST, "collectSlurmJobLogEvidence", "uploadResults")
-
-        self.assertIn(
-            'script: "timeout 300s " + Utils.sshUserCmd',
-            scanner,
-        )
-        self.assertIn("timeout: false,", scanner)
-        self.assertNotIn("isActualInterruption()", scanner)
-
-    def test_slurm_job_log_evidence_ignores_pytest_nodeids(self) -> None:
-        scanner = _function_body(L0_TEST, "collectSlurmJobLogEvidence", "uploadResults")
-
-        self.assertIn("""awk 'index($0, \".py::\") == 0'""", scanner)
-        self.assertIn("seen[key]++ == 0", scanner)
-        self.assertNotIn("!seen[key]++", scanner)
-
     def test_cbts_parser_preserves_coverage_preflight_fields(self) -> None:
         parser = _function_body(
             L0_PARENT, "_cbtsParseSelectionResult", "getOssComplianceFileChanged"

From 39954d6b1377e73890fd51ab766b46648b1ed863 Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:58:42 +0800
Subject: [PATCH 17/19] Restrict SLURM job log UCX classification

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 7254c1c411ac..08723107aedf 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -163,7 +163,6 @@ SLURM_INFRA_RETRY_MAX = 1
 // stdout/stderr in the SLURM job log. Keep the concrete terms and their retry
 // policy owned by FailureClassifier.
 SLURM_JOB_LOG_EVIDENCE_QUERY_IDS = [
-    "slurm-ucx-worker-address",
     "cuda-device-unavailable",
     "cuda-buffer-mapping",
     "nvidia-driver-unavailable",

From 2357ad18a009449c52bc17144a3d02bf83338958 Mon Sep 17 00:00:00 2001
From: Hanjing Tian <hanjingt@nvidia.com>
Date: Wed, 21 Oct 2026 14:38:09 +0800
Subject: [PATCH 18/19] [TRTLLMINF-384][fix] preserve terminal rerun infra
 evidence

Signed-off-by: Hanjing Tian <hanjingt@nvidia.com>
---
 jenkins/L0_Test.groovy | 50 +++++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 08723107aedf..e8b51127279c 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -471,11 +471,12 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String
     }
 }
 
-def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName, postTag="") {
+def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName, postTag="", Map retryContext=null) {
     // Run the isolated tests one by one to avoid any potential conflicts
     def isolateTestList = preprocessedLists.isolate
     def isolateTestLines = readFile(file: isolateTestList).readLines()
     def rerunFailed = false
+    def rerunInfraFailure = null
     def hasUnrerunFailure = false
 
     for (int i = 0; i < isolateTestLines.size(); i++) {
@@ -504,14 +505,15 @@ def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName
         } catch (InterruptedException e) {
             throw e
         } catch (Exception e) {
-            def isRerunFailed = rerunFailedTests(
-                stageName, llmSrc, isolateTestCmdLine, "results_isolated_${i}.xml", "isolated_${i}", postTag)
-            if (isRerunFailed) {
+            def rerunResult = rerunFailedTests(
+                pipeline, stageName, llmSrc, isolateTestCmdLine, "results_isolated_${i}.xml", "isolated_${i}", postTag, retryContext)
+            if (rerunResult.failed) {
                 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                     error "Isolated test ${i} (${isolateTestName}) failed after rerun attempt"
                 }
                 // Mark that at least one isolated test failed, but continue processing other tests
                 rerunFailed = true
+                rerunInfraFailure = rerunResult.infraFailure ?: rerunInfraFailure
             } else {
                 // Strip trailing " TIMEOUT (N)" / "TIMEOUT(N)" / " ISOLATION" turtle
                 // directives the same way generate_duration.py's _TURTLE_DIRECTIVE_RE does.
@@ -558,7 +560,11 @@ def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName
         }
     }
 
-    return [rerunFailed: rerunFailed, hasUnrerunFailure: hasUnrerunFailure]
+    return [
+        rerunFailed: rerunFailed,
+        rerunInfraFailure: rerunInfraFailure,
+        hasUnrerunFailure: hasUnrerunFailure,
+    ]
 }
 
 def getInfraDryRunPytestTargets(testListPath) {
@@ -1538,7 +1544,7 @@ def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
         // TODO: refactor the finallyRunner to reuse within slurm or nonslurm job.
         cacheErrorAndUploadResult(pipeline, stageName, {
             try {
-                runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations)
+                runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations, retryContext)
             } catch (InterruptedException e) {
                 throw e
             } catch (Exception e) {
@@ -4542,8 +4548,10 @@ def fileHasContent(path) {
     return fileExists(path) && readFile(path).trim()
 }
 
-// Return true means the test rerun also fails. Return false otherwise.
-def rerunFailedTests(stageName, llmSrc, testCmdLine, resultFileName="results.xml", testType="regular", postTag="") {
+// Return whether the rerun failed and any typed infra failure classified from
+// the rerun's originating shell step. The typed failure lets the outer retry
+// layer preserve bounded evidence after result/report generation finishes.
+def rerunFailedTests(pipeline, stageName, llmSrc, testCmdLine, resultFileName="results.xml", testType="regular", postTag="", Map retryContext=null) {
     if (!fileExists("${WORKSPACE}/${stageName}/${resultFileName}")) {
         // No results were ever flushed (e.g. the process was killed before any
         // test finished reporting). test_rerun.py treats a missing xml as "no
@@ -4589,6 +4597,7 @@ def rerunFailedTests(stageName, llmSrc, testCmdLine, resultFileName="results.xml
 
     // Rerun tests
     def isRerunFailed = false
+    def rerunInfraFailure = null
     for (times in [1, 2]) {
         def currentRerunTestList = "${rerunDir}/rerun_${times}.txt"
         if (!fileExists(currentRerunTestList)) {
@@ -4654,13 +4663,18 @@ def rerunFailedTests(stageName, llmSrc, testCmdLine, resultFileName="results.xml
                 echo "The ${testType} tests crashed when rerun attempt."
                 throw e
             }
+            String scope = retryContext?.scope ?: InfraFailure.K8S
+            def classified = classifyFailure(pipeline, e, scope, retryContext)
+            if (classified instanceof InfraFailure) {
+                rerunInfraFailure = classified
+            }
             echo "The ${testType} tests still failed after rerun attempt."
             isRerunFailed = true
         }
     }
 
     echo "isRerunFailed for ${testType}: ${isRerunFailed}"
-    return isRerunFailed
+    return [failed: isRerunFailed, infraFailure: rerunInfraFailure]
 }
 
 def generateRerunReport(stageName, llmSrc) {
@@ -5068,7 +5082,7 @@ def priorAttemptTags(String postTag) {
     return priors
 }
 
-def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false)
+def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map retryContext=null)
 {
     // Step 1: create LLM_ROOT dir and clean up the workspace
     def llmRootConfig = "${LLM_ROOT}${config}"
@@ -5229,6 +5243,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         def noRegularTests = false
         def noIsolateTests = false
         def rerunFailed = false
+        def rerunInfraFailure = null
         def hasUnrerunFailure = false
         def infraDryRun = isInfraDryRun()
         if (infraDryRun) {
@@ -5419,18 +5434,19 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                     } catch (InterruptedException e) {
                         throw e
                     } catch (Exception e) {
-                        def isRerunFailed = rerunFailedTests(
-                            stageName, llmSrc, pytestCommand, "results.xml", "regular", postTag)
+                        def rerunResult = rerunFailedTests(
+                            pipeline, stageName, llmSrc, pytestCommand, "results.xml", "regular", postTag, retryContext)
                         // Unconditional: a test can crash again during the rerun itself
                         // (isRerunFailed=true), and unfinished_test.txt is the only
                         // record of that crash. junit()'s "results*.xml" glob picks up
                         // results-timeout.xml directly, so this is what surfaces it.
                         def hadUnfinishedTests = generateTimeoutTestResultXml(pipeline, stageName)
-                        if (isRerunFailed) {
+                        if (rerunResult.failed) {
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                                 error "Regular tests failed after rerun attempt"
                             }
                             rerunFailed = true
+                            rerunInfraFailure = rerunResult.infraFailure ?: rerunInfraFailure
                         } else if (hadUnfinishedTests) {
                             // Rerun passed but the first run had a timeout: mark this
                             // stage FAILURE so "[${stageName}] Run Pytest" turns red,
@@ -5456,8 +5472,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                         stage ("[${stageName}] Run Pytest (Isolated)") {
                             echo "There are ${preprocessedLists.isolateCount} isolated tests to run"
                             def isolatedResult = runIsolatedTests(
-                                pipeline, preprocessedLists, pytestCommand, llmSrc, stageName, postTag)
+                                pipeline, preprocessedLists, pytestCommand, llmSrc, stageName, postTag, retryContext)
                             rerunFailed = isolatedResult.rerunFailed || rerunFailed
+                            rerunInfraFailure = isolatedResult.rerunInfraFailure ?: rerunInfraFailure
                             hasUnrerunFailure = isolatedResult.hasUnrerunFailure || hasUnrerunFailure
                         }
                     } else {
@@ -5536,6 +5553,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         // Unguarded error() on purpose -- catchError here would swallow the
         // throw that cacheErrorAndUploadResult relies on to fail the build.
         if (rerunFailed) {
+            if (rerunInfraFailure != null) {
+                throw rerunInfraFailure
+            }
             error "Some tests still failed after rerun attempts, please check the test report."
         }
 
@@ -5577,7 +5597,7 @@ def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG
             "PMIX_HOSTNAME=mpi-node0",
         ]
         withEnv(testEnv) {
-            runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations)
+            runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations, retryContext)
         }
     }, {
         if (testFilter[(DEBUG_MODE)]) {

From eb0879da4171bc6d2cb0f9ef6d8f3165b2fe033f Mon Sep 17 00:00:00 2001
From: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:29:48 +0800
Subject: [PATCH 19/19] [TRTLLMINF-384][fix] preserve no-rerun failure evidence

Signed-off-by: hanjingtian <312433810+hanjingtian@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                        | 17 +++++++++++++++++
 .../tools/test_infra_dry_run_pipeline.py      | 19 +++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index b60d7321a5df..17a0b363569c 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -477,6 +477,7 @@ def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName
     def isolateTestLines = readFile(file: isolateTestList).readLines()
     def rerunFailed = false
     def rerunInfraFailure = null
+    def unrerunInfraFailure = null
     def hasUnrerunFailure = false
 
     for (int i = 0; i < isolateTestLines.size(); i++) {
@@ -532,6 +533,11 @@ def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName
                         error "Isolated test ${i} (${isolateTestName}) terminated unexpectedly, please check the test report."
                     }
                 } else if (fileHasContent("${WORKSPACE}/${stageName}/rerun/isolated_${i}/rerun_0.txt")) {
+                    String scope = retryContext?.scope ?: InfraFailure.K8S
+                    def classified = classifyFailure(pipeline, e, scope, retryContext)
+                    if (classified instanceof InfraFailure) {
+                        unrerunInfraFailure = classified
+                    }
                     // Same duration/no-signature gap as the regular-test path: this
                     // finished but failed, and was never actually rerun, so
                     // results_isolated_${i}.xml still carries the original
@@ -563,6 +569,7 @@ def runIsolatedTests(pipeline, preprocessedLists, testCmdLine, llmSrc, stageName
     return [
         rerunFailed: rerunFailed,
         rerunInfraFailure: rerunInfraFailure,
+        unrerunInfraFailure: unrerunInfraFailure,
         hasUnrerunFailure: hasUnrerunFailure,
     ]
 }
@@ -5327,6 +5334,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         def noIsolateTests = false
         def rerunFailed = false
         def rerunInfraFailure = null
+        def unrerunInfraFailure = null
         def hasUnrerunFailure = false
         def infraDryRun = isInfraDryRun()
         if (infraDryRun) {
@@ -5537,6 +5545,11 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                                 error "Some tests terminated unexpectedly, please check the test report."
                             }
                         } else if (fileHasContent("${WORKSPACE}/${stageName}/rerun/regular/rerun_0.txt")) {
+                            String scope = retryContext?.scope ?: InfraFailure.K8S
+                            def classified = classifyFailure(pipeline, e, scope, retryContext)
+                            if (classified instanceof InfraFailure) {
+                                unrerunInfraFailure = classified
+                            }
                             // Failures that finished (not a timeout) but were never
                             // rerun because duration > 10 min and no known failure
                             // signature matched: results.xml still carries their
@@ -5557,6 +5570,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                                 pipeline, preprocessedLists, pytestCommand, llmSrc, stageName, postTag, retryContext)
                             rerunFailed = isolatedResult.rerunFailed || rerunFailed
                             rerunInfraFailure = isolatedResult.rerunInfraFailure ?: rerunInfraFailure
+                            unrerunInfraFailure = isolatedResult.unrerunInfraFailure ?: unrerunInfraFailure
                             hasUnrerunFailure = isolatedResult.hasUnrerunFailure || hasUnrerunFailure
                         }
                     } else {
@@ -5647,6 +5661,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         }
 
         if (hasUnrerunFailure) {
+            if (unrerunInfraFailure != null) {
+                throw unrerunInfraFailure
+            }
             error "Some tests failed and were not eligible for rerun (duration > 10 min, no matching failure signature), please check the test report."
         }
 
diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py
index 6b261d642496..1f21f303f2f6 100644
--- a/tests/unittest/tools/test_infra_dry_run_pipeline.py
+++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py
@@ -115,6 +115,25 @@ def test_failure_evidence_match_is_logged_before_caching(self) -> None:
             classifier.index("retryContext.failureEvidence = evidence"),
         )
 
+    def test_no_rerun_failure_preserves_initial_infra_classification(self) -> None:
+        isolated = _function_body(L0_TEST, "runIsolatedTests", "getInfraDryRunPytestTargets")
+        platform = _function_body(
+            L0_TEST, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform"
+        )
+        classify_initial_failure = "classifyFailure(pipeline, e, scope, retryContext)"
+
+        self.assertIn(classify_initial_failure, isolated)
+        self.assertIn("unrerunInfraFailure: unrerunInfraFailure", isolated)
+        self.assertIn(classify_initial_failure, platform)
+        self.assertIn(
+            "unrerunInfraFailure = isolatedResult.unrerunInfraFailure ?: unrerunInfraFailure",
+            platform,
+        )
+        self.assertLess(
+            platform.index("throw unrerunInfraFailure"),
+            platform.rindex('error "Some tests failed and were not eligible for rerun'),
+        )
+
     def test_post_actions_keep_interruptions_higher_priority_than_primary_failures(self) -> None:
         result_handler = _function_body(
             L0_TEST, "cacheErrorAndUploadResult", "createKubernetesPodConfig"