Retry testkit fixture execs on the transient ETXTBSY fork race

internal/testkit/exec_fixture_dir.go

@@ -4,15 +4,75 @@
 package testkit
 
 import (
+	"errors"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
+	"syscall"
 	"testing"
+	"time"
 )
 
+// execRetryETXTBSY runs newCmd's freshly built *exec.Cmd and returns its
+// combined output, retrying up to 20 times with a short capped backoff when
+// the fork/exec fails with ETXTBSY: the transient Linux race (golang/go#22315)
+// where a concurrent goroutine's dup'd fd briefly holds a just-written test
+// fixture script open for write, even though that script's own writer closed
+// it before making it executable. exec.Cmd cannot be re-run, so newCmd builds
+// a fresh one each attempt. Any other error - including a genuinely noexec
+// fixture directory, which surfaces as EACCES or ENOEXEC rather than ETXTBSY -
+// returns immediately without retrying.
+func execRetryETXTBSY(newCmd func() *exec.Cmd) ([]byte, error) {
+	return execRetryOn(newCmd, func(out []byte, err error) bool {
+		return errors.Is(err, syscall.ETXTBSY)
+	})
+}
+
+// execRetryETXTBSYOrNestedBusy is execRetryETXTBSY plus one more retryable
+// shape: newCmd's target is a shell script that itself execs a second
+// test-written fixture (e.g. run-hosted-controls-audit.sh launching the
+// verifier fixture it was just handed). The outer process starts fine - Go
+// never sees ETXTBSY directly - but the same golang/go#22315 race can hit the
+// script's own inner execve, and bash reports that as exit 126 with "text
+// file busy" on stderr rather than propagating an errno Go can unwrap. Only
+// that exact message is treated as retryable, so an unrelated exit 126 (a
+// missing interpreter, a real permission error) still fails immediately.
+func execRetryETXTBSYOrNestedBusy(newCmd func() *exec.Cmd) ([]byte, error) {
+	return execRetryOn(newCmd, func(out []byte, err error) bool {
+		if errors.Is(err, syscall.ETXTBSY) {
+			return true
+		}
+		var exitErr *exec.ExitError
+		return errors.As(err, &exitErr) && exitErr.ExitCode() == 126 &&
+			strings.Contains(strings.ToLower(string(out)), "text file busy")
+	})
+}
+
+// execRetryOn is the shared retry loop behind execRetryETXTBSY and
+// execRetryETXTBSYOrNestedBusy: up to 20 attempts, backoff starting at 5ms and
+// doubling up to a 100ms cap, retrying only while retryable reports true.
+func execRetryOn(newCmd func() *exec.Cmd, retryable func(out []byte, err error) bool) ([]byte, error) {
+	const maxBackoff = 100 * time.Millisecond
+	backoff := 5 * time.Millisecond
+	for attempt := 0; ; attempt++ {
+		out, err := newCmd().CombinedOutput()
+		if !retryable(out, err) || attempt >= 19 {
+			return out, err
+		}
+		time.Sleep(backoff)
+		// Double the backoff for the next attempt, capped at maxBackoff: the
+		// cap must apply after doubling, or the sleep just before it hits the
+		// cap overshoots to double the cap.
+		backoff *= 2
+		if backoff > maxBackoff {
+			backoff = maxBackoff
+		}
+	}
+}
+
 // shellSafePath matches a candidate base that is safe to splice as literal
 // text into generated shell script source (a double-quoted assignment, a
 // redirect target) as well as safe for a tool's own naive space-splitting.
@@ -111,7 +171,7 @@ func ExecFixtureDir(tb testing.TB) string {
 			tried = append(tried, c.name+" ("+c.path+"): not writable: "+err.Error())
 			continue
 		}
-		if err := exec.Command(probe).Run(); err != nil {
+		if _, err := execRetryETXTBSY(func() *exec.Cmd { return exec.Command(probe) }); err != nil {
 			_ = os.RemoveAll(dir)
 			tried = append(tried, c.name+" ("+c.path+"): noexec: "+err.Error())
 			continue

internal/testkit/exec_fixture_dir_test.go

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 Omkhar Arasaratnam
+
+package testkit
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+// A non-ETXTBSY error (here, a missing executable) must return on the first
+// attempt: retrying it would just be a 20x slower version of the same failure.
+func TestExecRetryETXTBSYReturnsImmediatelyOnOtherErrors(t *testing.T) {
+	calls := 0
+	_, err := execRetryETXTBSY(func() *exec.Cmd {
+		calls++
+		return exec.Command(filepath.Join(t.TempDir(), "does-not-exist"))
+	})
+	if err == nil {
+		t.Fatal("expected an error for a missing executable")
+	}
+	if calls != 1 {
+		t.Fatalf("newCmd calls = %d, want 1 (no retry on a non-ETXTBSY error)", calls)
+	}
+}
+
+// Reproduces the golang/go#22315 shape directly and deterministically: a file
+// held open for write is ETXTBSY to exec on Linux, with no timing dependence
+// (the kernel checks this unconditionally, not as a race), so gating the
+// close on newCmd's own call count - rather than a sleep - guarantees the
+// first attempt sees ETXTBSY and the second sees a closed, executable file.
+// A goroutine + sleep would not: under a loaded scheduler the closer could
+// run before the first attempt ever executes, and the test would pass
+// without ever exercising the retry path.
+func TestExecRetryETXTBSYRetriesThroughTheTransientRace(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("ETXTBSY on a file open for write is Linux-specific (golang/go#22315)")
+	}
+	// ExecFixtureDir, not t.TempDir(): TMPDIR can be a noexec mount (a layout
+	// this package's own hostile-env axis exercises elsewhere), which would
+	// fail this exec with EACCES instead of proving ETXTBSY recovery.
+	path := filepath.Join(ExecFixtureDir(t), "busy.sh")
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = f.Close() }() // safety net if attempts never reaches 2 below
+	if _, err := f.WriteString("#!/bin/sh\nexit 0\n"); err != nil {
+		t.Fatal(err)
+	}
+	attempts := 0
+	_, err = execRetryETXTBSY(func() *exec.Cmd {
+		attempts++
+		// Close only once the first attempt's exec has already been issued
+		// against this exact newCmd call - the loop always calls newCmd
+		// immediately before running it, so by the second call the first
+		// attempt is guaranteed to have already seen the file open.
+		if attempts == 2 {
+			if closeErr := f.Close(); closeErr != nil {
+				t.Fatal(closeErr)
+			}
+		}
+		return exec.Command(path)
+	})
+	if err != nil {
+		t.Fatalf("execRetryETXTBSY did not recover from the transient ETXTBSY race: %v", err)
+	}
+	if attempts < 2 {
+		t.Fatalf("attempts = %d, want >1: the file was never held open through a real attempt, so this proves nothing about the retry path", attempts)
+	}
+}
+
+// execRetryETXTBSYOrNestedBusy's extra shape (exit 126 with "text file busy"
+// on stderr, the way bash reports a nested exec hitting the same
+// golang/go#22315 race) is deterministic to reproduce without a real kernel
+// race: a fixture script that reports that exact shape on its first run and
+// succeeds afterward.
+func TestExecRetryETXTBSYOrNestedBusyRetriesOnlyOnTheBusyMessage(t *testing.T) {
+	dir := ExecFixtureDir(t)
+
+	busyThenOK := filepath.Join(dir, "busy-then-ok.sh")
+	writeExecFile(t, busyThenOK, []byte(`#!/bin/sh
+if [ ! -e "$1" ]; then
+  : >"$1"
+  echo "bash: ./verify-github-hosted-controls.sh: Text file busy" >&2
+  exit 126
+fi
+exit 0
+`), 0o755)
+	marker := filepath.Join(t.TempDir(), "ran-once")
+	if _, err := execRetryETXTBSYOrNestedBusy(func() *exec.Cmd { return exec.Command(busyThenOK, marker) }); err != nil {
+		t.Fatalf("execRetryETXTBSYOrNestedBusy did not retry past the nested-busy shape: %v", err)
+	}
+
+	unrelated126 := filepath.Join(dir, "unrelated-126.sh")
+	writeExecFile(t, unrelated126, []byte("#!/bin/sh\necho permission denied >&2\nexit 126\n"), 0o755)
+	calls := 0
+	_, err := execRetryETXTBSYOrNestedBusy(func() *exec.Cmd {
+		calls++
+		return exec.Command(unrelated126)
+	})
+	var exitErr *exec.ExitError
+	if !errors.As(err, &exitErr) || exitErr.ExitCode() != 126 {
+		t.Fatalf("expected an unretried exit 126, got %v", err)
+	}
+	if calls != 1 {
+		t.Fatalf("calls = %d, want 1: an exit 126 without the busy message must not be retried", calls)
+	}
+}

internal/testkit/hosted_controls_token_isolation_test.go

@@ -22,9 +22,15 @@ func TestHostedControlsRunnerRelaysTokenOnlyThroughStdin(t *testing.T) {
 	stdinPath := filepath.Join(root, "stdin")
 	argsPath := filepath.Join(root, "args")
 	envPath := filepath.Join(root, "environment")
-	cmd := exec.Command(filepath.Join(scriptsDir, "run-hosted-controls-audit.sh"), "owner/repo")
-	cmd.Env = hostedControlProbeEnvironment(stdinPath, argsPath, envPath, "37")
-	output, err := cmd.CombinedOutput()
+	// runner is a fixture this test just wrote, and it in turn execs the
+	// verifier fixture above, so a concurrent sibling test's forkExec can
+	// transiently ETXTBSY either the outer exec or that inner one
+	// (golang/go#22315); retry both shapes.
+	output, err := execRetryETXTBSYOrNestedBusy(func() *exec.Cmd {
+		cmd := exec.Command(filepath.Join(scriptsDir, "run-hosted-controls-audit.sh"), "owner/repo")
+		cmd.Env = hostedControlProbeEnvironment(stdinPath, argsPath, envPath, "37")
+		return cmd
+	})
 	requireExitStatus(t, err, 37, output)
 	requireHostedControlProbe(t, stdinPath, argsPath, envPath, "dummy-token\n")
 }
@@ -37,9 +43,15 @@ func TestHostedControlsRunnerRequiresNamespacedTokenInGitHubActions(t *testing.T
 	writeCanonicalFixture(t, filepath.Join(scriptsDir, "verify-github-hosted-controls.sh"), []byte(hostedControlChildProbe), 0o755)
 	environment := hostedControlProbeEnvironment(filepath.Join(root, "stdin"), filepath.Join(root, "args"), filepath.Join(root, "environment"), "0")
 	environment = environmentWithout(environment, "WORKCELL_HOSTED_CONTROLS_TOKEN")
-	cmd := exec.Command(runner, "owner/repo")
-	cmd.Env = environment
-	output, err := cmd.CombinedOutput()
+	// runner is a fixture this test just wrote, and it in turn execs the
+	// verifier fixture above, so a concurrent sibling test's forkExec can
+	// transiently ETXTBSY either the outer exec or that inner one
+	// (golang/go#22315); retry both shapes.
+	output, err := execRetryETXTBSYOrNestedBusy(func() *exec.Cmd {
+		cmd := exec.Command(runner, "owner/repo")
+		cmd.Env = environment
+		return cmd
+	})
 	requireExitStatus(t, err, 1, output)
 	if !strings.Contains(string(output), "requires WORKCELL_HOSTED_CONTROLS_TOKEN") {
 		t.Fatalf("runner output = %q, want namespaced-token requirement", output)
@@ -57,9 +69,15 @@ func TestHostedControlsRunnerUsesGitHubTokenOutsideActions(t *testing.T) {
 	envPath := filepath.Join(root, "environment")
 	environment := hostedControlProbeEnvironment(stdinPath, argsPath, envPath, "31")
 	environment = environmentWithout(environment, "GITHUB_ACTIONS")
-	cmd := exec.Command(runner, "owner/repo")
-	cmd.Env = environment
-	output, err := cmd.CombinedOutput()
+	// runner is a fixture this test just wrote, and it in turn execs the
+	// verifier fixture above, so a concurrent sibling test's forkExec can
+	// transiently ETXTBSY either the outer exec or that inner one
+	// (golang/go#22315); retry both shapes.
+	output, err := execRetryETXTBSYOrNestedBusy(func() *exec.Cmd {
+		cmd := exec.Command(runner, "owner/repo")
+		cmd.Env = environment
+		return cmd
+	})
 	requireExitStatus(t, err, 31, output)
 	requireHostedControlProbe(t, stdinPath, argsPath, envPath, "wrong-gh-token\n")
 }
@@ -83,9 +101,13 @@ func TestHostedControlsVerifierRelaysAmbientTokenOnce(t *testing.T) {
 	stdinPath := filepath.Join(root, "stdin")
 	argsPath := filepath.Join(root, "args")
 	envPath := filepath.Join(root, "environment")
-	cmd := exec.Command(verifier, "owner/repo")
-	cmd.Env = hostedControlProbeEnvironment(stdinPath, argsPath, envPath, "29")
-	output, err := cmd.CombinedOutput()
+	// verifier is a fixture this test just wrote, so a concurrent sibling test's
+	// forkExec can transiently ETXTBSY it (golang/go#22315); retry that alone.
+	output, err := execRetryETXTBSY(func() *exec.Cmd {
+		cmd := exec.Command(verifier, "owner/repo")
+		cmd.Env = hostedControlProbeEnvironment(stdinPath, argsPath, envPath, "29")
+		return cmd
+	})
 	requireExitStatus(t, err, 29, output)
 	requireHostedControlProbe(t, stdinPath, argsPath, envPath, "wrong-gh-token\n")
 }
@@ -126,13 +148,17 @@ func TestHostedControlsVerifierScopesDummyTokenToGitHubCommands(t *testing.T) {
 	root := t.TempDir()
 	prepareHostedControlCommandGraphFixture(t, root)
 	verifier := filepath.Join(root, "scripts", "verify-github-hosted-controls.sh")
-	cmd := exec.Command(verifier, "--token-stdin", "owner/repo")
-	cmd.Env = append(canonicalBuildEnv(nil),
-		"WORKCELL_TEST_LOG_DIR="+filepath.Join(root, "logs"),
-		"WORKCELL_TEST_CITOOLS_TEMPLATE="+filepath.Join(root, "citools-template"),
-	)
-	cmd.Stdin = strings.NewReader("dummy-token\n")
-	output, err := cmd.CombinedOutput()
+	// verifier is a fixture this test just wrote, so a concurrent sibling test's
+	// forkExec can transiently ETXTBSY it (golang/go#22315); retry that alone.
+	output, err := execRetryETXTBSY(func() *exec.Cmd {
+		cmd := exec.Command(verifier, "--token-stdin", "owner/repo")
+		cmd.Env = append(canonicalBuildEnv(nil),
+			"WORKCELL_TEST_LOG_DIR="+filepath.Join(root, "logs"),
+			"WORKCELL_TEST_CITOOLS_TEMPLATE="+filepath.Join(root, "citools-template"),
+		)
+		cmd.Stdin = strings.NewReader("dummy-token\n")
+		return cmd
+	})
 	if err != nil {
 		t.Fatalf("command graph fixture: %v: %s", err, output)
 	}

internal/testkit/release_outputs_verify_test.go

@@ -313,9 +313,13 @@ func runVerifyDriver(t *testing.T, driver string, args, env []string) (int, stri
 			full = append(full, name+"=")
 		}
 	}
-	cmd := exec.Command(driver, args...)
-	cmd.Env = full
-	out, err := cmd.CombinedOutput()
+	// The driver is a fixture the test just wrote, so a concurrent sibling test's
+	// forkExec can transiently ETXTBSY it (golang/go#22315); retry that alone.
+	out, err := execRetryETXTBSY(func() *exec.Cmd {
+		cmd := exec.Command(driver, args...)
+		cmd.Env = full
+		return cmd
+	})
 	if err == nil {
 		return 0, string(out)
 	}
@@ -1119,17 +1123,21 @@ func runReleaseOutputsInventoryGuard(t *testing.T, dir string) (int, string) {
 	driver := writeExecutable(t, t.TempDir(), "release-outputs-inventory-driver.sh", script)
 
 	digest := strings.Repeat("c", 40)
-	cmd := exec.Command(driver,
-		"--assets-dir", dir,
-		"--repo", "omkhar/workcell",
-		"--tag", "v1.2.3",
-		"--image-repository", "ghcr.io/omkhar/workcell",
-		"--source-digest", digest,
-		"--workflow-digest", digest,
-	)
-	// Bash startup files are cleared so caller state cannot reach the verifier.
-	cmd.Env = []string{"BASH_ENV=", "ENV="}
-	out, err := cmd.CombinedOutput()
+	// The driver is a fixture the test just wrote, so a concurrent sibling test's
+	// forkExec can transiently ETXTBSY it (golang/go#22315); retry that alone.
+	out, err := execRetryETXTBSY(func() *exec.Cmd {
+		cmd := exec.Command(driver,
+			"--assets-dir", dir,
+			"--repo", "omkhar/workcell",
+			"--tag", "v1.2.3",
+			"--image-repository", "ghcr.io/omkhar/workcell",
+			"--source-digest", digest,
+			"--workflow-digest", digest,
+		)
+		// Bash startup files are cleared so caller state cannot reach the verifier.
+		cmd.Env = []string{"BASH_ENV=", "ENV="}
+		return cmd
+	})
 	if err == nil {
 		return 0, string(out)
 	}