diff --git a/scripts/test/cloud-vitest-wrapper.test.sh b/scripts/test/cloud-vitest-wrapper.test.sh index 5f8f2ff70..e7c700195 100755 --- a/scripts/test/cloud-vitest-wrapper.test.sh +++ b/scripts/test/cloud-vitest-wrapper.test.sh @@ -139,6 +139,39 @@ else FAILURES=$((FAILURES + 1)) fi +# Check 10: transient execution-status query failures are retried. +# Regression: a single flaky `gcloud run jobs executions describe` right after +# `execute --wait` completed must not fail an otherwise-green run (observed +# live 2026-08-18: execution succeeded on all shards, wrapper exited 1 because +# the status query errored once). +cat > "$FAKE_GCLOUD_DIR/gcloud" << 'FAKE2' +#!/usr/bin/env bash +echo "FAKE_GCLOUD: $@" >> "${FAKE_GCLOUD_LOG:-/dev/null}" +if [[ "$*" == *"artifacts docker images describe"* ]] || [[ "$*" == *"artifacts repositories describe"* ]] || [[ "$*" == *"builds submit"* ]]; then exit 0; fi +if [[ "$*" == *"auth print-access-token"* ]]; then echo "fake-token"; exit 0; fi +if [[ "$*" == *"info"* ]]; then echo "/usr/lib/google-cloud-sdk"; exit 0; fi +if [[ "$*" == *"logs read"* ]]; then echo "Test Files 1 passed (1)"; exit 0; fi +if [[ "$*" == *"executions describe"* ]]; then + if [[ "$*" == *"succeededCount"* ]]; then + CC=$(cat "${FAKE_GCLOUD_LOG}.desccount" 2>/dev/null || echo 0); CC=$((CC+1)); echo "$CC" > "${FAKE_GCLOUD_LOG}.desccount" + if [ "$CC" -le 2 ]; then exit 1; fi + echo "1" # one succeeded shard + else + echo "0" # zero failed shards + fi + exit 0 +fi +if [[ "$*" == *"executions list"* ]]; then echo "test-execution-1"; exit 0; fi +if [[ "$*" == *"run jobs execute"* ]]; then echo "Execution test-execution-1"; exit 0; fi +if [[ "$*" == *"run jobs"* ]]; then exit 0; fi +exit 0 +FAKE2 +chmod +x "$FAKE_GCLOUD_DIR/gcloud" +rm -f "${FAKE_GCLOUD_LOG}.desccount" +rm -f "$FAKE_GCLOUD_LOG"; touch "$FAKE_GCLOUD_LOG" +check "transient describe failures retried (2 failures, then success)" bash -c "bash '$SCRIPT' run --cloud --config=default >/dev/null 2>&1" +check "describe was retried (>=3 describe calls logged)" bash -c "[ \$(grep -c 'executions describe' '$FAKE_GCLOUD_LOG') -ge 3 ]" + # Cleanup rm -rf "$FAKE_GCLOUD_DIR" diff --git a/scripts/vitest-cloud.sh b/scripts/vitest-cloud.sh index f1a6e5859..73f8072b7 100755 --- a/scripts/vitest-cloud.sh +++ b/scripts/vitest-cloud.sh @@ -423,10 +423,15 @@ cmd_run() { exit 1 fi - # Fetch logs + # Fetch logs (one short retry: right after --wait completes, log reads can + # transiently return empty; observed live 2026-08-18). echo "[vitest-cloud] Fetching logs..." local log_output log_output=$(gcloud beta run jobs executions logs read $(gcloud_flags) "$execution_id" 2>/dev/null || true) + if [ -z "$log_output" ]; then + sleep 3 + log_output=$(gcloud beta run jobs executions logs read $(gcloud_flags) "$execution_id" 2>/dev/null || true) + fi # Print full log output from ALL shards. echo "$log_output" @@ -437,23 +442,33 @@ cmd_run() { echo "$log_output" | grep -E '(\[vitest-entrypoint\]|Test Files|Tests )' || true # Check execution status — propagate query errors instead of normalizing to 0. + # Transient describe failures right after `execute --wait` returns are a real + # flake class (observed live 2026-08-18: execution succeeded on all 4 shards + # while a single describe errored, failing the wrapper); retry briefly before + # declaring the run failed. + query_count() { + local field="$1" val attempt + for attempt in 1 2 3 4 5; do + if val=$(gcloud run jobs executions describe $(gcloud_flags) "$execution_id" \ + --format="value($field)" 2>/dev/null); then + echo "${val:-0}" + return 0 + fi + sleep 3 + done + return 1 + } local succeeded local failed - if ! succeeded=$(gcloud run jobs executions describe $(gcloud_flags) "$execution_id" \ - --format="value(status.succeededCount)" 2>/dev/null); then + if ! succeeded=$(query_count status.succeededCount); then echo "[vitest-cloud] ERROR: failed to query execution status" exit 1 fi - if ! failed=$(gcloud run jobs executions describe $(gcloud_flags) "$execution_id" \ - --format="value(status.failedCount)" 2>/dev/null); then + if ! failed=$(query_count status.failedCount); then echo "[vitest-cloud] ERROR: failed to query execution status" exit 1 fi - # Normalize empty/null to 0 - succeeded="${succeeded:-0}" - failed="${failed:-0}" - echo "" echo "[vitest-cloud] Succeeded tasks: $succeeded" echo "[vitest-cloud] Failed tasks: $failed"