Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions scripts/test/cloud-vitest-wrapper.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,39 @@ else
FAILURES=$((FAILURES + 1))
fi

# Check 10: transient execution-status query failures are retried.
# Regression: a single flaky `gcloud run jobs executions describe` right after
# `execute --wait` completed must not fail an otherwise-green run (observed
# live 2026-08-18: execution succeeded on all shards, wrapper exited 1 because
# the status query errored once).
cat > "$FAKE_GCLOUD_DIR/gcloud" << 'FAKE2'
#!/usr/bin/env bash
echo "FAKE_GCLOUD: $@" >> "${FAKE_GCLOUD_LOG:-/dev/null}"
if [[ "$*" == *"artifacts docker images describe"* ]] || [[ "$*" == *"artifacts repositories describe"* ]] || [[ "$*" == *"builds submit"* ]]; then exit 0; fi
if [[ "$*" == *"auth print-access-token"* ]]; then echo "fake-token"; exit 0; fi
if [[ "$*" == *"info"* ]]; then echo "/usr/lib/google-cloud-sdk"; exit 0; fi
if [[ "$*" == *"logs read"* ]]; then echo "Test Files 1 passed (1)"; exit 0; fi
if [[ "$*" == *"executions describe"* ]]; then
if [[ "$*" == *"succeededCount"* ]]; then
CC=$(cat "${FAKE_GCLOUD_LOG}.desccount" 2>/dev/null || echo 0); CC=$((CC+1)); echo "$CC" > "${FAKE_GCLOUD_LOG}.desccount"
if [ "$CC" -le 2 ]; then exit 1; fi
echo "1" # one succeeded shard
else
echo "0" # zero failed shards
fi
exit 0
fi
if [[ "$*" == *"executions list"* ]]; then echo "test-execution-1"; exit 0; fi
if [[ "$*" == *"run jobs execute"* ]]; then echo "Execution test-execution-1"; exit 0; fi
if [[ "$*" == *"run jobs"* ]]; then exit 0; fi
exit 0
FAKE2
chmod +x "$FAKE_GCLOUD_DIR/gcloud"
rm -f "${FAKE_GCLOUD_LOG}.desccount"
rm -f "$FAKE_GCLOUD_LOG"; touch "$FAKE_GCLOUD_LOG"
check "transient describe failures retried (2 failures, then success)" bash -c "bash '$SCRIPT' run --cloud --config=default >/dev/null 2>&1"
check "describe was retried (>=3 describe calls logged)" bash -c "[ \$(grep -c 'executions describe' '$FAKE_GCLOUD_LOG') -ge 3 ]"

# Cleanup
rm -rf "$FAKE_GCLOUD_DIR"

Expand Down
33 changes: 24 additions & 9 deletions scripts/vitest-cloud.sh
Original file line number Diff line number Diff line change
Expand Up @@ -423,10 +423,15 @@ cmd_run() {
exit 1
fi

# Fetch logs
# Fetch logs (one short retry: right after --wait completes, log reads can
# transiently return empty; observed live 2026-08-18).
echo "[vitest-cloud] Fetching logs..."
local log_output
log_output=$(gcloud beta run jobs executions logs read $(gcloud_flags) "$execution_id" 2>/dev/null || true)
if [ -z "$log_output" ]; then
sleep 3
log_output=$(gcloud beta run jobs executions logs read $(gcloud_flags) "$execution_id" 2>/dev/null || true)
fi

# Print full log output from ALL shards.
echo "$log_output"
Expand All @@ -437,23 +442,33 @@ cmd_run() {
echo "$log_output" | grep -E '(\[vitest-entrypoint\]|Test Files|Tests )' || true

# Check execution status — propagate query errors instead of normalizing to 0.
# Transient describe failures right after `execute --wait` returns are a real
# flake class (observed live 2026-08-18: execution succeeded on all 4 shards
# while a single describe errored, failing the wrapper); retry briefly before
# declaring the run failed.
query_count() {
local field="$1" val attempt
for attempt in 1 2 3 4 5; do
if val=$(gcloud run jobs executions describe $(gcloud_flags) "$execution_id" \
--format="value($field)" 2>/dev/null); then
echo "${val:-0}"
return 0
fi
sleep 3
done
return 1
}
local succeeded
local failed
if ! succeeded=$(gcloud run jobs executions describe $(gcloud_flags) "$execution_id" \
--format="value(status.succeededCount)" 2>/dev/null); then
if ! succeeded=$(query_count status.succeededCount); then
echo "[vitest-cloud] ERROR: failed to query execution status"
exit 1
fi
if ! failed=$(gcloud run jobs executions describe $(gcloud_flags) "$execution_id" \
--format="value(status.failedCount)" 2>/dev/null); then
if ! failed=$(query_count status.failedCount); then
echo "[vitest-cloud] ERROR: failed to query execution status"
exit 1
fi

# Normalize empty/null to 0
succeeded="${succeeded:-0}"
failed="${failed:-0}"

echo ""
echo "[vitest-cloud] Succeeded tasks: $succeeded"
echo "[vitest-cloud] Failed tasks: $failed"
Expand Down
Loading