From d082a29bd4f43f83e3b6899605f8b0464b1bbce2 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 30 Sep 2026 10:43:00 +0200 Subject: [PATCH] test(e2e): Retry a hung SAM start in aws-serverless tests SAM occasionally hangs on its first start in CI, while a fresh start comes up within seconds. Healthy starts take about 20s, so wait at most 90s per attempt and restart SAM once. Also log SAM's output for failed attempts, remove the runtime containers a killed SAM leaves behind, and actually send SIGKILL when SAM ignores SIGTERM (`killed` is already true after the first signal). Co-Authored-By: Claude Opus 5.5 --- .../tests/lambda-fixtures.ts | 88 +++++++++++++++---- .../aws-serverless/tests/lambda-fixtures.ts | 88 +++++++++++++++---- 2 files changed, 140 insertions(+), 36 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts index 0c15ef39d04e..0d4d123f05c3 100644 --- a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts +++ b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/lambda-fixtures.ts @@ -2,12 +2,18 @@ import { test as base, expect } from '@playwright/test'; import { App } from 'aws-cdk-lib'; import { LocalLambdaStack, SAM_PORT, getHostIp } from '../src/stack'; import { writeFileSync } from 'node:fs'; -import { execSync, spawn } from 'node:child_process'; +import { type ChildProcess, execSync, spawn } from 'node:child_process'; import { LambdaClient } from '@aws-sdk/client-lambda'; const DOCKER_NETWORK_NAME = 'lambda-test-network'; const SAM_TEMPLATE_FILE = 'sam.template.yml'; +// A healthy SAM start takes about 20s on CI, but SAM occasionally hangs on its first start while a +// fresh one comes up in seconds, so give up early and retry instead of waiting out one long timeout. +const SAM_START_TIMEOUT_MS = 90_000; +const SAM_START_ATTEMPTS = 2; +const SAM_OUTPUT_MAX_CHARS = 20_000; + /** Major Node for SAM `--invoke-image`; default matches root `package.json` `volta.node` and `pull-sam-image.sh`. */ const DEFAULT_NODE_VERSION_MAJOR = '20'; @@ -55,34 +61,23 @@ export const test = base.extend<{ testEnvironment: LocalLambdaStack; lambdaClien console.log(`[testEnvironment fixture] Running SAM with args: ${args.join(' ')}`); - const samProcess = spawn('sam', args, { - stdio: process.env.DEBUG ? 'inherit' : 'ignore', - env: envForSamChild(), - }); + let samProcess: ChildProcess | undefined; try { - await LocalLambdaStack.waitForStack(); + samProcess = await startSam(args); await use(stack); } finally { console.log('[testEnvironment fixture] Tearing down AWS Lambda test infrastructure'); - samProcess.kill('SIGTERM'); - await new Promise(resolve => { - samProcess.once('exit', resolve); - setTimeout(() => { - if (!samProcess.killed) { - samProcess.kill('SIGKILL'); - } - resolve(void 0); - }, 5000); - }); - + if (samProcess) { + await stopSam(samProcess); + } removeDockerNetwork(); } }, // Own timeout so slow SAM stack startup does not eat into the first test's budget. - { scope: 'worker', auto: true, timeout: 240_000 }, + { scope: 'worker', auto: true, timeout: 300_000 }, ], lambdaClient: async ({}, use) => { const lambdaClient = new LambdaClient({ @@ -98,6 +93,54 @@ export const test = base.extend<{ testEnvironment: LocalLambdaStack; lambdaClien }, }); +async function startSam(args: string[]): Promise { + for (let attempt = 1; ; attempt++) { + let output = ''; + const samProcess = spawn('sam', args, { + stdio: process.env.DEBUG ? 'inherit' : ['ignore', 'pipe', 'pipe'], + env: envForSamChild(), + }); + const collectOutput = (chunk: Buffer): void => { + output = (output + chunk.toString()).slice(-SAM_OUTPUT_MAX_CHARS); + }; + samProcess.stdout?.on('data', collectOutput); + samProcess.stderr?.on('data', collectOutput); + + try { + await LocalLambdaStack.waitForStack(SAM_START_TIMEOUT_MS); + return samProcess; + } catch (error) { + console.warn(`[testEnvironment fixture] SAM output of failed start attempt ${attempt}:\n${output}`); + await stopSam(samProcess); + + if (attempt >= SAM_START_ATTEMPTS) { + throw error; + } + console.warn(`[testEnvironment fixture] Restarting SAM (attempt ${attempt + 1}/${SAM_START_ATTEMPTS})`); + } + } +} + +async function stopSam(samProcess: ChildProcess): Promise { + if (samProcess.exitCode === null && samProcess.signalCode === null) { + samProcess.kill('SIGTERM'); + await new Promise(resolve => { + const timer = setTimeout(() => { + samProcess.kill('SIGKILL'); + resolve(void 0); + }, 5000); + samProcess.once('exit', () => { + clearTimeout(timer); + resolve(void 0); + }); + }); + } + + // A SAM process killed mid-start leaves its runtime containers behind, which would keep the + // docker network in use and hold on to their ports. + removeNetworkContainers(); +} + /** Avoid forcing linux/amd64 on Apple Silicon when `DOCKER_DEFAULT_PLATFORM` is set globally. */ function envForSamChild(): NodeJS.ProcessEnv { const env = { ...process.env }; @@ -128,6 +171,15 @@ function createDockerNetwork() { } } +function removeNetworkContainers() { + const containerIds = execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' }) + .split('\n') + .filter(Boolean); + if (containerIds.length) { + execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'ignore' }); + } +} + function removeDockerNetwork() { try { execSync(`docker network rm ${DOCKER_NETWORK_NAME}`); diff --git a/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts b/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts index 0c15ef39d04e..0d4d123f05c3 100644 --- a/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts +++ b/dev-packages/e2e-tests/test-applications/aws-serverless/tests/lambda-fixtures.ts @@ -2,12 +2,18 @@ import { test as base, expect } from '@playwright/test'; import { App } from 'aws-cdk-lib'; import { LocalLambdaStack, SAM_PORT, getHostIp } from '../src/stack'; import { writeFileSync } from 'node:fs'; -import { execSync, spawn } from 'node:child_process'; +import { type ChildProcess, execSync, spawn } from 'node:child_process'; import { LambdaClient } from '@aws-sdk/client-lambda'; const DOCKER_NETWORK_NAME = 'lambda-test-network'; const SAM_TEMPLATE_FILE = 'sam.template.yml'; +// A healthy SAM start takes about 20s on CI, but SAM occasionally hangs on its first start while a +// fresh one comes up in seconds, so give up early and retry instead of waiting out one long timeout. +const SAM_START_TIMEOUT_MS = 90_000; +const SAM_START_ATTEMPTS = 2; +const SAM_OUTPUT_MAX_CHARS = 20_000; + /** Major Node for SAM `--invoke-image`; default matches root `package.json` `volta.node` and `pull-sam-image.sh`. */ const DEFAULT_NODE_VERSION_MAJOR = '20'; @@ -55,34 +61,23 @@ export const test = base.extend<{ testEnvironment: LocalLambdaStack; lambdaClien console.log(`[testEnvironment fixture] Running SAM with args: ${args.join(' ')}`); - const samProcess = spawn('sam', args, { - stdio: process.env.DEBUG ? 'inherit' : 'ignore', - env: envForSamChild(), - }); + let samProcess: ChildProcess | undefined; try { - await LocalLambdaStack.waitForStack(); + samProcess = await startSam(args); await use(stack); } finally { console.log('[testEnvironment fixture] Tearing down AWS Lambda test infrastructure'); - samProcess.kill('SIGTERM'); - await new Promise(resolve => { - samProcess.once('exit', resolve); - setTimeout(() => { - if (!samProcess.killed) { - samProcess.kill('SIGKILL'); - } - resolve(void 0); - }, 5000); - }); - + if (samProcess) { + await stopSam(samProcess); + } removeDockerNetwork(); } }, // Own timeout so slow SAM stack startup does not eat into the first test's budget. - { scope: 'worker', auto: true, timeout: 240_000 }, + { scope: 'worker', auto: true, timeout: 300_000 }, ], lambdaClient: async ({}, use) => { const lambdaClient = new LambdaClient({ @@ -98,6 +93,54 @@ export const test = base.extend<{ testEnvironment: LocalLambdaStack; lambdaClien }, }); +async function startSam(args: string[]): Promise { + for (let attempt = 1; ; attempt++) { + let output = ''; + const samProcess = spawn('sam', args, { + stdio: process.env.DEBUG ? 'inherit' : ['ignore', 'pipe', 'pipe'], + env: envForSamChild(), + }); + const collectOutput = (chunk: Buffer): void => { + output = (output + chunk.toString()).slice(-SAM_OUTPUT_MAX_CHARS); + }; + samProcess.stdout?.on('data', collectOutput); + samProcess.stderr?.on('data', collectOutput); + + try { + await LocalLambdaStack.waitForStack(SAM_START_TIMEOUT_MS); + return samProcess; + } catch (error) { + console.warn(`[testEnvironment fixture] SAM output of failed start attempt ${attempt}:\n${output}`); + await stopSam(samProcess); + + if (attempt >= SAM_START_ATTEMPTS) { + throw error; + } + console.warn(`[testEnvironment fixture] Restarting SAM (attempt ${attempt + 1}/${SAM_START_ATTEMPTS})`); + } + } +} + +async function stopSam(samProcess: ChildProcess): Promise { + if (samProcess.exitCode === null && samProcess.signalCode === null) { + samProcess.kill('SIGTERM'); + await new Promise(resolve => { + const timer = setTimeout(() => { + samProcess.kill('SIGKILL'); + resolve(void 0); + }, 5000); + samProcess.once('exit', () => { + clearTimeout(timer); + resolve(void 0); + }); + }); + } + + // A SAM process killed mid-start leaves its runtime containers behind, which would keep the + // docker network in use and hold on to their ports. + removeNetworkContainers(); +} + /** Avoid forcing linux/amd64 on Apple Silicon when `DOCKER_DEFAULT_PLATFORM` is set globally. */ function envForSamChild(): NodeJS.ProcessEnv { const env = { ...process.env }; @@ -128,6 +171,15 @@ function createDockerNetwork() { } } +function removeNetworkContainers() { + const containerIds = execSync(`docker ps -aq --filter network=${DOCKER_NETWORK_NAME}`, { encoding: 'utf-8' }) + .split('\n') + .filter(Boolean); + if (containerIds.length) { + execSync(`docker rm -f ${containerIds.join(' ')}`, { stdio: 'ignore' }); + } +} + function removeDockerNetwork() { try { execSync(`docker network rm ${DOCKER_NETWORK_NAME}`);