From 69d4126427686cde904e94188fb50f00d8ea913f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 17:28:00 -0700 Subject: [PATCH 1/3] fix(e2e): re-signal session stragglers on a monotonic clock and report HTTP E2E failures once, by route - stop-session.sh times its grace period from /proc/uptime, which is monotonic and free of the locale's decimal separator, re-signals every 0.1s so a process that joins the session after the first signal is stopped too, and explains why zombies don't count - version compare: a request whose body times out names its route like a request that never answered - desktop inbox: callers that join an executor pull in flight share it, so a failed pull is logged once --- .github/scripts/stop-session.sh | 36 +++++++++++-------- apps/sim/scripts/test-desktop-inbox-e2e.ts | 21 +++++------ .../test-workflow-version-compare-e2e.ts | 16 ++++++--- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/.github/scripts/stop-session.sh b/.github/scripts/stop-session.sh index 1dc62d1dab4..ab5938fd671 100755 --- a/.github/scripts/stop-session.sh +++ b/.github/scripts/stop-session.sh @@ -5,8 +5,11 @@ # # `next dev` runs its server and workers as child processes. Waiting on `next dev` alone returns # while those can still be running, and the next app in the job reuses the same .next directory. -# Every member of the session is signalled, including one that moved to its own process group. -# The leader stays a zombie until the shell that started it waits on it, so zombies don't count. +# Every member of the session is signalled, including one that moved to its own process group, +# and one that started after the first signal. +# +# Zombies don't count as running: they have exited and only await reaping. The leader is one +# until the shell that started it waits on it, which that shell does after this returns. set -u leader=$1 @@ -30,33 +33,38 @@ leader_running() { ps -p "$leader" -o stat= 2>/dev/null | grep -qv '^Z' } -now_us() { - echo "${EPOCHREALTIME/./}" +# Centiseconds since boot: monotonic, and independent of the locale's decimal separator. +now_cs() { + local uptime + read -r uptime _ < /proc/uptime + echo "${uptime/./}" } -# Waits while the given check holds, until the shared deadline. Succeeds once it stops holding. -wait_while() { - while (($(now_us) < deadline)); do +# Re-sends the signal every 0.1s while the check holds, until the shared deadline. +# Succeeds once the check stops holding. +signal_while() { + local signal=$1 + shift + while ((10#$(now_cs) < deadline)); do "$@" || return 0 + signal_session "$signal" sleep 0.1 done ! "$@" } -deadline=$(($(now_us) + grace_seconds * 1000000)) -signal_session TERM -wait_while leader_running +deadline=$((10#$(now_cs) + grace_seconds * 100)) +signal_while TERM leader_running if ! leader_running && running; then echo "Processes from session $leader outlived its leader:" ps -s "$leader" -o pid,stat,etimes,args 2>/dev/null | cut -c1-200 || true fi -wait_while running && exit 0 +signal_while TERM running && exit 0 echo "::warning::Processes from session $leader were still running ${grace_seconds}s after SIGTERM:" ps -s "$leader" -o pid,stat,etimes,args 2>/dev/null | cut -c1-200 || true -deadline=$(($(now_us) + grace_seconds * 1000000)) -signal_session KILL -wait_while running && exit 0 +deadline=$((10#$(now_cs) + grace_seconds * 100)) +signal_while KILL running && exit 0 echo "::error::Processes from session $leader survived SIGKILL for ${grace_seconds}s." exit 1 diff --git a/apps/sim/scripts/test-desktop-inbox-e2e.ts b/apps/sim/scripts/test-desktop-inbox-e2e.ts index 32c296e7424..02d0246a780 100644 --- a/apps/sim/scripts/test-desktop-inbox-e2e.ts +++ b/apps/sim/scripts/test-desktop-inbox-e2e.ts @@ -350,22 +350,19 @@ function startExecutor(desktop: Desktop, doorbell: ReturnType { if (inFlight) { rerun = true - return inFlight + return } - inFlight = drain() - return inFlight + inFlight = drain().catch((error) => + logger.error('executor pull failed', { error: getErrorMessage(error) }) + ) } - const offDoorbell = doorbell.onEvent( - () => void pull().catch((error) => logger.error('pull failed', error)) - ) - const timer = setInterval( - () => void pull().catch((error) => logger.error('reconcile failed', error)), - RECONCILE_MS - ) - void pull().catch((error) => logger.error('pull failed', error)) + const offDoorbell = doorbell.onEvent(pull) + const timer = setInterval(pull, RECONCILE_MS) + pull() return { ran, completed, @@ -377,7 +374,7 @@ function startExecutor(desktop: Desktop, doorbell: ReturnType {}) + await inFlight }, } } diff --git a/apps/sim/scripts/test-workflow-version-compare-e2e.ts b/apps/sim/scripts/test-workflow-version-compare-e2e.ts index a59b7890e0e..da3accc1730 100644 --- a/apps/sim/scripts/test-workflow-version-compare-e2e.ts +++ b/apps/sim/scripts/test-workflow-version-compare-e2e.ts @@ -356,10 +356,18 @@ async function get(path: string, auth: { key?: string; session?: string } = {}, assert.match(response.headers.get('content-type') ?? '', /application\/json/) if (path.startsWith('/api/v2/')) assert.equal(response.headers.get('cache-control'), 'private, no-store') - const text = await readResponseTextWithLimit(response, { - maxBytes: MAX_RESPONSE_BYTES, - label: 'Version comparison E2E response', - }) + let text: string + try { + text = await readResponseTextWithLimit(response, { + maxBytes: MAX_RESPONSE_BYTES, + label: 'Version comparison E2E response', + }) + } catch (error) { + /** `get` passes no signal of its own, so an abort here is the request timeout. */ + if (!(error instanceof DOMException && ['TimeoutError', 'AbortError'].includes(error.name))) + throw error + throw new Error(`GET ${path}: response body not complete within ${REQUEST_TIMEOUT_MS / 1000}s`) + } return record(JSON.parse(text)) } From 43bab1fdd8f46f40c2e8f0f2d0a7c9642c9d776b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 17:45:15 -0700 Subject: [PATCH 2/3] fix(e2e): log a failed executor pull with its stack --- apps/sim/scripts/test-desktop-inbox-e2e.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/sim/scripts/test-desktop-inbox-e2e.ts b/apps/sim/scripts/test-desktop-inbox-e2e.ts index 02d0246a780..cf6607f94bf 100644 --- a/apps/sim/scripts/test-desktop-inbox-e2e.ts +++ b/apps/sim/scripts/test-desktop-inbox-e2e.ts @@ -356,9 +356,7 @@ function startExecutor(desktop: Desktop, doorbell: ReturnType - logger.error('executor pull failed', { error: getErrorMessage(error) }) - ) + inFlight = drain().catch((error) => logger.error('executor pull failed', error)) } const offDoorbell = doorbell.onEvent(pull) const timer = setInterval(pull, RECONCILE_MS) From aa43cdd2c510631a12d43b495f7d42e9aa79ae4e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 18:38:01 -0700 Subject: [PATCH 3/3] fix(ci): also stop the detached telemetry flush an HTTP end-to-end app leaves behind `next dev` spawns its telemetry flush detached, in a session of its own, even with telemetry disabled. That process outlives the app's session and writes .next/dev/trace after stop-session.sh returned, so the next app's `rm -rf .next/dev` could fail with "Directory not empty". Each app now starts under an E2E_APP tag that every process it spawns inherits, and stop-session.sh stops and waits for tagged processes as well as the session's members. --- .github/scripts/stop-session.sh | 48 +++++++++++++++++++++----------- .github/workflows/test-build.yml | 22 ++++++++------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/.github/scripts/stop-session.sh b/.github/scripts/stop-session.sh index ab5938fd671..0125f786a0a 100755 --- a/.github/scripts/stop-session.sh +++ b/.github/scripts/stop-session.sh @@ -1,38 +1,52 @@ #!/usr/bin/env bash -# Stops every process in a session started with `setsid`, and returns only once none is left. +# Stops every process an app started, and returns only once none is left. # -# Usage: stop-session.sh +# Usage: stop-session.sh # -# `next dev` runs its server and workers as child processes. Waiting on `next dev` alone returns -# while those can still be running, and the next app in the job reuses the same .next directory. -# Every member of the session is signalled, including one that moved to its own process group, -# and one that started after the first signal. +# Start the app as `E2E_APP= setsid &`. Its processes are the members of the +# leader's session, plus any process that inherited `E2E_APP=` but left the session. +# `next dev` needs both: its server and workers stay in the session, while its telemetry flush is +# spawned detached, in a session of its own, and writes .next/dev/trace after `next dev` exits. +# The next app in the job wipes that directory, so it must not start while any of them runs. +# +# Every member is signalled, including one that moved to its own process group, and one that +# started after the first signal. # # Zombies don't count as running: they have exited and only await reaping. The leader is one # until the shell that started it waits on it, which that shell does after this returns. set -u leader=$1 +tag=$2 grace_seconds=10 -session_pids() { - ps -s "$leader" -o pid=,stat= 2>/dev/null | awk '$2 !~ /^Z/ { print $1 }' +app_pids() { + { + ps -s "$leader" -o pid=,stat= 2>/dev/null | awk '$2 !~ /^Z/ { print $1 }' + grep -lzx "E2E_APP=$tag" /proc/[0-9]*/environ 2>/dev/null | cut -d/ -f3 + } | sort -u } -signal_session() { +signal_app() { local pids - pids=$(session_pids) + pids=$(app_pids) [ -z "$pids" ] || kill "-$1" $pids 2>/dev/null || true } running() { - [ -n "$(session_pids)" ] + [ -n "$(app_pids)" ] } leader_running() { ps -p "$leader" -o stat= 2>/dev/null | grep -qv '^Z' } +list_app() { + local pids + pids=$(app_pids | paste -sd, -) + [ -z "$pids" ] || ps -p "$pids" -o pid,sid,stat,etimes,args 2>/dev/null | cut -c1-200 || true +} + # Centiseconds since boot: monotonic, and independent of the locale's decimal separator. now_cs() { local uptime @@ -47,7 +61,7 @@ signal_while() { shift while ((10#$(now_cs) < deadline)); do "$@" || return 0 - signal_session "$signal" + signal_app "$signal" sleep 0.1 done ! "$@" @@ -56,15 +70,15 @@ signal_while() { deadline=$((10#$(now_cs) + grace_seconds * 100)) signal_while TERM leader_running if ! leader_running && running; then - echo "Processes from session $leader outlived its leader:" - ps -s "$leader" -o pid,stat,etimes,args 2>/dev/null | cut -c1-200 || true + echo "Processes from app $tag outlived its leader:" + list_app fi signal_while TERM running && exit 0 -echo "::warning::Processes from session $leader were still running ${grace_seconds}s after SIGTERM:" -ps -s "$leader" -o pid,stat,etimes,args 2>/dev/null | cut -c1-200 || true +echo "::warning::Processes from app $tag were still running ${grace_seconds}s after SIGTERM:" +list_app deadline=$((10#$(now_cs) + grace_seconds * 100)) signal_while KILL running && exit 0 -echo "::error::Processes from session $leader survived SIGKILL for ${grace_seconds}s." +echo "::error::Processes from app $tag survived SIGKILL for ${grace_seconds}s." exit 1 diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 1909f3676d4..2430529a1a2 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -211,8 +211,10 @@ jobs: # same lifecycle. It starts from an empty Turbopack dev cache: a cache written under # other NEXT_PUBLIC_* values, by a server that `next dev` SIGKILLs 100ms after # SIGTERM, can panic Turbopack or wedge a route compile on restore. It runs in its - # own session, and stop-session.sh returns only once all of it has exited, so no app - # starts beside one still writing .next/dev. + # own session under an E2E_APP tag, and stop-session.sh returns only once every + # process in that session or carrying that tag has exited (Next's telemetry flush + # runs detached and still writes .next/dev), so no app starts beside one still + # writing .next/dev. - name: Verify SCIM, administration and workflow comparisons over real HTTP working-directory: apps/sim env: @@ -240,11 +242,11 @@ jobs: server_log="$report_dir/scim-next.log" mkdir -p "$report_dir" rm -rf .next/dev - setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 & + E2E_APP=scim setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 & server_pid=$! finish() { status=$? - bash ../../.github/scripts/stop-session.sh "$server_pid" || status=1 + bash ../../.github/scripts/stop-session.sh "$server_pid" scim || status=1 wait "$server_pid" 2>/dev/null || true awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/scim-http-status.log" if [ "$status" -ne 0 ]; then @@ -293,11 +295,11 @@ jobs: server_log="$report_dir/cli-next.log" mkdir -p "$report_dir" rm -rf .next/dev - setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 & + E2E_APP=cli setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 & server_pid=$! finish() { status=$? - bash ../../.github/scripts/stop-session.sh "$server_pid" || status=1 + bash ../../.github/scripts/stop-session.sh "$server_pid" cli || status=1 wait "$server_pid" 2>/dev/null || true if [ "$status" -ne 0 ]; then tail -n 200 "$server_log" @@ -344,11 +346,11 @@ jobs: server_log="$report_dir/stop-after-next.log" mkdir -p "$report_dir" rm -rf .next/dev - setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 & + E2E_APP=stop-after setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 & server_pid=$! finish() { status=$? - bash ../../.github/scripts/stop-session.sh "$server_pid" || status=1 + bash ../../.github/scripts/stop-session.sh "$server_pid" stop-after || status=1 wait "$server_pid" 2>/dev/null || true awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/stop-after-http-status.log" if [ "$status" -ne 0 ]; then @@ -395,11 +397,11 @@ jobs: server_log="$report_dir/desktop-inbox-next.log" mkdir -p "$report_dir" rm -rf .next/dev - setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3019 > "$server_log" 2>&1 & + E2E_APP=desktop-inbox setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3019 > "$server_log" 2>&1 & server_pid=$! finish() { status=$? - bash ../../.github/scripts/stop-session.sh "$server_pid" || status=1 + bash ../../.github/scripts/stop-session.sh "$server_pid" desktop-inbox || status=1 wait "$server_pid" 2>/dev/null || true awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/desktop-inbox-http-status.log" if [ "$status" -ne 0 ]; then