From c5bac68395aaa524ed67082b35bb2d380334d7b4 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 22 Oct 2026 09:32:16 +0200 Subject: [PATCH 1/2] T-22072 [CI] Name the test in the E2E Linear issue and include the failure summary The issue title names the test (E2E failure: AWS CloudFormation stack) and its body carries the failed step's errors and the job summary. The read-back step writes a summary however it fails: what arrived against what was expected and, after a timeout, what CloudWatch Logs forwarded per log group. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/e2e.yml | 81 ++++++++++++++++++--- .github/workflows/notify_linear_failure.yml | 72 ++++++++++++++---- 2 files changed, 129 insertions(+), 24 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8ea7ed3..8cc4fd9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -268,6 +268,32 @@ jobs: - name: "[Test] Read everything back through the SQL API connection" run: | set -o pipefail + mkdir -p out + fail() { FAILURE="$1"; echo "::error::$1"; exit 1; } + # Every way this step fails still leaves a summary: what arrived against what was expected, and, after a + # timeout, what CloudWatch Logs did with each group. + summarize_failure() { + local status=$? + [ "$status" = 0 ] && return + { + echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" + echo + echo "❌ ${FAILURE:-The read-back stopped on an error, see the log of this step}" + if [ -n "${LOGS:-}" ]; then + echo + echo "| read back through \`$CONNECT_HOST\` | arrived | expected |" + echo "|---|---|---|" + echo "| logs of \`$LOG_GROUP\` | $LOGS | 20 |" + echo "| logs of \`$STATIC_LOG_GROUP\` | $STATIC_LOGS | 20 |" + echo "| spans | $SPANS | 2 |" + echo "| metric points | $METRICS | 15 |" + echo "| CloudTrail event | $CLOUDTRAIL | 1 |" + fi + [ ! -f out/forwarding.md ] || { echo; cat out/forwarding.md; } + } > out/summary.md + exit "$status" + } + trap summarize_failure EXIT sql() { CODE=$(curl -sS -o response.json -w '%{http_code}' -u "$TEST_TEAM_CONNECTION_USERNAME:$TEST_TEAM_CONNECTION_PASSWORD" -H 'Content-Type: plain/text' --data-binary "$1 FORMAT JSONEachRow" "https://$CONNECT_HOST/") # A source's logs, spans and metrics collections each exist only from their first stored row on; @@ -295,13 +321,21 @@ jobs: # What CloudWatch Logs did with each group in the last 40 min; DeliveryErrors mean the filter paused for 10 min # and skipped what arrived meanwhile. forwarding() { + { + echo "| CloudWatch Logs, last 40 min | incoming | forwarded | delivery errors |" + echo "|---|---|---|---|" + } > out/forwarding.md for group in "$LOG_GROUP" "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + ROW="| \`$group\` |" for metric in IncomingLogEvents ForwardedLogEvents DeliveryErrors; do DIMS="Name=LogGroupName,Value=$group" [ "$metric" = IncomingLogEvents ] || DIMS="$DIMS Name=DestinationType,Value=Firehose Name=FilterName,Value=BetterStackLogs" - echo "$group $metric $(aws cloudwatch get-metric-statistics --namespace AWS/Logs --metric-name "$metric" --dimensions $DIMS \ - --start-time "$(date -u -d '-40 minutes' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 2400 --statistics Sum --query 'sum(Datapoints[].Sum)' --output text)" + VALUE=$(aws cloudwatch get-metric-statistics --namespace AWS/Logs --metric-name "$metric" --dimensions $DIMS \ + --start-time "$(date -u -d '-40 minutes' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 2400 --statistics Sum --query 'sum(Datapoints[].Sum)' --output text) + echo "$group $metric $VALUE" + ROW="$ROW ${VALUE%.0} |" done + echo "$ROW" >> out/forwarding.md done } # Within an hour of the previous stack deletion the static group, aws/spans and CloudTrail may deliver nothing @@ -316,7 +350,7 @@ jobs: while :; do counts arrived && break - [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 20 min"; forwarding; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } + [ "$SECONDS" -lt "$DEADLINE" ] || { forwarding; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; fail "Not everything arrived within 20 min"; } sleep 20 done # A last look 60 s later catches duplicate delivery. Every count must be exact; within that hour a fixed-name @@ -328,8 +362,7 @@ jobs: IFS=: read -r name got want <<<"$check" [ "$got" = "$want" ] && continue if [ -n "${RECENT_STACK_MIN:-}" ] && [ "$name" != logs ] && [ "$got" = 0 ]; then MISSING+=("$name"); continue; fi - echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)" - exit 1 + fail "Unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)" done # Each minute must hold exactly the value sent in it: 1, 2 and 3 for the metric, _max, _min and _sum, 1 for _count. # The lowest and highest stored value per point catch a re-sent minute with a different value. Points are @@ -339,20 +372,19 @@ jobs: | {name, minute: $t, min: .v, max: .v}] | sort_by(.name, .minute)') ACTUAL=$(sql "SELECT name, toUnixTimestamp(dt) AS t, minMerge(value_min) AS min, maxMerge(value_max) AS max FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt" \ | jq -sc 'map({name, minute: (((.t | tonumber) / 60 | floor) * 60), min: (.min | tonumber), max: (.max | tonumber)}) | sort_by(.name, .minute)') - [ "$ACTUAL" = "$EXPECTED" ] || { echo "::error::metric points differ from the datapoints sent: expected $EXPECTED, got $ACTUAL"; exit 1; } + [ "$ACTUAL" = "$EXPECTED" ] || fail "Metric points differ from the datapoints sent: expected $EXPECTED, got $ACTUAL" if [ "${#MISSING[@]}" -gt 0 ]; then NAMES=$(printf '%s\n' "${MISSING[@]}" | sed -e "s#^static\$#the logs of $STATIC_LOG_GROUP#" -e 's#^spans$#the spans#' -e 's#^cloudtrail$#the CloudTrail event#' | paste -sd, - | sed 's/,/, /g') WARNING="Not verified: $NAMES. The previous run deleted its stack $RECENT_STACK_MIN min before this one, and for up to an hour AWS keeps using sessions of the IAM roles every run recreates under the same names, so CloudWatch Logs and CloudTrail can skip this data. A run more than an hour after the previous one checks it." echo "::warning::$WARNING" forwarding fi - mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json if [ "$SPANS" = 2 ]; then sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json # The X-Ray trace id 1-- should be stored as the 32-hex id the mapper takes from Transaction Search. STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') - [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } + [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || fail "Stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX" fi sql "SELECT name, dt, tags, avgMerge(value_avg) AS value FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND tags['run_id'] = '$RUN_ID' AND name = '$METRIC' GROUP BY name, dt, tags ORDER BY dt" > out/sample_metric.json [ "$CLOUDTRAIL" = 0 ] || sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json @@ -366,6 +398,8 @@ jobs: if [ "${#MISSING[@]}" -gt 0 ]; then echo "⚠️ $WARNING" echo + cat out/forwarding.md + echo echo "✅ everything else round-tripped through the stack" else echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" @@ -378,15 +412,44 @@ jobs: - name: "[Report] Job summary" if: ${{ always() }} + env: + GH_TOKEN: ${{ github.token }} run: | - if [ -f out/summary.md ]; then cat out/summary.md >> "$GITHUB_STEP_SUMMARY"; else echo "No summary, the run failed before verification." >> "$GITHUB_STEP_SUMMARY"; fi + mkdir -p report + if [ -f out/summary.md ]; then + cp out/summary.md report/summary.md + else + # The read-back writes the summary however it ends, so a step before it failed (or the job was cancelled). + FAILED_STEP=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs" \ + --jq ".jobs[] | select(.runner_name == \"$RUNNER_NAME\") | .steps[] | select(.conclusion == \"failure\") | .name" | head -1) || true + { echo "### AWS CloudFormation stack E2E \`$RUN_ID\`"; echo; echo "❌ Stopped${FAILED_STEP:+ at **$FAILED_STEP**}, before anything was read back."; } > report/summary.md + fi + cat report/summary.md >> "$GITHUB_STEP_SUMMARY" + # The Linear issue gets the cleanup only when it left something behind. + grep -q '❌' out/cleanup.md 2>/dev/null && { echo; cat out/cleanup.md; } >> report/summary.md # Rows come back as JSONEachRow with the stored event as a JSON string in raw; render them as a pretty array. for f in out/sample_*.json; do [ -f "$f" ] && { echo; echo "
$(basename "$f")"; echo; echo '```json'; jq -s 'map(if has("raw") then .raw |= (fromjson? // .) else . end)' "$f"; echo '```'; echo '
'; echo; } >> "$GITHUB_STEP_SUMMARY"; done [ -f out/cleanup.md ] && { echo; cat out/cleanup.md; } >> "$GITHUB_STEP_SUMMARY" true + - name: "[Report] Hand the summary to the Linear issue" + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: e2e-summary-Stack round trip + path: report/summary.md + retention-days: 7 + overwrite: true + notify-linear: if: ${{ failure() && (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }} needs: [test] + permissions: + contents: read + actions: read + checks: read uses: ./.github/workflows/notify_linear_failure.yml + with: + test_name: AWS CloudFormation stack + job_summaries: true secrets: inherit diff --git a/.github/workflows/notify_linear_failure.yml b/.github/workflows/notify_linear_failure.yml index 529be28..49813ab 100644 --- a/.github/workflows/notify_linear_failure.yml +++ b/.github/workflows/notify_linear_failure.yml @@ -1,7 +1,19 @@ name: "Notify Linear on failure" +# A caller that sets neither input gets the plain issue: "E2E failure: " listing the failed jobs and steps. on: workflow_call: + inputs: + test_name: + description: "Names the test in the issue title instead of the workflow name" + type: string + default: "" + job_summaries: + description: >- + Adds each failed job's errors and summary to the issue. A job hands its summary over as an artifact named + "e2e-summary-" holding summary.md. The calling job needs checks: read for the errors. + type: boolean + default: false jobs: create-linear-issue: @@ -9,26 +21,56 @@ jobs: permissions: contents: read actions: read + checks: read steps: + - name: "Download the jobs' summaries" + if: ${{ inputs.job_summaries }} + continue-on-error: true + uses: actions/download-artifact@v8 + with: + pattern: e2e-summary-* + path: summaries + run-id: ${{ github.event.workflow_run.id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Gather failure context" id: ctx env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} RUN_ID: ${{ github.event.workflow_run.id || github.run_id }} RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }} + JOB_SUMMARIES: ${{ inputs.job_summaries }} run: | - FAILED=$(gh api \ + JOBS=$(gh api \ "repos/${{ github.repository }}/actions/runs/$RUN_ID/attempts/$RUN_ATTEMPT/jobs" \ --paginate \ - --jq '[.jobs[] + --jq '.jobs[] | select(.conclusion == "failure") - | "- **\(.name)** — failed at _\([.steps[] | select(.conclusion == "failure") | .name] | join(", "))_"] - | join("\n")') - + | {id, name, steps: ([.steps[] | select(.conclusion == "failure") | .name] | join(", "))} + | @json') + + FAILED="" + while read -r job; do + [ -n "$job" ] || continue + NAME=$(jq -r .name <<<"$job") + STEPS=$(jq -r .steps <<<"$job") + if [ "$JOB_SUMMARIES" != true ]; then + FAILED="$FAILED- **$NAME** — failed at _${STEPS}_"$'\n' + continue + fi + # The job's ::error:: lines, without the runner's closing "Process completed with exit code 1." + ERRORS=$(gh api "repos/${{ github.repository }}/check-runs/$(jq -r .id <<<"$job")/annotations" \ + --jq '.[] | select(.annotation_level == "failure" and (.message | startswith("Process completed with exit code") | not)) | "- \(.message | gsub("\n"; " "))"' || true) + FAILED="$FAILED### $NAME"$'\n\n'"Failed at **${STEPS}**"$'\n\n'"${ERRORS:+$ERRORS$'\n\n'}" + SUMMARY="summaries/e2e-summary-$NAME/summary.md" + [ ! -f "$SUMMARY" ] || FAILED="$FAILED$(cat "$SUMMARY")"$'\n\n' + done <<<"$JOBS" + if [ -z "$FAILED" ]; then FAILED="_(no failed jobs reported by the API yet)_" + elif [ "${#FAILED}" -gt 20000 ]; then + FAILED="${FAILED:0:20000}"$'\n\n'"_(cut at 20,000 characters, the rest is in the run)_" fi - + { echo "failed_jobs< Date: Thu, 22 Oct 2026 09:42:47 +0200 Subject: [PATCH 2/2] Fetch each failed job's summary by its artifact name download-artifact unpacks a lone matching artifact straight into its path, so in a run with one job the summary was never found. Also keep an annotations API error out of the issue, and keep the plain issue text byte-identical to before. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/notify_linear_failure.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/notify_linear_failure.yml b/.github/workflows/notify_linear_failure.yml index 49813ab..72f9f2d 100644 --- a/.github/workflows/notify_linear_failure.yml +++ b/.github/workflows/notify_linear_failure.yml @@ -11,27 +11,19 @@ on: job_summaries: description: >- Adds each failed job's errors and summary to the issue. A job hands its summary over as an artifact named - "e2e-summary-" holding summary.md. The calling job needs checks: read for the errors. + "e2e-summary-" holding summary.md. type: boolean default: false jobs: create-linear-issue: runs-on: ubuntu-latest + # A caller must grant checks: read as well (the errors come from the annotations), or this job cannot start. permissions: contents: read actions: read checks: read steps: - - name: "Download the jobs' summaries" - if: ${{ inputs.job_summaries }} - continue-on-error: true - uses: actions/download-artifact@v8 - with: - pattern: e2e-summary-* - path: summaries - run-id: ${{ github.event.workflow_run.id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Gather failure context" id: ctx env: @@ -59,12 +51,15 @@ jobs: fi # The job's ::error:: lines, without the runner's closing "Process completed with exit code 1." ERRORS=$(gh api "repos/${{ github.repository }}/check-runs/$(jq -r .id <<<"$job")/annotations" \ - --jq '.[] | select(.annotation_level == "failure" and (.message | startswith("Process completed with exit code") | not)) | "- \(.message | gsub("\n"; " "))"' || true) + --jq '.[] | select(.annotation_level == "failure" and (.message | startswith("Process completed with exit code") | not)) | "- \(.message | gsub("\n"; " "))"') || ERRORS="" FAILED="$FAILED### $NAME"$'\n\n'"Failed at **${STEPS}**"$'\n\n'"${ERRORS:+$ERRORS$'\n\n'}" - SUMMARY="summaries/e2e-summary-$NAME/summary.md" - [ ! -f "$SUMMARY" ] || FAILED="$FAILED$(cat "$SUMMARY")"$'\n\n' + # A job that failed before writing its summary has no artifact and gets the errors alone. + SUMMARY_DIR="summaries/$(jq -r .id <<<"$job")" + gh run download "$RUN_ID" -R "${{ github.repository }}" -n "e2e-summary-$NAME" -D "$SUMMARY_DIR" >/dev/null 2>&1 || true + [ ! -f "$SUMMARY_DIR/summary.md" ] || FAILED="$FAILED$(cat "$SUMMARY_DIR/summary.md")"$'\n\n' done <<<"$JOBS" + while [[ "$FAILED" == *$'\n' ]]; do FAILED="${FAILED%$'\n'}"; done if [ -z "$FAILED" ]; then FAILED="_(no failed jobs reported by the API yet)_" elif [ "${#FAILED}" -gt 20000 ]; then