Retry E2E tests once in CI and report flaky results

.github/workflows/tests-pr.yml

@@ -260,6 +260,10 @@ jobs:
           E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
           E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
         run: pnpm exec playwright test --shard ${{ matrix.shard }}
+      - name: Report flaky tests
+        if: ${{ !cancelled() }}
+        working-directory: packages/e2e
+        run: node scripts/report-flaky.js
       - name: Upload Playwright report
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}

packages/e2e/playwright.config.ts

@@ -12,10 +12,12 @@ export default defineConfig({
   testDir: './tests',
   fullyParallel: true,
   forbidOnly: isCI,
-  retries: 0,
+  retries: isCI ? 1 : 0, // One retry in CI; flaky-on-retry is reported, not ignored (scripts/report-flaky.js)
   workers: 10,
-  maxFailures: isCI ? 3 : 0, // Stop early in CI after 3 failures
-  reporter: isCI ? [['html', {open: 'never'}], ['list']] : [['list']],
+  maxFailures: isCI ? 5 : 0, // Stop early in CI; first attempts of flaky tests count, so leave room for retries
+  reporter: isCI
+    ? [['html', {open: 'never'}], ['list'], ['json', {outputFile: 'test-results/results.json'}]]
+    : [['list']],
   timeout: TEST_TIMEOUT.default, // Heavy tests override via test.setTimeout()
   globalTimeout: 20 * 60 * 1000,
 

packages/e2e/scripts/report-flaky.js

@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+
+// Surfaces tests that failed and then passed on retry ("flaky" in Playwright's
+// JSON report). Retries keep flake from failing the shard; this keeps it from
+// disappearing. Each flaky test is printed as a grep-able `FLAKY:` log line
+// and listed in the GitHub job summary.
+
+import {appendFileSync, existsSync, readFileSync} from 'node:fs'
+import * as path from 'node:path'
+import {fileURLToPath} from 'node:url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const resultsPath = path.join(__dirname, '../test-results/results.json')
+
+if (!existsSync(resultsPath)) {
+  console.log('no results.json found, skipping flaky report')
+  process.exit(0)
+}
+
+const report = JSON.parse(readFileSync(resultsPath, 'utf8'))
+
+const flakyTests = []
+function walkSuite(suite, titlePath) {
+  for (const spec of suite.specs ?? []) {
+    for (const test of spec.tests ?? []) {
+      if (test.status === 'flaky') {
+        flakyTests.push({title: [...titlePath, spec.title].join(' › '), file: spec.file})
+      }
+    }
+  }
+  for (const child of suite.suites ?? []) {
+    walkSuite(child, [...titlePath, child.title])
+  }
+}
+for (const suite of report.suites ?? []) {
+  // Root suites are titled with the file name, which spec.file already carries.
+  walkSuite(suite, [])
+}
+
+if (flakyTests.length === 0) {
+  console.log('no flaky tests in this shard')
+  process.exit(0)
+}
+
+for (const test of flakyTests) {
+  console.log(`FLAKY: ${test.file} › ${test.title}`)
+}
+
+if (process.env.GITHUB_STEP_SUMMARY) {
+  const summary = [
+    '## Flaky tests (failed, then passed on retry)',
+    '',
+    ...flakyTests.map((test) => `- \`${test.file}\` › ${test.title}`),
+    '',
+  ].join('\n')
+  appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`)
+}