ci: duration-balanced integration shards, warm desktop-live, CI on every PR (#8763)

* ci: duration-balanced integration shards, warm desktop-live, CI on every PR

- integration: a custom Vitest sequencer packs `--shard` files by measured duration from a
  committed `vitest.integration-durations.json` (refreshed with
  scripts/update-integration-durations.ts) instead of by path hash. On the measured run the
  apps/sim shards go from 183-384s to 254s each, so the slowest job drops from ~7.4 to ~5.3 min
  at the same cost. Every shard computes the same partition, so files stay disjoint and covered.
- desktop-live: Turbopack's dev cache is kept on a sticky disk keyed by event, fork and the
  installed Next version, so the spec's warm-up restores instead of cold-compiling. The app is
  stopped with SIGINT so the cache write completes, a cache Turbopack reports as corrupt is
  dropped and the app restarted from empty, and the apps boot while apt and the Electron
  bundle run.
- ci.yml: no paths-ignore on pull_request. `checks / ci` is going to be the required check,
  and a pull request whose run never starts never reports it.

* ci: clear the dev cache by absolute path, always run session cleanup, floor shard weights, log through @sim/logger

* ci: a malformed durations file names its fix, and non-finite weights count as unknown

Author: waleedlatif1

.github/workflows/checks.yml

@@ -15,8 +15,9 @@ jobs:
     #
     # The two paths build different schemas (migrations add triggers, checks and NOT VALID
     # constraints that `db:push` does not), so each runs the whole suite. Vitest splits each
-    # suite's files across four shards; a shard runs its files one at a time against its own
-    # database. Files run serially, so a shard barely uses more than one core: 4 vCPU is enough.
+    # suite's files across four shards by measured duration (DurationBalancedSequencer in vitest.shared.ts); a
+    # shard runs its files one at a time against its own database. Files run serially, so a shard
+    # barely uses more than one core: 4 vCPU is enough.
     name: integration (${{ matrix.provision }}, ${{ matrix.shard }}/4)
     runs-on: &runner-4vcpu ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 20
@@ -272,6 +273,23 @@ jobs:
         working-directory: packages/db
         run: bun run db:migrate
 
+      # Turbopack's dev cache turns the spec's route warm-up from a cold compile (~4 min) into a
+      # restore. It is content-addressed, so a pull request's changed modules still recompile; the
+      # key carries the installed Next version so an upgrade starts from an empty cache, and the
+      # event and fork segments keep untrusted runs off the cache trusted runs read.
+      - name: Resolve Turbopack dev cache key
+        id: next-cache
+        run: echo "key=${GITHUB_REPOSITORY}-next-dev-desktop-live-${GITHUB_EVENT_NAME}${FORK_SUFFIX}-$(jq -r .version node_modules/next/package.json)" >> "$GITHUB_OUTPUT"
+        env:
+          FORK_SUFFIX: ${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+
+      - name: Mount Turbopack dev cache
+        uses: ./.github/actions/cache
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+          key: ${{ steps.next-cache.outputs.key }}
+          path: ./apps/sim/.next/dev
+
       # Chat switches, Stop, sign-out, approval and the flag-off foreground round trip. The spec
       # runs the recording proxy (the app's public origin) and the stand-in worker.
       - name: Verify desktop tools in the Electron app against a local app
@@ -292,39 +310,84 @@ jobs:
           READY_TIMEOUT_SECONDS: 300
         run: |
           report_dir="$RUNNER_TEMP/e2e"
+          next_log="$report_dir/desktop-tools-next.log"
           mkdir -p "$report_dir"
-          sudo apt-get update -q
-          sudo apt-get install -yq xvfb libgtk-3-0t64 libnss3 libasound2t64 libgbm1 libxss1 \
-            libxtst6 libatk-bridge2.0-0t64 libxkbcommon0 > /dev/null
-          # Bundle only: `bun run build` also fetches the macOS node-pty prebuilds for packaging,
-          # which a Linux run does not use.
-          (cd apps/desktop && bun run scripts/build.ts)
           # Each app runs in its own session under an E2E_APP tag, and stop-session.sh returns once
           # every process it started has exited.
           realtime_tag="desktop-realtime-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
           server_tag="desktop-tools-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
+          # Keeps the restored cache under the cap the dev scripts apply locally.
+          (cd apps/sim && bun run dev:cache:cap)
+          start_sim() {
+            (cd apps/sim && E2E_APP="$server_tag" exec setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \
+              --port 3021 >> "$next_log" 2>&1) &
+            server_pid=$!
+          }
+          # A corrupted cache aborts Turbopack instead of falling back. Only then is the shared cache
+          # dropped: a failing test must not cost every later run its warm cache.
+          cache_broken() {
+            grep -qiE 'cache corruption|turbopack.*panic|panicked' "$next_log" 2>/dev/null
+          }
+          # The cache directory is a mount point: empty it rather than remove it. Absolute, because
+          # the EXIT trap runs after the step has moved into apps/desktop.
+          clear_cache() {
+            find "$GITHUB_WORKSPACE/apps/sim/.next/dev" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+          }
+          # SIGINT first: `next dev` SIGKILLs its server 100ms after SIGTERM, which discards a cache
+          # write in flight. stop-session.sh then removes anything still running.
+          # SIGINT is best-effort; the cleanup always runs, since workers and the detached telemetry
+          # flush can outlive a server that has already exited.
+          stop_sim() {
+            if kill -INT "$server_pid" 2>/dev/null; then
+              for _ in $(seq 1 30); do kill -0 "$server_pid" 2>/dev/null || break; sleep 1; done
+            fi
+            bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag"
+          }
           (cd apps/realtime && PORT=3023 SIM_DB_ROLE=realtime ALLOWED_ORIGINS="$NEXT_PUBLIC_APP_URL" \
             E2E_APP="$realtime_tag" exec setsid bun src/index.ts > "$report_dir/desktop-tools-realtime.log" 2>&1) &
           realtime_pid=$!
-          (cd apps/sim && E2E_APP="$server_tag" exec setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \
-            --port 3021 > "$report_dir/desktop-tools-next.log" 2>&1) &
-          server_pid=$!
+          start_sim
           finish() {
             status=$?
-            bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag" || status=1
+            stop_sim || status=1
             bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$realtime_pid" "$realtime_tag" || status=1
             wait "$server_pid" "$realtime_pid" 2>/dev/null || true
+            if cache_broken; then
+              echo "::warning::Turbopack reported a broken dev cache; clearing it for the next run."
+              clear_cache
+            fi
             exit "$status"
           }
           trap finish EXIT
+          # The apps boot while the runner installs Electron's libraries and bundles the shell.
+          sudo apt-get update -q
+          sudo apt-get install -yq xvfb libgtk-3-0t64 libnss3 libasound2t64 libgbm1 libxss1 \
+            libxtst6 libatk-bridge2.0-0t64 libxkbcommon0 > /dev/null
+          # Bundle only: `bun run build` also fetches the macOS node-pty prebuilds for packaging,
+          # which a Linux run does not use.
+          (cd apps/desktop && bun run scripts/build.ts)
           started=$SECONDS
+          retried=0
           until curl --fail --silent --max-time 10 http://127.0.0.1:3021/api/health > /dev/null &&
             curl --fail --silent --max-time 10 http://127.0.0.1:3023/health > /dev/null; do
-            kill -0 "$server_pid" 2>/dev/null || { tail -n 200 "$report_dir/desktop-tools-next.log"; exit 1; }
+            if ! kill -0 "$server_pid" 2>/dev/null; then
+              tail -n 200 "$next_log"
+              if [ "$retried" = 0 ] && cache_broken; then
+                echo "::warning::Turbopack rejected the restored dev cache; restarting from an empty cache."
+                bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag"
+                clear_cache
+                : > "$next_log"
+                retried=1
+                start_sim
+                continue
+              fi
+              exit 1
+            fi
             kill -0 "$realtime_pid" 2>/dev/null || { tail -n 200 "$report_dir/desktop-tools-realtime.log"; exit 1; }
             [ $((SECONDS - started)) -lt "$READY_TIMEOUT_SECONDS" ] || { echo '::error::Local app did not become ready'; exit 1; }
             sleep 2
           done
+          echo "Local apps ready $((SECONDS - started))s after the Electron bundle"
           cd apps/desktop
           SIM_DESKTOP_E2E_SIM_URL=http://127.0.0.1:3021 \
           SIM_DESKTOP_E2E_PROXY_PORT=3020 \

.github/workflows/ci.yml

@@ -18,13 +18,11 @@ name: ci
 on:
   push:
     branches: [main, staging, dev]
+  # No path filter: `checks / ci` is the required status check, and a pull request whose run
+  # never starts never reports it, so it could not merge. Docs-only pull requests are rare enough
+  # that running CI on them costs nothing measurable.
   pull_request:
     branches: [main, staging, dev]
-    # Docs content and markdown don't affect the app build or images; push
-    # runs stay unfiltered because they feed the deploy pipeline.
-    paths-ignore:
-      - 'apps/docs/content/**'
-      - '**/*.md'
 
 concurrency:
   group: ci-${{ github.ref }}

scripts/update-integration-durations.ts

@@ -0,0 +1,79 @@
+#!/usr/bin/env bun
+/**
+ * Rewrites `vitest.integration-durations.json`, the per-file weights the integration shards are
+ * balanced by (`DurationBalancedSequencer` in `vitest.shared.ts`), from a CI run's integration reports.
+ *
+ *   gh run download <run-id> -p 'integration-reports-*' -D /tmp/integration-reports
+ *   bun run scripts/update-integration-durations.ts /tmp/integration-reports
+ *
+ * A file's weight is its wall time including the import and collection that precede its first
+ * test (the gap since the previous file in the same shard ended), averaged across the push and
+ * migrate provisioning runs. apps/sim runs its files one at a time, so those gaps are real shard
+ * time; where files overlap, a file counts only its own time. Weights only steer balance; a stale
+ * file never changes what runs.
+ */
+import { readdir, readFile, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import { createLogger } from '@sim/logger'
+
+interface JsonReport {
+  testResults: { name: string; startTime: number; endTime: number }[]
+}
+
+const logger = createLogger('UpdateIntegrationDurations')
+const REPORT_NAME = 'integration.json'
+const ROOT = path.resolve(import.meta.dir, '..')
+const OUTPUT = path.join(ROOT, 'vitest.integration-durations.json')
+
+async function findReports(dir: string): Promise<string[]> {
+  const entries = await readdir(dir, { withFileTypes: true, recursive: true })
+  return entries
+    .filter((entry) => entry.isFile() && entry.name === REPORT_NAME)
+    .map((entry) => path.join(entry.parentPath, entry.name))
+}
+
+/** `/home/runner/_work/sim/sim/apps/sim/lib/x.integration.ts` → `apps/sim/lib/x.integration.ts` */
+function repoPath(absolute: string): string | null {
+  const match = absolute.match(/\/((?:apps|packages)\/.+)$/)
+  return match ? match[1] : null
+}
+
+const source = process.argv[2]
+if (!source) {
+  logger.error('Usage: bun run scripts/update-integration-durations.ts <downloaded-reports-dir>')
+  process.exit(1)
+}
+
+const samples = new Map<string, number[]>()
+const reports = await findReports(path.resolve(source))
+for (const reportPath of reports) {
+  const report = JSON.parse(await readFile(reportPath, 'utf8')) as JsonReport
+  const files = [...report.testResults].sort((a, b) => a.startTime - b.startTime)
+  let previousEnd = Number.NEGATIVE_INFINITY
+  for (const file of files) {
+    const key = repoPath(file.name)
+    // The gap before a file is its import time only when nothing else was running: in a run
+    // with file parallelism (packages/db) files overlap, and a file then counts its own time.
+    const start =
+      previousEnd > Number.NEGATIVE_INFINITY && previousEnd <= file.startTime
+        ? previousEnd
+        : file.startTime
+    if (key) samples.set(key, [...(samples.get(key) ?? []), (file.endTime - start) / 1000])
+    previousEnd = Math.max(previousEnd, file.endTime)
+  }
+}
+
+if (samples.size === 0) {
+  logger.error(`No ${REPORT_NAME} reports with test results under ${source}`)
+  process.exit(1)
+}
+
+const durations = Object.fromEntries(
+  [...samples.keys()].sort().map((key) => {
+    const values = samples.get(key) ?? []
+    const mean = values.reduce((sum, value) => sum + value, 0) / values.length
+    return [key, Math.round(mean * 10) / 10]
+  })
+)
+await writeFile(OUTPUT, `${JSON.stringify(durations, null, 2)}\n`)
+logger.info(`Wrote ${samples.size} file weights from ${reports.length} reports to ${OUTPUT}`)