Test and Build #295
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test and Build | |
| on: | |
| workflow_call: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| postgres-integration: | |
| # Runs the real-infrastructure test layer: every `*.integration.ts` in packages/db and | |
| # apps/sim, discovered by glob (`vitest run --mode integration`), against the database each | |
| # provisioning path produces. A new integration suite needs no workflow change. | |
| name: PostgreSQL integration (${{ matrix.provision }}) | |
| runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provision: [push, migrate] | |
| services: | |
| redis: | |
| image: redis:8.2-alpine | |
| ports: | |
| - 6379:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| postgres: | |
| image: pgvector/pgvector:pg17 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: sim_test | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d sim_test" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| postgres-legacy: | |
| image: postgres:16-alpine | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: sim_test | |
| ports: | |
| - 5433:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d sim_test" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test | |
| TEST_REDIS_URL: redis://127.0.0.1:6379 | |
| DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test | |
| BETTER_AUTH_SECRET: oauth-postgres-ci-secret-at-least-32-characters | |
| NEXT_PUBLIC_APP_URL: https://test.sim.ai | |
| ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| with: | |
| provider: ${{ vars.CI_PROVIDER }} | |
| - name: Provision a fresh database through the supported command | |
| working-directory: packages/db | |
| run: | | |
| bun -e 'import postgres from "postgres"; const sql = postgres(process.env.DATABASE_URL); for (const extension of ["vector", "btree_gin", "pg_trgm"]) await sql`CREATE EXTENSION IF NOT EXISTS ${sql(extension)}`; await sql.end()' | |
| bun run db:${{ matrix.provision }} | |
| - name: Verify migration replay is a no-op | |
| if: matrix.provision == 'migrate' | |
| working-directory: packages/db | |
| run: bun run db:migrate | |
| - name: Run packages/db integration tests | |
| working-directory: packages/db | |
| run: bun run test --mode integration | |
| - name: Run apps/sim integration tests | |
| working-directory: apps/sim | |
| # A non-UTC process zone keeps timestamp-without-time-zone handling honest. | |
| env: | |
| TZ: America/Los_Angeles | |
| run: bun run test --mode integration | |
| - name: Verify cumulative billing timeout recovery on PostgreSQL 16 | |
| if: matrix.provision == 'push' | |
| working-directory: apps/sim | |
| env: | |
| TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5433/sim_test | |
| run: >- | |
| bun run test --mode integration lib/billing/core/usage-log.integration.ts | |
| --outputFile.json=test-results/integration-pg16.json | |
| - name: Upload integration test reports | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: integration-reports-${{ matrix.provision }} | |
| path: | | |
| packages/db/test-results/*.json | |
| apps/sim/test-results/*.json | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| # Acceptance suites that cross a real HTTP boundary. Its own job, off the | |
| # integration legs' critical path and on its own database: the SCIM app boots | |
| # hosted, which starts background usage replay against DATABASE_URL, so it must | |
| # never share a database with suites asserting on billing rows. The suites | |
| # exercise HTTP behavior rather than a provisioning path, so they run once, | |
| # against the production (migrate) path. | |
| http-e2e: | |
| name: End-to-end over real HTTP | |
| runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} | |
| timeout-minutes: 20 | |
| services: | |
| postgres: | |
| image: pgvector/pgvector:pg17 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: sim_test | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d sim_test" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test | |
| BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters | |
| ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| with: | |
| provider: ${{ vars.CI_PROVIDER }} | |
| # Migrations create their own extensions, as on a fresh self-hosted install. | |
| - name: Provision the database through migrations | |
| working-directory: packages/db | |
| run: bun run db:migrate | |
| - name: Verify Google document reads over real HTTP | |
| working-directory: apps/sim | |
| env: | |
| NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 | |
| NEXT_PUBLIC_FORCE_HOSTED: 'false' | |
| SEARCH_GOOGLE_CONTENT_REPORT_PATH: ${{ runner.temp }}/e2e/search-google-content.json | |
| run: bun scripts/test-search-google-content-e2e.ts | |
| - name: Verify Lucid MCP search and complete diagram reads over real HTTP | |
| working-directory: apps/sim | |
| env: | |
| NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 | |
| NEXT_PUBLIC_FORCE_HOSTED: 'false' | |
| SEARCH_LUCID_REPORT_PATH: ${{ runner.temp }}/e2e/search-lucid.json | |
| run: bun scripts/test-search-lucid-e2e.ts | |
| - name: Verify Zoom search over real HTTP | |
| working-directory: apps/sim | |
| env: | |
| NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 | |
| NEXT_PUBLIC_FORCE_HOSTED: 'false' | |
| SEARCH_ZOOM_REPORT_PATH: ${{ runner.temp }}/e2e/search-zoom.json | |
| run: bun scripts/test-search-zoom-e2e.ts | |
| - name: Verify Google Meet search over real HTTP | |
| working-directory: apps/sim | |
| env: | |
| NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 | |
| NEXT_PUBLIC_FORCE_HOSTED: 'false' | |
| SEARCH_GOOGLE_MEET_REPORT_PATH: ${{ runner.temp }}/e2e/search-google-meet.json | |
| run: bun scripts/test-search-google-meet-e2e.ts | |
| # The first request cold-compiles the app under Turbopack, which took 42-150s | |
| # on this runner class: a fixed 120s readiness deadline failed on the slow | |
| # tail. The deadline only has to catch a hung boot; an exited server fails | |
| # immediately, and either way the server log tail lands in the job log. No | |
| # step timeout: the job's bound covers a hang without cutting a slow but | |
| # healthy suite short of writing its report. | |
| - name: Verify SCIM, administration and workflow comparisons over real HTTP | |
| working-directory: apps/sim | |
| env: | |
| NEXT_PUBLIC_APP_URL: http://127.0.0.1:3017 | |
| BETTER_AUTH_URL: http://127.0.0.1:3017 | |
| NEXT_PUBLIC_FORCE_HOSTED: 'true' | |
| BILLING_ENABLED: 'true' | |
| NEXT_PUBLIC_BILLING_ENABLED: 'true' | |
| ENTERPRISE_ENABLED: 'true' | |
| NEXT_PUBLIC_ENTERPRISE_ENABLED: 'true' | |
| SCIM_ENABLED: 'true' | |
| NEXT_PUBLIC_SCIM_ENABLED: 'true' | |
| SSO_ENABLED: 'true' | |
| NEXT_PUBLIC_SSO_ENABLED: 'true' | |
| ORGANIZATIONS_ENABLED: 'true' | |
| NEXT_PUBLIC_ORGANIZATIONS_ENABLED: 'true' | |
| INTERNAL_API_SECRET: scim-http-ci-local-secret-at-least-32-characters | |
| DB_TX_TRIPWIRE: throw | |
| DISABLE_TELEMETRY: 'true' | |
| NEXT_TELEMETRY_DISABLED: '1' | |
| NEXT_PUBLIC_CHAT_DISABLED: 'true' | |
| READY_TIMEOUT_SECONDS: 300 | |
| run: | | |
| report_dir="$RUNNER_TEMP/e2e" | |
| server_log="$report_dir/scim-next.log" | |
| mkdir -p "$report_dir" | |
| node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 & | |
| server_pid=$! | |
| finish() { | |
| kill "$server_pid" 2>/dev/null || true | |
| wait "$server_pid" 2>/dev/null || true | |
| awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/scim-http-status.log" | |
| } | |
| trap finish EXIT | |
| fail_startup() { | |
| echo "::error::$1" | |
| tail -n 200 "$server_log" | |
| exit 1 | |
| } | |
| started=$SECONDS | |
| until curl --fail --silent --max-time 10 http://127.0.0.1:3017/api/health > /dev/null; do | |
| kill -0 "$server_pid" 2>/dev/null || fail_startup 'Local SCIM app exited during startup.' | |
| [ $((SECONDS - started)) -lt "$READY_TIMEOUT_SECONDS" ] || | |
| fail_startup "Local SCIM app did not become ready within $READY_TIMEOUT_SECONDS seconds." | |
| sleep 2 | |
| done | |
| echo "Local SCIM app ready after $((SECONDS - started))s" | |
| SCIM_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \ | |
| SCIM_E2E_DATABASE_URL="$DATABASE_URL" \ | |
| SCIM_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ | |
| SCIM_E2E_REPORT_PATH="$report_dir/scim-e2e-report.json" \ | |
| bun run test:scim:e2e | |
| VERSION_COMPARE_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \ | |
| VERSION_COMPARE_E2E_DATABASE_URL="$DATABASE_URL" \ | |
| VERSION_COMPARE_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ | |
| VERSION_COMPARE_E2E_REPORT_PATH="$report_dir/version-compare-http-report.json" \ | |
| bun run test:workflow-version-compare:e2e | |
| - name: Upload end-to-end reports and server logs | |
| if: failure() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: http-e2e-reports | |
| path: ${{ runner.temp }}/e2e/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| test-build: | |
| name: Lint and Test | |
| runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} | |
| timeout-minutes: 15 | |
| steps: | |
| # The diff-based audits below need a base commit to read, and the default | |
| # depth of 1 clones a single commit with no parent. They normally fetch | |
| # their base by SHA (see "Resolve base ref"), so this depth only covers the | |
| # `HEAD~1` fallback — but without it that fallback resolves to nothing. | |
| # | |
| # Worth stating because the failure was invisible for so long: the migration | |
| # audit read the resulting `git diff` failure as "no migrations changed" and | |
| # exited 0, so it had never actually run on a push build. | |
| - name: Checkout code | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| with: | |
| fetch-depth: 2 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| with: | |
| provider: ${{ vars.CI_PROVIDER }} | |
| turbo-cache-key: turbo-cache | |
| # Surfaces known CVEs in the dependency tree. Non-blocking until the | |
| # existing advisory backlog is triaged, then flip to a required gate by | |
| # removing continue-on-error. | |
| - name: Security audit | |
| run: bun audit | |
| continue-on-error: true | |
| - name: Validate env flags | |
| run: | | |
| FILE="apps/sim/lib/core/config/env-flags.ts" | |
| ERRORS="" | |
| echo "Checking for hardcoded boolean env flags..." | |
| # Use perl for multiline matching to catch both: | |
| # export const isHosted = true | |
| # export const isHosted = | |
| # true | |
| HARDCODED=$(perl -0777 -ne 'while (/export const (is[A-Za-z]+)\s*=\s*\n?\s*(true|false)\b/g) { print " $1 = $2\n" }' "$FILE") | |
| if [ -n "$HARDCODED" ]; then | |
| ERRORS="${ERRORS}\n❌ Env flags must not be hardcoded to boolean literals!\n\nFound hardcoded flags:\n${HARDCODED}\n\nEnv flags should derive their values from environment variables.\n" | |
| fi | |
| echo "Checking env flag naming conventions..." | |
| # Check that all export const (except functions) start with 'is' | |
| # This finds exports like "export const someFlag" that don't start with "is" or "get" | |
| BAD_NAMES=$(grep -E "^export const [a-z]" "$FILE" | grep -vE "^export const (is|get)" | sed 's/export const \([a-zA-Z]*\).*/ \1/') | |
| if [ -n "$BAD_NAMES" ]; then | |
| ERRORS="${ERRORS}\n❌ Env flags must use 'is' prefix for boolean flags!\n\nFound incorrectly named flags:\n${BAD_NAMES}\n\nExample: 'hostedMode' should be 'isHostedMode'\n" | |
| fi | |
| if [ -n "$ERRORS" ]; then | |
| echo "" | |
| echo -e "$ERRORS" | |
| exit 1 | |
| fi | |
| echo "✅ All env flags are properly configured" | |
| # One fetch for both base-ref audits, and no `|| true`: a swallowed fetch leaves | |
| # the base ref absent, which neither audit can tell apart from a branch that | |
| # changed nothing. The block-registry check at least degrades to a visible | |
| # `⚠ … skipping` line; the migration audit printed `✓ No new migrations to | |
| # check` and exited 0, clearing the only guard on production DDL. | |
| # | |
| # Depth stays at 1 — without a merge-base the migration audit diffs the two | |
| # tips, which under `--diff-filter=AM` is exactly the migrations new here. | |
| # | |
| # On push the base is `github.event.before`, the tip the branch had before | |
| # this push — not `HEAD~1`, which names only the last commit and would let a | |
| # multi-commit push slip every earlier commit's migrations past the audit. | |
| # It is fetched by SHA at depth 1; the audits diff two tips and need no | |
| # common ancestry. An all-zero `before` means the branch is new and has no | |
| # predecessor to diff, so `HEAD~1` remains the fallback there. | |
| - name: Resolve base ref for diff-based audits | |
| id: audit_base | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| git fetch --depth=1 origin "${{ github.base_ref }}" | |
| echo "ref=origin/${{ github.base_ref }}" >> "$GITHUB_OUTPUT" | |
| elif [ -n "${{ github.event.before }}" ] && | |
| [ "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]; then | |
| git fetch --depth=1 origin "${{ github.event.before }}" | |
| echo "ref=${{ github.event.before }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "ref=HEAD~1" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Check block registry invariants | |
| run: bun run apps/sim/scripts/check-block-registry.ts "${{ steps.audit_base.outputs.ref }}" | |
| - name: Lint code | |
| run: bun run lint:check | |
| # Every zero-argument `check:*` script, run concurrently. The list is derived in | |
| # scripts/run-audits.ts, which also writes the per-audit timing table to the job | |
| # summary and annotates failures. Audits needing a base ref stay separate below. | |
| - name: Repo audits | |
| run: bun run check:audits | |
| - name: Verify docs manifest is in sync | |
| run: bun run docs-manifest:check | |
| - name: Migration safety (zero-downtime) audit | |
| run: bun run check:migrations "${{ steps.audit_base.outputs.ref }}" | |
| # Every workspace, not just realtime. packages/emcn, packages/utils, | |
| # apps/desktop and apps/docs had no type check in CI at all; apps/sim's | |
| # source was covered only as a side effect of `next build` in the separate | |
| # Build App job. Note this does NOT cover apps/sim's tests — its tsconfig | |
| # excludes *.test.ts(x), and including them today surfaces ~2.2k errors, | |
| # so that is its own cleanup rather than a gate to switch on here. | |
| - name: Type-check all workspaces | |
| run: bunx turbo run type-check | |
| # cloud-review-tools.test.ts runs the real helper on the runner, which shells | |
| # out to rg. Blacksmith's image ships it, GitHub's doesn't. | |
| - name: Install ripgrep | |
| run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep) | |
| # Runs the root scripts and each workspace's Vitest suite, | |
| # without `--coverage`. | |
| - name: Run tests | |
| env: | |
| NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' | |
| NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' | |
| DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' | |
| ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only | |
| TURBO_CACHE_DIR: .turbo | |
| run: bun run test | |
| - name: Check schema and migrations are in sync | |
| working-directory: packages/db | |
| run: | | |
| bunx drizzle-kit generate --config=./drizzle.config.ts | |
| if [ -n "$(git status --porcelain ./migrations)" ]; then | |
| echo "❌ Schema and migrations are out of sync!" | |
| echo "Run 'cd packages/db && bunx drizzle-kit generate' and commit the new migrations." | |
| git status --porcelain ./migrations | |
| git diff ./migrations | |
| exit 1 | |
| fi | |
| echo "✅ Schema and migrations are in sync" | |
| # Next.js production build, in parallel with lint + tests. Sticky disks are | |
| # cloned from the last committed snapshot per job and committed last-writer- | |
| # wins, so concurrent mounts are safe. The bun/node_modules disks are shared | |
| # with test-build (the lockfile-hashed key means they only ever share when the | |
| # dependency tree really is identical, so LWW loss is harmless), but the Turbo | |
| # cache gets its own key: with a shared key, only the last committer's new | |
| # entries survive each run, so the test and build Turbo entries would evict | |
| # each other nondeterministically. | |
| # | |
| # Runner is sized for the COLD-cache build, which is what OOM-killed the 8vcpu | |
| # tier (23 kills / 1074 runs at 98% of its 30.4 GB): warm peaks ~12 GB, cold | |
| # peaked 51 GB. NODE_OPTIONS' --max-old-space-size caps only Node's JS heap, | |
| # not the native Turbopack workers that dominate, so it cannot prevent this. | |
| build: | |
| name: Build App | |
| runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }} | |
| # Build durations crossed 15 minutes as the app grew (10m02 on Jul 29 AM, | |
| # 14m44 after the folders/desktop/library merges, then two straight | |
| # timeouts) — GitHub reports a job timeout as "cancelled". 25 keeps | |
| # headroom without masking a genuine hang. | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| with: | |
| provider: ${{ vars.CI_PROVIDER }} | |
| turbo-cache-key: turbo-cache-build | |
| # No `.next/cache` mount: the Turbopack persistent build cache is off. A | |
| # controlled A/B on one branch (PR #6078) with a byte-identical module graph | |
| # measured compile at 113s with the cache off, 162s cold with it on, and | |
| # 360s warm — the cache made the same build 3.2x slower, and it grew | |
| # 5.1 GB -> 12 GB across two runs of an unchanged tree, so a disk degrades | |
| # the more it is used. Mounting a disk nothing reads would only cost storage. | |
| # Running out of RAM kills the whole VM and surfaces only as "the runner | |
| # has received a shutdown signal" — no mention of memory, ~12 min in. Warn | |
| # with the real numbers so that failure is a one-line diagnosis instead of | |
| # a mystery. Warn, never fail: a warm build peaks ~12 GB and a partial one | |
| # ~28 GB, so a 32 GB runner still completes plenty of builds, and the | |
| # GitHub fallback is the break-glass path — degrading it to a guaranteed | |
| # failure would be worse than the risk this flags. | |
| - name: Check runner memory headroom | |
| run: | | |
| TOTAL_GB=$(awk '/MemTotal/ {printf "%d", $2/1048576}' /proc/meminfo) | |
| echo "Runner memory: ${TOTAL_GB} GB" | |
| if [ "$TOTAL_GB" -lt 40 ]; then | |
| echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit." | |
| fi | |
| - name: Build application | |
| env: | |
| NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' | |
| NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' | |
| DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' | |
| STRIPE_SECRET_KEY: 'dummy_key_for_ci_only' | |
| STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only' | |
| RESEND_API_KEY: 'dummy_key_for_ci_only' | |
| AWS_REGION: 'us-west-2' | |
| ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only | |
| TURBO_CACHE_DIR: .turbo | |
| run: bunx turbo run build --filter=@sim/app |