diff --git a/.github/actions/image/action.yml b/.github/actions/image/action.yml index ff32a025bdb..b4a3307c990 100644 --- a/.github/actions/image/action.yml +++ b/.github/actions/image/action.yml @@ -23,6 +23,10 @@ inputs: description: Newline-separated Docker build arguments. required: false default: '' + target: + description: Dockerfile stage to build. Empty builds the final stage. + required: false + default: '' max-cache-size-mb: description: >- Layer cache to retain after this action prunes, in MB. Must stay above one @@ -77,6 +81,7 @@ runs: push: true tags: ${{ inputs.tags }} build-args: ${{ inputs.build-args }} + target: ${{ inputs.target }} provenance: false sbom: false @@ -183,5 +188,6 @@ runs: push: true tags: ${{ inputs.tags }} build-args: ${{ inputs.build-args }} + target: ${{ inputs.target }} provenance: false sbom: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8986a857d27..db922131867 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,8 +209,11 @@ jobs: ecr_repo_secret: ECR_APP gh_runner: linux-x64-8-core bs_runner: blacksmith-16vcpu-ubuntu-2404 + # Dev runs `db:push`, which needs drizzle-kit; every other environment + # gets the final stage, which carries runtime dependencies only. - id: db dockerfile: ./docker/db.Dockerfile + target: dev ecr_repo_secret: ECR_MIGRATIONS gh_runner: ubuntu-latest bs_runner: blacksmith-2vcpu-ubuntu-2404 @@ -259,6 +262,7 @@ jobs: tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }}-dev build-args: | MSHIP_PLAN_MODE_DEFAULT=true + target: ${{ matrix.target }} max-cache-size-mb: ${{ matrix.cache_mb }} # Staging/production coordinate task releases with app traffic cutover. diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile index b9e59e50481..2a83a09c2bd 100644 --- a/docker/db.Dockerfile +++ b/docker/db.Dockerfile @@ -4,62 +4,68 @@ FROM oven/bun:1.4.4-alpine AS base # ======================================== -# Dependencies Stage: Install Dependencies +# Pruner Stage: Emit the monorepo subset @sim/db needs # ======================================== -FROM base AS deps +FROM base AS pruner WORKDIR /app -# Copy only package files needed for migrations (these change less frequently) -COPY package.json bun.lock turbo.json ./ -COPY patches ./patches -RUN mkdir -p packages/db packages/logger packages/security packages/tsconfig packages/utils -COPY packages/db/package.json ./packages/db/package.json -COPY packages/logger/package.json ./packages/logger/package.json -COPY packages/security/package.json ./packages/security/package.json -COPY packages/tsconfig/package.json ./packages/tsconfig/package.json -COPY packages/utils/package.json ./packages/utils/package.json - -# Install dependencies with cache mount for faster builds. This stage contains -# only the migration workspace manifests, so Bun must normalize the full root -# lockfile to that subset; full-repository CI owns frozen-lockfile validation. -RUN --mount=type=cache,id=bun-cache,target=/root/.bun/install/cache \ - bun install --ignore-scripts +COPY . . + +# out/ keeps @sim/db's development toolchain for the dev target's `db:push`; +# production/ narrows a copy to what `db:migrate` runs (see scripts/prune-production-workspace.ts). +RUN TURBO_VERSION="$(bun -e "console.log(require('./package.json').devDependencies.turbo)")" && \ + bunx --bun "turbo@${TURBO_VERSION}" prune @sim/db --docker && \ + cp -R out production && \ + bun scripts/prune-production-workspace.ts @sim/db production/full production/json # ======================================== -# Runner Stage: Production Environment +# Dependencies Stages # ======================================== -FROM base AS runner +# Neither install is --frozen-lockfile: each holds a subset of the workspace, so +# Bun normalizes the pruned lockfile to it while keeping every locked resolution. +# Full-repository CI owns frozen-lockfile validation. +FROM base AS dev-deps WORKDIR /app -# Create non-root user and group (cached separately) -RUN addgroup -g 1001 -S nodejs && \ - adduser -S nextjs -u 1001 - -# Copy only the necessary files from deps (cached if dependencies don't change) -COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules +COPY --from=pruner /app/out/json/ ./ +COPY --from=pruner /app/out/bun.lock ./bun.lock -# Copy root package.json for workspace resolution -COPY --chown=nextjs:nodejs package.json ./package.json +RUN --mount=type=cache,id=bun-cache,target=/root/.bun/install/cache \ + bun install --linker=hoisted --ignore-scripts -# Copy package configuration files (needed for migrations) -COPY --chown=nextjs:nodejs packages/db/drizzle.config.ts ./packages/db/drizzle.config.ts +FROM base AS deps +WORKDIR /app -# Copy tsconfig package (needed for workspace symlink resolution) -COPY --chown=nextjs:nodejs packages/tsconfig ./packages/tsconfig +COPY --from=pruner /app/production/json/ ./ +COPY --from=pruner /app/production/bun.lock ./bun.lock -# Copy utils package (needed by db scripts that import @sim/utils) -COPY --chown=nextjs:nodejs packages/utils ./packages/utils +RUN --mount=type=cache,id=bun-cache,target=/root/.bun/install/cache \ + bun install --linker=hoisted --omit=dev --ignore-scripts -# Copy logger package (needed by @sim/db's tx-tripwire at import time) -COPY --chown=nextjs:nodejs packages/logger ./packages/logger +FROM base AS user +RUN addgroup -g 1001 -S nodejs && \ + adduser -S nextjs -u 1001 -# Copy security package (the schema's sealed columns import its encryption) -COPY --chown=nextjs:nodejs packages/security ./packages/security +# ======================================== +# Dev Stage: also runs `db:push`, which needs drizzle-kit +# ======================================== +FROM user AS dev +WORKDIR /app -# Copy database package source code (changes most frequently - placed last) -COPY --chown=nextjs:nodejs packages/db ./packages/db +COPY --from=dev-deps --chown=nextjs:nodejs /app ./ +COPY --from=pruner --chown=nextjs:nodejs /app/out/full/ ./ -# Switch to non-root user USER nextjs +WORKDIR /app/packages/db +# ======================================== +# Runner Stage (default): runs `db:migrate` with runtime dependencies only +# ======================================== +FROM user AS runner +WORKDIR /app + +COPY --from=deps --chown=nextjs:nodejs /app ./ +COPY --from=pruner --chown=nextjs:nodejs /app/production/full/ ./ + +USER nextjs WORKDIR /app/packages/db diff --git a/docker/realtime.Dockerfile b/docker/realtime.Dockerfile index 1b3631c184a..8aea18f961e 100644 --- a/docker/realtime.Dockerfile +++ b/docker/realtime.Dockerfile @@ -6,15 +6,18 @@ FROM oven/bun:1.4.4-alpine AS base RUN apk add --no-cache libc6-compat curl # ======================================== -# Pruner Stage: Emit a minimal monorepo subset that @sim/realtime depends on +# Pruner Stage: Emit the monorepo subset @sim/realtime runs in production # ======================================== FROM base AS pruner WORKDIR /app COPY . . +# turbo keeps the workspace members @sim/realtime needs to build and test too; +# scripts/prune-production-workspace.ts drops those and their devDependencies. RUN TURBO_VERSION="$(bun -e "console.log(require('./package.json').devDependencies.turbo)")" && \ - bunx --bun "turbo@${TURBO_VERSION}" prune @sim/realtime --docker + bunx --bun "turbo@${TURBO_VERSION}" prune @sim/realtime --docker && \ + bun scripts/prune-production-workspace.ts @sim/realtime out/full out/json # ======================================== # Dependencies Stage: Install Dependencies @@ -25,8 +28,11 @@ WORKDIR /app COPY --from=pruner /app/out/json/ ./ COPY --from=pruner /app/out/bun.lock ./bun.lock +# Not --frozen-lockfile: the narrowed manifests no longer match the pruned +# lockfile. Bun keeps every locked resolution, and full-repository CI owns +# frozen-lockfile validation. RUN --mount=type=cache,id=bun-cache,target=/root/.bun/install/cache \ - bun install --frozen-lockfile --linker=hoisted --omit=dev --ignore-scripts + bun install --linker=hoisted --omit=dev --ignore-scripts # ======================================== # Runner Stage: Run the Socket Server diff --git a/scripts/prune-production-workspace.ts b/scripts/prune-production-workspace.ts new file mode 100644 index 00000000000..7095ef8f4ca --- /dev/null +++ b/scripts/prune-production-workspace.ts @@ -0,0 +1,127 @@ +/** + * Narrows a `turbo prune --docker` output to what one package runs in production, so a + * following `bun install --omit=dev` installs only its runtime dependencies. + * + * `--omit=dev` leaves out the root's devDependencies, but Bun still installs every workspace + * member, each member's devDependencies, and their peers. A pruned image therefore carried the + * build and test toolchain (drizzle-kit and its bundled esbuild, vitest, vite, tsx) that its + * members declare for development, along with members such as `@sim/testing` that only tests + * import. This keeps the members the target reaches through `dependencies`, + * `optionalDependencies`, or a `tsconfig.json` `extends` (Bun reads each member's tsconfig when it + * runs TypeScript source), drops every other member, and strips the devDependencies of the root + * and the kept members. + * + * Usage: `bun scripts/prune-production-workspace.ts []`. The closure is + * computed from ``, which holds the tsconfig files, and applied to every directory given. + * The lockfile no longer matches the narrowed manifests, so install without `--frozen-lockfile`: + * Bun keeps each locked resolution whose range still matches and only drops the entries the + * narrowed workspace no longer reaches, so no version changes. + */ +import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' + +interface PackageManifest { + name?: string + workspaces?: string[] + dependencies?: Record + optionalDependencies?: Record + devDependencies?: Record +} + +interface WorkspaceMember { + /** Member directory relative to the workspace root, e.g. `packages/db`. */ + dir: string + manifest: PackageManifest +} + +function readJson(path: string): T { + return JSON.parse(readFileSync(path, 'utf8')) as T +} + +function listMembers(rootDir: string): Map { + const root = readJson(join(rootDir, 'package.json')) + const members = new Map() + for (const pattern of root.workspaces ?? []) { + for (const manifestPath of new Bun.Glob(`${pattern}/package.json`).scanSync(rootDir)) { + const manifest = readJson(join(rootDir, manifestPath)) + if (manifest.name) members.set(manifest.name, { dir: dirname(manifestPath), manifest }) + } + } + return members +} + +/** The package a module specifier names, e.g. `@sim/tsconfig` for `@sim/tsconfig/base.json`. */ +function packageName(specifier: string): string { + const segments = specifier.split('/') + return segments.slice(0, specifier.startsWith('@') ? 2 : 1).join('/') +} + +function tsconfigParents(rootDir: string, member: WorkspaceMember): string[] { + const tsconfigPath = join(rootDir, member.dir, 'tsconfig.json') + if (!existsSync(tsconfigPath)) return [] + const { extends: parents } = readJson<{ extends?: string | string[] }>(tsconfigPath) + return [parents ?? []].flat().map(packageName) +} + +function productionClosure( + rootDir: string, + members: Map, + target: string +): Set { + const kept = new Set() + const queue = [target] + for (let name = queue.pop(); name; name = queue.pop()) { + if (kept.has(name)) continue + const member = members.get(name) + if (!member) throw new Error(`${name} is not a member of the workspace in ${rootDir}`) + kept.add(name) + const { dependencies = {}, optionalDependencies = {} } = member.manifest + const reached = [ + ...Object.keys(dependencies), + ...Object.keys(optionalDependencies), + ...tsconfigParents(rootDir, member), + ] + queue.push(...reached.filter((next) => members.has(next))) + } + return kept +} + +function writeJson(path: string, value: unknown) { + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`) +} + +/** Drops the members outside `kept`, from disk and from the root's workspace list. */ +function narrow(rootDir: string, kept: Set) { + const dropped = new Set() + for (const [name, member] of listMembers(rootDir)) { + const memberDir = join(rootDir, member.dir) + if (!kept.has(name)) { + rmSync(memberDir, { recursive: true, force: true }) + dropped.add(member.dir) + continue + } + const { devDependencies: _, ...manifest } = member.manifest + writeJson(join(memberDir, 'package.json'), manifest) + } + // `turbo prune` lists each member's path, and Bun refuses a listed path that no longer exists. + // The root's devDependencies go too: `--omit=dev` skips them, but while they stay in the + // lockfile Bun still links them to satisfy optional peers (better-auth's on vitest). + const rootPath = join(rootDir, 'package.json') + const { devDependencies: _, ...root } = readJson(rootPath) + writeJson(rootPath, { + ...root, + workspaces: (root.workspaces ?? []).filter((pattern) => !dropped.has(pattern)), + }) +} + +const [target, fullDir, ...otherDirs] = process.argv.slice(2) +if (!target || !fullDir) { + console.error( + 'Usage: bun scripts/prune-production-workspace.ts []' + ) + process.exit(1) +} + +const kept = productionClosure(fullDir, listMembers(fullDir), target) +for (const dir of [fullDir, ...otherDirs]) narrow(dir, kept) +console.log(`Production workspace for ${target}: ${[...kept].sort().join(', ')}`)