diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..bcca0b664cc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,41 @@ +# What a `docker build -f api/Dockerfile .` is allowed to see. +# +# This file has to live at the REPO ROOT. Docker reads `.dockerignore` from the +# build CONTEXT, and the context of both the deploy step (api/cloudbuild.yaml) +# and the pre-merge job (.github/workflows/ci-image.yml) is `.` — so the +# `api/.dockerignore` that used to hold these rules was never read by anything. +# The proof was inside the file itself: it excluded `*.md`, and yet +# `COPY pyproject.toml uv.lock README.md ./` in the builder stage has succeeded +# on every build since it was written. +# +# Narrowing the context to `api/` instead is not an option: the image needs +# `core/`, `pyproject.toml` and `uv.lock`, which live above it. +# +# Allowlist, not denylist. A denylist that misses a new directory only makes the +# context quietly fatter — that is how ~230 MB (a 120 MB `.git` plus a 69 MB +# `plots/`) came to be uploaded on every build; an allowlist that misses one +# fails at the COPY line, which is the loud failure. `app/` keeps its own +# `app/.dockerignore`: the frontend build's context is `app`, so nothing here +# applies to it. +* + +!api +!core +!pyproject.toml +!uv.lock +!README.md + +# …and the noise that lives inside the trees above. `__pycache__` matters twice +# over now that the image compiles its own bytecode: a stale host `.pyc` copied +# into a layer would shadow the source it no longer matches. +**/__pycache__ +**/*.py[cod] +**/.pytest_cache +**/.ruff_cache +**/.mypy_cache + +# `.env*` is already covered by the `*` above. It is spelled out anyway so that +# no future `!` line can let DATABASE_URL or ADMIN_TOKEN into a layer — the same +# rule the sibling repo kurrentschrift keeps in its own `.dockerignore`. +**/.env +**/.env.* diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index d581eee7b63..42c699ee543 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -129,10 +129,10 @@ jobs: # Dockerfiles, the dependency lock, the source the runtime stage # copies, README.md (the builder copies it next to pyproject.toml, so # a rename breaks the install) and a root .dockerignore, which decides - # what a `context: .` build can see at all. plots/ is copied too but - # nothing reads it at runtime (the implementations are served from - # Postgres), so the plot pipeline's PRs must not each pay for a - # container build. + # what a `context: .` build can see at all. plots/ is not among them + # and is no longer in the image either — nothing under api/ or core/ + # reads it at runtime, the implementations being served from Postgres — + # so the plot pipeline's PRs must not each pay for a container build. IMAGE_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(api/|core/|pyproject\.toml$|uv\.lock$|README\.md$|\.dockerignore$|app/Dockerfile$|\.github/workflows/ci-image\.yml$)' || true) if [[ "$EVENT_NAME" == "workflow_dispatch" && "$FORCE_RUN" == "true" ]]; then @@ -227,35 +227,23 @@ jobs: if: failure() && steps.check.outputs.should_build == 'true' run: docker logs api || true - # Threshold `warning` with three exceptions, rather than a non-blocking - # run: that way a warning of any other code blocks, which is the point of - # having the linter at all. All three are deliberate choices in - # api/Dockerfile, named at their line below. - # - # Note what `ignore` is and is not: hadolint applies it to the whole file, - # so a NEW DL3013, DL3008 or DL3025 somewhere else in api/Dockerfile is - # also suppressed. Line-scoping needs `# hadolint ignore=` comments - # in the Dockerfile itself, next to the three instructions they excuse — - # the better home for them, and the right follow-up, but a Dockerfile edit - # is not this change's to make. app/Dockerfile needs no exceptions at all, - # so nothing there is suppressed — verified against hadolint 2.15.1, the - # version this action pins. + # Threshold `warning` and NO file-wide exceptions: a warning of any code + # blocks, which is the point of having the linter at all. The two rules + # api/Dockerfile declines (DL3008 apt pinning, DL3025 shell-form + # HEALTHCHECK) now carry `# hadolint ignore=` comments at the exact + # instruction they excuse, so a NEW occurrence elsewhere in the file is + # still caught — which a file-wide `ignore:` silently swallowed. The third + # former exception, DL3013, is simply gone: `pip install uv==` is + # pinned. DL3066 (non-numeric USER) still fires at info level, below the + # threshold, so it stays visible without blocking. app/Dockerfile needs no + # exceptions at all — verified against hadolint 2.15.1, the version this + # action pins. - name: Hadolint (api/Dockerfile) if: steps.check.outputs.should_build == 'true' uses: hadolint/hadolint-action@06be81baf89a55ffd0e24b8f04a4185738dd3387 # v3.5.0 with: dockerfile: api/Dockerfile failure-threshold: warning - # DL3013 `pip install uv` unpinned — uv is the installer; the versions - # that matter are pinned in uv.lock, which the next line honours. - # DL3008 unpinned apt `curl`/`libraqm0` — pinning a Debian point release - # breaks the build on every security update of the base image. - # DL3025 shell-form HEALTHCHECK CMD — the `|| exit 1` fallback needs a - # shell; JSON form cannot express it. - # DL3066 (non-numeric USER) also fires but only at info level, so it - # stays visible in the log without blocking; `useradd -u 1000` - # already gives the user a fixed uid. - ignore: DL3013,DL3008,DL3025 - name: Hadolint (app/Dockerfile) if: steps.check.outputs.should_build == 'true' diff --git a/CHANGELOG.md b/CHANGELOG.md index 0475c20e969..494b903de3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -222,6 +222,33 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Changed +- **The API image build gets a `.dockerignore` that is actually read, loses 69 MB of dead + weight and ships its own bytecode** — `api/.dockerignore` never did anything: Docker reads + the ignore file from the build CONTEXT, and both the Cloud Build step and the pre-merge + image job build with `.` at the repo root. The file said so itself — it excluded `*.md` + while the builder's `COPY … README.md` succeeded on every build — and the result was a + context of roughly 230 MB per build, a 120 MB `.git` and a 69 MB `plots/` foremost. The + replacement lives at the root and is an **allowlist** (`api`, `core`, `pyproject.toml`, + `uv.lock`, `README.md`): a denylist that misses a new directory only makes the context + quietly fatter, while an allowlist that misses one fails at the COPY line. The context + cannot instead be narrowed to `api/`, which is why this shape: the image needs `core/` and + the lock files, and they live above it. `COPY plots/ ./plots/` is gone from the runtime + stage — it was 16.6 MB of every pulled image (69.2 MB unpacked, 9,166 files; layer 10 of + the `latest` manifest) for a directory nothing reads, because the implementations this API + serves come from Postgres; `ci-image.yml` already said as much where it explains why + `plots/**` is not a build trigger. `UV_COMPILE_BYTECODE=1` plus a `compileall` over `api` + and `core` in the runtime stage put `.pyc` in the image, which takes ~1.8 s off every cold + start (`import api.main` measured at 3.56–4.43 s with nothing cached against 1.79–2.26 s + with bytecode present) at the price of a bigger venv layer (686.6 MB unpacked / 210.3 MB + compressed against 493.1 / 142.3). `uv` itself is pinned — the resolver that reads + `uv.lock` was the unpinned link in the dependency chain, though the image as a whole + stays unreproducible on purpose: `python:3.13-slim` is a mutable tag and the apt packages + are deliberately unversioned — and `UV_PYTHON` names the interpreter so uv can never + quietly download a managed CPython that the runtime stage does not have at the same path. + With the pin, hadolint's DL3013 exception disappears; the remaining two (DL3008, DL3025) + move out of the workflow's file-wide `ignore:` and onto the exact instructions they excuse, + so a new occurrence elsewhere in the file is caught instead of swallowed. (#11211) + - **The frontend deploys through a candidate revision instead of straight onto live traffic** — `app/cloudbuild.yaml` now follows the same candidate-rollout pattern as `api/cloudbuild.yaml`: deploy with `--no-traffic --tag=candidate diff --git a/api/.dockerignore b/api/.dockerignore deleted file mode 100644 index 41c0c35f812..00000000000 --- a/api/.dockerignore +++ /dev/null @@ -1,49 +0,0 @@ -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -*.egg-info/ -dist/ -build/ -.venv/ -venv/ -env/ - -# Testing -.pytest_cache/ -.coverage -htmlcov/ -.tox/ -.hypothesis/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# Git -.git/ -.gitignore -.gitattributes - -# Documentation -*.md -docs/ - -# Temporary files -*.log -*.tmp -temp/ - -# Development -.env -.env.local -*.test.py - -# Other -.DS_Store -Thumbs.db diff --git a/api/Dockerfile b/api/Dockerfile index ec8fa148036..21a7fc0c39b 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -11,6 +11,11 @@ # path in .venv/pyvenv.cfg and writes /app/.venv/bin shebangs, so the runtime # stage has to offer the same interpreter at the same path (hence WORKDIR /app # on both sides too). +# +# The build CONTEXT is the repo root — `core/`, `pyproject.toml` and `uv.lock` +# all live above `api/` — and what that context may contain is decided by the +# root `.dockerignore`, not by one under `api/`. Read that file before adding a +# COPY: it is an allowlist, so a path it does not name simply is not there. FROM python:3.13-slim AS builder @@ -22,7 +27,31 @@ WORKDIR /app # `plotting` extra, which this image does not install.) The toolchain was # ~110 MB that only ever sat there. If a future dependency arrives sdist-only, # the build fails loudly at this step — the right place to notice. -RUN pip install --no-cache-dir uv +# +# uv is PINNED — the unpinned link in the DEPENDENCY chain, nothing wider than +# that. uv.lock fixes every package, but the resolver that reads it floated on +# whatever PyPI served that morning, and a resolver change is exactly the kind +# of difference that makes "it built yesterday" true and useless. The image as +# a whole is still not reproducible: `python:3.13-slim` is a mutable tag, and +# the apt packages in the runtime stage are deliberately unpinned for the reason +# given at their DL3008 line. Pinning uv also retires the DL3013 exception. +RUN pip install --no-cache-dir uv==0.10.9 + +# Write .pyc next to every installed module. Without this the venv ships source +# only — and the root `.dockerignore` excludes `__pycache__`, so the COPYs bring +# none either — which leaves every cold start compiling the whole import graph. +# Measured on this dependency set (2026-09-03): `import api.main` takes +# 3.56–4.43 s with nothing cached against 1.79–2.26 s with bytecode present, +# i.e. ~1.8 s off every cold start. It is not free — the compiled venv is +# 686.6 MB unpacked / 210.3 MB compressed against 493.1 / 142.3 — so the trade +# is ~68 MB of pulled layer against ~1.8 s of every container's first request. +ENV UV_COMPILE_BYTECODE=1 +# Which interpreter the venv is built on, said out loud. uv otherwise picks one +# by discovery and may DOWNLOAD a managed CPython — a different build of 3.13 +# than the runtime stage offers at the same path, which is the one thing the +# two-stage split cannot survive (see the note above about pyvenv.cfg). +ENV UV_PYTHON=/usr/local/bin/python3.13 +ENV UV_PYTHON_DOWNLOADS=never # Copy requirements first for better caching. COPY pyproject.toml uv.lock README.md ./ @@ -44,6 +73,11 @@ WORKDIR /app # #10813's regression — and why the guard it added has failed every build of # this image since (the deploy-api trigger has been red since 2026-08-30). # The Debian package is 32 KB plus its HarfBuzz/FriBiDi dependencies. +# +# DL3008 (pin the apt versions) is declined here, not repo-wide: pinning a +# Debian point release breaks the build on every security update of the base +# image, and these two packages carry no version-sensitive behaviour. +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libraqm0 \ @@ -59,10 +93,14 @@ RUN useradd -m -u 1000 appuser # so the `chmod -R 755 .venv/bin` that used to ride along is gone too. COPY --from=builder --chown=appuser:appuser /app/.venv ./.venv -# Copy application code +# Copy application code. `plots/` used to be copied here too and was never read: +# the implementations this API serves come from Postgres (core/database), and +# nothing under api/ or core/ opens a path below plots/ — ci-image.yml already +# says so where it explains why plots/** is not a build trigger. It cost 16.6 MB +# of every pulled image (69.2 MB unpacked, 9,166 files), measured as layer 10 of +# the `latest` manifest on 2026-09-03. COPY --chown=appuser:appuser api/ ./api/ COPY --chown=appuser:appuser core/ ./core/ -COPY --chown=appuser:appuser plots/ ./plots/ # Fail the build if Pillow cannot shape text. Without it Pillow falls back to # its BASIC layout engine: the OG cards lose MonoLisa's italic `ss02` swashes @@ -79,6 +117,16 @@ assert features.check('raqm'), \ # Switch to non-root user USER appuser +# The other half of UV_COMPILE_BYTECODE above: that one covers the venv, this +# one covers OUR code, which arrives through the COPYs and therefore has no +# .pyc of its own. Run as appuser, after the USER switch, so the __pycache__ +# directories belong to the process that will read them — a root-owned cache in +# an appuser tree works, but only by accident of file modes. +# `-q` keeps the build log readable; a compile error still fails the build, +# which is the second thing this line buys: a syntax error in a module that +# only some request path imports now stops the IMAGE instead of a request. +RUN .venv/bin/python -m compileall -q api core + # Expose port 8000 for FastAPI EXPOSE 8000 @@ -86,7 +134,9 @@ EXPOSE 8000 ENV PYTHONUNBUFFERED=1 ENV PORT=8000 -# Health check +# Health check. DL3025 wants the JSON exec form, which cannot express the +# `|| exit 1` fallback — that needs a shell. +# hadolint ignore=DL3025 HEALTHCHECK CMD curl --fail http://localhost:8000/health || exit 1 # Run FastAPI app with uvicorn