python-tests.yml

# Runs python tests on a matrix of python versions and platforms.
#
# Checks for changes to python files before running the tests.
# If always_run is true, always runs the tests.

name: 'python tests'

on:
  push:
    branches:
      - 'main'
  pull_request:
    types:
      - 'ready_for_review'
      - 'opened'
      - 'synchronize'
  merge_group:
  workflow_dispatch:
    inputs:
      always_run:
        description: 'Always run the tests'
        required: true
        type: boolean
        default: true
  workflow_call:
    inputs:
      always_run:
        description: 'Always run the tests'
        required: true
        type: boolean
        default: true

# One run per pull request: a new push cancels the superseded run. The PR number is used rather
# than the head branch, which two forks can share. Pushes to main and merge groups fall back to
# the run id and never cancel each other. The group names the workflow literally
# because `github.workflow` is the caller's name inside a reusable workflow, and release.yml calls
# several of these at once.
permissions:
  contents: read

concurrency:
  group: python-tests-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  matrix:
    strategy:
      # Let the other platforms finish when one runner fails, so a rerun repeats only the failed job.
      fail-fast: false
      matrix:
        python-version:
          - '3.12'
        platform:
          - linux-cpu
          - macos-default
          - windows-cpu
          - windows-ffmpeg
          - windows-arm64
        include:
          - platform: linux-cpu
            os: blacksmith-4vcpu-ubuntu-2404
            extra-index-url: 'https://download.pytorch.org/whl/cpu'
            github-env: $GITHUB_ENV
            pytest-workers: logical
            timeout-minutes: 30
          - platform: macos-default
            os: blacksmith-6vcpu-macos-15
            github-env: $GITHUB_ENV
            pytest-workers: logical
            timeout-minutes: 30
          - platform: windows-cpu
            os: blacksmith-8vcpu-windows-2025
            github-env: $env:GITHUB_ENV
            pytest-workers: logical
            # This runner is Windows Server 2025 Core, which lacks the Video for Windows DLLs
            # (AVICAP32, MSVFW32) every standard Windows ffmpeg build links against, so ffmpeg
            # cannot start here. The `ffmpeg` tests run in windows-ffmpeg instead.
            # Windows installs linux's dependency set, so a `platform_independent` test cannot fail differently
            # here than on linux, which runs it. macOS runs a different torch without bitsandbytes and keeps them.
            pytest-markers: 'not slow and not platform_independent and not ffmpeg'
            prepare-server-core: true
            # Job takes 10-15 min (runner prep ~3.5, pytest startup ~2, tests ~5-8); 30 leaves linux's and macOS's room.
            timeout-minutes: 30
          - platform: windows-ffmpeg
            os: windows-2022
            github-env: $env:GITHUB_ENV
            # Half the vCPUs, not one worker each. The ffmpeg tests spawn decoders and encoders that
            # bring their own threads, which the per-worker `OMP_NUM_THREADS=1` below does not cap,
            # so a full-width run leaves this runner with no spare core and the deadline-sensitive
            # tests miss their budgets non-deterministically. Two workers keep that headroom.
            pytest-workers: '2'
            pytest-markers: 'not slow and not platform_independent and ffmpeg'
            # Roughly 1.5x the pytest step's two-worker wall time (~6 min), plus the job's ~1 min
            # of checkout and install, plus margin.
            timeout-minutes: 15
          # Windows on ARM64 (NVIDIA RTX Spark). The lockfile's ARM64 fork is exercised nowhere else:
          # torch 2.14.0+cu134 from NVIDIA's index (a 1.85 GB wheel, CPU-only on this runner), numpy 2,
          # onnxruntime 1.31, sentencepiece 0.2.2 and the self-built opencv/blake3/tiktoken wheels. uv
          # installs an x86_64 CPython on ARM64 Windows unless the request names the architecture.
          - platform: windows-arm64
            os: windows-11-arm
            uv-python: cpython-3.12-windows-aarch64-none
            github-env: $env:GITHUB_ENV
            # Two workers for the same reason as windows-ffmpeg. The job also downloads the NVIDIA torch
            # wheel and builds imageio-ffmpeg from its sdist (~18 min at four workers).
            pytest-workers: '2'
            # A per-test timeout turns a hang into a failure with a traceback, so pytest still reaches its summary
            # inside the job timeout.
            pytest-args: '--timeout=600'
            # About 1.6x the slowest measured job (37 min, 31 of them pytest; another run took 28).
            timeout-minutes: 60
    name: 'py${{ matrix.python-version }}: ${{ matrix.platform }}'
    runs-on: ${{ matrix.os }}
    # Per platform: expected run time is 8-14 min on linux and macos. See the matrix for windows.
    timeout-minutes: ${{ matrix.timeout-minutes }}
    env:
      PIP_USE_PEP517: '1'
      # One compute thread per worker. Torch otherwise sizes its thread pools to the whole
      # machine, so one worker per vCPU puts vCPUs^2 threads on a runner that fight for the same
      # cores. This caps torch only; helper processes the tests spawn thread themselves.
      OMP_NUM_THREADS: '1'
      MKL_NUM_THREADS: '1'

    steps:
      - name: checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: restore LFS cache
        uses: actions/cache@v5
        with:
          path: .git/lfs
          # Checkout leaves LFS pointer files, so their hash is the content key.
          key: lfs-${{ hashFiles('tests/test_model_probe/stripped_models/**', 'tests/model_identification/stripped_models/**') }}
          restore-keys: lfs-

      - name: pull LFS files
        run: git lfs pull
        shell: bash

      - name: check for changed python files
        if: ${{ inputs.always_run != true }}
        id: changed-files
        # Pinned to the _hash_ for v45.0.9 to prevent supply-chain attacks.
        # See:
        # - CVE-2025-30066
        # - https://www.stepsecurity.io/blog/harden-runner-detection-tj-actions-changed-files-action-is-compromised
        # - https://github.com/tj-actions/changed-files/issues/2463
        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96
        with:
          files_yaml: |
            python:
              - 'pyproject.toml'
              # The lockfile pins pytest-xdist, which this workflow's pytest command depends on.
              - 'uv.lock'
              - '.github/workflows/python-tests.yml'
              - 'invokeai/**'
              - '!invokeai/frontend/webv1/**'
              - '!invokeai/frontend/webv2/**'
              # Re-includes must follow the exclusions above.
              # test_imports.py walks every package, including the frontend ones.
              - 'invokeai/frontend/webv1/**/*.py'
              - 'invokeai/frontend/webv2/**/*.py'
              # test_workflow_graph_builder.py validates these editor fixtures against the backend.
              - 'invokeai/frontend/webv2/src/features/workflow/core/fixtures/**'
              # Cross-stack generation contracts: the frontend writes these, Python validates them
              # against the invocation and architecture registries. A frontend-only change that
              # regenerates one would otherwise pass vitest while skipping the backend half.
              # `__fixtures__/` does not exist yet -- it arrives with the graph-builder fixtures.
              # A pattern matching nothing costs nothing here, and listing it now means the branch
              # that adds the directory does not also have to remember to add this line.
              - 'invokeai/frontend/webv2/src/features/generation/core/__snapshots__/**'
              - 'invokeai/frontend/webv2/src/features/generation/core/__fixtures__/**'
              - 'invokeai/frontend/webv2/src/features/video/core/__snapshots__/**'
              - 'invokeai/frontend/webv2/src/features/upscale/core/__snapshots__/**'
              # test_shared_numerical_fixtures.py checks the Python image adjustments against the
              # canvas spatial-adjustment kernels' shared fixtures.
              - 'invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/**'
              # test_ideogram4_frontend_bounds.py reads the Ideogram 4 control bounds from here.
              - 'invokeai/frontend/webv2/src/features/generation/core/settings.ts'
              - 'tests/**'
              # tests/ covers a few of these (e.g. test_check_pins.py, test_docs_json_export.py)
              - 'scripts/**'
              - 'pins.json'
              # test_check_pr_metadata.py parses the template as a body.
              - '.github/pull_request_template.md'

      - name: setup uv
        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true }}
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
        with:
          version: '0.11.28'  # When updating this, make sure the version in launcher's package.json gets updated too.
          # The ARM64 lane's torch wheel unpacks to several GB of binaries; cached per lock hash it would crowd the
          # other lanes out of the repository's cache, and re-downloading it costs a couple of minutes.
          enable-cache: ${{ matrix.platform != 'windows-arm64' }}
          python-version: ${{ matrix.uv-python || matrix.python-version }}

      - name: prepare the ARM64 runner
        if: ${{ matrix.platform == 'windows-arm64' && (steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true) }}
        # The imageio-ffmpeg sdist bundles no ffmpeg binary; the video tests find one on PATH. Chocolatey's
        # package is an x64 build, which Windows on ARM runs under emulation.
        run: |
          uv run --no-project python -c "import platform, sys; assert platform.machine() == 'ARM64', platform.machine(); print(sys.version)"
          choco install ffmpeg --version 9.0.2 -y --no-progress
          ffmpeg -version | Select-Object -First 1

      - name: install dependencies
        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true }}
        env:
          UV_INDEX: ${{ matrix.extra-index-url }}
        run: uv sync --no-progress --locked --extra test

      # Windows Server Core lacks two things the suite imports. The non-headless opencv build needs
      # Media Foundation at `import cv2`; the feature installs in ~30 s without a reboot. There is
      # no audio stack either, so `import sounddevice` raises PortAudioError, which mediapipe does
      # not catch (only ImportError/OSError). Removing the module turns that into the ImportError
      # mediapipe falls back on. No test needs a real audio device.
      - name: prepare server core
        if: ${{ matrix.prepare-server-core && (steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true) }}
        run: |
          $result = Install-WindowsFeature -Name Server-Media-Foundation
          Write-Output "Success: $($result.Success)  RestartNeeded: $($result.RestartNeeded)"
          $result.FeatureResult | Format-Table -AutoSize | Out-String | Write-Output
          if (-not $result.Success) { throw 'Installing Server-Media-Foundation failed' }
          Remove-Item .venv\Lib\site-packages\sounddevice.py
          uv run --no-sync python -c "import cv2, mediapipe; print(cv2.__version__, mediapipe.__version__)"
          if ($LASTEXITCODE -ne 0) { throw 'cv2 or mediapipe still fails to import after preparing the runner' }

      - name: import the self-built and platform-specific wheels
        if: ${{ matrix.platform == 'windows-arm64' && (steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true) }}
        # The suite never imports tiktoken, so a broken self-built wheel would otherwise pass unnoticed.
        run: uv run --no-sync python -c "import blake3, cv2, onnxruntime, tiktoken, torch, torchvision; print(torch.__version__, torchvision.__version__, cv2.__version__, 'cuda available:', torch.cuda.is_available())"

      - name: run pytest
        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true }}
        # `logical`, not `auto`: xdist's `auto` asks psutil for *physical* cores, which is half
        # the vCPUs a standard runner exposes. Much of the suite waits on timers and polls, so
        # one worker per vCPU is the right level where the runner has the headroom for it. The
        # matrix carries the count, because windows-ffmpeg and windows-arm64 need a lower one.
        run: uv run --no-sync pytest -n ${{ matrix.pytest-workers }} --max-worker-restart=0 -m "${{ matrix.pytest-markers || 'not slow' }}" ${{ matrix.pytest-args }}

  # Decides whether the external-database job runs. That job's service containers start before any of its
  # steps, so the decision has to come from a job of its own: otherwise every pull request, frontend-only ones
  # included, would pull both server images and wait for them to become healthy.
  external-database-changes:
    name: 'external database: change detection'
    runs-on: blacksmith-4vcpu-ubuntu-2404
    timeout-minutes: 10
    outputs:
      run: ${{ steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true }}
    steps:
      - name: checkout
        if: ${{ inputs.always_run != true }}
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: restore LFS cache
        if: ${{ inputs.always_run != true }}
        uses: actions/cache@v5
        with:
          path: .git/lfs
          # Checkout leaves LFS pointer files, so their hash is the content key.
          key: lfs-${{ hashFiles('tests/test_model_probe/stripped_models/**', 'tests/model_identification/stripped_models/**') }}
          restore-keys: lfs-

      - name: pull LFS files
        if: ${{ inputs.always_run != true }}
        run: git lfs pull
        shell: bash

      - name: check for changed python files
        if: ${{ inputs.always_run != true }}
        id: changed-files
        # Pinned to the _hash_ for v45.0.9, see the matrix job above.
        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96
        with:
          files_yaml: |
            python:
              - 'pyproject.toml'
              - 'uv.lock'
              - '.github/workflows/python-tests.yml'
              - 'invokeai/**/*.py'
              # The library's tests store the bundled workflows and style presets.
              - 'invokeai/app/services/workflow_records/default_workflows/**'
              - 'invokeai/app/services/style_preset_records/default_style_presets.json'
              - 'tests/**'

  # The database layer against MySQL and MariaDB, at the oldest release of each that InvokeAI supports.
  # Only the tests on the database fixtures (marked `uses_database`) target the server; the matrix above
  # runs the whole suite, those tests included, on SQLite.
  external-database:
    needs: external-database-changes
    if: ${{ needs.external-database-changes.outputs.run == 'true' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - server: mysql
            image: 'mysql:8.4.11'
            health-cmd: 'mysqladmin ping -h 127.0.0.1 --silent'
            url: 'mysql+pymysql://root:invokeai@127.0.0.1:3306'
          - server: mariadb
            image: 'mariadb:10.11.19'
            health-cmd: 'mariadb-admin ping -h 127.0.0.1 --silent'
            url: 'mariadb+pymysql://root:invokeai@127.0.0.1:3306'
    name: 'py3.12: linux-${{ matrix.server }}'
    runs-on: blacksmith-4vcpu-ubuntu-2404
    timeout-minutes: 30
    services:
      database:
        image: ${{ matrix.image }}
        env:
          MYSQL_ROOT_PASSWORD: invokeai
          MARIADB_ROOT_PASSWORD: invokeai
        ports:
          - '3306:3306'
        options: >-
          --health-cmd="${{ matrix.health-cmd }}"
          --health-interval=5s
          --health-timeout=5s
          --health-retries=30
    env:
      OMP_NUM_THREADS: '1'
      MKL_NUM_THREADS: '1'
      INVOKEAI_TEST_DB_URL: ${{ matrix.url }}

    steps:
      - name: checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: restore LFS cache
        uses: actions/cache@v5
        with:
          path: .git/lfs
          # Checkout leaves LFS pointer files, so their hash is the content key.
          key: lfs-${{ hashFiles('tests/test_model_probe/stripped_models/**', 'tests/model_identification/stripped_models/**') }}
          restore-keys: lfs-

      - name: pull LFS files
        run: git lfs pull
        shell: bash

      - name: setup uv
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
        with:
          version: '0.11.28'  # When updating this, make sure the version in launcher's package.json gets updated too.
          enable-cache: true
          python-version: '3.12'

      - name: install dependencies
        env:
          UV_INDEX: 'https://download.pytorch.org/whl/cpu'
        run: uv sync --no-progress --locked --extra test --extra mysql

      - name: run pytest
        run: uv run --no-sync pytest -n logical --max-worker-restart=0 -m "uses_database and not sqlite_only and not slow"

      - name: server log
        if: ${{ failure() }}
        run: docker logs "${{ job.services.database.id }}"