diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index f9a4dc5a6833a..cb49d1d6d57d0 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -21,7 +21,7 @@ description: 'Sets up Python and Breeze' inputs: python-version: description: 'Python version to use' - default: "3.10" + default: "3.11" outputs: host-python-version: description: Python version used in host diff --git a/.github/actions/prepare_all_ci_images/action.yml b/.github/actions/prepare_all_ci_images/action.yml index 7e2e8395636eb..5adf48bd81a79 100644 --- a/.github/actions/prepare_all_ci_images/action.yml +++ b/.github/actions/prepare_all_ci_images/action.yml @@ -34,9 +34,9 @@ runs: # TODO: Currently we cannot loop through the list of python versions and have dynamic list of # tasks. Instead we hardcode all possible python versions and they - but # this should be implemented in stash action as list of keys to download. - # That includes 3.9 - 3.12 as we are backporting it to v3-0-test branch + # That includes 3.9 - 3.14 as we are backporting to the v3-*-test branches # This is captured in https://github.com/apache/airflow/issues/45268 - # So we actually need 3.9 even if 3.9 support on main is dropped! + # So we actually need 3.9 and 3.10 even if their support on main is dropped! - name: "Restore CI docker image ${{ inputs.platform }}:3.9" uses: ./.github/actions/prepare_single_ci_image with: diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 42b71e1f32bb4..de26e37fbfc62 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -58,7 +58,9 @@ jobs: # Check that the image builds quickly from the registry cache. This build is slow from # scratch, so we only run it on regular PRs. check-that-image-builds-quickly: - timeout-minutes: 25 + # A PR that touches install_os_dependencies.sh or scripts/docker/keys misses the cache for the + # layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners. + timeout-minutes: 35 name: Check that image builds quickly runs-on: ${{ fromJSON(inputs.runners) }} env: @@ -83,5 +85,5 @@ jobs: uses: ./.github/actions/breeze - name: "Check that image builds quickly" # Synchronize to be a little bit shorter than above timeout-minutes to make sure that - # if the build takes too long the job will fail with logs. 22 minutes * 60 s = 1320 seconds - run: breeze shell --max-time 1320 --platform "${PLATFORM}" + # if the build takes too long the job will fail with logs. 33 minutes * 60 s = 1980 seconds + run: breeze shell --max-time 1980 --platform "${PLATFORM}" diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 9dfdc866274b1..eea84eb86b538 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -39,7 +39,7 @@ on: # yamllint disable-line rule:truthy default-python-version: description: "Which version of python should be used by default" type: string - default: '3.10' + default: '3.11' use-uv: description: "Whether to use uv to build the image (true/false)" type: string diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 7ff13294e1c86..f8f75cc01327f 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -127,6 +127,9 @@ jobs: tests-shared-distributions: timeout-minutes: 10 name: Shared ${{ matrix.shared-distribution }} tests + env: + # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. + UV_PYTHON: "${{ inputs.default-python-version }}" strategy: fail-fast: false matrix: @@ -157,6 +160,9 @@ jobs: name: Scripts tests runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.run-scripts-tests == 'true' + env: + # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. + UV_PYTHON: "${{ inputs.default-python-version }}" steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 33db1c2433342..8165c981cb863 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -154,14 +154,10 @@ jobs: run: > breeze release-management constraints-version-check --python "${MATRIX_PYTHON_VERSION}" - --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" "${EXPLAIN_WHY_FLAG}" + --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" --explain-why env: MATRIX_PYTHON_VERSION: "${{ matrix.python-version }}" MATRIX_CONSTRAINTS_MODE: "${{ matrix.constraints-mode }}" - # Explaining an outdated package resolves its dependency tree; on Python 3.10 more and more - # dependencies have dropped support, so there is far more to explain and the job takes much - # longer. Not worth it weeks before 3.10 support is dropped - remove this along with 3.10. - EXPLAIN_WHY_FLAG: ${{ matrix.python-version == '3.10' && '--no-explain-why' || '--explain-why' }} VERBOSE: "false" push-buildx-cache-to-github-registry: diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 389da5010c6e7..4c65a55381588 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -29,7 +29,7 @@ on: # yamllint disable-line rule:truthy default: false limitPythonVersions: type: string - description: 'Force python versions (e.g. "3.10 3.11")' + description: 'Force python versions (e.g. "3.11 3.12")' default: '' permissions: contents: read diff --git a/.github/workflows/release_single_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml index 9099919daba67..19315247a4c4a 100644 --- a/.github/workflows/release_single_dockerhub_image.yml +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -29,7 +29,7 @@ on: # yamllint disable-line rule:truthy type: string required: true pythonVersion: - description: 'Python version (e.g. 3.10, 3.11)' + description: 'Python version (e.g. 3.11, 3.12)' type: string required: true skipLatest: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index b55ec0445622b..519f0f1174673 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -39,7 +39,7 @@ on: # yamllint disable-line rule:truthy default-python-version: description: "Which version of python should be used by default" type: string - default: '3.10' + default: '3.11' use-uv: description: "Whether to use uv to build the image (true/false)" type: string @@ -90,7 +90,7 @@ jobs: name: ${{ inputs.workflow-name || 'UI E2E Tests' }} runs-on: ${{ fromJSON(inputs.runners || '["ubuntu-24.04"]') }} env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.10' }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11' }}" GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} diff --git a/.readthedocs.yml b/.readthedocs.yml index ddc2ffd3681fe..315d27e23e0c9 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,7 +20,7 @@ formats: [] sphinx: configuration: devel-common/src/docs/rtd-deprecation/conf.py python: - version: "3.10" + version: "3.11" install: - method: pip path: . diff --git a/Dockerfile b/Dockerfile index 08e1f19dc334a..4048b64a5bd19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,7 +119,7 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.11.16} PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} @@ -398,56 +398,35 @@ function install_python() { wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="pablogsal@python.org" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="thomas@python.org" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="thomas@python.org" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="hugo@python.org" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com" - [3.12]="https://accounts.google.com" - [3.13]="https://accounts.google.com" - [3.14]="https://github.com/login/oauth" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado " imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc" - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi + # Sigstore verification (PEP 761) + declare -A sigstore_identities=( + # https://peps.python.org/pep-0664/#release-manager-and-crew + [3.11]="pablogsal@python.org" + # https://peps.python.org/pep-0693/#release-manager-and-crew + [3.12]="thomas@python.org" + # https://peps.python.org/pep-0719/#release-manager-and-crew + [3.13]="thomas@python.org" + # https://peps.python.org/pep-0745/#release-manager-and-crew + [3.14]="hugo@python.org" + ) + declare -A sigstore_issuers=( + [3.11]="https://accounts.google.com" + [3.12]="https://accounts.google.com" + [3.13]="https://accounts.google.com" + [3.14]="https://github.com/login/oauth" + ) + wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ + "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" + install_cosign + local identity="${sigstore_identities[${major_minor_version}]}" + local issuer="${sigstore_issuers[${major_minor_version}]}" + /tmp/cosign verify-blob \ + --bundle python.tar.xz.sigstore \ + --certificate-identity "${identity}" \ + --certificate-oidc-issuer "${issuer}" \ + python.tar.xz + rm -f python.tar.xz.sigstore /tmp/cosign mkdir -p /usr/src/python tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz rm python.tar.xz diff --git a/Dockerfile.ci b/Dockerfile.ci index b2125f7c38d67..3549e5a994c51 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -59,7 +59,7 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.11.16} PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} @@ -338,56 +338,35 @@ function install_python() { wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="pablogsal@python.org" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="thomas@python.org" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="thomas@python.org" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="hugo@python.org" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com" - [3.12]="https://accounts.google.com" - [3.13]="https://accounts.google.com" - [3.14]="https://github.com/login/oauth" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado " imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc" - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi + # Sigstore verification (PEP 761) + declare -A sigstore_identities=( + # https://peps.python.org/pep-0664/#release-manager-and-crew + [3.11]="pablogsal@python.org" + # https://peps.python.org/pep-0693/#release-manager-and-crew + [3.12]="thomas@python.org" + # https://peps.python.org/pep-0719/#release-manager-and-crew + [3.13]="thomas@python.org" + # https://peps.python.org/pep-0745/#release-manager-and-crew + [3.14]="hugo@python.org" + ) + declare -A sigstore_issuers=( + [3.11]="https://accounts.google.com" + [3.12]="https://accounts.google.com" + [3.13]="https://accounts.google.com" + [3.14]="https://github.com/login/oauth" + ) + wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ + "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" + install_cosign + local identity="${sigstore_identities[${major_minor_version}]}" + local issuer="${sigstore_issuers[${major_minor_version}]}" + /tmp/cosign verify-blob \ + --bundle python.tar.xz.sigstore \ + --certificate-identity "${identity}" \ + --certificate-oidc-issuer "${issuer}" \ + python.tar.xz + rm -f python.tar.xz.sigstore /tmp/cosign mkdir -p /usr/src/python tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz rm python.tar.xz @@ -1187,7 +1166,7 @@ chmod 1777 /tmp AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd) -PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.10} +PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.11} export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} diff --git a/README.md b/README.md index 98fd1ff6e16f8..8e7f8f70b4101 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Apache Airflow is tested with: | | Main version (dev) | Stable version (3.3.2) | Deprecate version (2.11.2) | |------------|------------------------------------------|------------------------------------|------------------------------| -| Python | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12 | +| Python | 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64(\*) | | Kubernetes | 1.31, 1.32, 1.33, 1.34, 1.35, 1.36, 1.37 | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.26, 1.27, 1.28, 1.29, 1.30 | | PostgreSQL | 14, 15, 16, 17, 18 | 14, 15, 16, 17, 18 | 12, 13, 14, 15, 16 | diff --git a/airflow-core/docs/installation/prerequisites.rst b/airflow-core/docs/installation/prerequisites.rst index 7431082f781d9..88f5ed40698b7 100644 --- a/airflow-core/docs/installation/prerequisites.rst +++ b/airflow-core/docs/installation/prerequisites.rst @@ -22,7 +22,7 @@ Airflow® is tested with: .. Beginning of the auto-generated tested versions -* Python: 3.10, 3.11, 3.12, 3.13, 3.14 +* Python: 3.11, 3.12, 3.13, 3.14 * Databases: diff --git a/airflow-core/newsfragments/74151.significant.rst b/airflow-core/newsfragments/74151.significant.rst new file mode 100644 index 0000000000000..dbfa8d889d0b2 --- /dev/null +++ b/airflow-core/newsfragments/74151.significant.rst @@ -0,0 +1,17 @@ +Support for Python 3.10 has been removed + +Airflow now requires Python 3.11 or newer. Python 3.10 reaches its end of life in October 2026, +and Airflow drops support for a Python version in ``main`` as soon as it reaches end of life. +The ``apache-airflow`` distribution, the Task SDK, ``airflowctl`` and all providers released +from ``main`` declare ``requires-python = ">=3.11"``. + +* Types of change + + * [ ] Dag changes + * [ ] Config changes + * [ ] API changes + * [ ] CLI changes + * [ ] Behaviour changes + * [ ] Plugin changes + * [x] Dependency changes + * [ ] Code interface changes diff --git a/airflow-ctl-tests/tests/airflowctl_tests/constants.py b/airflow-ctl-tests/tests/airflowctl_tests/constants.py index 8a8bf812ac019..a15af7f60c554 100644 --- a/airflow-ctl-tests/tests/airflowctl_tests/constants.py +++ b/airflow-ctl-tests/tests/airflowctl_tests/constants.py @@ -21,7 +21,7 @@ AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3] -DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10" +DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.11" DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest" DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py index 4a1a2f7be3880..c259022d35e2f 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py @@ -22,7 +22,7 @@ AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3] DOCKER_COMPOSE_HOST_PORT = os.environ.get("HOST_PORT", "localhost:8080") -DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10" +DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.11" DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest" DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE os.environ["AIRFLOW_UID"] = str(os.getuid()) diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index e6405bcad55fb..93c54b8a97cc4 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -1,4 +1,4 @@ - +