From 16f7b3904d47ba3153684bb8dce4c06b3bdb27d1 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:03:23 +0300 Subject: [PATCH 1/2] Drop Python 3.10 from CI, Breeze and the container images Python 3.10 reaches end of life in October 2026 and more and more of Airflow's dependencies have already dropped it, which makes the 3.10 leg the slowest and most fragile part of CI. This is the first layer of the stack that moves the baseline to Python 3.11: Breeze stops offering 3.10, the default Python for CI, Breeze and the images moves to 3.11, and the PGP path for verifying the Python tarball goes away because every remaining version ships Sigstore bundles. Python 3.10 stays in the historical version lists that Breeze uses to reason about older Airflow releases, and in the CI image restore action that release branches still rely on. The distributions still declare Python 3.10 in this layer; layer 7 bumps them once the code no longer carries 3.10-only constructs. Until then the provider metadata generator keeps its own 3.10-inclusive version list, and the prek minimum-Python check keeps 3.10 because its script header still pins it to a 3.10 interpreter until layer 2. The shared-distribution and scripts test jobs run uv without an interpreter pin, so uv picks the runner's system Python, which is 3.10 on the current images. Pinning them to the default CI Python keeps them on a supported interpreter regardless of the runner image. The image build speed check builds the CI image from the registry cache within a fixed budget. A PR that changes install_os_dependencies.sh or the embedded GPG keys misses the cache for the layer that compiles Python from source. That compile took about 4.5 minutes for Python 3.10 but 11 to 18 minutes for 3.11 and newer on hosted runners, so the budget is raised to fit a cold Python 3.11+ build. --- .github/actions/breeze/action.yml | 2 +- .../actions/prepare_all_ci_images/action.yml | 4 +- .../workflows/additional-ci-image-checks.yml | 8 +- .github/workflows/airflow-e2e-tests.yml | 2 +- .github/workflows/basic-tests.yml | 6 + .github/workflows/finalize-tests.yml | 6 +- .github/workflows/release_dockerhub_image.yml | 2 +- .../release_single_dockerhub_image.yml | 2 +- .github/workflows/ui-e2e-tests.yml | 4 +- .readthedocs.yml | 2 +- Dockerfile | 79 ++--- Dockerfile.ci | 81 ++--- README.md | 2 +- .../docs/installation/prerequisites.rst | 2 +- .../newsfragments/74151.significant.rst | 17 + .../tests/airflowctl_tests/constants.py | 2 +- .../tests/airflow_e2e_tests/constants.py | 2 +- dev/breeze/doc/images/output-commands.svg | 192 +++++------ .../doc/images/output_ci-image_build.svg | 4 +- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_ci-image_load.svg | 4 +- .../doc/images/output_ci-image_load.txt | 2 +- .../doc/images/output_ci-image_pull.svg | 6 +- .../doc/images/output_ci-image_pull.txt | 2 +- .../doc/images/output_ci-image_save.svg | 4 +- .../doc/images/output_ci-image_save.txt | 2 +- .../doc/images/output_ci-image_verify.svg | 6 +- .../doc/images/output_ci-image_verify.txt | 2 +- .../doc/images/output_k8s_build-k8s-image.svg | 4 +- .../doc/images/output_k8s_build-k8s-image.txt | 2 +- .../images/output_k8s_configure-cluster.svg | 6 +- .../images/output_k8s_configure-cluster.txt | 2 +- .../doc/images/output_k8s_create-cluster.svg | 6 +- .../doc/images/output_k8s_create-cluster.txt | 2 +- .../doc/images/output_k8s_delete-cluster.svg | 4 +- .../doc/images/output_k8s_delete-cluster.txt | 2 +- .../doc/images/output_k8s_deploy-airflow.svg | 6 +- .../doc/images/output_k8s_deploy-airflow.txt | 2 +- .../doc/images/output_k8s_deploy-cluster.svg | 4 +- .../doc/images/output_k8s_deploy-cluster.txt | 2 +- dev/breeze/doc/images/output_k8s_dev.svg | 4 +- dev/breeze/doc/images/output_k8s_dev.txt | 2 +- dev/breeze/doc/images/output_k8s_k9s.svg | 4 +- dev/breeze/doc/images/output_k8s_k9s.txt | 2 +- dev/breeze/doc/images/output_k8s_logs.svg | 4 +- dev/breeze/doc/images/output_k8s_logs.txt | 2 +- .../images/output_k8s_run-complete-tests.svg | 6 +- .../images/output_k8s_run-complete-tests.txt | 2 +- .../images/output_k8s_setup-lang-sdk-test.svg | 4 +- .../images/output_k8s_setup-lang-sdk-test.txt | 2 +- dev/breeze/doc/images/output_k8s_shell.svg | 4 +- dev/breeze/doc/images/output_k8s_shell.txt | 2 +- .../images/output_k8s_smoke-test-overlay.svg | 4 +- .../images/output_k8s_smoke-test-overlay.txt | 2 +- dev/breeze/doc/images/output_k8s_status.svg | 4 +- dev/breeze/doc/images/output_k8s_status.txt | 2 +- dev/breeze/doc/images/output_k8s_tests.svg | 6 +- dev/breeze/doc/images/output_k8s_tests.txt | 2 +- .../images/output_k8s_upload-k8s-image.svg | 6 +- .../images/output_k8s_upload-k8s-image.txt | 2 +- .../doc/images/output_prod-image_build.svg | 6 +- .../doc/images/output_prod-image_build.txt | 2 +- .../doc/images/output_prod-image_load.svg | 4 +- .../doc/images/output_prod-image_load.txt | 2 +- .../doc/images/output_prod-image_pull.svg | 6 +- .../doc/images/output_prod-image_pull.txt | 2 +- .../doc/images/output_prod-image_save.svg | 4 +- .../doc/images/output_prod-image_save.txt | 2 +- .../doc/images/output_prod-image_verify.svg | 6 +- .../doc/images/output_prod-image_verify.txt | 2 +- .../doc/images/output_registry_backfill.svg | 4 +- .../doc/images/output_registry_backfill.txt | 2 +- .../images/output_registry_extract-data.svg | 4 +- .../images/output_registry_extract-data.txt | 2 +- ...e-management_constraints-version-check.svg | 4 +- ...e-management_constraints-version-check.txt | 2 +- ...elease-management_generate-constraints.svg | 8 +- ...elease-management_generate-constraints.txt | 2 +- ...agement_install-provider-distributions.svg | 4 +- ...agement_install-provider-distributions.txt | 2 +- ...t_release-management_merge-prod-images.svg | 2 +- ...t_release-management_merge-prod-images.txt | 2 +- ...release-management_release-prod-images.svg | 2 +- ...release-management_release-prod-images.txt | 2 +- ...nagement_verify-provider-distributions.svg | 4 +- ...nagement_verify-provider-distributions.txt | 2 +- dev/breeze/doc/images/output_run.svg | 4 +- dev/breeze/doc/images/output_run.txt | 2 +- ...put_sbom_export-dependency-information.svg | 4 +- ...put_sbom_export-dependency-information.txt | 2 +- dev/breeze/doc/images/output_setup_config.svg | 4 +- dev/breeze/doc/images/output_setup_config.txt | 2 +- dev/breeze/doc/images/output_shell.svg | 322 +++++++++--------- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 278 ++++++++------- .../doc/images/output_start-airflow.txt | 2 +- ..._testing_airflow-ctl-integration-tests.svg | 4 +- ..._testing_airflow-ctl-integration-tests.txt | 2 +- .../output_testing_airflow-ctl-tests.svg | 4 +- .../output_testing_airflow-ctl-tests.txt | 2 +- .../output_testing_airflow-e2e-tests.svg | 4 +- .../output_testing_airflow-e2e-tests.txt | 2 +- .../output_testing_core-integration-tests.svg | 2 +- .../output_testing_core-integration-tests.txt | 2 +- .../doc/images/output_testing_core-tests.svg | 2 +- .../doc/images/output_testing_core-tests.txt | 2 +- .../output_testing_docker-compose-tests.svg | 4 +- .../output_testing_docker-compose-tests.txt | 2 +- ...ut_testing_providers-integration-tests.svg | 2 +- ...ut_testing_providers-integration-tests.txt | 2 +- .../images/output_testing_providers-tests.svg | 2 +- .../images/output_testing_providers-tests.txt | 2 +- ...output_testing_python-api-client-tests.svg | 2 +- ...output_testing_python-api-client-tests.txt | 2 +- .../images/output_testing_system-tests.svg | 2 +- .../images/output_testing_system-tests.txt | 2 +- ...put_testing_task-sdk-integration-tests.svg | 4 +- ...put_testing_task-sdk-integration-tests.txt | 2 +- .../images/output_testing_task-sdk-tests.svg | 2 +- .../images/output_testing_task-sdk-tests.txt | 2 +- .../images/output_testing_ui-e2e-tests.svg | 4 +- .../images/output_testing_ui-e2e-tests.txt | 2 +- .../commands/ci_image_commands.py | 2 +- .../commands/minor_release_command.py | 2 +- .../src/airflow_breeze/global_constants.py | 19 +- .../params/common_build_params.py | 2 +- .../utils/check_release_files.py | 14 +- .../src/airflow_breeze/utils/packages.py | 9 +- .../utils/provider_dependencies.py | 10 - dev/breeze/tests/test_cache.py | 2 +- dev/breeze/tests/test_ci_image_commands.py | 4 +- dev/breeze/tests/test_docker_command_utils.py | 2 +- .../test_kubernetes_lang_sdk_commands.py | 4 +- .../tests/test_release_management_commands.py | 2 +- dev/breeze/tests/test_reproduce_ci.py | 4 +- dev/breeze/tests/test_sbom_commands.py | 6 +- dev/breeze/tests/test_shell_params.py | 16 +- dev/refresh_images.sh | 4 +- dev/registry/tests/test_extract_metadata.py | 6 +- .../tests/test_merge_registry_data.py | 2 +- .../tests/test_registry_contract_models.py | 2 +- dev/retag_docker_images.py | 2 +- docker-tests/tests/docker_tests/constants.py | 2 +- .../tests/docker_tests/docker_utils.py | 4 +- generated/PYPI_README.md | 2 +- scripts/ci/docker-compose/devcontainer.env | 4 +- scripts/ci/docker-compose/devcontainer.yml | 2 +- scripts/ci/prek/common_prek_utils.py | 10 +- scripts/ci/prek/mypy.py | 2 +- scripts/ci/prek/mypy_folder.py | 4 +- scripts/ci/prek/update_docker_gpg_keys.py | 2 - scripts/ci/prek/upgrade_important_versions.py | 2 +- .../get_min_airflow_version_for_python.py | 2 +- scripts/docker/entrypoint_ci.sh | 2 +- scripts/docker/install_os_dependencies.sh | 79 ++--- scripts/docker/keys/python-3.10.asc | 109 ------ .../install_airflow_and_providers.py | 4 +- .../tests/ci/prek/test_supported_versions.py | 4 +- .../ci/prek/test_update_tested_versions.py | 4 +- .../tests/ci/test_analyze_ci_job_durations.py | 12 +- ...test_get_min_airflow_version_for_python.py | 6 +- .../docker/test_get_distribution_specs.py | 16 +- .../test_run_generate_constraints.py | 10 +- .../tests/task_sdk_tests/constants.py | 2 +- 164 files changed, 773 insertions(+), 952 deletions(-) create mode 100644 airflow-core/newsfragments/74151.significant.rst delete mode 100644 scripts/docker/keys/python-3.10.asc diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index f9a4dc5a6833a..cb49d1d6d57d0 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -21,7 +21,7 @@ description: 'Sets up Python and Breeze' inputs: python-version: description: 'Python version to use' - default: "3.10" + default: "3.11" outputs: host-python-version: description: Python version used in host diff --git a/.github/actions/prepare_all_ci_images/action.yml b/.github/actions/prepare_all_ci_images/action.yml index 7e2e8395636eb..5adf48bd81a79 100644 --- a/.github/actions/prepare_all_ci_images/action.yml +++ b/.github/actions/prepare_all_ci_images/action.yml @@ -34,9 +34,9 @@ runs: # TODO: Currently we cannot loop through the list of python versions and have dynamic list of # tasks. Instead we hardcode all possible python versions and they - but # this should be implemented in stash action as list of keys to download. - # That includes 3.9 - 3.12 as we are backporting it to v3-0-test branch + # That includes 3.9 - 3.14 as we are backporting to the v3-*-test branches # This is captured in https://github.com/apache/airflow/issues/45268 - # So we actually need 3.9 even if 3.9 support on main is dropped! + # So we actually need 3.9 and 3.10 even if their support on main is dropped! - name: "Restore CI docker image ${{ inputs.platform }}:3.9" uses: ./.github/actions/prepare_single_ci_image with: diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 42b71e1f32bb4..de26e37fbfc62 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -58,7 +58,9 @@ jobs: # Check that the image builds quickly from the registry cache. This build is slow from # scratch, so we only run it on regular PRs. check-that-image-builds-quickly: - timeout-minutes: 25 + # A PR that touches install_os_dependencies.sh or scripts/docker/keys misses the cache for the + # layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners. + timeout-minutes: 35 name: Check that image builds quickly runs-on: ${{ fromJSON(inputs.runners) }} env: @@ -83,5 +85,5 @@ jobs: uses: ./.github/actions/breeze - name: "Check that image builds quickly" # Synchronize to be a little bit shorter than above timeout-minutes to make sure that - # if the build takes too long the job will fail with logs. 22 minutes * 60 s = 1320 seconds - run: breeze shell --max-time 1320 --platform "${PLATFORM}" + # if the build takes too long the job will fail with logs. 33 minutes * 60 s = 1980 seconds + run: breeze shell --max-time 1980 --platform "${PLATFORM}" diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 9dfdc866274b1..eea84eb86b538 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -39,7 +39,7 @@ on: # yamllint disable-line rule:truthy default-python-version: description: "Which version of python should be used by default" type: string - default: '3.10' + default: '3.11' use-uv: description: "Whether to use uv to build the image (true/false)" type: string diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 7ff13294e1c86..f8f75cc01327f 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -127,6 +127,9 @@ jobs: tests-shared-distributions: timeout-minutes: 10 name: Shared ${{ matrix.shared-distribution }} tests + env: + # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. + UV_PYTHON: "${{ inputs.default-python-version }}" strategy: fail-fast: false matrix: @@ -157,6 +160,9 @@ jobs: name: Scripts tests runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.run-scripts-tests == 'true' + env: + # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. + UV_PYTHON: "${{ inputs.default-python-version }}" steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 33db1c2433342..8165c981cb863 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -154,14 +154,10 @@ jobs: run: > breeze release-management constraints-version-check --python "${MATRIX_PYTHON_VERSION}" - --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" "${EXPLAIN_WHY_FLAG}" + --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" --explain-why env: MATRIX_PYTHON_VERSION: "${{ matrix.python-version }}" MATRIX_CONSTRAINTS_MODE: "${{ matrix.constraints-mode }}" - # Explaining an outdated package resolves its dependency tree; on Python 3.10 more and more - # dependencies have dropped support, so there is far more to explain and the job takes much - # longer. Not worth it weeks before 3.10 support is dropped - remove this along with 3.10. - EXPLAIN_WHY_FLAG: ${{ matrix.python-version == '3.10' && '--no-explain-why' || '--explain-why' }} VERBOSE: "false" push-buildx-cache-to-github-registry: diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 389da5010c6e7..4c65a55381588 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -29,7 +29,7 @@ on: # yamllint disable-line rule:truthy default: false limitPythonVersions: type: string - description: 'Force python versions (e.g. "3.10 3.11")' + description: 'Force python versions (e.g. "3.11 3.12")' default: '' permissions: contents: read diff --git a/.github/workflows/release_single_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml index 9099919daba67..19315247a4c4a 100644 --- a/.github/workflows/release_single_dockerhub_image.yml +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -29,7 +29,7 @@ on: # yamllint disable-line rule:truthy type: string required: true pythonVersion: - description: 'Python version (e.g. 3.10, 3.11)' + description: 'Python version (e.g. 3.11, 3.12)' type: string required: true skipLatest: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index b55ec0445622b..519f0f1174673 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -39,7 +39,7 @@ on: # yamllint disable-line rule:truthy default-python-version: description: "Which version of python should be used by default" type: string - default: '3.10' + default: '3.11' use-uv: description: "Whether to use uv to build the image (true/false)" type: string @@ -90,7 +90,7 @@ jobs: name: ${{ inputs.workflow-name || 'UI E2E Tests' }} runs-on: ${{ fromJSON(inputs.runners || '["ubuntu-24.04"]') }} env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.10' }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11' }}" GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} diff --git a/.readthedocs.yml b/.readthedocs.yml index ddc2ffd3681fe..315d27e23e0c9 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,7 +20,7 @@ formats: [] sphinx: configuration: devel-common/src/docs/rtd-deprecation/conf.py python: - version: "3.10" + version: "3.11" install: - method: pip path: . diff --git a/Dockerfile b/Dockerfile index 08e1f19dc334a..4048b64a5bd19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,7 +119,7 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.11.16} PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} @@ -398,56 +398,35 @@ function install_python() { wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="pablogsal@python.org" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="thomas@python.org" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="thomas@python.org" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="hugo@python.org" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com" - [3.12]="https://accounts.google.com" - [3.13]="https://accounts.google.com" - [3.14]="https://github.com/login/oauth" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado " imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc" - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi + # Sigstore verification (PEP 761) + declare -A sigstore_identities=( + # https://peps.python.org/pep-0664/#release-manager-and-crew + [3.11]="pablogsal@python.org" + # https://peps.python.org/pep-0693/#release-manager-and-crew + [3.12]="thomas@python.org" + # https://peps.python.org/pep-0719/#release-manager-and-crew + [3.13]="thomas@python.org" + # https://peps.python.org/pep-0745/#release-manager-and-crew + [3.14]="hugo@python.org" + ) + declare -A sigstore_issuers=( + [3.11]="https://accounts.google.com" + [3.12]="https://accounts.google.com" + [3.13]="https://accounts.google.com" + [3.14]="https://github.com/login/oauth" + ) + wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ + "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" + install_cosign + local identity="${sigstore_identities[${major_minor_version}]}" + local issuer="${sigstore_issuers[${major_minor_version}]}" + /tmp/cosign verify-blob \ + --bundle python.tar.xz.sigstore \ + --certificate-identity "${identity}" \ + --certificate-oidc-issuer "${issuer}" \ + python.tar.xz + rm -f python.tar.xz.sigstore /tmp/cosign mkdir -p /usr/src/python tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz rm python.tar.xz diff --git a/Dockerfile.ci b/Dockerfile.ci index b2125f7c38d67..3549e5a994c51 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -59,7 +59,7 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.11.16} PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} @@ -338,56 +338,35 @@ function install_python() { wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="pablogsal@python.org" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="thomas@python.org" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="thomas@python.org" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="hugo@python.org" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com" - [3.12]="https://accounts.google.com" - [3.13]="https://accounts.google.com" - [3.14]="https://github.com/login/oauth" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado " imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc" - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi + # Sigstore verification (PEP 761) + declare -A sigstore_identities=( + # https://peps.python.org/pep-0664/#release-manager-and-crew + [3.11]="pablogsal@python.org" + # https://peps.python.org/pep-0693/#release-manager-and-crew + [3.12]="thomas@python.org" + # https://peps.python.org/pep-0719/#release-manager-and-crew + [3.13]="thomas@python.org" + # https://peps.python.org/pep-0745/#release-manager-and-crew + [3.14]="hugo@python.org" + ) + declare -A sigstore_issuers=( + [3.11]="https://accounts.google.com" + [3.12]="https://accounts.google.com" + [3.13]="https://accounts.google.com" + [3.14]="https://github.com/login/oauth" + ) + wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ + "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" + install_cosign + local identity="${sigstore_identities[${major_minor_version}]}" + local issuer="${sigstore_issuers[${major_minor_version}]}" + /tmp/cosign verify-blob \ + --bundle python.tar.xz.sigstore \ + --certificate-identity "${identity}" \ + --certificate-oidc-issuer "${issuer}" \ + python.tar.xz + rm -f python.tar.xz.sigstore /tmp/cosign mkdir -p /usr/src/python tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz rm python.tar.xz @@ -1187,7 +1166,7 @@ chmod 1777 /tmp AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd) -PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.10} +PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.11} export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} diff --git a/README.md b/README.md index 98fd1ff6e16f8..8e7f8f70b4101 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Apache Airflow is tested with: | | Main version (dev) | Stable version (3.3.2) | Deprecate version (2.11.2) | |------------|------------------------------------------|------------------------------------|------------------------------| -| Python | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12 | +| Python | 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64(\*) | | Kubernetes | 1.31, 1.32, 1.33, 1.34, 1.35, 1.36, 1.37 | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.26, 1.27, 1.28, 1.29, 1.30 | | PostgreSQL | 14, 15, 16, 17, 18 | 14, 15, 16, 17, 18 | 12, 13, 14, 15, 16 | diff --git a/airflow-core/docs/installation/prerequisites.rst b/airflow-core/docs/installation/prerequisites.rst index 7431082f781d9..88f5ed40698b7 100644 --- a/airflow-core/docs/installation/prerequisites.rst +++ b/airflow-core/docs/installation/prerequisites.rst @@ -22,7 +22,7 @@ Airflow® is tested with: .. Beginning of the auto-generated tested versions -* Python: 3.10, 3.11, 3.12, 3.13, 3.14 +* Python: 3.11, 3.12, 3.13, 3.14 * Databases: diff --git a/airflow-core/newsfragments/74151.significant.rst b/airflow-core/newsfragments/74151.significant.rst new file mode 100644 index 0000000000000..dbfa8d889d0b2 --- /dev/null +++ b/airflow-core/newsfragments/74151.significant.rst @@ -0,0 +1,17 @@ +Support for Python 3.10 has been removed + +Airflow now requires Python 3.11 or newer. Python 3.10 reaches its end of life in October 2026, +and Airflow drops support for a Python version in ``main`` as soon as it reaches end of life. +The ``apache-airflow`` distribution, the Task SDK, ``airflowctl`` and all providers released +from ``main`` declare ``requires-python = ">=3.11"``. + +* Types of change + + * [ ] Dag changes + * [ ] Config changes + * [ ] API changes + * [ ] CLI changes + * [ ] Behaviour changes + * [ ] Plugin changes + * [x] Dependency changes + * [ ] Code interface changes diff --git a/airflow-ctl-tests/tests/airflowctl_tests/constants.py b/airflow-ctl-tests/tests/airflowctl_tests/constants.py index 8a8bf812ac019..a15af7f60c554 100644 --- a/airflow-ctl-tests/tests/airflowctl_tests/constants.py +++ b/airflow-ctl-tests/tests/airflowctl_tests/constants.py @@ -21,7 +21,7 @@ AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3] -DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10" +DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.11" DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest" DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py index 4a1a2f7be3880..c259022d35e2f 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py @@ -22,7 +22,7 @@ AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3] DOCKER_COMPOSE_HOST_PORT = os.environ.get("HOST_PORT", "localhost:8080") -DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10" +DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.11" DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest" DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE os.environ["AIRFLOW_UID"] = str(os.getuid()) diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index e6405bcad55fb..93c54b8a97cc4 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -1,4 +1,4 @@ - +