Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 159 additions & 0 deletions .github/workflows/wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Build Python development wheels

on:
workflow_dispatch:
inputs:
version:
description: "Python development version, e.g. 2.5.1.dev0 or 2.5.1.dev2"
required: true
type: string

permissions:
contents: read

jobs:
build:
name: Build development wheels on ${{ matrix.name }}
runs-on: ${{ matrix.os }}
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
include:
- name: manylinux_2_28-x86_64
os: ubuntu-22.04
arch: x86_64
- name: manylinux_2_28-aarch64
os: ubuntu-22.04-arm
arch: aarch64
- name: macos12-x86_64
os: macos-15-intel
arch: x86_64
- name: macos12-arm64
os: macos-latest
arch: arm64
- name: windows-msvc-x86_64
os: windows-2022
arch: AMD64
steps:
- name: Checkout repository
uses: actions/checkout@v7

- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: "3.11"

- name: Set the requested development version
id: version
env:
PYTHON_DEV_VERSION: ${{ inputs.version }}
shell: bash
run: |
python packaging/scripts/prepare_python_dev_version.py \
--version "$PYTHON_DEV_VERSION" --github-output "$GITHUB_OUTPUT"

- name: Set up Java 17
if: runner.os != 'Linux'
uses: actions/setup-java@v6.1.0
with:
distribution: temurin
java-version: "17"

- name: Install macOS build prerequisites
if: runner.os == 'macOS'
run: |
command -v cmake > /dev/null || brew install cmake
command -v pkg-config > /dev/null || brew install pkgconf

- name: Configure the MSVC environment
if: runner.os == 'Windows'
uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64

- name: Build the Windows Python SDK
if: runner.os == 'Windows'
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
$PSNativeCommandUseErrorActionPreference = $true
$projectInclude = (Resolve-Path packaging/cmake/PythonSDK.cmake).Path
python packaging/scripts/run_maven_with_retry.py -Pwith-cpp package `
"-Dcpp.toolchain=msvc" `
"-Dbuild.type=Release" `
"-Dbuild.test=OFF" `
"-DskipTests" `
"-Dspotless.skip=true" `
"-Dtsfile.dependency.source=BUNDLED" `
"-Dtsfile.msvc.static.runtime=ON" `
"-Denable.lzma2=ON" `
"-Dtsfile.project.include=$projectInclude" `
"-Dtsfile.version.sync.skip=true"
$sdkRoot = Join-Path $PWD 'python-sdk'
cmake --install cpp/target/build --config Release --prefix $sdkRoot
python packaging/scripts/verify_windows_runtime.py "$sdkRoot"
# setup.py consumes DLLs and import libraries together under lib/.
Copy-Item "$sdkRoot/bin/*.dll" "$sdkRoot/lib/"
if (-not (Test-Path "$sdkRoot/lib/tsfile.lib")) { throw 'Missing MSVC import library' }
if (-not (Test-Path "$sdkRoot/lib/tsfile.dll")) { throw 'Missing TsFile DLL' }

- name: Install wheel build tools
run: python -m pip install cibuildwheel==3.4.1

- name: Build and smoke-test development wheels
env:
CIBW_ARCHS: ${{ matrix.arch }}
CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*"
CIBW_SKIP: "pp* *-musllinux*"
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
CIBW_BEFORE_ALL_LINUX: >-
bash packaging/scripts/build_python_sdk.sh
${{ matrix.name }} ${{ steps.version.outputs.python_version }}
CIBW_BEFORE_ALL_MACOS: >-
bash packaging/scripts/build_python_sdk.sh
${{ matrix.name }} ${{ steps.version.outputs.python_version }}
CIBW_ENVIRONMENT_LINUX: TSFILE_CPP_BUILD=/project/python-sdk
CIBW_ENVIRONMENT_MACOS: >-
TSFILE_CPP_BUILD="${{ github.workspace }}/python-sdk"
MACOSX_DEPLOYMENT_TARGET=12.0
CIBW_ENVIRONMENT_WINDOWS: >-
TSFILE_CPP_BUILD="${{ github.workspace }}/python-sdk"
MACOSX_DEPLOYMENT_TARGET: "12.0"
CIBW_TEST_COMMAND: python "{project}/packaging/tests/python_wheel_smoke.py"
CIBW_BUILD_VERBOSITY: "1"
run: python -m cibuildwheel --output-dir wheelhouse python

- name: Check wheel contents and development version
env:
PYTHON_DEV_VERSION: ${{ steps.version.outputs.python_version }}
shell: bash
run: |
python packaging/scripts/check_python_wheel.py \
--wheelhouse wheelhouse --expected-version "$PYTHON_DEV_VERSION"

- name: Upload development wheels
uses: actions/upload-artifact@v7
with:
name: tsfile-wheels-${{ steps.version.outputs.python_version }}-${{ matrix.name }}
path: wheelhouse/*.whl
if-no-files-found: error
retention-days: 14
23 changes: 18 additions & 5 deletions packaging/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,18 +305,31 @@ the same manually installed native package that Go users consume.

## Python wheels

The native workflow is the single packaging entry point. It replaces the former
`.github/workflows/wheels.yml` workflow; pushing an `rc/**` branch no longer
starts a separate wheel build. Dispatch the native workflow to build the full
package bundle, including all 30 Python wheels:
The native workflow builds the full release package bundle, including all 30
Python wheels. For Python development versions only, use **Build Python
development wheels** (`.github/workflows/wheels.yml`): select **Run workflow**,
choose the source branch, and enter an explicit version such as `2.5.1.dev0`
or `2.5.1.dev2`. This workflow is manual-only; pushing an `rc/**` branch does
not trigger it.

The development workflow updates both Python version declarations in its
temporary checkout and preserves the requested version through the C++ build.
It builds the C++ library needed by the bindings, then builds and smoke-tests
the installed wheels on the same five platforms and six CPython versions below.
It checks that wheel filenames and distribution metadata both carry the exact
requested development version. Download the five
`tsfile-wheels-<version>-<platform>` artifacts from the run; they contain only
wheel files and are retained for 14 days. This workflow does not build native
release packages or publish to PyPI. The native release workflow still accepts
only final source versions.

| Wheel platform | CPython | C++ SDK source |
| --- | --- | --- |
| manylinux_2_28 x86_64 | 3.9–3.14 | One bundled build inside the manylinux container |
| manylinux_2_28 aarch64 | 3.9–3.14 | One bundled build inside the native ARM container |
| macOS 12 x86_64 | 3.9–3.14 | One bundled build with deployment target 12.0 |
| macOS 12 arm64 | 3.9–3.14 | One bundled build with deployment target 12.0 |
| Windows x86_64 | 3.9–3.14 | The existing native MSVC SDK |
| Windows x86_64 | 3.9–3.14 | Bundled MSVC SDK; release builds reuse the native job's SDK |

Each SDK installs `share/tsfile/tsfile-build-config.json`, recording its compiler,
configuration, resolved MSVC runtime and runtime library filename. The same
Expand Down
24 changes: 22 additions & 2 deletions packaging/scripts/check_python_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"""Reject build-only files, repeated native libraries and ELF debug sections."""

import argparse
from email.parser import BytesParser
import hashlib
from pathlib import Path
import struct
Expand Down Expand Up @@ -47,12 +48,30 @@ def elf_debug_sections(data):
return result


def check_wheel(path):
def check_wheel(path, expected_version=None):
with ZipFile(path) as wheel:
files = [info for info in wheel.infolist() if not info.is_dir()]
names = [info.filename for info in files]
if len(names) != len(set(names)):
raise ValueError(f"Duplicate ZIP entries in {path}")
if expected_version is not None:
metadata_path = f"tsfile-{expected_version}.dist-info/METADATA"
metadata_files = [
name for name in names if name.endswith(".dist-info/METADATA")
]
if not Path(path).name.startswith(
f"tsfile-{expected_version}-"
) or metadata_files != [metadata_path]:
raise ValueError(
f"Wheel filename and metadata must use version {expected_version}: {path}"
)
metadata = BytesParser().parsebytes(wheel.read(metadata_path))
if metadata.get_all("Name") != ["tsfile"] or metadata.get_all(
"Version"
) != [expected_version]:
raise ValueError(
f"Wheel METADATA must declare tsfile version {expected_version}: {path}"
)
forbidden = [
name
for name in names
Expand Down Expand Up @@ -103,6 +122,7 @@ def check_wheel(path):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--wheelhouse", type=Path)
parser.add_argument("--expected-version")
parser.add_argument("wheels", type=Path, nargs="*")
arguments = parser.parse_args()
wheels = arguments.wheels
Expand All @@ -111,7 +131,7 @@ def main():
if not wheels:
raise ValueError("No wheels found for content validation")
for wheel in wheels:
check_wheel(wheel)
check_wheel(wheel, arguments.expected_version)


if __name__ == "__main__":
Expand Down
80 changes: 80 additions & 0 deletions packaging/scripts/prepare_python_dev_version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

"""Set an explicit Python development wheel version in a build checkout."""

import argparse
from pathlib import Path
import re

DEV_VERSION = re.compile(
r"(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*)){2}\.dev(?:0|[1-9][0-9]*)"
)
VERSION_ASSIGNMENT = re.compile(r"""(?m)^(version[ \t]*=[ \t]*)(["'])([^"'\r\n]+)\2""")
PROJECT_TABLE = re.compile(r"(?ms)^\[project\][ \t]*\r?\n(?P<body>.*?)(?=^\[|\Z)")


def _replace_version(text, version, path):
matches = list(VERSION_ASSIGNMENT.finditer(text))
if len(matches) != 1:
raise ValueError(f"Expected exactly one version declaration in {path}")
match = matches[0]
return text[: match.start(3)] + version + text[match.end(3) :]


def prepare_version(python_root, version):
if DEV_VERSION.fullmatch(version) is None:
raise ValueError(
"Python development version must be MAJOR.MINOR.PATCH.devN "
"(for example 2.5.1.dev0 or 2.5.1.dev2)"
)
setup_path = python_root / "setup.py"
project_path = python_root / "pyproject.toml"
setup = _replace_version(
setup_path.read_text(encoding="utf-8"), version, setup_path
)
project = project_path.read_text(encoding="utf-8")
table = PROJECT_TABLE.search(project)
if table is None:
raise ValueError(f"Missing [project] table in {project_path}")
body = _replace_version(table.group("body"), version, project_path)
project = project[: table.start("body")] + body + project[table.end("body") :]
# Validate both declarations before changing either file. Only the Python
# package metadata changes; the native library keeps its source/ABI version.
setup_path.write_text(setup, encoding="utf-8")
project_path.write_text(project, encoding="utf-8")


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True)
parser.add_argument(
"--python-root",
type=Path,
default=Path(__file__).resolve().parents[2] / "python",
)
parser.add_argument("--github-output", type=Path)
args = parser.parse_args()
prepare_version(args.python_root, args.version)
if args.github_output is not None:
with args.github_output.open("a", encoding="utf-8") as output:
output.write(f"\npython_version={args.version}\n")
print(f"Python wheel version: {args.version}")


if __name__ == "__main__":
main()
16 changes: 14 additions & 2 deletions packaging/tests/check_native_workflow.rb
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,19 @@ def run_text(job)
raise "Windows SDKs must retain Python wheel codec coverage" unless windows.include?("-Denable.lzma2=ON")
output, status = Open3.capture2e("bash", "-n", sdk_builder_path)
raise "Invalid Python SDK shell script: #{output}" unless status.success?
raise "The old wheel workflow must be retired" if File.exist?(File.expand_path("../../.github/workflows/wheels.yml", __dir__))
dev_workflow = YAML.load_file(File.expand_path("../../.github/workflows/wheels.yml", __dir__))
dev_triggers = dev_workflow["on"] || dev_workflow[true]
raise "Development wheels must be manual-only" unless dev_triggers.keys == ["workflow_dispatch"]
raise "Development wheels require an explicit version" unless dev_triggers.fetch("workflow_dispatch").fetch("inputs").fetch("version").fetch("required")
raise "Development wheels must remain read-only" unless dev_workflow.fetch("permissions") == { "contents" => "read" }
dev_jobs = dev_workflow.fetch("jobs")
dev_jobs.each_value do |job|
raise "Development wheels must not depend on native release jobs" unless Array(job["needs"]).all? { |name| dev_jobs.key?(name) }
job.fetch("steps").each do |step|
next unless step["uses"].to_s.start_with?("actions/upload-artifact@")
raise "Development workflow must upload only wheels" unless step.fetch("with").fetch("path") == "wheelhouse/*.whl"
end
end

homebrew = jobs.fetch("build-homebrew")
raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}"
Expand Down Expand Up @@ -283,7 +295,7 @@ def run_text(job)
raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle")
raise "workflow must not hard-code a fork repository" if File.read(path).include?("ColinLeeo/tsfile")

jobs.each_value do |job|
(jobs.values + dev_jobs.values).each do |job|
job.fetch("steps").each do |step|
artifact = step["with"].to_h["name"].to_s
raise "artifact names must not repeat the macOS matrix prefix" if artifact.include?("macos-macos")
Expand Down
Loading
Loading