diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..124e9cb85 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Python development wheels + +on: + workflow_dispatch: + inputs: + version: + description: "Python development version, e.g. 2.5.1.dev0 or 2.5.1.dev2" + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: Build development wheels on ${{ matrix.name }} + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - name: manylinux_2_28-x86_64 + os: ubuntu-22.04 + arch: x86_64 + - name: manylinux_2_28-aarch64 + os: ubuntu-22.04-arm + arch: aarch64 + - name: macos12-x86_64 + os: macos-15-intel + arch: x86_64 + - name: macos12-arm64 + os: macos-latest + arch: arm64 + - name: windows-msvc-x86_64 + os: windows-2022 + arch: AMD64 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Set the requested development version + id: version + env: + PYTHON_DEV_VERSION: ${{ inputs.version }} + shell: bash + run: | + python packaging/scripts/prepare_python_dev_version.py \ + --version "$PYTHON_DEV_VERSION" --github-output "$GITHUB_OUTPUT" + + - name: Set up Java 17 + if: runner.os != 'Linux' + uses: actions/setup-java@v6.1.0 + with: + distribution: temurin + java-version: "17" + + - name: Install macOS build prerequisites + if: runner.os == 'macOS' + run: | + command -v cmake > /dev/null || brew install cmake + command -v pkg-config > /dev/null || brew install pkgconf + + - name: Configure the MSVC environment + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Build the Windows Python SDK + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $projectInclude = (Resolve-Path packaging/cmake/PythonSDK.cmake).Path + python packaging/scripts/run_maven_with_retry.py -Pwith-cpp package ` + "-Dcpp.toolchain=msvc" ` + "-Dbuild.type=Release" ` + "-Dbuild.test=OFF" ` + "-DskipTests" ` + "-Dspotless.skip=true" ` + "-Dtsfile.dependency.source=BUNDLED" ` + "-Dtsfile.msvc.static.runtime=ON" ` + "-Denable.lzma2=ON" ` + "-Dtsfile.project.include=$projectInclude" ` + "-Dtsfile.version.sync.skip=true" + $sdkRoot = Join-Path $PWD 'python-sdk' + cmake --install cpp/target/build --config Release --prefix $sdkRoot + python packaging/scripts/verify_windows_runtime.py "$sdkRoot" + # setup.py consumes DLLs and import libraries together under lib/. + Copy-Item "$sdkRoot/bin/*.dll" "$sdkRoot/lib/" + if (-not (Test-Path "$sdkRoot/lib/tsfile.lib")) { throw 'Missing MSVC import library' } + if (-not (Test-Path "$sdkRoot/lib/tsfile.dll")) { throw 'Missing TsFile DLL' } + + - name: Install wheel build tools + run: python -m pip install cibuildwheel==3.4.1 + + - name: Build and smoke-test development wheels + env: + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_SKIP: "pp* *-musllinux*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL_LINUX: >- + bash packaging/scripts/build_python_sdk.sh + ${{ matrix.name }} ${{ steps.version.outputs.python_version }} + CIBW_BEFORE_ALL_MACOS: >- + bash packaging/scripts/build_python_sdk.sh + ${{ matrix.name }} ${{ steps.version.outputs.python_version }} + CIBW_ENVIRONMENT_LINUX: TSFILE_CPP_BUILD=/project/python-sdk + CIBW_ENVIRONMENT_MACOS: >- + TSFILE_CPP_BUILD="${{ github.workspace }}/python-sdk" + MACOSX_DEPLOYMENT_TARGET=12.0 + CIBW_ENVIRONMENT_WINDOWS: >- + TSFILE_CPP_BUILD="${{ github.workspace }}/python-sdk" + MACOSX_DEPLOYMENT_TARGET: "12.0" + CIBW_TEST_COMMAND: python "{project}/packaging/tests/python_wheel_smoke.py" + CIBW_BUILD_VERBOSITY: "1" + run: python -m cibuildwheel --output-dir wheelhouse python + + - name: Check wheel contents and development version + env: + PYTHON_DEV_VERSION: ${{ steps.version.outputs.python_version }} + shell: bash + run: | + python packaging/scripts/check_python_wheel.py \ + --wheelhouse wheelhouse --expected-version "$PYTHON_DEV_VERSION" + + - name: Upload development wheels + uses: actions/upload-artifact@v7 + with: + name: tsfile-wheels-${{ steps.version.outputs.python_version }}-${{ matrix.name }} + path: wheelhouse/*.whl + if-no-files-found: error + retention-days: 14 diff --git a/packaging/README.md b/packaging/README.md index f6282d491..449a1ee1c 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -305,10 +305,23 @@ the same manually installed native package that Go users consume. ## Python wheels -The native workflow is the single packaging entry point. It replaces the former -`.github/workflows/wheels.yml` workflow; pushing an `rc/**` branch no longer -starts a separate wheel build. Dispatch the native workflow to build the full -package bundle, including all 30 Python wheels: +The native workflow builds the full release package bundle, including all 30 +Python wheels. For Python development versions only, use **Build Python +development wheels** (`.github/workflows/wheels.yml`): select **Run workflow**, +choose the source branch, and enter an explicit version such as `2.5.1.dev0` +or `2.5.1.dev2`. This workflow is manual-only; pushing an `rc/**` branch does +not trigger it. + +The development workflow updates both Python version declarations in its +temporary checkout and preserves the requested version through the C++ build. +It builds the C++ library needed by the bindings, then builds and smoke-tests +the installed wheels on the same five platforms and six CPython versions below. +It checks that wheel filenames and distribution metadata both carry the exact +requested development version. Download the five +`tsfile-wheels--` artifacts from the run; they contain only +wheel files and are retained for 14 days. This workflow does not build native +release packages or publish to PyPI. The native release workflow still accepts +only final source versions. | Wheel platform | CPython | C++ SDK source | | --- | --- | --- | @@ -316,7 +329,7 @@ package bundle, including all 30 Python wheels: | manylinux_2_28 aarch64 | 3.9–3.14 | One bundled build inside the native ARM container | | macOS 12 x86_64 | 3.9–3.14 | One bundled build with deployment target 12.0 | | macOS 12 arm64 | 3.9–3.14 | One bundled build with deployment target 12.0 | -| Windows x86_64 | 3.9–3.14 | The existing native MSVC SDK | +| Windows x86_64 | 3.9–3.14 | Bundled MSVC SDK; release builds reuse the native job's SDK | Each SDK installs `share/tsfile/tsfile-build-config.json`, recording its compiler, configuration, resolved MSVC runtime and runtime library filename. The same diff --git a/packaging/scripts/check_python_wheel.py b/packaging/scripts/check_python_wheel.py index aad85b8a2..1ff57ec1b 100644 --- a/packaging/scripts/check_python_wheel.py +++ b/packaging/scripts/check_python_wheel.py @@ -19,6 +19,7 @@ """Reject build-only files, repeated native libraries and ELF debug sections.""" import argparse +from email.parser import BytesParser import hashlib from pathlib import Path import struct @@ -47,12 +48,30 @@ def elf_debug_sections(data): return result -def check_wheel(path): +def check_wheel(path, expected_version=None): with ZipFile(path) as wheel: files = [info for info in wheel.infolist() if not info.is_dir()] names = [info.filename for info in files] if len(names) != len(set(names)): raise ValueError(f"Duplicate ZIP entries in {path}") + if expected_version is not None: + metadata_path = f"tsfile-{expected_version}.dist-info/METADATA" + metadata_files = [ + name for name in names if name.endswith(".dist-info/METADATA") + ] + if not Path(path).name.startswith( + f"tsfile-{expected_version}-" + ) or metadata_files != [metadata_path]: + raise ValueError( + f"Wheel filename and metadata must use version {expected_version}: {path}" + ) + metadata = BytesParser().parsebytes(wheel.read(metadata_path)) + if metadata.get_all("Name") != ["tsfile"] or metadata.get_all( + "Version" + ) != [expected_version]: + raise ValueError( + f"Wheel METADATA must declare tsfile version {expected_version}: {path}" + ) forbidden = [ name for name in names @@ -103,6 +122,7 @@ def check_wheel(path): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wheelhouse", type=Path) + parser.add_argument("--expected-version") parser.add_argument("wheels", type=Path, nargs="*") arguments = parser.parse_args() wheels = arguments.wheels @@ -111,7 +131,7 @@ def main(): if not wheels: raise ValueError("No wheels found for content validation") for wheel in wheels: - check_wheel(wheel) + check_wheel(wheel, arguments.expected_version) if __name__ == "__main__": diff --git a/packaging/scripts/prepare_python_dev_version.py b/packaging/scripts/prepare_python_dev_version.py new file mode 100644 index 000000000..da39484d9 --- /dev/null +++ b/packaging/scripts/prepare_python_dev_version.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Set an explicit Python development wheel version in a build checkout.""" + +import argparse +from pathlib import Path +import re + +DEV_VERSION = re.compile( + r"(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*)){2}\.dev(?:0|[1-9][0-9]*)" +) +VERSION_ASSIGNMENT = re.compile(r"""(?m)^(version[ \t]*=[ \t]*)(["'])([^"'\r\n]+)\2""") +PROJECT_TABLE = re.compile(r"(?ms)^\[project\][ \t]*\r?\n(?P.*?)(?=^\[|\Z)") + + +def _replace_version(text, version, path): + matches = list(VERSION_ASSIGNMENT.finditer(text)) + if len(matches) != 1: + raise ValueError(f"Expected exactly one version declaration in {path}") + match = matches[0] + return text[: match.start(3)] + version + text[match.end(3) :] + + +def prepare_version(python_root, version): + if DEV_VERSION.fullmatch(version) is None: + raise ValueError( + "Python development version must be MAJOR.MINOR.PATCH.devN " + "(for example 2.5.1.dev0 or 2.5.1.dev2)" + ) + setup_path = python_root / "setup.py" + project_path = python_root / "pyproject.toml" + setup = _replace_version( + setup_path.read_text(encoding="utf-8"), version, setup_path + ) + project = project_path.read_text(encoding="utf-8") + table = PROJECT_TABLE.search(project) + if table is None: + raise ValueError(f"Missing [project] table in {project_path}") + body = _replace_version(table.group("body"), version, project_path) + project = project[: table.start("body")] + body + project[table.end("body") :] + # Validate both declarations before changing either file. Only the Python + # package metadata changes; the native library keeps its source/ABI version. + setup_path.write_text(setup, encoding="utf-8") + project_path.write_text(project, encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument( + "--python-root", + type=Path, + default=Path(__file__).resolve().parents[2] / "python", + ) + parser.add_argument("--github-output", type=Path) + args = parser.parse_args() + prepare_version(args.python_root, args.version) + if args.github_output is not None: + with args.github_output.open("a", encoding="utf-8") as output: + output.write(f"\npython_version={args.version}\n") + print(f"Python wheel version: {args.version}") + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index f1be69aec..a91261afa 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -127,7 +127,19 @@ def run_text(job) raise "Windows SDKs must retain Python wheel codec coverage" unless windows.include?("-Denable.lzma2=ON") output, status = Open3.capture2e("bash", "-n", sdk_builder_path) raise "Invalid Python SDK shell script: #{output}" unless status.success? -raise "The old wheel workflow must be retired" if File.exist?(File.expand_path("../../.github/workflows/wheels.yml", __dir__)) +dev_workflow = YAML.load_file(File.expand_path("../../.github/workflows/wheels.yml", __dir__)) +dev_triggers = dev_workflow["on"] || dev_workflow[true] +raise "Development wheels must be manual-only" unless dev_triggers.keys == ["workflow_dispatch"] +raise "Development wheels require an explicit version" unless dev_triggers.fetch("workflow_dispatch").fetch("inputs").fetch("version").fetch("required") +raise "Development wheels must remain read-only" unless dev_workflow.fetch("permissions") == { "contents" => "read" } +dev_jobs = dev_workflow.fetch("jobs") +dev_jobs.each_value do |job| + raise "Development wheels must not depend on native release jobs" unless Array(job["needs"]).all? { |name| dev_jobs.key?(name) } + job.fetch("steps").each do |step| + next unless step["uses"].to_s.start_with?("actions/upload-artifact@") + raise "Development workflow must upload only wheels" unless step.fetch("with").fetch("path") == "wheelhouse/*.whl" + end +end homebrew = jobs.fetch("build-homebrew") raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" @@ -283,7 +295,7 @@ def run_text(job) raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle") raise "workflow must not hard-code a fork repository" if File.read(path).include?("ColinLeeo/tsfile") -jobs.each_value do |job| +(jobs.values + dev_jobs.values).each do |job| job.fetch("steps").each do |step| artifact = step["with"].to_h["name"].to_s raise "artifact names must not repeat the macOS matrix prefix" if artifact.include?("macos-macos") diff --git a/packaging/tests/test_python_dev_version.py b/packaging/tests/test_python_dev_version.py new file mode 100644 index 000000000..37a6cda4d --- /dev/null +++ b/packaging/tests/test_python_dev_version.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Exercise explicit development versions without building the native library.""" + +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/prepare_python_dev_version.py" +SPEC = importlib.util.spec_from_file_location("python_dev_version", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + +SETUP = 'version = "2.5.1.dev" # package version\nname = "tsfile"\n' +PROJECT = ( + '[project]\nname = "tsfile"\nversion = "2.5.1.dev"\n' + '[tool.example]\nversion = "leave-this-alone"\n' +) + + +class PythonDevVersionTest(unittest.TestCase): + def write_fixture(self, root): + (root / "setup.py").write_text(SETUP) + (root / "pyproject.toml").write_text(PROJECT) + (root / "CMakeLists.txt").write_text("set(TsFile_CPP_VERSION 2.5.1.dev)\n") + + def test_sets_dev0_and_dev2_in_both_python_declarations(self): + for version in ("2.5.1.dev0", "2.5.1.dev2"): + with self.subTest( + version=version + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_fixture(root) + MODULE.prepare_version(root, version) + self.assertEqual( + (root / "setup.py").read_text(), SETUP.replace("2.5.1.dev", version) + ) + self.assertEqual( + (root / "pyproject.toml").read_text(), + PROJECT.replace("2.5.1.dev", version), + ) + self.assertEqual( + (root / "CMakeLists.txt").read_text(), + "set(TsFile_CPP_VERSION 2.5.1.dev)\n", + ) + # A rerun must keep the explicit version instead of appending a suffix. + MODULE.prepare_version(root, version) + self.assertEqual( + (root / "setup.py").read_text(), SETUP.replace("2.5.1.dev", version) + ) + + def test_rejects_noncanonical_versions_without_changing_sources(self): + for version in ( + "2.5.1", + "2.5.1.dev", + "2.5.1.dev-1", + "2.5.1.dev02", + "2.5.1rc1", + "02.5.1.dev0", + "2.5.1.dev0+local", + '2.5.1.dev0"\nother=value', + "", + ): + with self.subTest( + version=version + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_fixture(root) + with self.assertRaisesRegex(ValueError, "MAJOR.MINOR.PATCH.devN"): + MODULE.prepare_version(root, version) + self.assertEqual((root / "setup.py").read_text(), SETUP) + self.assertEqual((root / "pyproject.toml").read_text(), PROJECT) + + def test_validates_all_source_declarations_before_writing(self): + cases = ( + (SETUP + 'version = "duplicate"\n', PROJECT), + (SETUP.replace("version =", "other ="), PROJECT), + (SETUP, PROJECT.replace("[project]", "[other]")), + (SETUP, PROJECT.replace('version = "2.5.1.dev"', 'other = "2.5.1.dev"')), + ( + SETUP, + PROJECT.replace( + 'version = "2.5.1.dev"', + 'version = "2.5.1.dev"\nversion = "duplicate"', + ), + ), + ) + for setup, project in cases: + with self.subTest( + setup=setup, project=project + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "setup.py").write_text(setup) + (root / "pyproject.toml").write_text(project) + with self.assertRaises(ValueError): + MODULE.prepare_version(root, "2.5.1.dev2") + self.assertEqual((root / "setup.py").read_text(), setup) + self.assertEqual((root / "pyproject.toml").read_text(), project) + + def test_cli_preserves_existing_github_outputs_and_sets_requested_version(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_fixture(root) + output = root / "github-output" + output.write_text("existing=value") + subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--python-root", + str(root), + "--version", + "2.5.1.dev2", + "--github-output", + str(output), + ], + check=True, + ) + self.assertEqual( + output.read_text(), "existing=value\npython_version=2.5.1.dev2\n" + ) + self.assertIn('version = "2.5.1.dev2"', (root / "setup.py").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_python_wheel_contents.py b/packaging/tests/test_python_wheel_contents.py index 6d015862b..16898491c 100644 --- a/packaging/tests/test_python_wheel_contents.py +++ b/packaging/tests/test_python_wheel_contents.py @@ -51,6 +51,42 @@ def test_runtime_only_wheel_is_accepted(self): } ) + def test_development_wheel_filename_and_metadata_must_match_requested_version(self): + for expected in ("2.5.1.dev0", "2.5.1.dev2"): + for filename_version, directory_version, metadata_version in ( + (expected, expected, expected), + ("2.5.1", expected, expected), + (expected, "2.5.1", expected), + (expected, expected, "2.5.1"), + (expected, expected, "2.5.1.dev1"), + ): + with self.subTest( + expected=expected, + filename=filename_version, + directory=directory_version, + metadata=metadata_version, + ), tempfile.TemporaryDirectory() as temporary: + path = ( + Path(temporary) + / f"tsfile-{filename_version}-cp311-cp311-manylinux_2_28_x86_64.whl" + ) + with ZipFile(path, "w") as wheel: + wheel.writestr("tsfile/libtsfile.so.1", b"native") + wheel.writestr( + f"tsfile-{directory_version}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: tsfile\nVersion: {metadata_version}\n", + ) + if ( + filename_version + == directory_version + == metadata_version + == expected + ): + checker.check_wheel(path, expected) + else: + with self.assertRaisesRegex(ValueError, "version"): + checker.check_wheel(path, expected) + def test_sdk_headers_and_import_libraries_are_rejected(self): for name in [ "tsfile/include/tsfile/cwrapper/tsfile_cwrapper.h",