diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 1c6e38bfa62..177b0108d22 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -185,3 +185,64 @@ jobs: with: name: wheel path: python/dist/* + + # Wraps the signed binaries from the cli job into PyPI wheels (project + # "databricks"). Kept as a separate artifact from "wheel" (databricks-bundles) + # so each artifact maps to one PyPI project in the databricks-cli release workflow. + cli-wheels: + needs: cli + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + permissions: + id-token: write + contents: read + + steps: + # The script and LICENSE come from the workflow's own ref, like the + # local actions in the other jobs, so building an older tag still works. + - name: Checkout workflow ref + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + sparse-checkout: | + .github + tools/build_cli_wheels.py + LICENSE + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.6.5" + + - name: Download signed CLI binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: cli + path: cli-artifact + + - name: Build wheels + run: python3 tools/build_cli_wheels.py cli-artifact cli-wheels + + - name: Check wheel metadata + run: uvx twine@6.2.0 check --strict cli-wheels/* + + - name: Smoke test linux-amd64 wheel + run: | + cli_version=$(basename cli-artifact/databricks_cli_*_SHA256SUMS) + cli_version=${cli_version#databricks_cli_} + cli_version=${cli_version%_SHA256SUMS} + uv venv "$RUNNER_TEMP/venv" + uv pip install --python "$RUNNER_TEMP/venv" cli-wheels/*manylinux*x86_64.whl + "$RUNNER_TEMP/venv/bin/databricks" --version | tee /dev/stderr | grep -F "$cli_version" + + - name: Upload CLI wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-wheels + path: cli-wheels/*.whl + if-no-files-found: error diff --git a/Taskfile.yml b/Taskfile.yml index 8d59145d8a9..d3668a50d3b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -265,6 +265,12 @@ tasks: cmds: - 'uv run -p ">=3.11" --no-project python tools/validate_codeowners.py' + test-cli-wheels: + desc: Unit-test the script that packages CLI binaries as PyPI wheels + dir: tools + cmds: + - 'uv run -p ">=3.11" --no-project python -m unittest -q test_build_cli_wheels' + changelog-preview: desc: Print the CHANGELOG.md section the next release would add from .nextchanges/ env: @@ -314,6 +320,7 @@ tasks: - task: check-codeowners - task: check-lockfiles - task: check-agents-setup + - task: test-cli-wheels install-pythons: desc: Install Python 3.9-3.13 via uv diff --git a/acceptance/cmd/pypi-wheel/make_archives.py b/acceptance/cmd/pypi-wheel/make_archives.py new file mode 100644 index 00000000000..e784866ff9c --- /dev/null +++ b/acceptance/cmd/pypi-wheel/make_archives.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Fake goreleaser output: every platform archive holds the given CLI binary.""" + +import sys +import tarfile +import zipfile +from pathlib import Path + +cli, out = Path(sys.argv[1]), Path(sys.argv[2]) +out.mkdir() +for goos in ("linux", "darwin", "windows"): + for goarch in ("amd64", "arm64"): + if goos == "windows": + path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.zip" + with zipfile.ZipFile(path, "w") as z: + z.write(cli, "databricks.exe") + else: + path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz" + with tarfile.open(path, "w:gz") as t: + t.add(cli, "databricks") +# Only the file name is used: it carries the version. +(out / "databricks_cli_1.2.3_SHA256SUMS").touch() diff --git a/acceptance/cmd/pypi-wheel/out.test.toml b/acceptance/cmd/pypi-wheel/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/pypi-wheel/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt new file mode 100644 index 00000000000..01457a08a26 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -0,0 +1,10 @@ + +>>> python3 make_archives.py [CLI] archives + +>>> python3 [TESTROOT]/../tools/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE + +>>> ls wheels +6 + +>>> uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version +Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script new file mode 100644 index 00000000000..e87df944008 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/script @@ -0,0 +1,10 @@ +# Package the CLI under test the way release-build does, then run it through uvx +# to check that a wheel built by build_cli_wheels.py is usable by Python tooling. +trace python3 make_archives.py $CLI archives +trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null +trace ls wheels | grep -c '^databricks-1.2.3-py3-none-.*\.whl$' + +# uv picks the wheel matching this platform from the directory. +trace uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version + +rm -fr archives wheels diff --git a/acceptance/cmd/pypi-wheel/test.toml b/acceptance/cmd/pypi-wheel/test.toml new file mode 100644 index 00000000000..23eaa49b0c6 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/test.toml @@ -0,0 +1,5 @@ +Cloud = false + +# uvx may need to create a Python environment. +Timeout = '180s' +TimeoutWindows = '240s' diff --git a/tools/build_cli_wheels.py b/tools/build_cli_wheels.py new file mode 100644 index 00000000000..afedd541759 --- /dev/null +++ b/tools/build_cli_wheels.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Package the released Databricks CLI binaries as PyPI wheels. + +Input is the directory of the `cli` release-build artifact: the goreleaser +archives (databricks_cli_*__.{tar.gz,zip}) and the SHA256SUMS file, whose name carries the version. +The binaries are taken from the archives as-is, so the signed Windows binaries +are what ends up in the wheels. + +Each wheel is py3-none-: platform-specific but independent of the +Python version. It contains only the binary, installed to the environment's +scripts directory (bin/ or Scripts/) via the wheel `.data/scripts` mechanism, +and no Python package, so `databricks` and `uvx databricks` run the binary +directly without a Python bootstrapper. + +Usage: build_cli_wheels.py [license-file] +""" + +import base64 +import hashlib +import io +import re +import sys +import tarfile +import zipfile +from pathlib import Path + +DIST_NAME = "databricks" +SUMMARY = "Databricks CLI" +DESCRIPTION = """# Databricks CLI + +Command-line interface for Databricks. This package installs the prebuilt `databricks` binary. + +Run it with `uvx databricks` or `pipx run databricks`, or install it with `uv tool install databricks`. + +Documentation: https://docs.databricks.com/dev-tools/cli/ +Source: https://github.com/databricks/cli +""" + +# (goreleaser os, goreleaser arch) -> wheel platform tag. Compressed tag sets +# (dot-separated) make one wheel serve glibc and musl Linux: the binary is +# statically linked (CGO_ENABLED=0). +PLATFORMS = { + ("linux", "amd64"): "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64", + ("linux", "arm64"): "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64", + ("darwin", "amd64"): "macosx_11_0_x86_64", + ("darwin", "arm64"): "macosx_11_0_arm64", + ("windows", "amd64"): "win_amd64", + ("windows", "arm64"): "win_arm64", +} + +# Fixed timestamp so rebuilding from the same artifact yields identical wheels. +ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0) + + +def read_version(artifact_dir: Path) -> str: + """Return the CLI version from the databricks_cli__SHA256SUMS file name.""" + sums_files = list(artifact_dir.glob("databricks_cli_*_SHA256SUMS")) + if len(sums_files) != 1: + raise SystemExit(f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir}, found {sums_files}") + return sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")] + + +def pep440(cli_version: str) -> str: + """Convert a goreleaser version to PEP 440. + + Releases (1.23.0) are unchanged. Snapshots (1.23.1-dev+abc123) become + 1.23.1.dev0+abc123. + """ + m = re.fullmatch(r"(\d+\.\d+\.\d+)(?:-dev(?:\+([0-9A-Za-z.]+))?)?", cli_version) + if not m: + raise SystemExit(f"unsupported CLI version {cli_version!r}") + base, local = m.groups() + if cli_version == base: + return base + return f"{base}.dev0" + (f"+{local}" if local else "") + + +def find_archive(artifact_dir: Path, goos: str, goarch: str) -> Path: + ext = "zip" if goos == "windows" else "tar.gz" + matches = list(artifact_dir.glob(f"databricks_cli_*{goos}_{goarch}.{ext}")) + if len(matches) != 1: + raise SystemExit(f"expected exactly one {goos}/{goarch} .{ext} archive in {artifact_dir}, found {matches}") + return matches[0] + + +def extract_binary(archive: Path, binary_name: str) -> bytes: + if archive.name.endswith(".zip"): + with zipfile.ZipFile(archive) as z: + return z.read(binary_name) + with tarfile.open(archive) as t: + f = t.extractfile(binary_name) + if f is None: + raise SystemExit(f"{binary_name} is not a regular file in {archive}") + return f.read() + + +def record_hash(data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + return f"sha256={digest},{len(data)}" + + +def build_wheel( + out_dir: Path, + version: str, + platform_tag: str, + binary_name: str, + binary: bytes, + license_text: bytes, +) -> Path: + dist_info = f"{DIST_NAME}-{version}.dist-info" + data_dir = f"{DIST_NAME}-{version}.data" + + metadata = ( + "Metadata-Version: 2.1\n" + f"Name: {DIST_NAME}\n" + f"Version: {version}\n" + f"Summary: {SUMMARY}\n" + "Home-page: https://github.com/databricks/cli\n" + "License: Databricks License\n" + "License-File: LICENSE\n" + "Description-Content-Type: text/markdown\n" + f"\n{DESCRIPTION}" + ).encode() + wheel_meta = ( + "Wheel-Version: 1.0\n" + "Generator: databricks-cli build_cli_wheels.py\n" + "Root-Is-Purelib: false\n" + "".join(f"Tag: py3-none-{t}\n" for t in platform_tag.split(".")) + ).encode() + + # name -> (content, unix mode) + files = { + f"{data_dir}/scripts/{binary_name}": (binary, 0o755), + f"{dist_info}/METADATA": (metadata, 0o644), + f"{dist_info}/WHEEL": (wheel_meta, 0o644), + f"{dist_info}/licenses/LICENSE": (license_text, 0o644), + } + record = "".join(f"{name},{record_hash(data)}\n" for name, (data, _) in files.items()) + record += f"{dist_info}/RECORD,,\n" + files[f"{dist_info}/RECORD"] = (record.encode(), 0o644) + + out = out_dir / f"{DIST_NAME}-{version}-py3-none-{platform_tag}.whl" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + for name, (data, mode) in files.items(): + info = zipfile.ZipInfo(name, ZIP_DATE_TIME) + info.external_attr = (0o100000 | mode) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, data) + out.write_bytes(buf.getvalue()) + return out + + +def main() -> None: + if len(sys.argv) not in (3, 4): + raise SystemExit(__doc__) + artifact_dir = Path(sys.argv[1]) + out_dir = Path(sys.argv[2]) + license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[1] / "LICENSE" + + cli_version = read_version(artifact_dir) + version = pep440(cli_version) + license_text = license_file.read_bytes() + out_dir.mkdir(parents=True, exist_ok=True) + + for (goos, goarch), platform_tag in PLATFORMS.items(): + archive = find_archive(artifact_dir, goos, goarch) + binary_name = "databricks.exe" if goos == "windows" else "databricks" + binary = extract_binary(archive, binary_name) + print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text)) + + +if __name__ == "__main__": + main() diff --git a/tools/test_build_cli_wheels.py b/tools/test_build_cli_wheels.py new file mode 100644 index 00000000000..1086c0ea922 --- /dev/null +++ b/tools/test_build_cli_wheels.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import io +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +import build_cli_wheels as b + + +class TestBuildCliWheels(unittest.TestCase): + def test_pep440(self): + self.assertEqual(b.pep440("1.23.0"), "1.23.0") + self.assertEqual(b.pep440("1.23.1-dev+abc123"), "1.23.1.dev0+abc123") + with self.assertRaises(SystemExit): + b.pep440("v1.23.0") + + def test_end_to_end(self): + with tempfile.TemporaryDirectory() as tmp: + art, out = Path(tmp, "art"), Path(tmp, "out") + art.mkdir() + for (goos, goarch), _ in b.PLATFORMS.items(): + if goos == "windows": + path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.zip" + with zipfile.ZipFile(path, "w") as z: + z.writestr("databricks.exe", b"MZ") + else: + path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.tar.gz" + with tarfile.open(path, "w:gz") as t: + info = tarfile.TarInfo("databricks") + info.size = 3 + t.addfile(info, io.BytesIO(b"ELF")) + (art / "databricks_cli_1.23.0_SHA256SUMS").touch() + lic = Path(tmp, "LICENSE") + lic.write_text("license") + + b.sys.argv = ["x", str(art), str(out), str(lic)] + b.main() + + wheels = sorted(p.name for p in out.glob("*.whl")) + self.assertEqual(len(wheels), len(b.PLATFORMS)) + self.assertIn("databricks-1.23.0-py3-none-win_amd64.whl", wheels) + with zipfile.ZipFile(out / "databricks-1.23.0-py3-none-macosx_11_0_arm64.whl") as z: + script = z.getinfo("databricks-1.23.0.data/scripts/databricks") + self.assertEqual(script.external_attr >> 16, 0o100755) + self.assertIn("Tag: py3-none-macosx_11_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode()) + + +if __name__ == "__main__": + unittest.main()