diff --git a/.github/workflows/modal-deploy.yml b/.github/workflows/modal-deploy.yml index ad8bb19d..821c4dd1 100644 --- a/.github/workflows/modal-deploy.yml +++ b/.github/workflows/modal-deploy.yml @@ -14,6 +14,13 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + # Set to 0 for rollback; ordinary submissions use the shared cache automatically. + KERNELBOT_NVCC_TOOLCHAIN: ${{ vars.KERNELBOT_NVCC_TOOLCHAIN || '1' }} + KERNELBOT_NVCC_PCH: ${{ vars.KERNELBOT_NVCC_PCH || '1' }} + KERNELBOT_NVCC_PCH_PROFILES: ${{ vars.KERNELBOT_NVCC_PCH_PROFILES }} + KERNELBOT_NVCC_TOOLCHAIN_VOLUME: ${{ vars.KERNELBOT_NVCC_TOOLCHAIN_VOLUME || 'kernelbot-nvcc-toolchains' }} + KERNELBOT_NVCC_PCH_VOLUME: ${{ vars.KERNELBOT_NVCC_PCH_VOLUME || 'kernelbot-nvcc-pch' }} + KERNELBOT_PCH_VOLUME: ${{ vars.KERNELBOT_PCH_VOLUME || 'kernelbot-torch-pch' }} steps: - name: Checkout Repository @@ -33,6 +40,11 @@ jobs: run: | PYTHONPATH=${{ github.workspace }}/src:${{ github.workspace }}/src/runners modal run src/runners/modal_runner.py::warm_pch + - name: Warm NVCC framework header cache + if: env.KERNELBOT_NVCC_TOOLCHAIN == '1' + run: | + PYTHONPATH=${{ github.workspace }}/src:${{ github.workspace }}/src/runners modal run src/runners/modal_runner.py::warm_nvcc_pch --profiles "$KERNELBOT_NVCC_PCH_PROFILES" + - name: Deploy job run: | # TODO: Remove pythonpath here diff --git a/docs/modal-pch-cache.md b/docs/modal-pch-cache.md index 845900cb..56c98fd5 100644 --- a/docs/modal-pch-cache.md +++ b/docs/modal-pch-cache.md @@ -5,7 +5,9 @@ The runner sets `CXX` to a GCC wrapper that loads a precompiled changes. Leave `use_pch=False`; PyTorch's separate native PCH option is unnecessary. The submission path assumes `load_inline()`. -Only the Torch host header is cached. CUDA/CUTLASS templates, user headers, kernel +NVCC also caches the implicit CUDA-side framework headers by default (see below). + +The host cache only precompiles the Torch host header. CUDA/CUTLASS templates, user headers, kernel objects, and shared libraries still compile in each submission container. Different sources and module names share a PCH when their host flags match; changing a user header after the Torch include still changes the compiled code. @@ -28,6 +30,36 @@ speedup. Set `CXX=/usr/bin/g++` before building to bypass the wrapper entirely. ## Deployment +### CUDA-side framework cache + +The runner uses NVIDIA's checksum-pinned NVCC 13.4.99 drop with the complete CUDA +13.4 toolkit. `PYTORCH_NVCC` automatically reuses `torch/types.h`, `cuda.h`, and +`cuda_runtime.h` from `kernelbot-nvcc-pch`. CUTLASS and user headers stay outside +this cache. The trusted warmer builds 17 profiles from five observed flag tuples; +submission workers mount them read-only. Unknown flags, custom compiler inputs, +and sources without the implicit header prefix compile normally. Corrupt caches +fall back to normal compilation. CPU artifact replay checks compiler identity +and user-header dependencies. + +Upload the private compiler drop once, without committing binaries: + +```sh +PYTHONPATH=src/runners uv run python scripts/upload_nvcc_toolchain.py --source /private/path/to/binaries +``` + +CI warms both caches before deployment. `KERNELBOT_NVCC_PCH_PROFILES` selects a +comma-separated subset; empty warms all 17. Manual warmup uses +`modal run src/runners/modal_runner.py::warm_nvcc_pch --profiles ...`. +Set `KERNELBOT_NVCC_PCH=0` to disable device PCH. Set both +`KERNELBOT_NVCC_TOOLCHAIN=0` and `KERNELBOT_NVCC_PCH=0` to restore CUDA 13.3. +`KERNELBOT_NVCC_PCH_DISABLE=1` also disables lookup within a submission. +The corresponding `KERNELBOT_NVCC_TOOLCHAIN_VOLUME`, `KERNELBOT_NVCC_PCH_VOLUME`, +and `KERNELBOT_PCH_VOLUME` variables override the private input, device, and host +volumes. Retain old cache generations for rollback and replace workers after +warming so they see the new Volume snapshot. + +### Host cache + The main/dev workflow warms profiles before deploying GPU runners. Manually, run both commands in the same Modal profile/environment: diff --git a/scripts/upload_nvcc_toolchain.py b/scripts/upload_nvcc_toolchain.py new file mode 100644 index 00000000..57863608 --- /dev/null +++ b/scripts/upload_nvcc_toolchain.py @@ -0,0 +1,52 @@ +"""Upload the checksum-pinned NVIDIA drop to KernelBot's private Modal volume. + +PYTHONPATH=src/runners python scripts/upload_nvcc_toolchain.py --source /path/to/binaries +This does not deploy a runner or publish the binaries outside the Modal workspace. +""" + +import argparse +import hashlib +import json +import os +import tempfile +from pathlib import Path + +import modal +from nvcc_toolchain import BINARY_SHA256, RELEASE + + +def upload(source: Path, volume_name: str): + for name, expected in BINARY_SHA256.items(): + if hashlib.sha256((source / name).read_bytes()).hexdigest() != expected: + raise ValueError(f"Compiler checksum mismatch: {name}") + manifest = json.dumps({"release": RELEASE, "sha256": BINARY_SHA256}, sort_keys=True) + "\n" + volume = modal.Volume.from_name(volume_name, create_if_missing=True) + manifest_path = f"/{RELEASE}/manifest.json" + try: + existing = b"".join(volume.read_file(manifest_path)).decode() + except FileNotFoundError: + existing = None + if existing is not None: + if existing != manifest: + raise ValueError("Existing compiler release has a different manifest") + print(f"{RELEASE} is already present in {volume_name}") + return + with tempfile.TemporaryDirectory() as directory: + metadata = Path(directory) / "manifest.json" + metadata.write_text(manifest) + with volume.batch_upload(force=False) as batch: + for name in BINARY_SHA256: + batch.put_file(source / name, f"/{RELEASE}/{name}") + batch.put_file(metadata, manifest_path) + print(f"Uploaded verified {RELEASE} to private volume {volume_name}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument( + "--volume", + default=os.environ.get("KERNELBOT_NVCC_TOOLCHAIN_VOLUME", "kernelbot-nvcc-toolchains"), + ) + args = parser.parse_args() + upload(args.source, args.volume) diff --git a/src/libkernelbot/inline_artifacts.py b/src/libkernelbot/inline_artifacts.py index e7f2ece5..ce03a631 100644 --- a/src/libkernelbot/inline_artifacts.py +++ b/src/libkernelbot/inline_artifacts.py @@ -6,15 +6,96 @@ import json import os import platform +import re import subprocess import sysconfig from pathlib import Path +NVCC_FINGERPRINT = Path("/opt/kernelbot-pch/nvcc-fingerprint") +NVCC_PCH_ROOT = Path("/kernelbot-nvcc-pch") +MAX_CACHE_METADATA_BYTES = 64 * 1024 + def _digest(value: bytes) -> str: return hashlib.sha256(value).hexdigest() +def nvcc_fingerprint() -> str | None: + """Bind artifacts to the immutable compiler/wrapper manifest, when installed.""" + try: + contents = NVCC_FINGERPRINT.read_bytes() + except FileNotFoundError: + return None + if not contents.strip(): + raise RuntimeError("NVCC toolchain fingerprint is empty") + return _digest(contents) + + +def _small_cache_file(path: Path) -> bytes: + if path.is_symlink() or path.stat().st_size > MAX_CACHE_METADATA_BYTES: + raise RuntimeError("Invalid NVCC PCH cache metadata") + return path.read_bytes() + + +def _pch_dependency_digest(path: Path) -> str: + """Use a trusted, immutable cache manifest instead of rehashing GB-sized PCHs.""" + path = path.resolve() + root = NVCC_PCH_ROOT.resolve() + if not path.is_relative_to(root): + raise RuntimeError("PCH dependency is outside the trusted NVCC cache") + parts = path.relative_to(root).parts + if ( + len(parts) != 4 + or not all(re.fullmatch(r"[0-9a-f]{64}", part) for part in parts[:2]) + or parts[2] != "pch" + ): + raise RuntimeError("Invalid NVCC PCH dependency path") + generation, key, _, _ = parts + if generation != NVCC_FINGERPRINT.read_text().strip(): + raise RuntimeError("NVCC PCH dependency belongs to a different toolchain") + entry = root / generation / key + manifest_bytes = _small_cache_file(entry / "manifest.json") + manifest = json.loads(manifest_bytes) + if ( + manifest.get("version") != 1 + or manifest.get("generation") != generation + or manifest.get("key") != key + or manifest.get("header_sha256") != _digest(_small_cache_file(entry / "gpumode.cuh")) + ): + raise RuntimeError("NVCC PCH dependency manifest does not match its cache entry") + files = manifest.get("files", []) + if len(files) < 2: + raise RuntimeError("NVCC PCH dependency manifest is incomplete") + frontends, names = set(), set() + for item in files: + name, size = item.get("path", ""), item.get("bytes") + match = re.fullmatch(r"pch/(cicc|cudafe)_[A-Za-z0-9_]+\.pch", name) + if not match or name in names or type(size) is not int or size <= 0: + raise RuntimeError("Invalid NVCC PCH dependency file metadata") + cache_file = entry / name + if ( + cache_file.resolve() != cache_file + or not cache_file.is_file() + or cache_file.stat().st_size != size + ): + raise RuntimeError("NVCC PCH dependency file changed") + names.add(name) + frontends.add(match[1]) + if frontends != {"cicc", "cudafe"} or path.relative_to(entry).as_posix() not in names: + raise RuntimeError("NVCC PCH dependency is not covered by the cache manifest") + return f"nvcc-pch-v1:{_digest(manifest_bytes)}:{path.stat().st_size}" + + +def _dependency_digest(path: Path) -> str: + if path.suffix == ".pch": + return _pch_dependency_digest(path) + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + def _save_module(directory: Path, module, work: Path) -> None: directory.mkdir(exist_ok=True) binary = Path(module.__file__).read_bytes() @@ -64,7 +145,7 @@ def _dependencies(build: Path, work: Path) -> dict[str, str]: if path.parent == build.resolve() and path.name in {"main.cpp", "cuda.cu", "sycl.sycl"}: continue # Generated from the source strings already in the request key. name = str(path.relative_to(work)) if path.is_relative_to(work) else str(path) - dependencies[name] = _digest(path.read_bytes()) + dependencies[name] = _dependency_digest(path) return dependencies @@ -73,7 +154,7 @@ def _check_dependencies(dependencies: dict[str, str] | None, work: Path) -> None raise RuntimeError("Artifact has no compiler dependency information") for name, digest in dependencies.items(): path = work / name - if _digest(path.read_bytes()) != digest: + if _dependency_digest(path) != digest: raise RuntimeError(f"Build dependency changed: {path}") @@ -124,18 +205,30 @@ def dispatch(*args, **kwargs): current_runtime = { **runtime, "cuda_home": getattr(cpp_extension, "CUDA_HOME", None), + "nvcc_toolchain": nvcc_fingerprint(), "build_environment": { name: os.environ.get(name) for name in ( "TORCH_CUDA_ARCH_LIST", "CXX", "CC", + "PYTORCH_NVCC", "CUDA_HOME", + "CUDA_PATH", + "CUDA_ROOT", "CPATH", "CPLUS_INCLUDE_PATH", "C_INCLUDE_PATH", + "GCC_EXEC_PREFIX", + "COMPILER_PATH", + "PATH", + "NVCC_CCBIN", + "NVVMIR_LIBRARY_DIR", "NVCC_PREPEND_FLAGS", "NVCC_APPEND_FLAGS", + "KERNELBOT_NVCC_PCH", + "KERNELBOT_NVCC_PCH_DISABLE", + "KERNELBOT_NVCC_PCH_ROOT", "LIBRARY_PATH", "LD_LIBRARY_PATH", ) diff --git a/src/libkernelbot/python_precompile.py b/src/libkernelbot/python_precompile.py index 6d2ea1cd..558c20cf 100644 --- a/src/libkernelbot/python_precompile.py +++ b/src/libkernelbot/python_precompile.py @@ -14,7 +14,7 @@ from pathlib import Path from libkernelbot.consts import Timeout -from libkernelbot.inline_artifacts import prepare_environment +from libkernelbot.inline_artifacts import nvcc_fingerprint, prepare_environment from libkernelbot.run_eval import CPUCompileInfo MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 @@ -67,11 +67,12 @@ def should_precompile(config: dict) -> bool: def build_identity(config: dict) -> dict: return { - "version": 2, + "version": 3, "sources": hashlib.sha256( json.dumps(config["sources"], sort_keys=True).encode() ).hexdigest(), "image": Path("/opt/kernelbot-pch/fingerprint").read_text().strip(), + "nvcc_toolchain": nvcc_fingerprint(), } diff --git a/src/runners/modal_runner.py b/src/runners/modal_runner.py index a457a44c..b9812b26 100644 --- a/src/runners/modal_runner.py +++ b/src/runners/modal_runner.py @@ -6,6 +6,7 @@ from pathlib import Path from modal import App, Image, Volume +from nvcc_toolchain import BINARY_SHA256, RELEASE, install_nvcc_toolchain from libkernelbot.consts import Timeout from libkernelbot.python_precompile import compile_python, run_with_cpu_artifacts @@ -15,6 +16,10 @@ # IMPORTANT: This has to stay in separate file or modal breaks app = App(os.environ.get("KERNELBOT_MODAL_APP", "discord-bot-runner")) cuda_version = "13.3.0" +NVCC_TOOLCHAIN_ENABLED = os.environ.get("KERNELBOT_NVCC_TOOLCHAIN", "1") == "1" +NVCC_PCH_ENABLED = os.environ.get("KERNELBOT_NVCC_PCH", "1" if NVCC_TOOLCHAIN_ENABLED else "0") == "1" +if NVCC_PCH_ENABLED and not NVCC_TOOLCHAIN_ENABLED: + raise ValueError("NVCC PCH requires KERNELBOT_NVCC_TOOLCHAIN=1") flavor = "devel" operating_sys = "ubuntu24.04" tag = f"{cuda_version}-{flavor}-{operating_sys}" @@ -98,14 +103,16 @@ f"rm /tmp/{mathdx_archive}" ), ) - .env({ - "CUTLASS_PATH": "/opt/cutlass", - "MATHDX_HOME": "/opt/mathdx", - "CPLUS_INCLUDE_PATH": ( - "/opt/mathdx/include:/opt/mathdx/external/cutlass/include:" - "/opt/cutlass/include:/opt/cutlass/tools/util/include" - ), - }) + .env( + { + "CUTLASS_PATH": "/opt/cutlass", + "MATHDX_HOME": "/opt/mathdx", + "CPLUS_INCLUDE_PATH": ( + "/opt/mathdx/include:/opt/mathdx/external/cutlass/include:" + "/opt/cutlass/include:/opt/cutlass/tools/util/include" + ), + } + ) .run_commands( "python -m pip check", 'python -c "import cuda.tile, nvmath"', @@ -143,10 +150,75 @@ "ncu --version", ) +# This switch selects a separately versioned runner image. Keeping it separate +# from PCH enablement allows a same-compiler comparison and independent rollback. +NVCC_PCH_MOUNT = "/kernelbot-nvcc-pch" +nvcc_pch_volume = Volume.from_name( + os.environ.get("KERNELBOT_NVCC_PCH_VOLUME", "kernelbot-nvcc-pch"), create_if_missing=True +) +submission_volumes = {PCH_MOUNT: pch_volume.with_mount_options(read_only=True)} +if NVCC_TOOLCHAIN_ENABLED: + binary_volume = Volume.from_name( + os.environ.get("KERNELBOT_NVCC_TOOLCHAIN_VOLUME", "kernelbot-nvcc-toolchains") + ) + cuda_image = cuda_image.apt_install( + # Preserve the full devel-image surface (tileiras, cuBLAS, sanitizer, + # profiler tools, and all library headers) when switching CUDA_HOME. + "cuda-toolkit-13-4=13.4.3-1", + "cuda-nvcc-13-4=13.4.92-1", + "cuda-cudart-dev-13-4=13.4.92-1", + "cuda-driver-dev-13-4=13.4.92-1", + "libnvvm-13-4=13.4.92-1", + "cuda-crt-13-4=13.4.92-1", + "libnvptxcompiler-13-4=13.4.92-1", + ).run_commands( + "ln -sfn /usr/local/cuda-13.4 /usr/local/cuda", + # The toolkit meta-package may install a newer Nsight. Keep the tested + # profiler on both CUDA's and the system's command lookup paths. + "ln -sfn /opt/nvidia/nsight-compute/2025.2.1/ncu /usr/local/cuda/bin/ncu", + "ln -sfn /opt/nvidia/nsight-compute/2025.2.1/ncu /usr/local/bin/ncu", + "ncu --version", + "tileiras --version", + "compute-sanitizer --version", + ( + "printf '#include \\n#include \\n" + "int main() { cublasHandle_t h; return cublasCreate(&h); }\\n' | " + "nvcc -std=c++17 -x cu - -lcublas -o /tmp/cublas-toolkit-smoke && " + "rm /tmp/cublas-toolkit-smoke" + ), + "python /opt/kernelbot-pch/compiler.py --kernelbot-fingerprint", + ) + for filename in ("nvcc.py", "nvcc_common.py", "nvcc_warm.py"): + cuda_image = cuda_image.add_local_file( + pch_files / filename, f"/opt/kernelbot-pch/{filename}", copy=True + ) + cuda_image = ( + cuda_image.run_commands("chmod +x /opt/kernelbot-pch/nvcc.py") + .run_function( + install_nvcc_toolchain, + volumes={"/nvcc-toolchain-input": binary_volume.with_mount_options(read_only=True)}, + kwargs={"release": RELEASE, "expected_hashes": BINARY_SHA256}, + cpu=4, + memory=8192, + timeout=600, + ) + .env( + { + "PYTORCH_NVCC": "/opt/kernelbot-pch/nvcc.py", + "KERNELBOT_NVCC_TOOLCHAIN": "1", + "KERNELBOT_NVCC_PCH": "1" if NVCC_PCH_ENABLED else "0", + "KERNELBOT_NVCC_PCH_ROOT": NVCC_PCH_MOUNT, + } + ) + ) + submission_volumes[NVCC_PCH_MOUNT] = nvcc_pch_volume.with_mount_options(read_only=True) + +cuda_image_base = cuda_image cuda_image = cuda_image.add_local_python_source( "libkernelbot", "modal_runner", "modal_runner_archs", + "nvcc_toolchain", ) MODAL_RUN_TIMEOUT_SECONDS = 60 * 60 @@ -207,7 +279,7 @@ def modal_run_config( # noqa: C901 timeout=Timeout.COMPILE + 30, single_use_containers=True, restrict_modal_access=True, - volumes={PCH_MOUNT: pch_volume.with_mount_options(read_only=True)}, + volumes=submission_volumes, ) def compile_python_submission(config: dict) -> dict: return compile_python(config) @@ -236,3 +308,26 @@ def warm_pch(profiles: str = ""): subprocess.run(["python3", "/opt/kernelbot-pch/warm.py", *selected], check=True) pch_volume.commit() return "Torch PCH profiles committed" + + +@app.function( + image=cuda_image, + cpu=4, + memory=16384, + timeout=1800, + max_containers=1, + volumes={NVCC_PCH_MOUNT: nvcc_pch_volume}, +) +def warm_nvcc_pch(profiles: str = ""): + """Publish fixed, trusted profiles before replacing submission workers.""" + if not NVCC_TOOLCHAIN_ENABLED: + return "NVCC toolchain disabled; no CUDA profiles warmed" + nvcc_pch_volume.reload() + subprocess.run( + ["python3", "/opt/kernelbot-pch/nvcc_warm.py", *profiles.split(",")] + if profiles + else ["python3", "/opt/kernelbot-pch/nvcc_warm.py"], + check=True, + ) + nvcc_pch_volume.commit() + return "NVCC PCH profiles committed" diff --git a/src/runners/modal_runner_archs.py b/src/runners/modal_runner_archs.py index 64f327f1..04fcfb90 100644 --- a/src/runners/modal_runner_archs.py +++ b/src/runners/modal_runner_archs.py @@ -1,6 +1,12 @@ # This file contains wrapper functions for running # Modal apps on specific devices. We will fix this later. -from modal_runner import MODAL_RUN_TIMEOUT_SECONDS, PCH_MOUNT, app, cuda_image, modal_run_config, pch_volume +from modal_runner import ( + MODAL_RUN_TIMEOUT_SECONDS, + app, + cuda_image, + modal_run_config, + submission_volumes, +) from libkernelbot.validation_runtime import run_validation_config @@ -14,7 +20,7 @@ name=f"run_cuda_script_{gpu_slug}", serialized=True, restrict_modal_access=True, - volumes={PCH_MOUNT: pch_volume.with_mount_options(read_only=True)}, + volumes=submission_volumes, timeout=MODAL_RUN_TIMEOUT_SECONDS, )(modal_run_config) pytorch_functions[gpu_slug] = app.function( @@ -23,7 +29,7 @@ name=f"run_pytorch_script_{gpu_slug}", serialized=True, restrict_modal_access=True, - volumes={PCH_MOUNT: pch_volume.with_mount_options(read_only=True)}, + volumes=submission_volumes, timeout=MODAL_RUN_TIMEOUT_SECONDS, )(modal_run_config) @@ -33,6 +39,6 @@ name="run_validation_script_b200", serialized=True, restrict_modal_access=True, - volumes={PCH_MOUNT: pch_volume.with_mount_options(read_only=True)}, + volumes=submission_volumes, timeout=MODAL_RUN_TIMEOUT_SECONDS, )(run_validation_config) diff --git a/src/runners/nvcc_toolchain.py b/src/runners/nvcc_toolchain.py new file mode 100644 index 00000000..54764add --- /dev/null +++ b/src/runners/nvcc_toolchain.py @@ -0,0 +1,114 @@ +"""Pinned private NVIDIA compiler drop used by the Modal runner image.""" + +RELEASE = "nvidia-pch-13.4.99-20261017" +BINARY_SHA256 = { + "nvcc": "254c308d113e2cdd0159ddfc02b54630b36be2013c0b928b6513e4bae723a298", + "cudafe++": "f64ef97248879dccf4cd92aef02d56a8ea915fbc20abd288e63dd9c199f24442", + "nvvm/bin/cicc": "9b890cf57ba86bc2d3c56714cf24e7c5fe52d0919c3b13512a3eea75ac2c3050", + "nvvm-next/bin/cicc": "5cebc486361825fe2625031922b8b8d481556cc4051006f1abd0c8f440e8a1c6", +} + + +def install_nvcc_toolchain(release: str, expected_hashes: dict[str, str]): # noqa: C901 + """Image-build step: consume a private read-only volume, never submission input.""" + import hashlib + import json + import re + import shutil + import subprocess + import tempfile + from pathlib import Path + + source = Path("/nvcc-toolchain-input") / release + cuda = Path("/usr/local/cuda") + targets = { + "nvcc": "bin/nvcc", + "cudafe++": "bin/cudafe++", + "nvvm/bin/cicc": "nvvm/bin/cicc", + "nvvm-next/bin/cicc": "nvvm-next/bin/cicc", + } + if expected_hashes.keys() != targets.keys(): + raise ValueError("Expected checksums for exactly the four NVIDIA compiler binaries") + for name, expected in expected_hashes.items(): + data = (source / name).read_bytes() + if hashlib.sha256(data).hexdigest() != expected: + raise ValueError(f"NVIDIA compiler checksum mismatch: {name}") + for name, target_name in targets.items(): + target = cuda / target_name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source / name, target) + target.chmod(0o755) + + # All supported Modal runner GPUs are SM75 or newer. The vendor profile + # otherwise keeps selecting the old backend despite the second binary. + profile_path = cuda / "bin/nvcc.profile" + profile, count = re.subn( + r"^CICC_PATH\s*=.*$", + "CICC_PATH = $(TOP)/nvvm-next/bin", + profile_path.read_text(), + flags=re.MULTILINE, + ) + if count != 1: + raise ValueError("Expected exactly one CICC_PATH in the CUDA compiler profile") + profile_path.write_text(profile) + + compiler = str(cuda / "bin/nvcc") + version = subprocess.check_output([compiler, "--version"], text=True) + if "V13.4.99" not in version: + raise ValueError(f"Unexpected NVIDIA compiler version: {version}") + smoke = [] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_file = root / "smoke.cu" + source_file.write_text("__global__ void kernel(int* x) { x[0] = 7; }\n") + for arch in ("75", "80", "89", "90a", "100", "100a"): + command = [compiler, f"-arch=sm_{arch}", "-c", str(source_file), "-o"] + command.append(str(root / f"sm_{arch}.o")) + dryrun = subprocess.check_output( + [*command, "--dryrun"], stderr=subprocess.STDOUT, text=True + ) + if "nvvm-next/bin" not in dryrun: + raise ValueError(f"Wrong CUDA compiler backend for SM{arch}") + subprocess.run(command, check=True, timeout=60) + smoke.append(arch) + + package_output = subprocess.check_output( + [ + "dpkg-query", + "--show", + "--showformat=${Package}\t${Version}\t${Status}\n", + "*13-4*", + ], + text=True, + ) + cuda_packages = {} + for line in package_output.splitlines(): + name, package_version, status = line.split("\t", 2) + if status == "install ok installed": + cuda_packages[name] = package_version + if not cuda_packages: + raise ValueError("No installed CUDA 13.4 packages found for the toolchain fingerprint") + + artifact_root = Path("/opt/kernelbot-pch") + files = [ + *(cuda / name for name in targets.values()), + profile_path, + *(cuda / "bin" / name for name in ("ptxas", "nvlink", "fatbinary")), + cuda / "nvvm/libdevice/libdevice.10.bc", + *(artifact_root / name for name in ("nvcc.py", "nvcc_common.py", "nvcc_warm.py")), + ] + metadata = { + "version": 1, + "release": release, + "nvcc_version": version, + "smoke_architectures": smoke, + "cuda_packages": cuda_packages, + "headers_fingerprint": (artifact_root / "fingerprint").read_text().strip(), + "files": {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in files}, + } + fingerprint = hashlib.sha256(json.dumps(metadata, sort_keys=True).encode()).hexdigest() + (artifact_root / "nvcc-toolchain.json").write_text( + json.dumps({**metadata, "fingerprint": fingerprint}, indent=2) + "\n" + ) + (artifact_root / "nvcc-fingerprint").write_text(fingerprint + "\n") + print(f"Installed {release}; toolchain fingerprint {fingerprint}", flush=True) diff --git a/src/runners/pch/nvcc.py b/src/runners/pch/nvcc.py new file mode 100644 index 00000000..1f11a234 --- /dev/null +++ b/src/runners/pch/nvcc.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Read-only, opt-in NVCC PCH wrapper for compatible load_inline CUDA sources.""" + +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +import nvcc_common as common + +USED = re.compile(r'using precompiled header file "([^"]+)"') +CREATED = re.compile(r'creating precompiled header file "([^"]+)"') +REJECTION = re.compile( + r"invalid (?:PCH|precompiled header)|cannot open precompiled header|" + r"precompiled header[^\n]*(?:invalid|incomplete|not generated|have changed)|" + r"memory usage conflict with precompiled header|" + r"command line options do not match those used when precompiled header|" + r"initial sequence of preprocessing directives is not compatible|" + r"could not publish precompiled header|" + r"source preceding the #pragma hdrstop does not match|" + r"incompatible precompiled header \(PCH\)", + re.IGNORECASE, +) + + +def event(status, started, code, **details): + record = { + "status": status, + "seconds": time.perf_counter() - started, + "returncode": code, + **details, + } + print("[kernelbot-nvcc-pch] " + json.dumps(record, sort_keys=True), file=sys.stderr) + if destination := os.environ.get("KERNELBOT_NVCC_PCH_EVENTS"): + try: + with open(destination, "a") as output: + output.write(json.dumps(record, sort_keys=True) + "\n") + except OSError: + pass # Observability must not break a submission compilation. + + +def run_compiler(args, capture=False): + return subprocess.run([common.COMPILER, *args], capture_output=capture, text=capture) + + +def passthrough(args, status, reason, started, **details): + result = run_compiler(args) + if "-c" in args: + event(status, started, result.returncode, reason=reason, **details) + return result.returncode + + +def consumed(log, entry): + paths = USED.findall(log) + frontends = {Path(path).name.split("_", 1)[0] for path in paths} + valid = ( + frontends == {"cicc", "cudafe"} + and not CREATED.search(log) + and all(Path(path).is_relative_to(entry / "pch") for path in paths) + ) + return valid, sorted(frontends) + + +def main(args): + started = time.perf_counter() + if ( + os.environ.get("KERNELBOT_NVCC_PCH") != "1" + or os.environ.get("KERNELBOT_NVCC_PCH_DISABLE") == "1" + ): + return passthrough(args, "disabled", "disabled", started) + flags = common.compile_flags(args) + if flags is None: + return passthrough(args, "bypass", "unsupported_invocation_or_headers", started) + details = {} + try: + fingerprint = common.generation() + key = common.cache_key(flags, fingerprint) + details = {"generation": fingerprint, "key": key} + entry = common.cache_root() / fingerprint / key + manifest = common.read_manifest(entry, flags, fingerprint) + except (OSError, ValueError, KeyError, TypeError): + return passthrough(args, "miss", "profile_unavailable", started, **details) + details["profile"] = manifest["profile"] + cached_args = [ + *args, + "-include", + str(entry / "gpumode.cuh"), + "--pch", + "--pch-dir", + str(entry / "pch"), + "--pch-verbose", + ] + tick = time.perf_counter() + result = run_compiler(cached_args, capture=True) + cached_seconds = time.perf_counter() - tick + log = result.stdout + result.stderr + hit, frontends = consumed(log, entry) + if result.returncode and (CREATED.search(log) or REJECTION.search(log)): + # Cache-specific failure only. Genuine user compiler errors remain errors. + # Original argv removes both the forwarding header and every PCH option. + print("[kernelbot-nvcc-pch] rejected profile; compiling normally", file=sys.stderr) + return passthrough( + args, + "fallback", + "cache_rejected", + started, + cached_attempt_seconds=cached_seconds, + **details, + ) + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + status = "error" if result.returncode else ("hit" if hit else "fallback") + event( + status, + started, + result.returncode, + frontends=frontends, + reason="consumed" if hit else "not_consumed", + **details, + ) + return result.returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/runners/pch/nvcc_common.py b/src/runners/pch/nvcc_common.py new file mode 100644 index 00000000..3948e66f --- /dev/null +++ b/src/runners/pch/nvcc_common.py @@ -0,0 +1,190 @@ +"""Shared, stdlib-only identity rules for trusted NVCC header profiles.""" + +import hashlib +import json +import os +import re +from pathlib import Path + +VERSION = 1 +COMPILER = "/usr/local/cuda/bin/nvcc" +FINGERPRINT = Path("/opt/kernelbot-pch/nvcc-fingerprint") +# Match PyTorch 2.12 load_inline's CUDA prefix exactly. In particular, ATen is +# included transitively by torch/types.h rather than inserted before it here. +HEADERS = ("torch/types.h", "cuda.h", "cuda_runtime.h") +HEADER = "#pragma once\n" + "".join(f"#include <{h}>\n" for h in HEADERS) +HEADER += "#pragma hdrstop\n" +HEADER_SHA256 = hashlib.sha256(HEADER.encode()).hexdigest() +ARCHES = ("75", "80", "89", "90a", "100") +# Ordered tuples observed in the 2026-11-13 submission snapshot. Preserve every +# option and its position; the warmer captures PyTorch's remaining flags. +FLAG_PROFILES = { + "default": [], + "O3": ["-O3"], + "O3-fast-math": ["-O3", "--use_fast_math"], +} +PROFILES = { + f"sm{arch}-{name}": (arch, flags) for arch in ARCHES for name, flags in FLAG_PROFILES.items() +} +EXPLICIT_PROFILES = { + "sm100a-O3-fast-math-arch": ("100a", ["-O3", "--use_fast_math", "-arch=sm_100a"]), + "sm100-O3-arch-fast-math": ("100", ["-O3", "-arch=sm_100", "--use_fast_math"]), +} +# Historical C++17 tuples cannot precompile the current Torch headers. +PROFILES.update(EXPLICIT_PROFILES) +DEFAULT_PROFILES = tuple(PROFILES) +ENVIRONMENT_KEYS = ( + "CPATH", + "CPLUS_INCLUDE_PATH", + "C_INCLUDE_PATH", + "GCC_EXEC_PREFIX", + "COMPILER_PATH", + "NVCC_CCBIN", + "NVVMIR_LIBRARY_DIR", + "PATH", + "LD_LIBRARY_PATH", + "LIBRARY_PATH", + "CUDA_HOME", + "CUDA_PATH", + "CUDA_ROOT", +) +HEX = re.compile(r"[0-9a-f]{64}\Z") +PCH_NAME = re.compile(r"(?:cicc|cudafe)_[0-9a-f_]+\.pch\Z") +PREFIX = re.compile( + r"\A[ \t\r\n]*" + + "".join(r"#[ \t]*include[ \t]*<" + re.escape(h) + r">[ \t]*\r?\n[ \t\r\n]*" for h in HEADERS) +) + + +def cache_root(): + return Path(os.environ.get("KERNELBOT_NVCC_PCH_ROOT", "/kernelbot-nvcc-pch")) + + +def generation(): + value = FINGERPRINT.read_text().strip() + if not HEX.fullmatch(value): + raise ValueError("Invalid NVCC toolchain fingerprint") + return value + + +def environment(): + return {key: os.environ.get(key) for key in ENVIRONMENT_KEYS} + + +def compile_flags(args): + """Return only compatible single CUDA compile flags; never mutate the command.""" + if args.count("-c") != 1 or args.count("-o") != 1: + return None + if any(os.environ.get(k) for k in ("NVCC_PREPEND_FLAGS", "NVCC_APPEND_FLAGS")): + return None + if any( + a.startswith( + ( + "@", + "-ccbin", + "--compiler-bindir", + "--pch", + "--use-pch", + "--create-pch", + "-include", + "--pre-include", + ) + ) + for a in args + ): + return None + try: + source = Path(args[args.index("-c") + 1]) + if source.suffix != ".cu": + return None + # PyTorch inserts this prefix before user code. Do not move user macros or + # introduce Torch/CUTLASS headers into deliberately minimal CUDA sources. + with source.open() as stream: + if not PREFIX.match(stream.read(4096)): + return None + except (IndexError, OSError, UnicodeError): + return None + return strip_outputs(args) + + +def strip_outputs(args, drop_module=True): + flags = [] + paired = {"-c", "-o", "-MF", "-MT", "-MQ", "--dependency-output"} + standalone = {"-MD", "-MMD", "--generate-dependencies-with-compile"} + index = 0 + while index < len(args): + arg = args[index] + if arg in paired: + if index + 1 >= len(args): + return None + index += 2 + continue + if drop_module and re.fullmatch(r"-DTORCH_EXTENSION_NAME=[A-Za-z_]\w*", arg): + # Keep this macro in the real invocation. NVIDIA validates whether + # it was used in a PCH; the curated forwarding header does not use it. + index += 1 + continue + if arg not in standalone and not arg.startswith("--dependency-output="): + flags.append(arg) + index += 1 + return flags + + +def identity(flags, fingerprint): + return { + "version": VERSION, + "generation": fingerprint, + "header_sha256": HEADER_SHA256, + "flags": flags, + "environment": environment(), + } + + +def cache_key(flags, fingerprint): + value = json.dumps(identity(flags, fingerprint), sort_keys=True).encode() + return hashlib.sha256(value).hexdigest() + + +def read_manifest(entry, flags, fingerprint): + """Validate ready metadata and cheap file stats, without reading multi-GB blobs.""" + manifest = json.loads((entry / "manifest.json").read_text()) + return validate_manifest(manifest, entry, flags, fingerprint) + + +def validate_manifest(manifest, entry, flags, fingerprint): + if not isinstance(manifest, dict): + raise ValueError("Invalid PCH manifest") + expected = identity(flags, fingerprint) + if any(manifest.get(k) != value for k, value in expected.items()): + raise ValueError("PCH profile identity changed") + if manifest.get("key") != entry.name or entry.name != cache_key(flags, fingerprint): + raise ValueError("PCH profile key changed") + if manifest.get("profile") not in PROFILES: + raise ValueError("Unrecognized trusted PCH profile") + if (entry / "gpumode.cuh").read_text() != HEADER: + raise ValueError("PCH forwarding header changed") + validate_files(entry, manifest.get("files", [])) + return manifest + + +def validate_files(entry, files): + if len(files) < 2: + raise ValueError("PCH profile must contain both frontends") + frontends, paths = set(), set() + for item in files: + path = Path(item["path"]) + if path in paths or path.parent != Path("pch") or not PCH_NAME.fullmatch(path.name): + raise ValueError("Invalid PCH manifest path") + paths.add(path) + target = entry / path + if not target.resolve().is_relative_to(entry.resolve()): + raise ValueError("PCH file escapes its profile") + size = item["bytes"] + if ( + type(size) is not int or size <= 0 + or not target.is_file() or target.stat().st_size != size + ): + raise ValueError("PCH file is missing or truncated") + frontends.add(path.name.split("_", 1)[0]) + if frontends != {"cicc", "cudafe"}: + raise ValueError("PCH profile is missing a frontend") diff --git a/src/runners/pch/nvcc_warm.py b/src/runners/pch/nvcc_warm.py new file mode 100644 index 00000000..5c3de7ab --- /dev/null +++ b/src/runners/pch/nvcc_warm.py @@ -0,0 +1,137 @@ +"""Trusted CPU-only NVCC PCH warmer; accepts curated profile names, never user flags.""" + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import nvcc_common as common + +CAPTURE = """#!/usr/bin/env python3 +import json, os, subprocess, sys +from pathlib import Path +compiler = "/usr/local/cuda/bin/nvcc" if sys.argv[0].endswith("nvcc") else "/usr/bin/g++" +if "-c" in sys.argv: + if compiler.endswith("nvcc"): + with open(os.environ["KERNELBOT_CAPTURE_NVCC"], "w") as output: + json.dump(sys.argv[1:], output) + sys.exit(1) + Path(sys.argv[sys.argv.index("-o") + 1]).touch() + sys.exit(0) +sys.exit(subprocess.call([compiler, *sys.argv[1:]])) +""" +BUILD = """ +import json, os +from torch.utils.cpp_extension import load_inline +load_inline(name="kernelbot_nvcc_warmer", cpp_sources="int warm() { return 1; }", + cuda_sources="__global__ void warm_cuda() {}", functions=["warm"], + extra_cuda_cflags=json.loads(os.environ["KERNELBOT_CAPTURE_FLAGS"]), + build_directory=os.environ["KERNELBOT_CAPTURE_BUILD"], verbose=True) +""" + + +def torch_arch(arch): + match = re.fullmatch(r"([1-9][0-9]*)([0-9])(a?)", arch) + if match is None: + raise ValueError("Invalid curated CUDA architecture") + major, minor, suffix = match.groups() + return f"{major}.{minor}{suffix}" + + +def capture(profile, work): + arch, extra = common.PROFILES[profile] + for kind in ("cxx", "nvcc"): + path = work / f"capture-{kind}" + path.write_text(CAPTURE) + path.chmod(0o755) + capture_path = work / "args.json" + environment = os.environ | { + "CXX": str(work / "capture-cxx"), + "PYTORCH_NVCC": str(work / "capture-nvcc"), + "TORCH_CUDA_ARCH_LIST": torch_arch(arch), + "MAX_JOBS": "2", + "KERNELBOT_CAPTURE_NVCC": str(capture_path), + "KERNELBOT_CAPTURE_FLAGS": json.dumps(extra), + "KERNELBOT_CAPTURE_BUILD": str(work), + } + result = subprocess.run( + ["python3", "-c", BUILD], env=environment, capture_output=True, text=True, timeout=180 + ) + if not capture_path.exists(): + raise RuntimeError("Could not capture PyTorch NVCC flags: " + result.stdout + result.stderr) + args = json.loads(capture_path.read_text()) + flags = common.compile_flags(args) + if flags is None: + raise RuntimeError("Trusted warm invocation did not pass runtime eligibility rules") + return flags, common.strip_outputs(args, drop_module=False) + + +def warm_profile(profile, fingerprint): + with tempfile.TemporaryDirectory(prefix="kernelbot-nvcc-warm-") as directory: + work = Path(directory) + flags, warm_flags = capture(profile, work) + key = common.cache_key(flags, fingerprint) + entry = common.cache_root() / fingerprint / key + if (entry / "manifest.json").exists(): + return common.read_manifest(entry, flags, fingerprint) + # Only incomplete, unpublished entries may be repaired by the single writer. + entry.mkdir(parents=True, exist_ok=True) + shutil.rmtree(entry / "pch", ignore_errors=True) + (entry / "pch").mkdir() + (entry / "gpumode.cuh").write_text(common.HEADER) + command = [ + common.COMPILER, + *warm_flags, + "--pch", + "--pch-verbose", + "--pch-dir", + str(entry / "pch"), + "-x", + "cu", + "-c", + str(entry / "gpumode.cuh"), + "-o", + str(work / "header.o"), + ] + print(f"Warming {profile}: {key}", flush=True) + started = time.perf_counter() + result = subprocess.run(command, capture_output=True, text=True, timeout=600) + if result.returncode: + raise RuntimeError(result.stdout + result.stderr) + files = [ + {"path": str(path.relative_to(entry)), "bytes": path.stat().st_size} + for path in sorted((entry / "pch").glob("*.pch")) + ] + manifest = common.identity(flags, fingerprint) | { + "key": key, + "profile": profile, + "files": files, + "creation_seconds": time.perf_counter() - started, + } + common.validate_manifest(manifest, entry, flags, fingerprint) + temporary = entry / "manifest.json.tmp" + temporary.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") + # Validate the exact ready metadata before making it visible to workers. + # The parent Modal warmer commits the completed volume snapshot afterward. + temporary.replace(entry / "manifest.json") + print(json.dumps(manifest), flush=True) + return manifest + + +def warm(profiles=()): + selected = list(profiles) or list(common.DEFAULT_PROFILES) + if not set(selected) <= common.PROFILES.keys(): + raise ValueError(f"Choose fixed profiles from {sorted(common.PROFILES)}") + if any(os.environ.get(k) for k in ("NVCC_PREPEND_FLAGS", "NVCC_APPEND_FLAGS")): + raise ValueError("Trusted warmup does not accept NVCC environment flags") + fingerprint = common.generation() + return [warm_profile(profile, fingerprint) for profile in selected] + + +if __name__ == "__main__": + warm(sys.argv[1:]) diff --git a/tests/test_inline_artifacts.py b/tests/test_inline_artifacts.py index 869c1640..99d590f3 100644 --- a/tests/test_inline_artifacts.py +++ b/tests/test_inline_artifacts.py @@ -30,6 +30,9 @@ def original(name, cpp_sources, cuda_sources=None, extra_cuda_cflags=None, extra monkeypatch.setenv("KERNELBOT_INLINE_WORKDIR", str(tmp_path)) monkeypatch.setenv("TORCH_CUDA_ARCH_LIST", "7.5") monkeypatch.setenv("KERNELBOT_INLINE_MODE", "capture") + fingerprint = tmp_path / "nvcc-fingerprint" + fingerprint.write_text("a" * 64 + "\n") + monkeypatch.setattr("libkernelbot.inline_artifacts.NVCC_FINGERPRINT", fingerprint) with patch("libkernelbot.inline_artifacts._dependencies", return_value={}): install() extension.load_inline("example", "source", cuda_sources="cuda") @@ -53,7 +56,18 @@ def test_loads_saved_module_once_without_compiling(extension, tmp_path): assert extension.builds.call_count == 1 -@pytest.mark.parametrize("change", ["source", "flags", "arch", "manifest", "binary", "linker"]) +@pytest.mark.parametrize( + "change", + [ + "source", + "flags", + "arch", + "manifest", + "binary", + "linker", + "nvcc_toolchain", + ], +) def test_incompatible_artifacts_fall_back_before_loading(extension, tmp_path, monkeypatch, change): arguments = {"name": "example", "cpp_sources": "source", "cuda_sources": "cuda"} manifest_path = next((tmp_path / "artifacts").glob("*/manifest.json")) @@ -69,11 +83,41 @@ def test_incompatible_artifacts_fall_back_before_loading(extension, tmp_path, mo manifest_path.write_text(json.dumps(manifest)) elif change == "binary": manifest_path.with_name("extension.so").write_bytes(b"corrupted") - else: + elif change == "linker": arguments["extra_ldflags"] = ["-lcustom"] + elif change == "nvcc_toolchain": + (tmp_path / "nvcc-fingerprint").write_text("b" * 64 + "\n") with patch("importlib.util.spec_from_file_location") as load: extension.load_inline(**arguments) load.assert_not_called() assert extension.builds.call_count == 2 event = json.loads((tmp_path / "artifacts/events.jsonl").read_text()) assert event["mode"] == "fallback" and event["reason"] + + +@pytest.mark.parametrize( + "name,value", + [ + ("PYTORCH_NVCC", "/different/compiler-wrapper"), + ("KERNELBOT_NVCC_PCH", "1"), + ("KERNELBOT_NVCC_PCH_DISABLE", "1"), + ("KERNELBOT_NVCC_PCH_ROOT", "/different/cache"), + ("CUDA_PATH", "/different/cuda"), + ("CUDA_ROOT", "/different/cuda"), + ("GCC_EXEC_PREFIX", "/different/gcc"), + ("COMPILER_PATH", "/different/compiler"), + ("PATH", "/different/bin"), + ("NVCC_CCBIN", "/different/cxx"), + ("NVVMIR_LIBRARY_DIR", "/different/nvvm"), + ], +) +def test_nvcc_environment_change_rebuilds_before_loading( + extension, tmp_path, monkeypatch, name, value +): + monkeypatch.setenv(name, value) + with patch("importlib.util.spec_from_file_location") as load: + extension.load_inline("example", "source", cuda_sources="cuda") + load.assert_not_called() + assert extension.builds.call_count == 2 + event = json.loads((tmp_path / "artifacts/events.jsonl").read_text()) + assert event["mode"] == "fallback" diff --git a/tests/test_nvcc_pch.py b/tests/test_nvcc_pch.py new file mode 100644 index 00000000..c668e1fb --- /dev/null +++ b/tests/test_nvcc_pch.py @@ -0,0 +1,194 @@ +"""NVCC eligibility, immutable cache consumption, and fallback without CUDA.""" + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def load(name, filename): + spec = importlib.util.spec_from_file_location( + name, Path(__file__).parents[1] / "src/runners/pch" / filename + ) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +common = load("nvcc_common", "nvcc_common.py") +wrapper = load("kernelbot_nvcc_wrapper", "nvcc.py") +warmer = load("kernelbot_nvcc_warmer", "nvcc_warm.py") +GENERATION = "a" * 64 + + +@pytest.fixture +def cache(tmp_path, monkeypatch): + for key in ("NVCC_PREPEND_FLAGS", "NVCC_APPEND_FLAGS", "KERNELBOT_NVCC_PCH_DISABLE"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("KERNELBOT_NVCC_PCH", "1") + source = tmp_path / "first.cu" + source.write_text("#include \n#include \n#include \n") + args = ["-MD", "-MF", "first.d", "-DTORCH_EXTENSION_NAME=first", "-isystem", "/torch", + "-gencode=arch=compute_75,code=sm_75", "-std=c++20", "-c", str(source), "-o", "first.o"] + flags = common.compile_flags(args) + assert flags is not None # Actual Torch 2.12 prefix, independent of implementation constants. + fingerprint = tmp_path / "fingerprint" + fingerprint.write_text(GENERATION) + monkeypatch.setattr(common, "FINGERPRINT", fingerprint) + monkeypatch.setenv("KERNELBOT_NVCC_PCH_ROOT", str(tmp_path / "volume")) + entry = common.cache_root() / GENERATION / common.cache_key(flags, GENERATION) + (entry / "pch").mkdir(parents=True) + (entry / "gpumode.cuh").write_text(common.HEADER) + files = [{"path": f"pch/{name}_ab.pch", "bytes": 4} for name in ("cicc", "cudafe")] + for item in files: + (entry / item["path"]).write_bytes(b"stub") + manifest = common.identity(flags, GENERATION) | { + "key": entry.name, "profile": "sm75-default", "files": files} + (entry / "manifest.json").write_text(json.dumps(manifest)) + events = tmp_path / "events.jsonl" + monkeypatch.setenv("KERNELBOT_NVCC_PCH_EVENTS", str(events)) + log = "\n".join(f'using precompiled header file "{entry / item["path"]}"' for item in files) + calls, responses = [], [] + + def run(args, capture=False): + calls.append((args, capture)) + code, stderr = responses.pop(0) if responses else (0, log) + return subprocess.CompletedProcess(args, code, "", stderr) + + monkeypatch.setattr(wrapper, "run_compiler", run) + return SimpleNamespace(**locals()) + + +def test_identity_reuses_modules_but_invalidates_compiler_inputs(cache, monkeypatch): + c = cache + other = c.args.copy() + other[other.index("-DTORCH_EXTENSION_NAME=first")] = "-DTORCH_EXTENSION_NAME=second" + c.source.write_text(c.source.read_text() + "__global__ void changed_kernel() {}\n") + assert common.compile_flags(other) == c.flags + key = common.cache_key(c.flags, GENERATION) + for flag in ("-O3", "-DUSER=1", "-I/custom", "--use_fast_math", "-std=c++17", "-arch=sm_100"): + assert common.cache_key(common.compile_flags([*other, flag]), GENERATION) != key + assert common.cache_key(c.flags, "b" * 64) != key + monkeypatch.setenv("CPATH", "/different") + assert common.cache_key(c.flags, GENERATION) != key + + +@pytest.mark.parametrize("extra", [["@flags.rsp"], ["-include", "custom.h"], + ["--compiler-bindir=/custom"], ["--pch"]]) +def test_hidden_options_bypass(cache, extra): + assert common.compile_flags([*cache.args, *extra]) is None + + +@pytest.mark.parametrize("prefix", ["#include \n", "#define USER_SETTING 1\n"]) +def test_header_order_is_preserved(cache, prefix): + cache.source.write_text(prefix + cache.source.read_text()) + assert common.compile_flags(cache.args) is None + + +@pytest.mark.parametrize("mode", ["disabled", "missing", "environment"]) +def test_passthrough_never_writes_cache(cache, monkeypatch, mode): + monkeypatch.setenv("KERNELBOT_NVCC_PCH_ROOT", str(cache.tmp_path / "absent")) + if mode == "disabled": + monkeypatch.setenv("KERNELBOT_NVCC_PCH_DISABLE", "1") + monkeypatch.setattr(common, "generation", lambda: pytest.fail("Disabled cache accessed")) + elif mode == "environment": + monkeypatch.setenv("NVCC_APPEND_FLAGS", "-include custom.h") + assert wrapper.main(cache.args) == 0 + assert cache.calls == [(cache.args, False)] + assert not common.cache_root().exists() + + +@pytest.mark.parametrize("kind,code,status", [ + ("hit", 0, "hit"), ("partial", 0, "fallback"), + ("rejected", 1, "fallback"), ("user-error", 2, "error")]) +def test_consumption_evidence_and_cache_only_retry(cache, kind, code, status): + log = {"partial": cache.log.splitlines()[0], "rejected": "invalid precompiled header file"} + log["user-error"] = cache.log + "\nerror: identifier typo undefined" + cache.responses.append((code, log.get(kind, cache.log))) + assert wrapper.main(cache.args) == (0 if kind == "rejected" else code) + assert cache.calls[0][0][:len(cache.args)] == cache.args + assert len(cache.calls) == (2 if kind == "rejected" else 1) + if kind == "rejected": + assert cache.calls[1] == (cache.args, False) + event = json.loads(cache.events.read_text()) + assert event["status"] == status + if status == "hit": + assert event["frontends"] == ["cicc", "cudafe"] + + +@pytest.mark.parametrize("change", [ + "truncated", "header", "generation", "escape", "partial", "duplicate", "duplicate-alias"]) +def test_bad_ready_metadata_falls_back_before_compiler(cache, change): + c = cache + if change == "truncated": + (c.entry / c.files[0]["path"]).write_bytes(b"bad") + elif change == "header": + (c.entry / "gpumode.cuh").write_text("#include \n") + elif change == "generation": + c.manifest["generation"] = "b" * 64 + elif change == "escape": + c.files[0]["path"] = "../foreign.pch" + elif change.startswith("duplicate"): + path = c.files[0]["path"] + path = path.replace("pch/", "pch/./") if change.endswith("alias") else path + c.files.append({"path": path, "bytes": 4}) + else: + c.files.pop() + (c.entry / "manifest.json").write_text(json.dumps(c.manifest)) + assert wrapper.main(c.args) == 0 and c.calls == [(c.args, False)] + + +@pytest.mark.parametrize("count", [2, 3]) +def test_validation_does_not_read_gigabyte_payloads(cache, monkeypatch, count): + if count == 3: + (cache.entry / "pch/cicc_cd.pch").write_bytes(b"stub") + cache.files.append({"path": "pch/cicc_cd.pch", "bytes": 4}) + (cache.entry / "manifest.json").write_text(json.dumps(cache.manifest)) + original = Path.open + + def guarded(path, *args, **kwargs): + assert path.suffix != ".pch" + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", guarded) + assert common.read_manifest(cache.entry, cache.flags, GENERATION)["files"] + + +@pytest.mark.parametrize("complete", [True, False]) +def test_warmer_retains_module_macro_and_publishes_atomically(cache, monkeypatch, complete): + c = cache + monkeypatch.setenv("KERNELBOT_NVCC_PCH_ROOT", str(c.tmp_path / "warm")) + captured = c.flags, common.strip_outputs(c.args, False) + monkeypatch.setattr(warmer, "capture", lambda *a: captured) + + def compile_header(argv, **kwargs): + assert "-DTORCH_EXTENSION_NAME=first" in argv + entry = Path(argv[argv.index("--pch-dir") + 1]).parent + assert not (entry / "manifest.json").exists() + for item in c.files[:2 if complete else 1]: + (entry / item["path"]).write_bytes(b"stub") + return subprocess.CompletedProcess(argv, 0, "", "") + + monkeypatch.setattr(warmer.subprocess, "run", compile_header) + entry = common.cache_root() / GENERATION / c.entry.name + if not complete: + with pytest.raises(ValueError): + warmer.warm_profile("sm75-default", GENERATION) + assert not (entry / "manifest.json").exists() + else: + result = warmer.warm_profile("sm75-default", GENERATION) + assert common.read_manifest(entry, c.flags, GENERATION) == result + + +def test_warmer_accepts_only_curated_profiles(monkeypatch): + monkeypatch.setattr(warmer, "warm_profile", lambda *a: pytest.fail("Unexpected compilation")) + with pytest.raises(ValueError, match="fixed profiles"): + warmer.warm(["-include /user/header.h"]) + assert common.HEADERS == ("torch/types.h", "cuda.h", "cuda_runtime.h") + for arch, expected in (("75", "7.5"), ("90a", "9.0a"), ("100", "10.0"), ("100a", "10.0a")): + assert warmer.torch_arch(arch) == expected diff --git a/tests/test_nvcc_toolchain.py b/tests/test_nvcc_toolchain.py new file mode 100644 index 00000000..7f4aefc8 --- /dev/null +++ b/tests/test_nvcc_toolchain.py @@ -0,0 +1,123 @@ +"""Compiler installation must fail closed and fingerprint all supporting inputs.""" + +import builtins +import hashlib +import importlib.util +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "kernelbot_nvcc_toolchain", Path(__file__).parents[1] / "src/runners/nvcc_toolchain.py") +toolchain = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(toolchain) + + +@pytest.fixture +def installation(tmp_path, monkeypatch): # noqa: C901 + roots = {name: tmp_path / str(i) for i, name in enumerate( + ("/nvcc-toolchain-input", "/usr/local/cuda", "/opt/kernelbot-pch"))} + source = roots["/nvcc-toolchain-input"] / toolchain.RELEASE + cuda, artifacts = roots["/usr/local/cuda"], roots["/opt/kernelbot-pch"] + targets = {name: name if "/" in name else "bin/" + name for name in toolchain.BINARY_SHA256} + hashes = {} + files = {cuda / "bin/nvcc.profile": "CICC_PATH = $(TOP)/nvvm/bin\n"} + for name, target in targets.items(): + files[source / name], files[cuda / target] = name, "original" + hashes[name] = hashlib.sha256(name.encode()).hexdigest() + for name in ("bin/ptxas", "bin/nvlink", "bin/fatbinary", "nvvm/libdevice/libdevice.10.bc"): + files[cuda / name] = name + for name in ("fingerprint", "nvcc.py", "nvcc_common.py", "nvcc_warm.py"): + files[artifacts / name] = name + for path, text in files.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + original_import = builtins.__import__ + + def sandbox_import(name, globals=None, locals=None, fromlist=(), level=0): + if name == "pathlib" and globals and globals.get("__name__") == toolchain.__name__: + return SimpleNamespace(Path=lambda path: roots.get(str(path), Path(path))) + return original_import(name, globals, locals, fromlist, level) + + state = SimpleNamespace(source=source, cuda=cuda, artifacts=artifacts, targets=targets, + hashes=hashes, calls=[], version="V13.4.99", + backend="nvvm-next/bin/cicc", + packages="cuda-toolkit-13-4\t13.4.3-1\tinstall ok installed\n", + fail_arch=None) + + def output(command, **kwargs): + state.calls.append(command) + if command[0] == "dpkg-query": + return state.packages + return state.version if "--version" in command else state.backend + + def run(command, **kwargs): + state.calls.append(command) + if state.fail_arch and f"-arch=sm_{state.fail_arch}" in command: + raise subprocess.CalledProcessError(1, command) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(builtins, "__import__", sandbox_import) + monkeypatch.setattr(subprocess, "check_output", output) + monkeypatch.setattr(subprocess, "run", run) + state.install = lambda: toolchain.install_nvcc_toolchain(toolchain.RELEASE, hashes) + return state + + +def test_install_checks_every_architecture_and_is_reproducible(installation): + s = installation + s.install() + for name, target in s.targets.items(): + assert (s.cuda / target).read_bytes() == (s.source / name).read_bytes() + assert (s.cuda / target).stat().st_mode & 0o777 == 0o755 + compiled = [c[1] for c in s.calls if "-c" in c and "--dryrun" not in c] + assert compiled == [f"-arch=sm_{a}" for a in ("75", "80", "89", "90a", "100", "100a")] + before = (s.artifacts / "nvcc-fingerprint").read_text() + manifest = json.loads((s.artifacts / "nvcc-toolchain.json").read_text()) + assert manifest["fingerprint"] == before.strip() + s.install() + assert (s.artifacts / "nvcc-fingerprint").read_text() == before + + +@pytest.mark.parametrize("failure", [*toolchain.BINARY_SHA256, "missing-hash", "extra-hash", + "profile", "version", "backend", "smoke", "packages"]) +def test_bad_installation_never_publishes_ready_fingerprint(installation, failure): + s = installation + if failure in s.hashes: + (s.source / failure).write_bytes(b"corrupt") + elif failure == "missing-hash": + del s.hashes["nvcc"] + elif failure == "extra-hash": + s.hashes["unexpected"] = "0" * 64 + elif failure == "profile": + (s.cuda / "bin/nvcc.profile").write_text("CICC_PATH=one\nCICC_PATH=two\n") + else: + field, value = {"version": ("version", "V13.3"), "backend": ("backend", "nvvm/bin/cicc"), + "smoke": ("fail_arch", "100a"), "packages": ("packages", "")}[failure] + setattr(s, field, value) + with pytest.raises((ValueError, subprocess.CalledProcessError)): + s.install() + assert not (s.artifacts / "nvcc-fingerprint").exists() + if failure in toolchain.BINARY_SHA256 or failure.endswith("hash"): + assert not s.calls + assert all((s.cuda / target).read_text() == "original" for target in s.targets.values()) + + +@pytest.mark.parametrize("root,name", [ + ("artifacts", name) for name in ("fingerprint", "nvcc.py", "nvcc_common.py", "nvcc_warm.py") +] + [("cuda", name) for name in ("bin/ptxas", "bin/nvlink", "bin/fatbinary", + "nvvm/libdevice/libdevice.10.bc")] + [(None, "packages")]) +def test_changed_supporting_input_invalidates_generation(installation, root, name): + s = installation + s.install() + before = (s.artifacts / "nvcc-fingerprint").read_text() + if root: + path = getattr(s, root) / name + path.write_bytes(path.read_bytes() + b" changed") + else: + s.packages = s.packages.replace("13.4.3-1", "13.4.3-2") + s.install() + assert (s.artifacts / "nvcc-fingerprint").read_text() != before diff --git a/tests/test_python_precompile.py b/tests/test_python_precompile.py index 1908f468..f3884319 100644 --- a/tests/test_python_precompile.py +++ b/tests/test_python_precompile.py @@ -1,4 +1,5 @@ import asyncio +import hashlib import json import os import subprocess @@ -9,9 +10,15 @@ import pytest from libkernelbot.consts import get_gpu_by_name -from libkernelbot.inline_artifacts import _check_dependencies, _dependencies +from libkernelbot.inline_artifacts import ( + _check_dependencies, + _dependencies, + _dependency_digest, + nvcc_fingerprint, +) from libkernelbot.launchers.modal import ModalLauncher from libkernelbot.python_precompile import ( + build_identity, compile_python, cuda_arch, pack_artifacts, @@ -133,7 +140,7 @@ def test_cpu_import_failures_return_fallback(config, monkeypatch, error): assert packet["artifacts"] == b"" -@pytest.mark.parametrize("changed", ["sources", "image", "version"]) +@pytest.mark.parametrize("changed", ["sources", "image", "version", "nvcc_toolchain"]) def test_gpu_rejects_stale_build_without_failing_submission(config, result, changed): packet = { "identity": {"image": "image"}, @@ -226,3 +233,128 @@ def test_missing_dependency_information_cannot_be_reused(tmp_path, monkeypatch): monkeypatch.setenv("KERNELBOT_INLINE_WORKDIR", str(tmp_path)) with pytest.raises(RuntimeError, match="no compiler dependency"): _check_dependencies(None, tmp_path) + + +def test_cpu_packet_identity_tracks_experimental_compiler(config, tmp_path, monkeypatch): + fingerprint = tmp_path / "nvcc-fingerprint" + monkeypatch.setattr("libkernelbot.inline_artifacts.NVCC_FINGERPRINT", fingerprint) + with patch("libkernelbot.python_precompile.Path.read_text", return_value="host headers"): + original = build_identity(config) + assert original["nvcc_toolchain"] is None + fingerprint.write_bytes(b"first compiler and wrapper digest\n") + first = build_identity(config) + fingerprint.write_bytes(b"second compiler and wrapper digest\n") + second = build_identity(config) + assert first["image"] == second["image"] == original["image"] + assert first["sources"] == second["sources"] == original["sources"] + assert len({first["nvcc_toolchain"], second["nvcc_toolchain"], None}) == 3 + + +def test_empty_nvcc_fingerprint_cannot_authorize_reuse(tmp_path, monkeypatch): + fingerprint = tmp_path / "nvcc-fingerprint" + fingerprint.write_bytes(b"\n") + monkeypatch.setattr("libkernelbot.inline_artifacts.NVCC_FINGERPRINT", fingerprint) + with pytest.raises(RuntimeError, match="fingerprint is empty"): + nvcc_fingerprint() + + +@pytest.fixture(params=[2, 3]) +def trusted_pch(tmp_path, monkeypatch, request): + generation, key = "a" * 64, "b" * 64 + root = tmp_path / "trusted-cache" + entry = root / generation / key + (entry / "pch").mkdir(parents=True) + header = entry / "gpumode.cuh" + header.write_text("#include \n#pragma hdrstop\n") + files = [] + for name in ("cicc_123456", "cudafe_123456", "cicc_654321")[:request.param]: + name = f"pch/{name}.pch" + (entry / name).write_bytes(b"compiler cache contents") + files.append({"path": name, "bytes": (entry / name).stat().st_size}) + manifest = { + "version": 1, + "generation": generation, + "key": key, + "profile": "sm75-default", + "flags": ["-std=c++20"], + "environment": {}, + "header_sha256": hashlib.sha256(header.read_bytes()).hexdigest(), + "files": files, + } + (entry / "manifest.json").write_text(json.dumps(manifest)) + fingerprint = tmp_path / "nvcc-fingerprint" + fingerprint.write_text(generation + "\n") + monkeypatch.setattr("libkernelbot.inline_artifacts.NVCC_FINGERPRINT", fingerprint) + monkeypatch.setattr("libkernelbot.inline_artifacts.NVCC_PCH_ROOT", root) + return entry, manifest, fingerprint + + +def test_trusted_pch_dependencies_do_not_read_cache_bodies(trusted_pch, tmp_path, monkeypatch): + entry, _, _ = trusted_pch + original_open = Path.open + + def small_files_only(path, *args, **kwargs): + assert path.suffix != ".pch", "Artifact replay must not read multi-GB PCH bodies" + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", small_files_only) + cpu, gpu = tmp_path / "cpu", tmp_path / "gpu" + build = cpu / "build" + build.mkdir(parents=True) + gpu.mkdir() + for work in (cpu, gpu): + (work / "user.h").write_text("#define VALUE 1\n") + pch = entry / "pch/cicc_123456.pch" + output = f"cuda.o: #deps 2\n {pch}\n {cpu / 'user.h'}\n" + with patch( + "libkernelbot.inline_artifacts.subprocess.run", return_value=SimpleNamespace(stdout=output) + ): + dependencies = _dependencies(build, cpu) + assert dependencies[str(pch)].startswith("nvcc-pch-v1:") + assert len(dependencies) == 2 + _check_dependencies(dependencies, gpu) + (gpu / "user.h").write_text("#define VALUE 2\n") + with pytest.raises(RuntimeError, match="Build dependency changed"): + _check_dependencies(dependencies, gpu) + + +@pytest.mark.parametrize("change", [ + "manifest", "header", "generation", "size", "missing", "duplicate"]) +def test_trusted_pch_dependency_changes_reject_replay(trusted_pch, tmp_path, change): + entry, manifest, fingerprint = trusted_pch + path = entry / "pch/cicc_123456.pch" + dependencies = {str(path): _dependency_digest(path)} + if change == "manifest": + manifest["flags"] = ["-std=c++20", "-O3"] + (entry / "manifest.json").write_text(json.dumps(manifest)) + elif change == "header": + (entry / "gpumode.cuh").write_text("#include \n") + elif change == "generation": + fingerprint.write_text("c" * 64 + "\n") + elif change == "size": + path.write_bytes(b"truncated") + elif change == "duplicate": + manifest["files"].append(manifest["files"][0].copy()) + (entry / "manifest.json").write_text(json.dumps(manifest)) + else: + (entry / "manifest.json").unlink() + message = "Invalid NVCC PCH dependency file metadata" if change == "duplicate" else None + with pytest.raises((RuntimeError, FileNotFoundError), match=message): + _check_dependencies(dependencies, tmp_path) + + +def test_untrusted_pch_is_not_skipped_or_read(tmp_path, monkeypatch): + path = tmp_path / "user.pch" + path.write_bytes(b"not trusted") + monkeypatch.setenv("KERNELBOT_NVCC_PCH_ROOT", str(tmp_path)) + with patch.object(Path, "open", side_effect=AssertionError("Must not read PCH contents")): + with pytest.raises(RuntimeError, match="outside the trusted"): + _dependency_digest(path) + + +def test_pch_file_must_be_named_by_the_complete_ready_manifest(trusted_pch): + entry, _, _ = trusted_pch + path = entry / "pch/unknown.pch" + path.write_bytes(b"unlisted cache") + with pytest.raises(RuntimeError, match="not covered"): + _dependency_digest(path)