diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 91575e67..807bbd18 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -17,7 +17,13 @@ from packaging.markers import Marker from packaging.requirements import InvalidRequirement, Requirement -from packaging.utils import canonicalize_name +from packaging.utils import ( + InvalidSdistFilename, + InvalidWheelFilename, + canonicalize_name, + parse_sdist_filename, + parse_wheel_filename, +) from .errors import AppError from .images import DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS @@ -115,6 +121,84 @@ def collect_requirements(project_root: Path, app) -> List[str]: return unique +# archive extensions a url or path requirement can point at, longest first +_ARCHIVE_SUFFIXES = (".tar.gz", ".tar.bz2", ".tgz", ".whl", ".zip", ".tar") + + +def _named_distribution(value: str) -> str: + """the canonical distribution name of a PEP 508 requirement string. + + handles a bare name, a version pin, extras (``runpod[all]``), and a + direct reference (``runpod @ https://.../sdk.tar.gz``). a value that is + not a valid requirement (a bare archive filename) is named by its file. + """ + try: + return canonicalize_name(Requirement(value).name) + except InvalidRequirement: + return _archive_distribution(posixpath.basename(value)) + + +def _archive_distribution(basename: str) -> str: + """the canonical distribution name a wheel or sdist filename declares. + + wheel and sdist names escape the version differently, so the pep 427 / + pep 625 parsers are used before falling back to the leading name token + for an archive that carries a name but no version (``runpod.tar.gz``). + """ + try: + return parse_wheel_filename(basename)[0] + except InvalidWheelFilename: + pass + try: + return parse_sdist_filename(basename)[0] + except InvalidSdistFilename: + pass + for suffix in _ARCHIVE_SUFFIXES: + if basename.endswith(suffix): + basename = basename[: -len(suffix)] + break + return canonicalize_name(basename.split("-", 1)[0]) + + +def _spec_distribution(spec: str) -> str: + """the canonical distribution name a requirement spec installs. + + a named spec (bare name, version pin, extras, or ``name @ url``) + resolves by its name; a direct url, vcs ref, or path resolves by its + ``#egg=`` fragment or the wheel/sdist filename it points at, so a + pinned runpod build is still recognised as runpod. + """ + spec = spec.strip() + base, _, fragment = spec.partition("#") + for part in fragment.split("&"): + if part.startswith("egg="): + return _named_distribution(part[len("egg=") :]) + + basename = posixpath.basename(base.split("?", 1)[0].rstrip("/")) + if basename.endswith(_ARCHIVE_SUFFIXES) and " @ " not in spec: + return _archive_distribution(basename) + return _named_distribution(spec) + + +def reject_runpod_dependency(requirements: List[str]) -> None: + """fail when an app pins the runpod sdk through its dependencies. + + deploy installs the app's dependencies and then overlays the client's + own runpod tree (``sync_running_package``), so a runpod entry in + ``dependencies`` or requirements.txt produces a worker running a + silent mix of the two builds. RUNPOD_PACKAGE_SPEC is the supported way + to choose the worker's runpod build. + """ + for req in requirements: + if _spec_distribution(req) == "runpod": + raise BuildError( + f"{req!r} pins the runpod sdk through dependencies, which " + f"would overlay the client build and run a mixed install on " + f"the worker. Remove it and set the RUNPOD_PACKAGE_SPEC " + f"environment variable to choose the worker's runpod build." + ) + + def split_exclusions( requirements: List[str], extra_excludes: Optional[List[str]] = None, @@ -635,6 +719,7 @@ def build_environment( env_dir.mkdir(parents=True, exist_ok=True) user_requirements = collect_requirements(project_root, app) + reject_runpod_dependency(user_requirements) # size-prohibitive packages come preinstalled only on the builtin # gpu images, and custom images carry no guarantee. builtin cpu # images ship without them, but a vendored pypi torch cannot serve diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index dde4e5aa..b100387b 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -17,6 +17,7 @@ collect_requirements, installed_closure, prune_excluded, + reject_runpod_dependency, requirement_name, runpod_requirement, split_exclusions, @@ -71,6 +72,51 @@ def q(): assert collect_requirements(tmp_path, app) == [] +class TestRejectRunpodDependency: + @pytest.mark.parametrize( + "spec", + [ + "runpod", + "runpod==1.8.0", + "runpod>=1.0,<2", + "Runpod==1.8.0", + "runpod[all]==1.8.0", + "runpod @ https://example.com/releases/sdk.tar.gz", + "runpod-1.8.0-py3-none-any.whl", + "https://files.pythonhosted.org/runpod-1.8.0-py3-none-any.whl", + "https://example.com/runpod.tar.gz", + "./dist/runpod-1.8.0.tar.gz", + "https://example.com/download?release=1.8.0#egg=runpod", + "https://example.com/x.git#egg=runpod[all]", + ], + ) + def test_runpod_entry_rejected_with_package_spec_hint(self, spec): + with pytest.raises(BuildError, match="RUNPOD_PACKAGE_SPEC") as exc: + reject_runpod_dependency(["numpy==1.26", spec, "requests"]) + assert spec in str(exc.value) + + @pytest.mark.parametrize( + "spec", + [ + "numpy==1.26", + "requests", + "runpod-cleanup==1.0", + "my-runpod-helper", + "run_pod==1.0", + "https://example.com/not-runpod-1.0-py3-none-any.whl", + "https://example.com/runpodctl-1.0.tar.gz", + "https://example.com/runpod-cleanup-1.0.tar.gz", + "runpod-cleanup-1.0.tar.gz", + ], + ) + def test_non_runpod_entries_pass(self, spec): + # must not raise + reject_runpod_dependency([spec]) + + def test_empty_is_noop(self): + reject_runpod_dependency([]) + + class TestSplitExclusions: def test_torch_family_auto_excluded(self): kept, excluded = split_exclusions( @@ -430,6 +476,20 @@ def test_gpu_builtin_image_prunes_transitive_torch(self, tmp_path): assert result.excluded == ["torch"] assert _installed(result.env_dir) == ["peft"] + def test_runpod_in_dependencies_fails_before_vendoring(self, tmp_path): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", dependencies=["runpod==1.8.0"]) + def q(): + pass + + def _no_vendor(*args, **kwargs): + raise AssertionError("vendor must not run when the app is rejected") + + with patch("runpod.apps.build.vendor", side_effect=_no_vendor): + with pytest.raises(BuildError, match="RUNPOD_PACKAGE_SPEC"): + build_environment(app, tmp_path, scratch_dir=tmp_path / "s") + def test_mixed_cpu_gpu_app_prunes_transitive_torch(self, tmp_path): app = self._app(gpu="AMPERE_16")