From 976f889af1a46a56c1dde322030a03db622e6866 Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 12:13:55 -0400 Subject: [PATCH 01/11] feat(settings): add version-specific prebuilt wheel configuration Allow per-version overrides of `pre_built` and `wheel_server_url` within a variant via a new `versions` mapping on `VariantInfo`. Also adds an `is_pre_built` plugin hook for dynamic override support. Closes: #801 Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- .../bootstrap_requirement_resolver.py | 20 +++++- src/fromager/bootstrapper/_bootstrapper.py | 4 +- .../bootstrapper/_process_install_deps.py | 2 +- src/fromager/bootstrapper/_start.py | 2 +- src/fromager/commands/build.py | 7 +- src/fromager/overrides.py | 1 + src/fromager/packagesettings/__init__.py | 2 + src/fromager/packagesettings/_models.py | 53 ++++++++++++++ src/fromager/packagesettings/_pbi.py | 49 ++++++++++++- src/fromager/wheels.py | 20 ++++-- tests/test_bootstrap_requirement_resolver.py | 3 + ...bootstrap_requirement_resolver_multiple.py | 1 + tests/test_bootstrapper_iterative.py | 20 ++++-- tests/test_packagesettings.py | 69 +++++++++++++++++++ .../context/overrides/settings/test_pkg.yaml | 9 +++ 15 files changed, 242 insertions(+), 20 deletions(-) diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index c4943096a..7b9daa18c 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -24,6 +24,17 @@ logger = logging.getLogger(__name__) +def _extract_pinned_version(req: Requirement) -> Version | None: + """Return the version if *req* is pinned to exactly one (``==``). + + Returns ``None`` for range specifiers, extras-only, or empty specifiers. + """ + specs = list(req.specifier) + if len(specs) == 1 and specs[0].operator == "==": + return Version(specs[0].version) + return None + + class BootstrapRequirementResolver: """Resolve package requirements from PyPI or dependency graph during bootstrap. @@ -113,7 +124,8 @@ def resolve( # Determine pre_built if not specified (needed for cache key) if pre_built is None: pbi = self.ctx.package_build_info(req) - pre_built = pbi.pre_built + pinned = _extract_pinned_version(req) + pre_built = pbi.is_pre_built(pinned) rule_key = (str(req), pre_built) @@ -181,8 +193,12 @@ def _resolve_and_extend( results = cached_resolution elif pre_built: # Resolve prebuilt wheel + pinned = _extract_pinned_version(req) wheel_server_urls = wheels.get_wheel_server_urls( - self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL + self.ctx, + req, + cache_wheel_server_url=resolver.PYPI_SERVER_URL, + version=pinned, ) results = wheels.resolve_all_prebuilt_wheels( ctx=self.ctx, diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index 8d33ad899..99bade774 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -181,7 +181,7 @@ def _resolve_and_add_top_level( req=req, req_version=version, download_url=source_url, - pre_built=pbi.pre_built, + pre_built=pbi.is_pre_built(version), constraint=self.ctx.constraints.get_constraint(req.name), ) @@ -600,7 +600,7 @@ def add_to_graph( req=req, req_version=req_version, download_url=download_url, - pre_built=pbi.pre_built, + pre_built=pbi.is_pre_built(req_version), constraint=self.ctx.constraints.get_constraint(req.name), ) self._write_graph_async() diff --git a/src/fromager/bootstrapper/_process_install_deps.py b/src/fromager/bootstrapper/_process_install_deps.py index a6ed4b58d..3ae5dc338 100644 --- a/src/fromager/bootstrapper/_process_install_deps.py +++ b/src/fromager/bootstrapper/_process_install_deps.py @@ -154,7 +154,7 @@ def run(self, bt: Bootstrapper) -> list[Phase]: version=wi.resolved_version, source_url=wi.source_url, source_type=wi.build_result.source_type, - prebuilt=pbi.pre_built, + prebuilt=pbi.is_pre_built(wi.resolved_version), constraint=constraint, ) diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index 2db1eac92..96c7d5518 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -72,6 +72,6 @@ def run(self, bt: Bootstrapper) -> list[Phase]: # Must set pbi_pre_built before constructing PrepareSource so that # PrepareSource.background_work() immediately sees the correct value. pbi = bt.ctx.package_build_info(wi.req) - wi.pbi_pre_built = pbi.pre_built + wi.pbi_pre_built = pbi.is_pre_built(wi.resolved_version) wi.exclusive_build = pbi.exclusive_build return [PrepareSource(wi)] diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 2e33f8f0e..8824d9b76 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -344,10 +344,13 @@ def _build( logger.info("starting processing") pbi = wkctx.package_build_info(req) - prebuilt = pbi.pre_built + prebuilt = pbi.is_pre_built(resolved_version) wheel_server_urls = wheels.get_wheel_server_urls( - wkctx, req, cache_wheel_server_url=cache_wheel_server_url + wkctx, + req, + cache_wheel_server_url=cache_wheel_server_url, + version=resolved_version, ) # See if we can reuse an existing wheel. diff --git a/src/fromager/overrides.py b/src/fromager/overrides.py index 7d4d02a0f..542701c94 100644 --- a/src/fromager/overrides.py +++ b/src/fromager/overrides.py @@ -30,6 +30,7 @@ "get_build_system_dependencies", "get_install_dependencies_of_sdist", "get_resolver_provider", + "is_pre_built", "prepare_source", "update_extra_environ", ) diff --git a/src/fromager/packagesettings/__init__.py b/src/fromager/packagesettings/__init__.py index abca2f21d..fc1a769eb 100644 --- a/src/fromager/packagesettings/__init__.py +++ b/src/fromager/packagesettings/__init__.py @@ -12,6 +12,7 @@ ResolverDist, SbomSettings, VariantInfo, + VersionSpecificSettings, ) from ._pbi import PackageBuildInfo from ._resolver import ( @@ -88,6 +89,7 @@ "Variant", "VariantChangelog", "VariantInfo", + "VersionSpecificSettings", "default_update_extra_environ", "get_extra_environ", "pep440_tag_matcher", diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index e87230613..589126ae8 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -22,6 +22,7 @@ BuildDirectory, EnvVars, Package, + PackageVersion, PurlType, RawAnnotations, Template, @@ -450,6 +451,32 @@ def validate_update_build_requires(cls, v: list[str]) -> list[str]: return v +class VersionSpecificSettings(pydantic.BaseModel): + """Per-version overrides within a variant. + + Allows overriding ``pre_built`` and ``wheel_server_url`` for + specific package versions. When a field is ``None``, the + variant-wide default is used. + + .. versionadded:: 0.90.0 + + :: + + versions: + "2.9.0": + pre_built: true + wheel_server_url: https://gitlab.example.com/simple + """ + + model_config = MODEL_CONFIG + + wheel_server_url: str | None = None + """Alternative package index for this version's pre-built wheel""" + + pre_built: bool | None = None + """Override pre-built flag for this version (None = inherit variant default)""" + + class VariantInfo(pydantic.BaseModel): """Variant information for a package @@ -460,6 +487,10 @@ class VariantInfo(pydantic.BaseModel): VAR2: "2.0 wheel_server_url: https://pypi.org/simple/ pre_built: False + versions: + "2.9.0": + pre_built: true + wheel_server_url: https://gitlab.example.com/simple """ model_config = MODEL_CONFIG @@ -480,10 +511,32 @@ class VariantInfo(pydantic.BaseModel): pre_built: bool = False """Use pre-built wheel from index server?""" + versions: Mapping[PackageVersion, VersionSpecificSettings] = Field( + default_factory=dict + ) + """Per-version overrides for ``pre_built`` and ``wheel_server_url``. + + Version-specific settings take precedence over variant defaults + when present. + + .. versionadded:: 0.90.0 + """ + # TODO # source: SourceResolver | None # """Source resolver and downloader""" + @pydantic.field_validator("versions", mode="before") + @classmethod + def before_none_versions( + cls, + v: dict[str, typing.Any] | None, + info: core_schema.ValidationInfo, + ) -> dict[str, typing.Any]: + if v is None: + return {} + return v + class GitOptions(pydantic.BaseModel): """Git repository cloning options diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 2e2620047..36b06c302 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -168,6 +168,34 @@ def pre_built(self) -> bool: return vi.pre_built return False + def is_pre_built(self, version: Version | None = None) -> bool: + """Version-aware pre-built check. + + Resolution order: + + 1. Plugin hook ``is_pre_built`` (if version given and hook exists) + 2. Version-specific YAML setting + 3. Variant-wide default + + .. versionadded:: 0.90.0 + """ + vi = self._ps.variants.get(self.variant) + if vi is None: + return False + if version is not None: + hook_fn = overrides.find_override_method(self.package, "is_pre_built") + if hook_fn is not None: + result = overrides.invoke( + hook_fn, version=version, variant=self.variant + ) + if result is not None: + return bool(result) + pv = typing.cast(PackageVersion, version) + vs = vi.versions.get(pv) + if vs is not None and vs.pre_built is not None: + return vs.pre_built + return vi.pre_built + @property def wheel_server_url(self) -> str | None: """Alternative package index for pre-build wheel""" @@ -176,6 +204,24 @@ def wheel_server_url(self) -> str | None: return str(vi.wheel_server_url) return None + def get_wheel_server_url(self, version: Version | None = None) -> str | None: + """Version-aware wheel server URL. + + Returns the version-specific URL if defined, otherwise + falls back to the variant-wide default. + + .. versionadded:: 0.90.0 + """ + vi = self._ps.variants.get(self.variant) + if vi is None: + return None + if version is not None: + pv = typing.cast(PackageVersion, version) + vs = vi.versions.get(pv) + if vs is not None and vs.wheel_server_url is not None: + return str(vs.wheel_server_url) + return str(vi.wheel_server_url) if vi.wheel_server_url is not None else None + @property def override_module_name(self) -> str: """Override module name from package name""" @@ -295,8 +341,7 @@ def build_tag(self, version: Version) -> BuildTag: the build tag from changelog, e.g. version `1.0.3+local.suffix` uses `1.0.3`. """ - if self.pre_built: - # pre-built wheels have no built tag + if self.is_pre_built(version): return () pv = typing.cast(PackageVersion, version) release = len(self.get_changelog(pv)) diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index dc9bd5241..24d7a07d0 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -459,13 +459,25 @@ def download_wheel( def get_wheel_server_urls( - ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None + ctx: context.WorkContext, + req: Requirement, + *, + cache_wheel_server_url: str | None, + version: Version | None = None, ) -> list[str]: + """Build ordered list of wheel server URLs for a package. + + When *version* is given, version-specific ``wheel_server_url`` + overrides are checked first. + + .. versionchanged:: 0.90.0 + Added *version* parameter for version-specific URL lookup. + """ pbi = ctx.package_build_info(req) + url = pbi.get_wheel_server_url(version) wheel_server_urls: list[str] = [] - if pbi.wheel_server_url: - # use only the wheel server from settings if it is defined. Do not fallback to other URLs - wheel_server_urls.append(pbi.wheel_server_url) + if url: + wheel_server_urls.append(url) else: if ctx.wheel_server_url: # local wheel server diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index 7222a6fbc..171dd4eb6 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -444,7 +444,9 @@ def test_resolve_auto_routes_to_prebuilt( # Mock package build info to return pre_built=True mock_pbi = MagicMock() mock_pbi.pre_built = True + mock_pbi.is_pre_built.return_value = True mock_pbi.wheel_server_url = None + mock_pbi.get_wheel_server_url.return_value = None mock_pbi.resolver_min_release_age = None with patch.object(tmp_context, "package_build_info", return_value=mock_pbi): @@ -485,6 +487,7 @@ def test_resolve_auto_routes_to_source( # Mock package build info to return pre_built=False mock_pbi = MagicMock() mock_pbi.pre_built = False + mock_pbi.is_pre_built.return_value = False mock_pbi.resolver_include_sdists = True mock_pbi.resolver_include_wheels = True mock_pbi.resolver_ignore_platform = True diff --git a/tests/test_bootstrap_requirement_resolver_multiple.py b/tests/test_bootstrap_requirement_resolver_multiple.py index ce9482def..c172919a2 100644 --- a/tests/test_bootstrap_requirement_resolver_multiple.py +++ b/tests/test_bootstrap_requirement_resolver_multiple.py @@ -25,6 +25,7 @@ def tmp_context(tmp_path: Path) -> WorkContext: ctx.package_build_info = MagicMock() pbi = MagicMock() pbi.pre_built = False + pbi.is_pre_built.return_value = False pbi.resolver_include_sdists = True pbi.resolver_include_wheels = False pbi.resolver_ignore_platform = False diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 79ff439ca..2a8151606 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -646,7 +646,7 @@ def test_sets_pbi_pre_built_before_prepare_source( with patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=True), + return_value=Mock(pre_built=True, is_pre_built=Mock(return_value=True)), ): result = item.run(bt) @@ -1914,7 +1914,9 @@ def test_normal_path_returns_item_and_dep_items( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order") as mock_build_order, @@ -1954,7 +1956,9 @@ def test_hook_error_test_mode_records_and_continues( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order") as mock_build_order, @@ -1999,7 +2003,9 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order") as mock_build_order, @@ -2051,7 +2057,9 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order"), @@ -2080,7 +2088,7 @@ def test_build_order_called_with_correct_args( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=True), + return_value=Mock(pre_built=True, is_pre_built=Mock(return_value=True)), ), patch.object( tmp_context.constraints, diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index a5107ad98..3f033d3c6 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -98,6 +98,16 @@ "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"}, "wheel_server_url": "https://wheel.test/simple", "pre_built": False, + "versions": { + Version("2.9.0"): { + "wheel_server_url": "https://mirror.test/simple", + "pre_built": True, + }, + Version("2.8.0"): { + "wheel_server_url": None, + "pre_built": False, + }, + }, }, "rocm": { "annotations": { @@ -106,12 +116,19 @@ "env": {"SPAM": ""}, "wheel_server_url": None, "pre_built": True, + "versions": { + Version("1.0.0"): { + "wheel_server_url": None, + "pre_built": False, + }, + }, }, "cuda": { "annotations": None, "env": {}, "wheel_server_url": None, "pre_built": False, + "versions": {}, }, }, } @@ -199,6 +216,7 @@ "env": {}, "pre_built": True, "wheel_server_url": None, + "versions": {}, }, }, } @@ -584,6 +602,57 @@ def test_global_changelog(testdata_context: context.WorkContext) -> None: assert pbi.build_tag(Version("1.0.1")) == () +def test_is_pre_built_version_specific( + testdata_context: context.WorkContext, +) -> None: + """Version-specific pre_built overrides variant default.""" + # cpu variant: pre_built=False by default, but 2.9.0 is pre_built=True + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.variant == "cpu" + assert pbi.pre_built is False + assert pbi.is_pre_built() is False + assert pbi.is_pre_built(Version("2.9.0")) is True + assert pbi.is_pre_built(Version("2.8.0")) is False + assert pbi.is_pre_built(Version("3.0.0")) is False + + # rocm variant: pre_built=True by default, but 1.0.0 is pre_built=False + testdata_context.settings.variant = Variant("rocm") + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.pre_built is True + assert pbi.is_pre_built() is True + assert pbi.is_pre_built(Version("1.0.0")) is False + assert pbi.is_pre_built(Version("2.0.0")) is True + + # cuda variant: no version-specific settings + testdata_context.settings.variant = Variant("cuda") + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.is_pre_built() is False + assert pbi.is_pre_built(Version("2.9.0")) is False + + +def test_get_wheel_server_url_version_specific( + testdata_context: context.WorkContext, +) -> None: + """Version-specific wheel_server_url overrides variant default.""" + # cpu variant: default wheel_server_url, 2.9.0 has override + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.wheel_server_url == "https://wheel.test/simple" + assert pbi.get_wheel_server_url() == "https://wheel.test/simple" + assert pbi.get_wheel_server_url(Version("2.9.0")) == "https://mirror.test/simple" + assert pbi.get_wheel_server_url(Version("2.8.0")) == "https://wheel.test/simple" + assert pbi.get_wheel_server_url(Version("3.0.0")) == "https://wheel.test/simple" + + +def test_build_tag_version_specific_prebuilt( + testdata_context: context.WorkContext, +) -> None: + """Pre-built versions return empty build tag even when variant default is source.""" + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.variant == "cpu" + # 2.9.0 is version-specific pre_built=True, so no build tag + assert pbi.build_tag(Version("2.9.0")) == () + + def test_settings_list(testdata_context: context.WorkContext) -> None: assert testdata_context.settings.list_overrides() == { TEST_COOLDOWN_PKG, diff --git a/tests/testdata/context/overrides/settings/test_pkg.yaml b/tests/testdata/context/overrides/settings/test_pkg.yaml index a1d11352c..1bb0c6302 100644 --- a/tests/testdata/context/overrides/settings/test_pkg.yaml +++ b/tests/testdata/context/overrides/settings/test_pkg.yaml @@ -50,10 +50,19 @@ variants: EGG: "spam ${EGG}" EGG_AGAIN: "$EGG" wheel_server_url: https://wheel.test/simple + versions: + "2.9.0": + pre_built: true + wheel_server_url: https://mirror.test/simple + "2.8.0": + pre_built: false rocm: annotations: fromager.test.override: amd override env: SPAM: "" pre_built: True + versions: + "1.0.0": + pre_built: false cuda: {} From f1d8bc3c8237fa658f9426426a50fb18ab42377d Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 12:21:22 -0400 Subject: [PATCH 02/11] fix(settings): guard against wildcard version specifiers Skip wildcard pins like `==1.*` in `_extract_pinned_version` to avoid `InvalidVersion` from `packaging.version.Version`. Add docstring to `before_none_versions` validator. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- src/fromager/bootstrap_requirement_resolver.py | 11 ++++++++--- src/fromager/packagesettings/_models.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 7b9daa18c..edec5b536 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -12,6 +12,7 @@ from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name +from packaging.version import InvalidVersion from packaging.version import Version from . import finders, resolver, sources, wheels @@ -27,11 +28,15 @@ def _extract_pinned_version(req: Requirement) -> Version | None: """Return the version if *req* is pinned to exactly one (``==``). - Returns ``None`` for range specifiers, extras-only, or empty specifiers. + Returns ``None`` for range specifiers, wildcard pins (``==1.*``), + extras-only, or empty specifiers. """ specs = list(req.specifier) - if len(specs) == 1 and specs[0].operator == "==": - return Version(specs[0].version) + if len(specs) == 1 and specs[0].operator == "==" and "*" not in specs[0].version: + try: + return Version(specs[0].version) + except InvalidVersion: + return None return None diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index 589126ae8..c9d3f24e3 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -533,6 +533,7 @@ def before_none_versions( v: dict[str, typing.Any] | None, info: core_schema.ValidationInfo, ) -> dict[str, typing.Any]: + """Coerce ``None`` to empty dict for bare ``versions:`` YAML key.""" if v is None: return {} return v From fd3d8dfe6d390905ef189f26c0414e3cff5e53fd Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 12:49:18 -0400 Subject: [PATCH 03/11] fix(settings): handle local version segments and re-resolve on mismatch Strip local version segment (e.g. `+cpu`) before version-specific lookup, matching the pattern used by `get_changelog` and `get_patches`. When version-specific `pre_built` differs from variant default, re-resolve the download URL in the START phase so the URL type (wheel vs sdist) matches the actual build mode. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- .../bootstrap_requirement_resolver.py | 3 +- src/fromager/bootstrapper/_start.py | 70 +++++++++++++++++++ src/fromager/packagesettings/_pbi.py | 4 +- tests/test_packagesettings.py | 5 ++ 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index edec5b536..ce6078985 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -12,8 +12,7 @@ from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name -from packaging.version import InvalidVersion -from packaging.version import Version +from packaging.version import InvalidVersion, Version from . import finders, resolver, sources, wheels from .dependency_graph import DependencyGraph diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index 96c7d5518..bc7af2ad1 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -3,17 +3,64 @@ import logging import typing +from packaging.requirements import Requirement +from packaging.version import Version + +from .. import resolver, sources, wheels from ..requirements_file import RequirementType from ._phase import Phase from ._prepare_source import PrepareSource from ._types import BootstrapPhase if typing.TYPE_CHECKING: + from .. import context from ._bootstrapper import Bootstrapper logger = logging.getLogger(__name__) +def _re_resolve_url( + ctx: context.WorkContext, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + pre_built: bool, + cache_wheel_server_url: str | None, +) -> str | None: + """Re-resolve the download URL when version-specific pre_built differs. + + Returns the new URL or ``None`` if re-resolution fails. + """ + pinned_req = Requirement(f"{req.name}=={resolved_version}") + if pre_built: + wheel_server_urls = wheels.get_wheel_server_urls( + ctx, + req, + cache_wheel_server_url=cache_wheel_server_url, + version=resolved_version, + ) + url, _ = wheels.resolve_prebuilt_wheel( + ctx=ctx, + req=pinned_req, + wheel_server_urls=wheel_server_urls, + req_type=req_type, + ) + return str(url) + else: + pbi = ctx.package_build_info(req) + sdist_server = pbi.resolver_sdist_server_url(resolver.PYPI_SERVER_URL) + provider = sources.get_source_provider( + ctx=ctx, + req=pinned_req, + sdist_server_url=sdist_server, + req_type=req_type, + ) + results = resolver.find_all_matching_from_provider(provider, pinned_req) + if results: + return str(results[0][0]) + return None + + class Start(Phase): """Record a resolved requirement in the dependency graph and deduplicate. @@ -74,4 +121,27 @@ def run(self, bt: Bootstrapper) -> list[Phase]: pbi = bt.ctx.package_build_info(wi.req) wi.pbi_pre_built = pbi.is_pre_built(wi.resolved_version) wi.exclusive_build = pbi.exclusive_build + + if wi.pbi_pre_built != pbi.pre_built: + logger.info( + f"{wi.req} {wi.resolved_version}: version-specific pre_built " + f"override ({wi.pbi_pre_built}) differs from variant default " + f"({pbi.pre_built}), re-resolving URL" + ) + new_url = _re_resolve_url( + bt.ctx, + wi.req, + wi.req_type, + wi.resolved_version, + wi.pbi_pre_built, + bt.cache_wheel_server_url, + ) + if new_url is not None: + wi.source_url = new_url + else: + logger.warning( + f"{wi.req} {wi.resolved_version}: could not re-resolve URL " + f"for pre_built={wi.pbi_pre_built}, using original" + ) + return [PrepareSource(wi)] diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 36b06c302..3e5f9a971 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -190,7 +190,7 @@ def is_pre_built(self, version: Version | None = None) -> bool: ) if result is not None: return bool(result) - pv = typing.cast(PackageVersion, version) + pv = typing.cast(PackageVersion, Version(version.public)) vs = vi.versions.get(pv) if vs is not None and vs.pre_built is not None: return vs.pre_built @@ -216,7 +216,7 @@ def get_wheel_server_url(self, version: Version | None = None) -> str | None: if vi is None: return None if version is not None: - pv = typing.cast(PackageVersion, version) + pv = typing.cast(PackageVersion, Version(version.public)) vs = vi.versions.get(pv) if vs is not None and vs.wheel_server_url is not None: return str(vs.wheel_server_url) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 3f033d3c6..99dde465c 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -612,7 +612,9 @@ def test_is_pre_built_version_specific( assert pbi.pre_built is False assert pbi.is_pre_built() is False assert pbi.is_pre_built(Version("2.9.0")) is True + assert pbi.is_pre_built(Version("2.9.0+cpu")) is True assert pbi.is_pre_built(Version("2.8.0")) is False + assert pbi.is_pre_built(Version("2.8.0+local")) is False assert pbi.is_pre_built(Version("3.0.0")) is False # rocm variant: pre_built=True by default, but 1.0.0 is pre_built=False @@ -639,6 +641,9 @@ def test_get_wheel_server_url_version_specific( assert pbi.wheel_server_url == "https://wheel.test/simple" assert pbi.get_wheel_server_url() == "https://wheel.test/simple" assert pbi.get_wheel_server_url(Version("2.9.0")) == "https://mirror.test/simple" + assert ( + pbi.get_wheel_server_url(Version("2.9.0+cpu")) == "https://mirror.test/simple" + ) assert pbi.get_wheel_server_url(Version("2.8.0")) == "https://wheel.test/simple" assert pbi.get_wheel_server_url(Version("3.0.0")) == "https://wheel.test/simple" From 5a169d5ac5bea45a57177c06c4114130ecc9f55a Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 13:00:50 -0400 Subject: [PATCH 04/11] fix(settings): correct version annotations, add hook test and docs Fix `versionadded` directives to use 0.95.0 (next release after 0.94.0). Add test for `is_pre_built` plugin hook precedence over YAML settings. Update package-settings and hooks-and-overrides concept docs. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- docs/concepts/hooks-and-overrides.rst | 3 +- docs/concepts/package-settings.rst | 3 ++ src/fromager/packagesettings/_models.py | 4 +-- src/fromager/packagesettings/_pbi.py | 4 +-- src/fromager/wheels.py | 2 +- tests/test_packagesettings.py | 44 +++++++++++++++++++++++++ 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/docs/concepts/hooks-and-overrides.rst b/docs/concepts/hooks-and-overrides.rst index 43d5f281c..af5b75860 100644 --- a/docs/concepts/hooks-and-overrides.rst +++ b/docs/concepts/hooks-and-overrides.rst @@ -48,7 +48,8 @@ name. If the module defines the requested method, it is called instead of the default; otherwise the default runs. Override hooks cover resolution, source acquisition, building, -dependency extraction, and build environment customization. Third-party +dependency extraction, build environment customization, and prebuilt +wheel selection (``is_pre_built``). Third-party packages register overrides via the ``fromager.project_overrides`` entry-point group in their ``pyproject.toml``, mapping a package name to a Python module. diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index 2f0a0d7f3..d3a2a8d88 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -78,6 +78,9 @@ layers override earlier ones: │ 4. Variant overrides (within package YAML) │ │ (env vars, pre_built, wheel_server_url) │ │ │ + │ 4a. Version-specific variant overrides │ + │ (per-version pre_built, wheel_server_url) │ + │ │ │ 5. Version-specific patches and changelog │ │ (patches/-/, changelog entries)│ │ │ diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index c9d3f24e3..8ee2bc027 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -458,7 +458,7 @@ class VersionSpecificSettings(pydantic.BaseModel): specific package versions. When a field is ``None``, the variant-wide default is used. - .. versionadded:: 0.90.0 + .. versionadded:: 0.95.0 :: @@ -519,7 +519,7 @@ class VariantInfo(pydantic.BaseModel): Version-specific settings take precedence over variant defaults when present. - .. versionadded:: 0.90.0 + .. versionadded:: 0.95.0 """ # TODO diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 3e5f9a971..fa0d2c7f8 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -177,7 +177,7 @@ def is_pre_built(self, version: Version | None = None) -> bool: 2. Version-specific YAML setting 3. Variant-wide default - .. versionadded:: 0.90.0 + .. versionadded:: 0.95.0 """ vi = self._ps.variants.get(self.variant) if vi is None: @@ -210,7 +210,7 @@ def get_wheel_server_url(self, version: Version | None = None) -> str | None: Returns the version-specific URL if defined, otherwise falls back to the variant-wide default. - .. versionadded:: 0.90.0 + .. versionadded:: 0.95.0 """ vi = self._ps.variants.get(self.variant) if vi is None: diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 24d7a07d0..477e5f0e3 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -470,7 +470,7 @@ def get_wheel_server_urls( When *version* is given, version-specific ``wheel_server_url`` overrides are checked first. - .. versionchanged:: 0.90.0 + .. versionchanged:: 0.95.0 Added *version* parameter for version-specific URL lookup. """ pbi = ctx.package_build_info(req) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 99dde465c..590b2aa76 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -658,6 +658,50 @@ def test_build_tag_version_specific_prebuilt( assert pbi.build_tag(Version("2.9.0")) == () +def test_is_pre_built_hook_overrides_yaml( + testdata_context: context.WorkContext, +) -> None: + """Plugin hook takes precedence over YAML version-specific settings.""" + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.variant == "cpu" + + # Hook returns True for 2.8.0 (YAML says False) + with patch( + "fromager.overrides.find_override_method", + return_value=lambda *, version, variant: True, + ): + assert pbi.is_pre_built(Version("2.8.0")) is True + + # Hook returns False for 2.9.0 (YAML says True) + with patch( + "fromager.overrides.find_override_method", + return_value=lambda *, version, variant: False, + ): + assert pbi.is_pre_built(Version("2.9.0")) is False + + # Hook returns None (defers to YAML) + with patch( + "fromager.overrides.find_override_method", + return_value=lambda *, version, variant: None, + ): + assert pbi.is_pre_built(Version("2.9.0")) is True + assert pbi.is_pre_built(Version("2.8.0")) is False + + # No hook (returns None from find_override_method) + with patch( + "fromager.overrides.find_override_method", + return_value=None, + ): + assert pbi.is_pre_built(Version("2.9.0")) is True + + # Without version, hook is not consulted + with patch( + "fromager.overrides.find_override_method", + ) as mock_find: + pbi.is_pre_built() + mock_find.assert_not_called() + + def test_settings_list(testdata_context: context.WorkContext) -> None: assert testdata_context.settings.list_overrides() == { TEST_COOLDOWN_PKG, From cc6e590f788e86250c53476a18c35302e890c9ae Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 13:07:51 -0400 Subject: [PATCH 05/11] refactor(settings): single code path, find_and_invoke pattern, more tests Make `pre_built` and `wheel_server_url` properties delegate to `is_pre_built()` and `get_wheel_server_url()` so there is one code path. Switch `is_pre_built` hook to `find_and_invoke` with `_default_is_pre_built`, matching the pattern used by all other override hooks. Add tests for `get_wheel_server_urls` with version parameter, `before_none_versions` validator with None/omitted YAML, and update hook test to use `find_and_invoke`. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- src/fromager/packagesettings/_pbi.py | 45 +++++++++++------- tests/test_packagesettings.py | 70 +++++++++++++++++++++------- 2 files changed, 82 insertions(+), 33 deletions(-) diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index fa0d2c7f8..8bf0c5fd5 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -31,6 +31,15 @@ logger = logging.getLogger(__name__) +def _default_is_pre_built( + *, + version: Version, + variant: str, +) -> bool | None: + """Default ``is_pre_built`` hook returns ``None`` to defer to YAML config.""" + return None + + def get_available_memory_gib() -> float: """available virtual memory in GiB""" return psutil.virtual_memory().available / (1024**3) @@ -162,11 +171,11 @@ def has_customizations(self) -> bool: @property def pre_built(self) -> bool: - """Does the variant use pre-build wheels?""" - vi = self._ps.variants.get(self.variant) - if vi is not None: - return vi.pre_built - return False + """Does the variant use pre-build wheels? + + Delegates to :meth:`is_pre_built` with no version. + """ + return self.is_pre_built() def is_pre_built(self, version: Version | None = None) -> bool: """Version-aware pre-built check. @@ -183,13 +192,15 @@ def is_pre_built(self, version: Version | None = None) -> bool: if vi is None: return False if version is not None: - hook_fn = overrides.find_override_method(self.package, "is_pre_built") - if hook_fn is not None: - result = overrides.invoke( - hook_fn, version=version, variant=self.variant - ) - if result is not None: - return bool(result) + result = overrides.find_and_invoke( + self.package, + "is_pre_built", + _default_is_pre_built, + version=version, + variant=self.variant, + ) + if result is not None: + return bool(result) pv = typing.cast(PackageVersion, Version(version.public)) vs = vi.versions.get(pv) if vs is not None and vs.pre_built is not None: @@ -198,11 +209,11 @@ def is_pre_built(self, version: Version | None = None) -> bool: @property def wheel_server_url(self) -> str | None: - """Alternative package index for pre-build wheel""" - vi = self._ps.variants.get(self.variant) - if vi is not None and vi.wheel_server_url is not None: - return str(vi.wheel_server_url) - return None + """Alternative package index for pre-build wheel. + + Delegates to :meth:`get_wheel_server_url` with no version. + """ + return self.get_wheel_server_url() def get_wheel_server_url(self, version: Version | None = None) -> str | None: """Version-aware wheel server URL. diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 590b2aa76..057f84948 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -667,39 +667,77 @@ def test_is_pre_built_hook_overrides_yaml( # Hook returns True for 2.8.0 (YAML says False) with patch( - "fromager.overrides.find_override_method", - return_value=lambda *, version, variant: True, + "fromager.overrides.find_and_invoke", + return_value=True, ): assert pbi.is_pre_built(Version("2.8.0")) is True # Hook returns False for 2.9.0 (YAML says True) with patch( - "fromager.overrides.find_override_method", - return_value=lambda *, version, variant: False, + "fromager.overrides.find_and_invoke", + return_value=False, ): assert pbi.is_pre_built(Version("2.9.0")) is False # Hook returns None (defers to YAML) with patch( - "fromager.overrides.find_override_method", - return_value=lambda *, version, variant: None, - ): - assert pbi.is_pre_built(Version("2.9.0")) is True - assert pbi.is_pre_built(Version("2.8.0")) is False - - # No hook (returns None from find_override_method) - with patch( - "fromager.overrides.find_override_method", + "fromager.overrides.find_and_invoke", return_value=None, ): assert pbi.is_pre_built(Version("2.9.0")) is True + assert pbi.is_pre_built(Version("2.8.0")) is False # Without version, hook is not consulted with patch( - "fromager.overrides.find_override_method", - ) as mock_find: + "fromager.overrides.find_and_invoke", + ) as mock_invoke: pbi.is_pre_built() - mock_find.assert_not_called() + mock_invoke.assert_not_called() + + +def test_get_wheel_server_urls_version_specific( + testdata_context: context.WorkContext, +) -> None: + """get_wheel_server_urls uses version-specific URL when version given.""" + from fromager import wheels + + req = Requirement("test-pkg") + # cpu variant: default URL is https://wheel.test/simple, + # version 2.9.0 overrides to https://mirror.test/simple + urls_default = wheels.get_wheel_server_urls( + testdata_context, req, cache_wheel_server_url=None + ) + assert urls_default == ["https://wheel.test/simple"] + + urls_versioned = wheels.get_wheel_server_urls( + testdata_context, req, cache_wheel_server_url=None, version=Version("2.9.0") + ) + assert urls_versioned == ["https://mirror.test/simple"] + + urls_other = wheels.get_wheel_server_urls( + testdata_context, req, cache_wheel_server_url=None, version=Version("3.0.0") + ) + assert urls_other == ["https://wheel.test/simple"] + + +def test_variant_info_versions_none_yaml() -> None: + """Bare ``versions:`` key in YAML (parsed as None) produces empty dict.""" + ps = PackageSettings.from_string( + "test-none-versions", + "variants:\n cpu:\n versions:\n", + ) + vi = ps.variants["cpu"] + assert vi.versions == {} + + +def test_variant_info_versions_omitted() -> None: + """Omitting ``versions`` entirely produces empty dict.""" + ps = PackageSettings.from_string( + "test-no-versions", + "variants:\n cpu:\n pre_built: true\n", + ) + vi = ps.variants["cpu"] + assert vi.versions == {} def test_settings_list(testdata_context: context.WorkContext) -> None: From 9c3cab8306206de151a34178684f06c8b7b0a531 Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 13:10:42 -0400 Subject: [PATCH 06/11] fix(tests): use Variant type for variants mapping index Fix mypy error: use `Variant("cpu")` instead of bare string `"cpu"` when indexing into `Mapping[Variant, VariantInfo]`. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- tests/test_packagesettings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 057f84948..595201a83 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -726,7 +726,7 @@ def test_variant_info_versions_none_yaml() -> None: "test-none-versions", "variants:\n cpu:\n versions:\n", ) - vi = ps.variants["cpu"] + vi = ps.variants[Variant("cpu")] assert vi.versions == {} @@ -736,7 +736,7 @@ def test_variant_info_versions_omitted() -> None: "test-no-versions", "variants:\n cpu:\n pre_built: true\n", ) - vi = ps.variants["cpu"] + vi = ps.variants[Variant("cpu")] assert vi.versions == {} From 075ad4fa1693798f1be8dfda2b0bca3a12d5c4a3 Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 13:28:31 -0400 Subject: [PATCH 07/11] fix(settings): re-resolve on wheel_server_url mismatch, doc directives Also trigger URL re-resolution when version-specific wheel_server_url differs from variant default (not just when pre_built differs). Add versionadded directives to RST docs. Move local import to module scope. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- docs/concepts/hooks-and-overrides.rst | 2 +- docs/concepts/package-settings.rst | 3 +++ src/fromager/bootstrapper/_start.py | 13 +++++++++---- tests/test_bootstrapper_iterative.py | 7 ++++++- tests/test_packagesettings.py | 4 +--- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/concepts/hooks-and-overrides.rst b/docs/concepts/hooks-and-overrides.rst index af5b75860..7a0b399e5 100644 --- a/docs/concepts/hooks-and-overrides.rst +++ b/docs/concepts/hooks-and-overrides.rst @@ -49,7 +49,7 @@ of the default; otherwise the default runs. Override hooks cover resolution, source acquisition, building, dependency extraction, build environment customization, and prebuilt -wheel selection (``is_pre_built``). Third-party +wheel selection (``is_pre_built``, added in version 0.95.0). Third-party packages register overrides via the ``fromager.project_overrides`` entry-point group in their ``pyproject.toml``, mapping a package name to a Python module. diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index d3a2a8d88..32f4b5f82 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -88,6 +88,9 @@ layers override earlier ones: │ (update_extra_environ can mutate env vars) │ └──────────────────────────────────────────────────┘ +.. versionadded:: 0.95.0 + Step 4a: version-specific variant overrides (``versions`` mapping). + For environment variables specifically, the merge order within a single ``get_extra_environ()`` call is: parallel-jobs settings, build environment paths, package-level ``env``, then variant-level ``env``. diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index bc7af2ad1..d5421d8cb 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -122,11 +122,16 @@ def run(self, bt: Bootstrapper) -> list[Phase]: wi.pbi_pre_built = pbi.is_pre_built(wi.resolved_version) wi.exclusive_build = pbi.exclusive_build - if wi.pbi_pre_built != pbi.pre_built: + version_url = pbi.get_wheel_server_url(wi.resolved_version) + variant_url = pbi.wheel_server_url + needs_re_resolve = wi.pbi_pre_built != pbi.pre_built or ( + wi.pbi_pre_built and version_url != variant_url + ) + if needs_re_resolve: logger.info( - f"{wi.req} {wi.resolved_version}: version-specific pre_built " - f"override ({wi.pbi_pre_built}) differs from variant default " - f"({pbi.pre_built}), re-resolving URL" + f"{wi.req} {wi.resolved_version}: version-specific override " + f"(pre_built={wi.pbi_pre_built}, url={version_url}) differs " + f"from variant default, re-resolving URL" ) new_url = _re_resolve_url( bt.ctx, diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 2a8151606..e4ab33895 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -646,7 +646,12 @@ def test_sets_pbi_pre_built_before_prepare_source( with patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=True, is_pre_built=Mock(return_value=True)), + return_value=Mock( + pre_built=True, + is_pre_built=Mock(return_value=True), + wheel_server_url=None, + get_wheel_server_url=Mock(return_value=None), + ), ): result = item.run(bt) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 595201a83..b1d31259b 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -8,7 +8,7 @@ from packaging.utils import NormalizedName from packaging.version import Version -from fromager import build_environment, context +from fromager import build_environment, context, wheels from fromager.packagesettings import ( Annotations, BuildDirectory, @@ -699,8 +699,6 @@ def test_get_wheel_server_urls_version_specific( testdata_context: context.WorkContext, ) -> None: """get_wheel_server_urls uses version-specific URL when version given.""" - from fromager import wheels - req = Requirement("test-pkg") # cpu variant: default URL is https://wheel.test/simple, # version 2.9.0 overrides to https://mirror.test/simple From e8e9fdf6fb9417135e4d21ed107649f0d5918e06 Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 15:17:07 -0400 Subject: [PATCH 08/11] fix(settings): graph insertion after re-resolution, remove concept doc directive Move `add_to_graph` call after URL re-resolution so the dependency graph always records the final download URL. Remove `versionadded` directive from concept doc (only belongs in API-level docs per repo guidelines). Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- docs/concepts/package-settings.rst | 3 --- src/fromager/bootstrapper/_start.py | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index 32f4b5f82..d3a2a8d88 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -88,9 +88,6 @@ layers override earlier ones: │ (update_extra_environ can mutate env vars) │ └──────────────────────────────────────────────────┘ -.. versionadded:: 0.95.0 - Step 4a: version-specific variant overrides (``versions`` mapping). - For environment variables specifically, the merge order within a single ``get_extra_environ()`` call is: parallel-jobs settings, build environment paths, package-level ``env``, then variant-level ``env``. diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index d5421d8cb..f5cbd83c0 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -91,16 +91,6 @@ def run(self, bt: Bootstrapper) -> list[Phase]: assert wi.resolved_version is not None assert wi.source_url is not None - # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) - if wi.req_type != RequirementType.TOP_LEVEL: - bt.add_to_graph( - wi.req, - wi.req_type, - wi.resolved_version, - wi.source_url, - wi.parent, - ) - wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): @@ -149,4 +139,14 @@ def run(self, bt: Bootstrapper) -> list[Phase]: f"for pre_built={wi.pbi_pre_built}, using original" ) + # Add to graph after re-resolution so the graph has the final URL + if wi.req_type != RequirementType.TOP_LEVEL: + bt.add_to_graph( + wi.req, + wi.req_type, + wi.resolved_version, + wi.source_url, + wi.parent, + ) + return [PrepareSource(wi)] From 29336514a24a165fcf9ac3f891352bb6eb479b4c Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 19:23:04 -0400 Subject: [PATCH 09/11] fix(settings): handle ExceptionGroup in re-resolution, restore graph edge ordering Wrap resolve_prebuilt_wheel in _re_resolve_url with try/except ExceptionGroup so a missing wheel returns None instead of crashing. Move add_to_graph back before has_been_seen so edges from duplicate parents are recorded. Add 8 tests for _re_resolve_url and two-parent graph edge regression. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- src/fromager/bootstrapper/_start.py | 36 ++-- tests/test_bootstrapper_iterative.py | 243 ++++++++++++++++++++++++++- 2 files changed, 262 insertions(+), 17 deletions(-) diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index f5cbd83c0..4393fef47 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -39,12 +39,15 @@ def _re_resolve_url( cache_wheel_server_url=cache_wheel_server_url, version=resolved_version, ) - url, _ = wheels.resolve_prebuilt_wheel( - ctx=ctx, - req=pinned_req, - wheel_server_urls=wheel_server_urls, - req_type=req_type, - ) + try: + url, _ = wheels.resolve_prebuilt_wheel( + ctx=ctx, + req=pinned_req, + wheel_server_urls=wheel_server_urls, + req_type=req_type, + ) + except ExceptionGroup: + return None return str(url) else: pbi = ctx.package_build_info(req) @@ -93,6 +96,17 @@ def run(self, bt: Bootstrapper) -> list[Phase]: wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() + # Add to graph before the seen-check so every parent-to-dep edge + # is recorded, even when the package was already processed. + if wi.req_type != RequirementType.TOP_LEVEL: + bt.add_to_graph( + wi.req, + wi.req_type, + wi.resolved_version, + wi.source_url, + wi.parent, + ) + if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): logger.debug( f"redundant {wi.req_type} dependency {wi.req} " @@ -139,14 +153,4 @@ def run(self, bt: Bootstrapper) -> list[Phase]: f"for pre_built={wi.pbi_pre_built}, using original" ) - # Add to graph after re-resolution so the graph has the final URL - if wi.req_type != RequirementType.TOP_LEVEL: - bt.add_to_graph( - wi.req, - wi.req_type, - wi.resolved_version, - wi.source_url, - wi.parent, - ) - return [PrepareSource(wi)] diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index e4ab33895..9560ce832 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -38,7 +38,7 @@ from fromager.bootstrapper._prepare_source import PrepareSource from fromager.bootstrapper._process_install_deps import ProcessInstallDeps from fromager.bootstrapper._resolve import Resolve -from fromager.bootstrapper._start import Start +from fromager.bootstrapper._start import Start, _re_resolve_url from fromager.bootstrapper._types import ( BootstrapPhase, PreparedSourceData, @@ -602,6 +602,52 @@ def test_skips_graph_for_toplevel(self, tmp_context: WorkContext) -> None: key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes + def test_graph_edge_recorded_for_already_seen_package( + self, tmp_context: WorkContext + ) -> None: + """Graph edge is recorded even when the package was already processed.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + + parent_a = (Requirement("parent-a"), Version("1.0")) + parent_b = (Requirement("parent-b"), Version("2.0")) + + # Add parent nodes so add_dependency can attach edges + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=parent_a[0], + req_version=parent_a[1], + ) + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=parent_b[0], + req_version=parent_b[1], + ) + + item1 = _make_start_item(req_type=RequirementType.INSTALL, parent=parent_a) + item2 = _make_start_item(req_type=RequirementType.INSTALL, parent=parent_b) + + result1 = item1.run(bt) + assert len(result1) == 1 + + result2 = item2.run(bt) + assert result2 == [] + + # Both parent edges should be in the graph + dep_key = f"{canonicalize_name('testpkg')}==1.0" + parent_a_key = f"{canonicalize_name('parent-a')}==1.0" + parent_b_key = f"{canonicalize_name('parent-b')}==2.0" + assert dep_key in tmp_context.dependency_graph.nodes + dep_node = tmp_context.dependency_graph.nodes[dep_key] + parent_a_node = tmp_context.dependency_graph.nodes[parent_a_key] + parent_b_node = tmp_context.dependency_graph.nodes[parent_b_key] + assert dep_node in [e.destination_node for e in parent_a_node.children] + assert dep_node in [e.destination_node for e in parent_b_node.children] + def test_sdist_only_set_for_non_build_requirement( self, tmp_context: WorkContext ) -> None: @@ -660,6 +706,201 @@ def test_sets_pbi_pre_built_before_prepare_source( assert result[0].work_item.pbi_pre_built is True +class TestReResolveUrl: + """Tests for _re_resolve_url used by Start.run for version-specific overrides.""" + + def test_prebuilt_returns_resolved_url(self, tmp_context: WorkContext) -> None: + """Pre-built path returns URL from resolve_prebuilt_wheel.""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.wheels.get_wheel_server_urls", + return_value=["https://wheels.test/simple/"], + ), + patch( + "fromager.bootstrapper._start.wheels.resolve_prebuilt_wheel", + return_value=( + "https://wheels.test/testpkg-1.0-py3-none-any.whl", + Version("1.0"), + ), + ), + ): + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=True, + cache_wheel_server_url=None, + ) + + assert result == "https://wheels.test/testpkg-1.0-py3-none-any.whl" + + def test_source_returns_resolved_url(self, tmp_context: WorkContext) -> None: + """Source path returns URL from find_all_matching_from_provider.""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.sources.get_source_provider", + ) as mock_provider, + patch( + "fromager.bootstrapper._start.resolver.find_all_matching_from_provider", + return_value=[("https://pypi.test/testpkg-1.0.tar.gz", Version("1.0"))], + ), + ): + mock_provider.return_value = Mock() + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=False, + cache_wheel_server_url=None, + ) + + assert result == "https://pypi.test/testpkg-1.0.tar.gz" + + def test_prebuilt_returns_none_on_exception_group( + self, tmp_context: WorkContext + ) -> None: + """Pre-built path returns None when no wheel found (ExceptionGroup).""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.wheels.get_wheel_server_urls", + return_value=["https://wheels.test/simple/"], + ), + patch( + "fromager.bootstrapper._start.wheels.resolve_prebuilt_wheel", + side_effect=ExceptionGroup( + "no wheel found", + [Exception("server 1 failed")], + ), + ), + ): + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=True, + cache_wheel_server_url=None, + ) + + assert result is None + + def test_source_returns_none_when_no_match(self, tmp_context: WorkContext) -> None: + """Source path returns None when find_all_matching returns empty.""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.sources.get_source_provider", + ) as mock_provider, + patch( + "fromager.bootstrapper._start.resolver.find_all_matching_from_provider", + return_value=[], + ), + ): + mock_provider.return_value = Mock() + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=False, + cache_wheel_server_url=None, + ) + + assert result is None + + def test_wheel_server_url_differs_triggers_re_resolve( + self, tmp_context: WorkContext + ) -> None: + """Start.run re-resolves when wheel_server_url differs but pre_built matches.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + + mock_pbi = Mock( + pre_built=True, + is_pre_built=Mock(return_value=True), + exclusive_build=False, + wheel_server_url="https://default.test/simple/", + get_wheel_server_url=Mock(return_value="https://version.test/simple/"), + ) + + with ( + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._start._re_resolve_url", + return_value="https://version.test/testpkg-1.0-py3-none-any.whl", + ) as mock_re_resolve, + ): + item.run(bt) + + mock_re_resolve.assert_called_once() + assert ( + item.work_item.source_url + == "https://version.test/testpkg-1.0-py3-none-any.whl" + ) + + def test_no_re_resolve_when_settings_match_defaults( + self, tmp_context: WorkContext + ) -> None: + """Start.run skips re-resolution when version settings match variant defaults.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + original_url = item.work_item.source_url + + mock_pbi = Mock( + pre_built=False, + is_pre_built=Mock(return_value=False), + exclusive_build=False, + wheel_server_url=None, + get_wheel_server_url=Mock(return_value=None), + ) + + with ( + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._start._re_resolve_url", + ) as mock_re_resolve, + ): + item.run(bt) + + mock_re_resolve.assert_not_called() + assert item.work_item.source_url == original_url + + def test_fallback_to_original_url_on_failed_re_resolve( + self, tmp_context: WorkContext + ) -> None: + """Start.run keeps original URL and logs warning when re-resolve returns None.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + original_url = item.work_item.source_url + + mock_pbi = Mock( + pre_built=True, + is_pre_built=Mock(return_value=True), + exclusive_build=False, + wheel_server_url=None, + get_wheel_server_url=Mock(return_value=None), + ) + + with ( + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._start._re_resolve_url", + return_value=None, + ), + ): + item.run(bt) + + assert item.work_item.source_url == original_url + + class TestComplete: def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) From 6d482bf2c0ed0ec0d76bb4a988e2cc7336dfb1e1 Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Tue, 25 Aug 2026 19:31:26 -0400 Subject: [PATCH 10/11] fix(settings): finalize URL before graph insertion, fix fallback test Move re-resolution before add_to_graph so the dependency graph stores the final URL instead of the pre-resolution one. Fix fallback test to actually trigger needs_re_resolve by setting pre_built != is_pre_built. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- src/fromager/bootstrapper/_start.py | 51 +++++++++++++++------------- tests/test_bootstrapper_iterative.py | 5 +-- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index 4393fef47..febd9602e 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -96,36 +96,15 @@ def run(self, bt: Bootstrapper) -> list[Phase]: wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() - # Add to graph before the seen-check so every parent-to-dep edge - # is recorded, even when the package was already processed. - if wi.req_type != RequirementType.TOP_LEVEL: - bt.add_to_graph( - wi.req, - wi.req_type, - wi.resolved_version, - wi.source_url, - wi.parent, - ) - - if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): - logger.debug( - f"redundant {wi.req_type} dependency {wi.req} " - f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " - f"for {bt.explain}" - ) - return [] - bt.mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) - - logger.info( - f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" - ) - # Must set pbi_pre_built before constructing PrepareSource so that # PrepareSource.background_work() immediately sees the correct value. pbi = bt.ctx.package_build_info(wi.req) wi.pbi_pre_built = pbi.is_pre_built(wi.resolved_version) wi.exclusive_build = pbi.exclusive_build + # Re-resolve URL before graph insertion so the graph stores the + # final URL, and before the seen-check so duplicate parents still + # get their edge recorded with the correct URL. version_url = pbi.get_wheel_server_url(wi.resolved_version) variant_url = pbi.wheel_server_url needs_re_resolve = wi.pbi_pre_built != pbi.pre_built or ( @@ -153,4 +132,28 @@ def run(self, bt: Bootstrapper) -> list[Phase]: f"for pre_built={wi.pbi_pre_built}, using original" ) + # Add to graph after URL finalization but before the seen-check + # so every parent-to-dep edge is recorded with the correct URL. + if wi.req_type != RequirementType.TOP_LEVEL: + bt.add_to_graph( + wi.req, + wi.req_type, + wi.resolved_version, + wi.source_url, + wi.parent, + ) + + if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): + logger.debug( + f"redundant {wi.req_type} dependency {wi.req} " + f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " + f"for {bt.explain}" + ) + return [] + bt.mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) + + logger.info( + f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" + ) + return [PrepareSource(wi)] diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 9560ce832..6ef366732 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -882,7 +882,7 @@ def test_fallback_to_original_url_on_failed_re_resolve( original_url = item.work_item.source_url mock_pbi = Mock( - pre_built=True, + pre_built=False, is_pre_built=Mock(return_value=True), exclusive_build=False, wheel_server_url=None, @@ -894,10 +894,11 @@ def test_fallback_to_original_url_on_failed_re_resolve( patch( "fromager.bootstrapper._start._re_resolve_url", return_value=None, - ), + ) as mock_re_resolve, ): item.run(bt) + mock_re_resolve.assert_called_once() assert item.work_item.source_url == original_url From 4a114ebf772591f5c3ec46b75606ab9ea82f3e9d Mon Sep 17 00:00:00 2001 From: Andre Lustosa Date: Wed, 26 Aug 2026 07:38:38 -0400 Subject: [PATCH 11/11] refactor(settings): remove is_pre_built hook, keep YAML config only Per maintainer feedback, the use case is contained enough for static YAML configuration. Remove the is_pre_built plugin hook, keeping only the versions mapping on VariantInfo for per-version overrides. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andre Lustosa --- docs/concepts/hooks-and-overrides.rst | 3 +-- src/fromager/overrides.py | 1 - src/fromager/packagesettings/_pbi.py | 23 ++--------------- tests/test_packagesettings.py | 37 --------------------------- 4 files changed, 3 insertions(+), 61 deletions(-) diff --git a/docs/concepts/hooks-and-overrides.rst b/docs/concepts/hooks-and-overrides.rst index 7a0b399e5..43d5f281c 100644 --- a/docs/concepts/hooks-and-overrides.rst +++ b/docs/concepts/hooks-and-overrides.rst @@ -48,8 +48,7 @@ name. If the module defines the requested method, it is called instead of the default; otherwise the default runs. Override hooks cover resolution, source acquisition, building, -dependency extraction, build environment customization, and prebuilt -wheel selection (``is_pre_built``, added in version 0.95.0). Third-party +dependency extraction, and build environment customization. Third-party packages register overrides via the ``fromager.project_overrides`` entry-point group in their ``pyproject.toml``, mapping a package name to a Python module. diff --git a/src/fromager/overrides.py b/src/fromager/overrides.py index 542701c94..7d4d02a0f 100644 --- a/src/fromager/overrides.py +++ b/src/fromager/overrides.py @@ -30,7 +30,6 @@ "get_build_system_dependencies", "get_install_dependencies_of_sdist", "get_resolver_provider", - "is_pre_built", "prepare_source", "update_extra_environ", ) diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 8bf0c5fd5..a318308df 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -31,15 +31,6 @@ logger = logging.getLogger(__name__) -def _default_is_pre_built( - *, - version: Version, - variant: str, -) -> bool | None: - """Default ``is_pre_built`` hook returns ``None`` to defer to YAML config.""" - return None - - def get_available_memory_gib() -> float: """available virtual memory in GiB""" return psutil.virtual_memory().available / (1024**3) @@ -182,9 +173,8 @@ def is_pre_built(self, version: Version | None = None) -> bool: Resolution order: - 1. Plugin hook ``is_pre_built`` (if version given and hook exists) - 2. Version-specific YAML setting - 3. Variant-wide default + 1. Version-specific YAML setting + 2. Variant-wide default .. versionadded:: 0.95.0 """ @@ -192,15 +182,6 @@ def is_pre_built(self, version: Version | None = None) -> bool: if vi is None: return False if version is not None: - result = overrides.find_and_invoke( - self.package, - "is_pre_built", - _default_is_pre_built, - version=version, - variant=self.variant, - ) - if result is not None: - return bool(result) pv = typing.cast(PackageVersion, Version(version.public)) vs = vi.versions.get(pv) if vs is not None and vs.pre_built is not None: diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index b1d31259b..8e5497c79 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -658,43 +658,6 @@ def test_build_tag_version_specific_prebuilt( assert pbi.build_tag(Version("2.9.0")) == () -def test_is_pre_built_hook_overrides_yaml( - testdata_context: context.WorkContext, -) -> None: - """Plugin hook takes precedence over YAML version-specific settings.""" - pbi = testdata_context.settings.package_build_info(TEST_PKG) - assert pbi.variant == "cpu" - - # Hook returns True for 2.8.0 (YAML says False) - with patch( - "fromager.overrides.find_and_invoke", - return_value=True, - ): - assert pbi.is_pre_built(Version("2.8.0")) is True - - # Hook returns False for 2.9.0 (YAML says True) - with patch( - "fromager.overrides.find_and_invoke", - return_value=False, - ): - assert pbi.is_pre_built(Version("2.9.0")) is False - - # Hook returns None (defers to YAML) - with patch( - "fromager.overrides.find_and_invoke", - return_value=None, - ): - assert pbi.is_pre_built(Version("2.9.0")) is True - assert pbi.is_pre_built(Version("2.8.0")) is False - - # Without version, hook is not consulted - with patch( - "fromager.overrides.find_and_invoke", - ) as mock_invoke: - pbi.is_pre_built() - mock_invoke.assert_not_called() - - def test_get_wheel_server_urls_version_specific( testdata_context: context.WorkContext, ) -> None: