From 83afe7967ce09a2b0de673c5026b8db479a143e9 Mon Sep 17 00:00:00 2001 From: Noor-ul-ain001 Date: Tue, 18 Aug 2026 21:48:09 +0500 Subject: [PATCH] fix(bundler): decode a downloaded (non-zip) bundle manifest as UTF-8 _download_remote_manifest's non-zip branch fed the downloaded bytes straight to `yaml.safe_load(io.BytesIO(raw))`. PyYAML's Reader auto-detects a UTF-16 BOM on a byte stream, so a well-formed UTF-16 bundle.yml (a realistic PowerShell `Out-File`/`>` output) was silently *accepted* here, while `yamlio.load_yaml` decodes local sources strictly as UTF-8 and rejects the identical content with "Could not read ...". BEFORE: a UTF-16 manifest downloaded via `bundle info`/`install` parses successfully -- exit code 0, no warning. AFTER: rejected with "... could not be read: ..." -- exit code 1, matching local directory and .zip sources. This is the same divergence, in the sibling branch of the same function, that was just fixed for the .zip case in commit 56aec8a (PR #3958): "feeding PyYAML the byte stream let its Reader honour a UTF-16 BOM and accept a manifest yamlio.load_yaml rejects, so zip and directory sources diverged." That fix covered `_local_manifest_source`'s `.zip` branch (which this same function calls for zip artifacts); the direct raw-YAML-download branch a few lines below it had the identical bug. Also drops the now-unused `import io` from this function. Co-Authored-By: Claude Sonnet 5 --- src/specify_cli/commands/bundle/__init__.py | 16 +++++++++-- tests/contract/test_bundle_cli.py | 32 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/commands/bundle/__init__.py b/src/specify_cli/commands/bundle/__init__.py index 1edbeef2ca..165f674a36 100644 --- a/src/specify_cli/commands/bundle/__init__.py +++ b/src/specify_cli/commands/bundle/__init__.py @@ -934,7 +934,6 @@ def _download_remote_manifest( expected_sha256: str | None = None, ): """Fetch a remote bundle artifact over HTTPS and extract its manifest.""" - import io import tempfile from pathlib import PurePosixPath from urllib.parse import urlparse as _urlparse @@ -1038,7 +1037,20 @@ def _validate_redirect(old_url: str, new_url: str) -> None: ) return manifest - data = _yaml.safe_load(io.BytesIO(raw)) + # Decode as UTF-8 explicitly -- matching yamlio.load_yaml's contract -- + # instead of feeding PyYAML the raw byte stream. PyYAML's Reader + # auto-detects a UTF-16 BOM and would silently *accept* a manifest + # that the local directory/bundle.yml sources reject, letting this + # remote-download path diverge from them (see the sibling .zip fix + # for _local_manifest_source, which had the identical bug). + try: + text = raw.decode("utf-8") + except UnicodeError as exc: + raise BundlerError( + f"Downloaded content for bundle '{entry_id}' from " + f"{_source_desc} could not be read: {exc}" + ) from exc + data = _yaml.safe_load(text) return BundleManifest.from_dict(data) except BundlerError: raise diff --git a/tests/contract/test_bundle_cli.py b/tests/contract/test_bundle_cli.py index c458a810ba..9d6024a277 100644 --- a/tests/contract/test_bundle_cli.py +++ b/tests/contract/test_bundle_cli.py @@ -786,6 +786,38 @@ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None assert asset_calls[0][1] == {"Accept": "application/octet-stream"} +def test_bundle_info_rejects_utf16_remote_manifest_like_local_sources(project: Path): + """A downloaded (non-zip) bundle.yml must be decoded strictly as UTF-8. + + ``yamlio.load_yaml`` decodes local ``bundle.yml`` sources strictly as + UTF-8, so a well-formed UTF-16 manifest (a realistic PowerShell + ``Out-File`` output) is rejected. Feeding the downloaded bytes straight + to ``yaml.safe_load(io.BytesIO(raw))`` let PyYAML's Reader honour the + UTF-16 BOM and silently *accept* the same manifest instead, diverging + from local/zip sources (the zip branch of this same download path was + already fixed for the identical bug). + """ + api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/99" + manifest_yaml_utf16 = yaml.safe_dump(valid_manifest_dict()).encode("utf-16") + + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): + return FakeBundleResponse(manifest_yaml_utf16, url=api_asset_url) + + catalog = project / "catalog.json" + write_catalog_file( + catalog, + {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=api_asset_url)}, + ) + _make_catalog_config(catalog, project) + + with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url): + result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"]) + + assert result.exit_code == 1 + output_flat = " ".join(result.output.split()) + assert "could not be read" in output_flat.lower() + + def test_bundle_info_passes_through_api_asset_url(project: Path): """bundle info passes a direct GitHub API asset URL through with octet-stream.""" api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/77"