From de68c0f089b08a13058c7548dd120ba4afae5946 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 21:26:57 +0000 Subject: [PATCH] NuGet: read .nuspec manifests in UTF-16 with a byte-order mark parse_nupkg refused a manifest in UTF-16 LE with a byte-order mark (Microsoft.Web.Infrastructure 1.0.0 on nuget.org), so it could neither be pushed nor served through a proxy of nuget.org. The manifest is now decoded the way NuGet's XDocument.Load reads it: a UTF-16 LE or BE mark is decoded, a UTF-8 mark is dropped, and every other check still runs on the decoded text. The stored manifest keeps the package's own bytes. Adds a UTF-16 manifest fixture (fixtures/nuget/make_fixtures.py). Fixes #597 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJaoNeeYjbTdvTzQ728 --- crates/formats/nuget/src/nuspec.rs | 88 ++++++++++++++++++++++- crates/formats/nuget/tests/parse.rs | 26 +++++++ docs/changes/2.2.0/597.fixed.md | 5 ++ fixtures/nuget/make_fixtures.py | 25 +++++++ fixtures/nuget/utf16.package.1.0.0.nupkg | Bin 0 -> 884 bytes 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 docs/changes/2.2.0/597.fixed.md create mode 100644 fixtures/nuget/utf16.package.1.0.0.nupkg diff --git a/crates/formats/nuget/src/nuspec.rs b/crates/formats/nuget/src/nuspec.rs index 5680642f..9500f1c3 100644 --- a/crates/formats/nuget/src/nuspec.rs +++ b/crates/formats/nuget/src/nuspec.rs @@ -967,9 +967,39 @@ struct RawNuspec { /// bounds the parser's path bookkeeping on hostile input. pub const MAX_NUSPEC_DEPTH: usize = 32; +/// The manifest's text as UTF-8, decoded the way NuGet's `XDocument.Load` +/// reads it when it starts with a byte-order mark: UTF-16 LE (`FF FE`) or BE +/// (`FE FF`) is decoded, and a UTF-8 mark (`EF BB BF`) is dropped. Older +/// manifests on nuget.org are UTF-16 (`Microsoft.Web.Infrastructure` 1.0.0). +/// Without a mark the bytes are returned as they are, read as UTF-8. +fn decode_manifest(bytes: &[u8]) -> Result, NupkgError> { + use std::borrow::Cow; + + let (body, little_endian) = match bytes { + [0xEF, 0xBB, 0xBF, rest @ ..] => return Ok(Cow::Borrowed(rest)), + [0xFF, 0xFE, rest @ ..] => (rest, true), + [0xFE, 0xFF, rest @ ..] => (rest, false), + _ => return Ok(Cow::Borrowed(bytes)), + }; + let malformed = |what: &str| { + NupkgError::MalformedNuspec(format!("cannot decode input using UTF-16: {what}")) + }; + let units = body.chunks(2).map(|pair| match *pair { + [a, b] if little_endian => Ok(u16::from_le_bytes([a, b])), + [a, b] => Ok(u16::from_be_bytes([a, b])), + _ => Err(malformed("odd number of bytes")), + }); + let mut text = String::with_capacity(body.len()); + for unit in char::decode_utf16(units.collect::, _>>()?) { + text.push(unit.map_err(|_| malformed("unpaired surrogate"))?); + } + Ok(Cow::Owned(text.into_bytes())) +} + /// Parse the `.nuspec` XML namespace-insensitively (by local name). fn parse_nuspec(bytes: &[u8]) -> Result { - let mut reader = Reader::from_reader(bytes); + let text = decode_manifest(bytes)?; + let mut reader = Reader::from_reader(text.as_ref()); // Text is kept verbatim (entity references split it into several // events, so trimming per event would eat inner spaces); values are // trimmed once parsing is complete. @@ -2126,6 +2156,62 @@ mod tests { assert!(err.to_string().contains("DOCTYPE"), "{err}"); } + /// `` with id `A.B` and a non-ASCII description. + const BOM_MANIFEST: &str = "\ + A.B1.0.0\ + \u{e9}t\u{e9} \u{1f4e6}"; + + #[test] + fn manifests_with_a_byte_order_mark_are_decoded_as_nuget_does() { + let le: Vec = [0xFF, 0xFE] + .into_iter() + .chain(BOM_MANIFEST.encode_utf16().flat_map(u16::to_le_bytes)) + .collect(); + let be: Vec = [0xFE, 0xFF] + .into_iter() + .chain(BOM_MANIFEST.encode_utf16().flat_map(u16::to_be_bytes)) + .collect(); + let utf8: Vec = [0xEF, 0xBB, 0xBF] + .into_iter() + .chain(BOM_MANIFEST.bytes()) + .collect(); + for (name, bytes) in [("utf-16 le", le), ("utf-16 be", be), ("utf-8", utf8)] { + let raw = parse_nuspec(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!((raw.id.as_str(), raw.version.as_str()), ("A.B", "1.0.0")); + assert_eq!( + raw.metadata.description.as_deref(), + Some("\u{e9}t\u{e9} \u{1f4e6}"), + "{name}" + ); + } + } + + #[test] + fn malformed_utf16_manifests_are_refused() { + let mut odd: Vec = [0xFF, 0xFE] + .into_iter() + .chain(BOM_MANIFEST.encode_utf16().flat_map(u16::to_le_bytes)) + .collect(); + odd.push(b'x'); + let err = parse_nuspec(&odd).unwrap_err(); + assert!(err.to_string().contains("odd number of bytes"), "{err}"); + + // A high surrogate (U+D83D) with no low one after it. + let unpaired: Vec = [0xFF, 0xFE, b'<', 0, 0x3D, 0xD8, b'a', 0].to_vec(); + let err = parse_nuspec(&unpaired).unwrap_err(); + assert!(err.to_string().contains("unpaired surrogate"), "{err}"); + + // The checks on the decoded text still apply. + let doctype = "\ + A.B1.0.0"; + let bytes: Vec = [0xFF, 0xFE] + .into_iter() + .chain(doctype.encode_utf16().flat_map(u16::to_le_bytes)) + .collect(); + let err = parse_nuspec(&bytes).unwrap_err(); + assert!(err.to_string().contains("DOCTYPE"), "{err}"); + } + #[test] fn missing_id_or_version_is_error() { let xml = br#"1.0.0"#; diff --git a/crates/formats/nuget/tests/parse.rs b/crates/formats/nuget/tests/parse.rs index fc01f597..1e30b782 100644 --- a/crates/formats/nuget/tests/parse.rs +++ b/crates/formats/nuget/tests/parse.rs @@ -72,6 +72,32 @@ fn parses_second_namespace_with_flat_deps_and_prerelease() { ); } +/// A manifest in UTF-16 LE with a byte-order mark (`encoding="utf-16"`), as +/// `Microsoft.Web.Infrastructure` 1.0.0 on nuget.org holds it, is read the way +/// NuGet reads it, and stored as it came (#597). +#[test] +fn parses_a_utf16_manifest_with_a_byte_order_mark() { + let fmt = Nuget::default(); + let mut f = fixture("utf16.package.1.0.0.nupkg"); + let parsed = fmt.parse_package(&mut f).expect("parse utf16.package"); + + assert_eq!(parsed.original_id, "Utf16.Package"); + assert_eq!(parsed.id, "utf16.package"); + assert_eq!(parsed.version_original, "1.0.0.0"); + assert_eq!(parsed.version, "1.0.0"); + assert_eq!(parsed.dependencies.len(), 1); + assert_eq!(parsed.dependencies[0].dependencies[0].id, "Newtonsoft.Json"); + // The stored manifest is the package's own bytes, still UTF-16. + assert_eq!(&parsed.manifest[..4], b"\xff\xfe<\x00"); + + let metadata = forge_format_nuget::nuspec::metadata_from_nuspec(&parsed.manifest) + .expect("metadata of the stored manifest"); + assert_eq!( + metadata.description.as_deref(), + Some("A manifest in UTF-16 \u{a9} Forge \u{1f4e6}") + ); +} + /// The `zip-bomb.nupkg` fixture holds a 64 MiB + 1 entry of zeros (ratio ~1000) /// next to a valid manifest. Only the `.nuspec` is ever decompressed, so that /// payload is harmless and must not fail the package — large native packages diff --git a/docs/changes/2.2.0/597.fixed.md b/docs/changes/2.2.0/597.fixed.md new file mode 100644 index 00000000..196aa3a5 --- /dev/null +++ b/docs/changes/2.2.0/597.fixed.md @@ -0,0 +1,5 @@ +A NuGet package whose `.nuspec` is in UTF-16 with a byte-order mark, as +older packages on nuget.org hold it (`Microsoft.Web.Infrastructure` 1.0.0), +was refused as a malformed manifest, on push and through a proxy of +nuget.org. The manifest is decoded the way NuGet reads it (UTF-16 LE or BE, +or UTF-8 with a byte-order mark), and every other check still applies. diff --git a/fixtures/nuget/make_fixtures.py b/fixtures/nuget/make_fixtures.py index 8db942f8..cf517a41 100644 --- a/fixtures/nuget/make_fixtures.py +++ b/fixtures/nuget/make_fixtures.py @@ -12,6 +12,9 @@ * two different nuspec `xmlns` schema versions (the parser must be namespace-insensitive); * grouped and flat dependency declarations; + * a manifest in UTF-16 LE with a byte-order mark and `encoding="utf-16"`, + as older nuget.org packages hold it (`Microsoft.Web.Infrastructure` + 1.0.0); NuGet reads it, so the parser must too; * a "zip bomb" archive with a hugely compressible payload entry next to a valid manifest. Only the .nuspec is ever decompressed, so the parser must ACCEPT it under the default limits (large native packages look like this) @@ -81,6 +84,23 @@ """ +# A manifest in UTF-16 LE with a byte-order mark, the way older nuget.org +# packages (Microsoft.Web.Infrastructure 1.0.0) hold it. The non-ASCII +# description checks that the text is decoded, not just stripped of zeros. +UTF16_PKG_NUSPEC = """ + + + Utf16.Package + 1.0.0.0 + forge + A manifest in UTF-16 \u00a9 Forge \U0001f4e6 + + + + + +""" + def write_nupkg(path, nuspec_name, nuspec_body, extra=None): with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z: @@ -129,6 +149,11 @@ def main(): "Other.Package.nuspec", OTHER_PKG_NUSPEC, ) + write_nupkg( + os.path.join(HERE, "utf16.package.1.0.0.nupkg"), + "Utf16.Package.nuspec", + b"\xff\xfe" + UTF16_PKG_NUSPEC.encode("utf-16-le"), + ) write_zip_bomb(os.path.join(HERE, "zip-bomb.nupkg")) print("fixtures written to", HERE) diff --git a/fixtures/nuget/utf16.package.1.0.0.nupkg b/fixtures/nuget/utf16.package.1.0.0.nupkg new file mode 100644 index 0000000000000000000000000000000000000000..39bb98c064fafba102687a8c43f8dc58d02e17ee GIT binary patch literal 884 zcmWIWW@Zs#U|`^2Fj}1)yL#b~*x5kdXCM{^;%Mjmypq(slK7Cyg4E(zy^7qN-hOYs zLk2u9>xC-s@7mzNVfvP72cvjL(0An^b+4F^Jb$n4IuU)(d)*o0l> zT$`>q&1swbg)8Q-&OUOvd%fmfcBN;d&WhK4dAU+P9C@4Ej2iE8accaT`EJf{?XLTx zXW0W#{BrTt)^Kk|1_m`|1_qE1Lrc;O&GZ5ild}`kQ}yyniwjbdTPL0MYYq@-J%4NY zj+b#8W@RL4^v>3ckr4mIu;jutjSC$AY$xSEIkdHZVO=>FuWByO&Oc}7SUx{vu-|^- zvZ~;P?v{HsE>F&Sx+)}V(Zpk1C5J9g)HGF}H%WKvmN%QFbW3<%%~`)PNmlbx6z2){ zciyXFPRAX3ogH??BtcKK=&C~a0ZDu1+z`BptM)7^xP3q0YinDHPKAp44@?*sJYg6BAUU=)LFUTeBvp=gmtI&G$ zyC1U@7sb!LArhWqqvUpTe`(g?oc%_u!MC>*S>X8RU0d$<-dXi)h4YE>&y#Q2-4UpEUA&3+<-ea>_fFqd z`Fq=^HFMwJ5O{bX`p0_f{+rvmFD9Hf>i)-w8u-f1OUq?|fi4BaqClLJnWUeWT4JGR zpdXx=Tac5gmy(l{{Gm6Cq)|@%%?~};D5a7+oB*Kh4Wdh9v zg9cy>V@;>%2B0Sgh;{~s2F5c$Cbq