NuGet: read .nuspec manifests in UTF-16 with a byte-order mark

crates/formats/nuget/src/nuspec.rs

@@ -967,9 +967,39 @@ struct RawNuspec {
 /// bounds the parser's path bookkeeping on hostile input.
 pub const MAX_NUSPEC_DEPTH: usize = 32;
 
+/// The manifest's text as UTF-8, decoded the way NuGet's `XDocument.Load`
+/// reads it when it starts with a byte-order mark: UTF-16 LE (`FF FE`) or BE
+/// (`FE FF`) is decoded, and a UTF-8 mark (`EF BB BF`) is dropped. Older
+/// manifests on nuget.org are UTF-16 (`Microsoft.Web.Infrastructure` 1.0.0).
+/// Without a mark the bytes are returned as they are, read as UTF-8.
+fn decode_manifest(bytes: &[u8]) -> Result<std::borrow::Cow<'_, [u8]>, NupkgError> {
+    use std::borrow::Cow;
+
+    let (body, little_endian) = match bytes {
+        [0xEF, 0xBB, 0xBF, rest @ ..] => return Ok(Cow::Borrowed(rest)),
+        [0xFF, 0xFE, rest @ ..] => (rest, true),
+        [0xFE, 0xFF, rest @ ..] => (rest, false),
+        _ => return Ok(Cow::Borrowed(bytes)),
+    };
+    let malformed = |what: &str| {
+        NupkgError::MalformedNuspec(format!("cannot decode input using UTF-16: {what}"))
+    };
+    let units = body.chunks(2).map(|pair| match *pair {
+        [a, b] if little_endian => Ok(u16::from_le_bytes([a, b])),
+        [a, b] => Ok(u16::from_be_bytes([a, b])),
+        _ => Err(malformed("odd number of bytes")),
+    });
+    let mut text = String::with_capacity(body.len());
+    for unit in char::decode_utf16(units.collect::<Result<Vec<u16>, _>>()?) {
+        text.push(unit.map_err(|_| malformed("unpaired surrogate"))?);
+    }
+    Ok(Cow::Owned(text.into_bytes()))
+}
+
 /// Parse the `.nuspec` XML namespace-insensitively (by local name).
 fn parse_nuspec(bytes: &[u8]) -> Result<RawNuspec, NupkgError> {
-    let mut reader = Reader::from_reader(bytes);
+    let text = decode_manifest(bytes)?;
+    let mut reader = Reader::from_reader(text.as_ref());
     // Text is kept verbatim (entity references split it into several
     // events, so trimming per event would eat inner spaces); values are
     // trimmed once parsing is complete.
@@ -2126,6 +2156,62 @@ mod tests {
         assert!(err.to_string().contains("DOCTYPE"), "{err}");
     }
 
+    /// `<package><metadata>` with id `A.B` and a non-ASCII description.
+    const BOM_MANIFEST: &str = "<?xml version=\"1.0\" encoding=\"utf-16\"?>\
+        <package><metadata><id>A.B</id><version>1.0.0</version>\
+        <description>\u{e9}t\u{e9} \u{1f4e6}</description></metadata></package>";
+
+    #[test]
+    fn manifests_with_a_byte_order_mark_are_decoded_as_nuget_does() {
+        let le: Vec<u8> = [0xFF, 0xFE]
+            .into_iter()
+            .chain(BOM_MANIFEST.encode_utf16().flat_map(u16::to_le_bytes))
+            .collect();
+        let be: Vec<u8> = [0xFE, 0xFF]
+            .into_iter()
+            .chain(BOM_MANIFEST.encode_utf16().flat_map(u16::to_be_bytes))
+            .collect();
+        let utf8: Vec<u8> = [0xEF, 0xBB, 0xBF]
+            .into_iter()
+            .chain(BOM_MANIFEST.bytes())
+            .collect();
+        for (name, bytes) in [("utf-16 le", le), ("utf-16 be", be), ("utf-8", utf8)] {
+            let raw = parse_nuspec(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
+            assert_eq!((raw.id.as_str(), raw.version.as_str()), ("A.B", "1.0.0"));
+            assert_eq!(
+                raw.metadata.description.as_deref(),
+                Some("\u{e9}t\u{e9} \u{1f4e6}"),
+                "{name}"
+            );
+        }
+    }
+
+    #[test]
+    fn malformed_utf16_manifests_are_refused() {
+        let mut odd: Vec<u8> = [0xFF, 0xFE]
+            .into_iter()
+            .chain(BOM_MANIFEST.encode_utf16().flat_map(u16::to_le_bytes))
+            .collect();
+        odd.push(b'x');
+        let err = parse_nuspec(&odd).unwrap_err();
+        assert!(err.to_string().contains("odd number of bytes"), "{err}");
+
+        // A high surrogate (U+D83D) with no low one after it.
+        let unpaired: Vec<u8> = [0xFF, 0xFE, b'<', 0, 0x3D, 0xD8, b'a', 0].to_vec();
+        let err = parse_nuspec(&unpaired).unwrap_err();
+        assert!(err.to_string().contains("unpaired surrogate"), "{err}");
+
+        // The checks on the decoded text still apply.
+        let doctype = "<?xml version=\"1.0\"?><!DOCTYPE package []>\
+            <package><metadata><id>A.B</id><version>1.0.0</version></metadata></package>";
+        let bytes: Vec<u8> = [0xFF, 0xFE]
+            .into_iter()
+            .chain(doctype.encode_utf16().flat_map(u16::to_le_bytes))
+            .collect();
+        let err = parse_nuspec(&bytes).unwrap_err();
+        assert!(err.to_string().contains("DOCTYPE"), "{err}");
+    }
+
     #[test]
     fn missing_id_or_version_is_error() {
         let xml = br#"<package><metadata><version>1.0.0</version></metadata></package>"#;

crates/formats/nuget/tests/parse.rs

@@ -72,6 +72,32 @@ fn parses_second_namespace_with_flat_deps_and_prerelease() {
     );
 }
 
+/// A manifest in UTF-16 LE with a byte-order mark (`encoding="utf-16"`), as
+/// `Microsoft.Web.Infrastructure` 1.0.0 on nuget.org holds it, is read the way
+/// NuGet reads it, and stored as it came (#597).
+#[test]
+fn parses_a_utf16_manifest_with_a_byte_order_mark() {
+    let fmt = Nuget::default();
+    let mut f = fixture("utf16.package.1.0.0.nupkg");
+    let parsed = fmt.parse_package(&mut f).expect("parse utf16.package");
+
+    assert_eq!(parsed.original_id, "Utf16.Package");
+    assert_eq!(parsed.id, "utf16.package");
+    assert_eq!(parsed.version_original, "1.0.0.0");
+    assert_eq!(parsed.version, "1.0.0");
+    assert_eq!(parsed.dependencies.len(), 1);
+    assert_eq!(parsed.dependencies[0].dependencies[0].id, "Newtonsoft.Json");
+    // The stored manifest is the package's own bytes, still UTF-16.
+    assert_eq!(&parsed.manifest[..4], b"\xff\xfe<\x00");
+
+    let metadata = forge_format_nuget::nuspec::metadata_from_nuspec(&parsed.manifest)
+        .expect("metadata of the stored manifest");
+    assert_eq!(
+        metadata.description.as_deref(),
+        Some("A manifest in UTF-16 \u{a9} Forge \u{1f4e6}")
+    );
+}
+
 /// The `zip-bomb.nupkg` fixture holds a 64 MiB + 1 entry of zeros (ratio ~1000)
 /// next to a valid manifest. Only the `.nuspec` is ever decompressed, so that
 /// payload is harmless and must not fail the package — large native packages

docs/changes/2.2.0/597.fixed.md

@@ -0,0 +1,5 @@
+A NuGet package whose `.nuspec` is in UTF-16 with a byte-order mark, as
+older packages on nuget.org hold it (`Microsoft.Web.Infrastructure` 1.0.0),
+was refused as a malformed manifest, on push and through a proxy of
+nuget.org. The manifest is decoded the way NuGet reads it (UTF-16 LE or BE,
+or UTF-8 with a byte-order mark), and every other check still applies.

fixtures/nuget/make_fixtures.py

@@ -12,6 +12,9 @@
   * two different nuspec `xmlns` schema versions (the parser must be
     namespace-insensitive);
   * grouped and flat dependency declarations;
+  * a manifest in UTF-16 LE with a byte-order mark and `encoding="utf-16"`,
+    as older nuget.org packages hold it (`Microsoft.Web.Infrastructure`
+    1.0.0); NuGet reads it, so the parser must too;
   * a "zip bomb" archive with a hugely compressible payload entry next to a
     valid manifest. Only the .nuspec is ever decompressed, so the parser must
     ACCEPT it under the default limits (large native packages look like this)
@@ -81,6 +84,23 @@
 </package>
 """
 
+# A manifest in UTF-16 LE with a byte-order mark, the way older nuget.org
+# packages (Microsoft.Web.Infrastructure 1.0.0) hold it. The non-ASCII
+# description checks that the text is decoded, not just stripped of zeros.
+UTF16_PKG_NUSPEC = """<?xml version="1.0" encoding="utf-16"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
+  <metadata>
+    <id>Utf16.Package</id>
+    <version>1.0.0.0</version>
+    <authors>forge</authors>
+    <description>A manifest in UTF-16 \u00a9 Forge \U0001f4e6</description>
+    <dependencies>
+      <dependency id="Newtonsoft.Json" version="13.0.1" />
+    </dependencies>
+  </metadata>
+</package>
+"""
+
 
 def write_nupkg(path, nuspec_name, nuspec_body, extra=None):
     with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z:
@@ -129,6 +149,11 @@ def main():
         "Other.Package.nuspec",
         OTHER_PKG_NUSPEC,
     )
+    write_nupkg(
+        os.path.join(HERE, "utf16.package.1.0.0.nupkg"),
+        "Utf16.Package.nuspec",
+        b"\xff\xfe" + UTF16_PKG_NUSPEC.encode("utf-16-le"),
+    )
     write_zip_bomb(os.path.join(HERE, "zip-bomb.nupkg"))
     print("fixtures written to", HERE)