From f0ab89bface448c5a3f8f0ffaff53c21498a0958 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Tue, 15 Sep 2026 18:14:38 -0500 Subject: [PATCH] [wasm] Package CoreCLR native symbol map Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/liveBuilds.targets | 1 + src/mono/browser/runtime/dotnet.d.ts | 1 + src/mono/browser/runtime/types/index.ts | 1 + .../CoreCLRWasmNativeDefaultsTests.cs | 275 ++++++++++++++++++ .../Wasm.Build.Tests/ModuleConfigTests.cs | 134 ++++++++- .../libs/Common/JavaScript/loader/dotnet.d.ts | 1 + .../Common/JavaScript/types/public-api.ts | 1 + .../BootJsonBuilderHelper.cs | 1 + .../BootJsonData.cs | 1 + 9 files changed, 410 insertions(+), 6 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index e494ca933bdbea..da24977023c101 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -287,6 +287,7 @@ Include=" $(HostSharedFrameworkDir)libBrowserHost.a; $(HostSharedFrameworkDir)dotnet.native.js; + $(HostSharedFrameworkDir)dotnet.native.js.symbols; $(HostSharedFrameworkDir)dotnet.native.wasm; " IsNative="true" /> diff --git a/src/mono/browser/runtime/dotnet.d.ts b/src/mono/browser/runtime/dotnet.d.ts index 85ed12291c531a..8e76f4129a8f08 100644 --- a/src/mono/browser/runtime/dotnet.d.ts +++ b/src/mono/browser/runtime/dotnet.d.ts @@ -323,6 +323,7 @@ type JsAsset = Asset & { }; type SymbolsAsset = Asset & { name: string; + hash?: string | null | ""; cache?: RequestCache; }; type VfsAsset = Asset & { diff --git a/src/mono/browser/runtime/types/index.ts b/src/mono/browser/runtime/types/index.ts index 1a06897003c298..aca5a771aa3965 100644 --- a/src/mono/browser/runtime/types/index.ts +++ b/src/mono/browser/runtime/types/index.ts @@ -291,6 +291,7 @@ export type JsAsset = Asset & { export type SymbolsAsset = Asset & { name: string; // actually URL + hash?: string | null | ""; cache?: RequestCache; } diff --git a/src/mono/wasm/Wasm.Build.Tests/CoreCLRWasmNativeDefaultsTests.cs b/src/mono/wasm/Wasm.Build.Tests/CoreCLRWasmNativeDefaultsTests.cs index c7e27fd6499bd3..a802fcbd5affe0 100644 --- a/src/mono/wasm/Wasm.Build.Tests/CoreCLRWasmNativeDefaultsTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/CoreCLRWasmNativeDefaultsTests.cs @@ -1,7 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.Buffers.Binary; +using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; using Xunit; using Xunit.Abstractions; @@ -247,4 +253,273 @@ public void NativeRelinkWithoutCrossgen2PackReportsMissingGenerator(bool publish return (output, line); } } + + internal sealed class NativeWasmSymbolMapInfo + { + public required int ImportedFunctionCount { get; init; } + public required int DefinedFunctionCount { get; init; } + public required int CodeFunctionCount { get; init; } + public required IReadOnlyDictionary Symbols { get; init; } + public required IReadOnlyDictionary FunctionExports { get; init; } + } + + internal static class NativeWasmSymbolMapValidator + { + private const byte FunctionExternalKind = 0; + private const byte TableExternalKind = 1; + private const byte MemoryExternalKind = 2; + private const byte GlobalExternalKind = 3; + private const byte TagExternalKind = 4; + + public static NativeWasmSymbolMapInfo Validate( + string wasmPath, + string symbolsPath, + string? expectedWasmIntegrity = null, + string? expectedSymbolsIntegrity = null) + => Validate( + File.ReadAllBytes(wasmPath), + File.ReadAllBytes(symbolsPath), + expectedWasmIntegrity, + expectedSymbolsIntegrity); + + public static NativeWasmSymbolMapInfo Validate( + byte[] wasmBytes, + byte[] symbolsBytes, + string? expectedWasmIntegrity = null, + string? expectedSymbolsIntegrity = null) + { + ValidateIntegrity(wasmBytes, expectedWasmIntegrity, "Wasm"); + ValidateIntegrity(symbolsBytes, expectedSymbolsIntegrity, "symbol map"); + + NativeWasmSymbolMapInfo module = ReadModule(wasmBytes); + Dictionary symbols = ReadSymbols(symbolsBytes); + int expectedFunctionCount = checked(module.ImportedFunctionCount + module.DefinedFunctionCount); + + if (module.CodeFunctionCount != module.DefinedFunctionCount) + { + throw new InvalidDataException( + $"Wasm function section contains {module.DefinedFunctionCount} entries, " + + $"but its code section contains {module.CodeFunctionCount}."); + } + + if (symbols.Count != expectedFunctionCount) + { + throw new InvalidDataException( + $"Symbol map contains {symbols.Count} entries for {module.ImportedFunctionCount} imported and " + + $"{module.DefinedFunctionCount} defined functions."); + } + + for (int expectedIndex = 0; expectedIndex < expectedFunctionCount; expectedIndex++) + { + if (!symbols.ContainsKey(expectedIndex)) + { + throw new InvalidDataException( + $"Symbol map expected absolute function index {expectedIndex}, " + + $"including {module.ImportedFunctionCount} function imports."); + } + } + + return new NativeWasmSymbolMapInfo + { + ImportedFunctionCount = module.ImportedFunctionCount, + DefinedFunctionCount = module.DefinedFunctionCount, + CodeFunctionCount = module.CodeFunctionCount, + Symbols = symbols, + FunctionExports = module.FunctionExports + }; + } + + public static string ComputeIntegrity(byte[] bytes) + => $"sha256-{Convert.ToBase64String(SHA256.HashData(bytes))}"; + + private static void ValidateIntegrity(byte[] bytes, string? expectedIntegrity, string artifactName) + { + if (expectedIntegrity is null) + return; + + string actualIntegrity = ComputeIntegrity(bytes); + if (!string.Equals(actualIntegrity, expectedIntegrity, StringComparison.Ordinal)) + { + throw new InvalidDataException( + $"{artifactName} SHA-256 mismatch. Expected '{expectedIntegrity}', actual '{actualIntegrity}'."); + } + } + + private static NativeWasmSymbolMapInfo ReadModule(byte[] wasmBytes) + { + ReadOnlySpan image = wasmBytes; + if (image.Length < 8 || + BinaryPrimitives.ReadUInt32LittleEndian(image) != 0x6D736100 || + BinaryPrimitives.ReadUInt32LittleEndian(image.Slice(4)) != 1) + { + throw new InvalidDataException("Invalid WebAssembly module header."); + } + + int importedFunctionCount = 0; + int definedFunctionCount = -1; + int codeFunctionCount = -1; + Dictionary functionExports = new(StringComparer.Ordinal); + int offset = 8; + + while (offset < image.Length) + { + byte sectionId = ReadByte(image, ref offset, image.Length); + uint sectionSize = ReadUleb32(image, ref offset, image.Length); + int sectionEnd = checked(offset + (int)sectionSize); + if (sectionEnd > image.Length) + throw new InvalidDataException($"WebAssembly section {sectionId} extends past the end of the module."); + + switch (sectionId) + { + case 2: + importedFunctionCount = ReadFunctionImports(image, ref offset, sectionEnd); + break; + case 3: + definedFunctionCount = checked((int)ReadUleb32(image, ref offset, sectionEnd)); + break; + case 7: + functionExports = ReadFunctionExports(image, ref offset, sectionEnd); + break; + case 10: + codeFunctionCount = checked((int)ReadUleb32(image, ref offset, sectionEnd)); + break; + } + + offset = sectionEnd; + } + + if (definedFunctionCount < 0 || codeFunctionCount < 0) + throw new InvalidDataException("WebAssembly module is missing its function or code section."); + + return new NativeWasmSymbolMapInfo + { + ImportedFunctionCount = importedFunctionCount, + DefinedFunctionCount = definedFunctionCount, + CodeFunctionCount = codeFunctionCount, + Symbols = new Dictionary(), + FunctionExports = functionExports + }; + } + + private static int ReadFunctionImports(ReadOnlySpan image, ref int offset, int end) + { + uint importCount = ReadUleb32(image, ref offset, end); + int functionCount = 0; + for (uint i = 0; i < importCount; i++) + { + ReadName(image, ref offset, end); + ReadName(image, ref offset, end); + byte kind = ReadByte(image, ref offset, end); + switch (kind) + { + case FunctionExternalKind: + ReadUleb32(image, ref offset, end); + functionCount++; + break; + case TableExternalKind: + ReadByte(image, ref offset, end); + SkipLimits(image, ref offset, end); + break; + case MemoryExternalKind: + SkipLimits(image, ref offset, end); + break; + case GlobalExternalKind: + ReadByte(image, ref offset, end); + ReadByte(image, ref offset, end); + break; + case TagExternalKind: + ReadByte(image, ref offset, end); + ReadUleb32(image, ref offset, end); + break; + default: + throw new InvalidDataException($"Unknown WebAssembly import kind {kind}."); + } + } + + return functionCount; + } + + private static Dictionary ReadFunctionExports( + ReadOnlySpan image, + ref int offset, + int end) + { + uint exportCount = ReadUleb32(image, ref offset, end); + Dictionary functionExports = new(StringComparer.Ordinal); + for (uint i = 0; i < exportCount; i++) + { + string name = ReadName(image, ref offset, end); + byte kind = ReadByte(image, ref offset, end); + int index = checked((int)ReadUleb32(image, ref offset, end)); + if (kind == FunctionExternalKind) + functionExports.Add(name, index); + } + + return functionExports; + } + + private static Dictionary ReadSymbols(byte[] symbolsBytes) + { + Dictionary symbols = new(); + string[] lines = Encoding.UTF8.GetString(symbolsBytes) + .Split('\n', StringSplitOptions.RemoveEmptyEntries); + + foreach (string rawLine in lines) + { + string line = rawLine.TrimEnd('\r'); + int separator = line.IndexOf(':'); + if (separator <= 0 || !int.TryParse(line.AsSpan(0, separator), out int index)) + throw new InvalidDataException($"Invalid Emscripten symbol map entry '{line}'."); + + if (!symbols.TryAdd(index, line[(separator + 1)..])) + throw new InvalidDataException($"Duplicate function index {index} in the Emscripten symbol map."); + } + + return symbols; + } + + private static void SkipLimits(ReadOnlySpan image, ref int offset, int end) + { + uint flags = ReadUleb32(image, ref offset, end); + ReadUleb32(image, ref offset, end); + if ((flags & 1) != 0) + ReadUleb32(image, ref offset, end); + } + + private static string ReadName(ReadOnlySpan image, ref int offset, int end) + { + int length = checked((int)ReadUleb32(image, ref offset, end)); + if (length > end - offset) + throw new InvalidDataException("WebAssembly name extends past the end of its section."); + + string value = Encoding.UTF8.GetString(image.Slice(offset, length)); + offset += length; + return value; + } + + private static uint ReadUleb32(ReadOnlySpan image, ref int offset, int end) + { + uint value = 0; + int shift = 0; + while (shift < 35) + { + byte current = ReadByte(image, ref offset, end); + value |= (uint)(current & 0x7F) << shift; + if ((current & 0x80) == 0) + return value; + + shift += 7; + } + + throw new InvalidDataException("Invalid WebAssembly ULEB128 value."); + } + + private static byte ReadByte(ReadOnlySpan image, ref int offset, int end) + { + if ((uint)offset >= (uint)end) + throw new InvalidDataException("Unexpected end of WebAssembly section."); + + return image[offset++]; + } + } } diff --git a/src/mono/wasm/Wasm.Build.Tests/ModuleConfigTests.cs b/src/mono/wasm/Wasm.Build.Tests/ModuleConfigTests.cs index 1629b859f015d4..0fc7c7a3d97814 100644 --- a/src/mono/wasm/Wasm.Build.Tests/ModuleConfigTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/ModuleConfigTests.cs @@ -5,8 +5,12 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.IO; +using System.IO.Compression; using System.Linq; +using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.NET.Sdk.WebAssembly; using Xunit; using Xunit.Abstractions; @@ -141,7 +145,7 @@ await RunForPublishWithWebServer(new BrowserRunOptions( [Theory] [InlineData(false)] [InlineData(true)] - [TestCategory("native"), TestCategory("mono")] + [TestCategory("native")] public void SymbolMapFileEmitted(bool isPublish) => SymbolMapFileEmittedCore(emitSymbolMap: true, isPublish); @@ -151,10 +155,106 @@ public void SymbolMapFileEmitted(bool isPublish) public void SymbolMapFileNotEmitted(bool isPublish) => SymbolMapFileEmittedCore(emitSymbolMap: false, isPublish); + [Fact] + [TestCategory("coreclr")] + public void RuntimePackSymbolMapMatchesFinalWasm() + { + (byte[] wasmBytes, byte[] symbolsBytes) = ReadRuntimePackNativeSymbols(); + NativeWasmSymbolMapInfo info = NativeWasmSymbolMapValidator.Validate(wasmBytes, symbolsBytes); + + Assert.True(info.ImportedFunctionCount > 0); + Assert.Equal(info.DefinedFunctionCount, info.CodeFunctionCount); + Assert.Equal(info.ImportedFunctionCount + info.DefinedFunctionCount, info.Symbols.Count); + Assert.Equal( + "InterpExecMethod(InterpreterFrame*, InterpMethodContextFrame*, InterpThreadContext*, ExceptionClauseArgs*)", + Assert.Single(info.Symbols, entry => entry.Value.StartsWith("InterpExecMethod(", StringComparison.Ordinal)).Value); + Assert.Contains(info.Symbols, entry => entry.Value == "ExecuteInterpretedMethod"); + Assert.Contains( + info.Symbols, + entry => entry.Value.StartsWith("ExecuteInterpretedMethodWithArgs_PortableEntryPoint(", StringComparison.Ordinal)); + Assert.Contains(info.Symbols, entry => Regex.IsMatch(entry.Value, @"^non-virtual thunk to .+_\d+$")); + Assert.DoesNotContain(info.Symbols, entry => entry.Value.Contains("WasmR2RToInterpreterThunk", StringComparison.Ordinal)); + Assert.DoesNotContain(info.Symbols, entry => entry.Value.Contains("WasmInterpreterToR2RThunk", StringComparison.Ordinal)); + + int browserHostIndex = info.FunctionExports["BrowserHost_InitializeDotnet"]; + Assert.Equal("BrowserHost_InitializeDotnet", info.Symbols[browserHostIndex]); + + int mallocIndex = info.FunctionExports["malloc"]; + Assert.Equal("emscripten_builtin_malloc", info.Symbols[mallocIndex]); + } + + [Fact] + [TestCategory("coreclr")] + public void RuntimePackSymbolMapRejectsIndexAndIdentityMismatches() + { + (byte[] wasmBytes, byte[] symbolsBytes) = ReadRuntimePackNativeSymbols(); + string wasmIntegrity = NativeWasmSymbolMapValidator.ComputeIntegrity(wasmBytes); + string symbolsIntegrity = NativeWasmSymbolMapValidator.ComputeIntegrity(symbolsBytes); + + byte[] shiftedSymbols = Encoding.UTF8.GetBytes( + string.Join( + Environment.NewLine, + Encoding.UTF8.GetString(symbolsBytes) + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(line => + { + int separator = line.IndexOf(':'); + int index = int.Parse(line.AsSpan(0, separator)); + return $"{index + 1}:{line[(separator + 1)..].TrimEnd('\r')}"; + })) + + Environment.NewLine); + + InvalidDataException shiftedException = Assert.Throws( + () => NativeWasmSymbolMapValidator.Validate( + wasmBytes, + shiftedSymbols, + wasmIntegrity, + NativeWasmSymbolMapValidator.ComputeIntegrity(shiftedSymbols))); + Assert.Contains("expected absolute function index 0", shiftedException.Message); + + byte[] staleWasm = (byte[])wasmBytes.Clone(); + staleWasm[^1] ^= 1; + InvalidDataException staleWasmException = Assert.Throws( + () => NativeWasmSymbolMapValidator.Validate(staleWasm, symbolsBytes, wasmIntegrity, symbolsIntegrity)); + Assert.Contains("Wasm SHA-256 mismatch", staleWasmException.Message); + + byte[] staleSymbols = (byte[])symbolsBytes.Clone(); + staleSymbols[^2] ^= 1; + InvalidDataException staleSymbolsException = Assert.Throws( + () => NativeWasmSymbolMapValidator.Validate(wasmBytes, staleSymbols, wasmIntegrity, symbolsIntegrity)); + Assert.Contains("symbol map SHA-256 mismatch", staleSymbolsException.Message); + } + + private static (byte[] WasmBytes, byte[] SymbolsBytes) ReadRuntimePackNativeSymbols() + { + string runtimePackVersion = s_buildEnv.GetRuntimePackVersion(DefaultTargetFramework); + string packagePath = Path.Combine( + s_buildEnv.BuiltNuGetsPath, + $"Microsoft.NETCore.App.Runtime.browser-wasm.{runtimePackVersion}.nupkg"); + + using ZipArchive package = ZipFile.OpenRead(packagePath); + return ( + ReadEntry("runtimes/browser-wasm/native/dotnet.native.wasm"), + ReadEntry("runtimes/browser-wasm/native/dotnet.native.js.symbols")); + + byte[] ReadEntry(string entryName) + { + ZipArchiveEntry? entry = package.GetEntry(entryName); + Assert.NotNull(entry); + using Stream stream = entry.Open(); + using MemoryStream buffer = new(checked((int)entry.Length)); + stream.CopyTo(buffer); + return buffer.ToArray(); + } + } + private void SymbolMapFileEmittedCore(bool emitSymbolMap, bool isPublish) { Configuration config = Configuration.Release; string extraProperties = $"{emitSymbolMap.ToString().ToLowerInvariant()}"; + if (IsCoreClrRuntime) + extraProperties += "false"; + ProjectInfo info = CopyTestAsset(config, aot: false, TestAsset.WasmBasicTestApp, $"SymbolMapFile_{emitSymbolMap}_{isPublish}", extraProperties: extraProperties); @@ -170,11 +270,11 @@ private void SymbolMapFileEmittedCore(bool emitSymbolMap, bool isPublish) // bin/{config}/{tfm}/publish/wwwroot/_framework/. // The file may be fingerprinted (e.g. dotnet.native..js.symbols), so use a glob. const string symbolsPattern = "dotnet.native*.js.symbols"; - bool symbolsFileExists; + string? symbolsFile; if (isPublish) { string frameworkDir = GetBinFrameworkDir(config, forPublish: true); - symbolsFileExists = Directory.EnumerateFiles(frameworkDir, symbolsPattern).Any(); + symbolsFile = Directory.EnumerateFiles(frameworkDir, symbolsPattern).SingleOrDefault(); } else { @@ -186,10 +286,32 @@ .. Directory.Exists(fxBaseDir) ? Directory.GetDirectories(fxBaseDir).Select(d => Path.Combine(d, "_framework")) : Array.Empty() ]; - symbolsFileExists = searchDirs + symbolsFile = searchDirs .Where(Directory.Exists) - .Any(d => Directory.EnumerateFiles(d, symbolsPattern).Any()); + .SelectMany(d => Directory.EnumerateFiles(d, symbolsPattern)) + .SingleOrDefault(); } - Assert.Equal(emitSymbolMap, symbolsFileExists); + + Assert.Equal(emitSymbolMap, symbolsFile is not null); + if (!emitSymbolMap || !isPublish) + return; + + string symbolsPath = Assert.IsType(symbolsFile); + string frameworkDirectory = GetBinFrameworkDir(config, forPublish: true); + WasmSdkBasedProjectProvider provider = GetProvider(); + BootJsonData bootJson = provider.GetBootJson(provider.GetBootConfigPath(frameworkDirectory)); + AssetsData assets = Assert.IsType(bootJson.resources); + SymbolsAsset symbolAsset = Assert.Single(assets.wasmSymbols); + WasmAsset wasmAsset = Assert.Single(assets.wasmNative); + + Assert.Equal(Path.GetFileName(symbolsPath), symbolAsset.name); + Assert.StartsWith("sha256-", symbolAsset.hash); + Assert.StartsWith("sha256-", wasmAsset.hash); + string wasmPath = Path.Combine(frameworkDirectory, wasmAsset.name); + NativeWasmSymbolMapValidator.Validate( + wasmPath, + symbolsPath, + expectedWasmIntegrity: wasmAsset.hash, + expectedSymbolsIntegrity: symbolAsset.hash); } } diff --git a/src/native/libs/Common/JavaScript/loader/dotnet.d.ts b/src/native/libs/Common/JavaScript/loader/dotnet.d.ts index 43637d44c19159..8241fd5b8c4dcf 100644 --- a/src/native/libs/Common/JavaScript/loader/dotnet.d.ts +++ b/src/native/libs/Common/JavaScript/loader/dotnet.d.ts @@ -303,6 +303,7 @@ type JsAsset = Asset & { }; type SymbolsAsset = Asset & { name: string; + hash?: string | null | ""; }; type VfsAsset = Asset & { virtualPath: string; diff --git a/src/native/libs/Common/JavaScript/types/public-api.ts b/src/native/libs/Common/JavaScript/types/public-api.ts index ce4f51ef9645ab..0ebb7c1e859502 100644 --- a/src/native/libs/Common/JavaScript/types/public-api.ts +++ b/src/native/libs/Common/JavaScript/types/public-api.ts @@ -281,6 +281,7 @@ export type JsAsset = Asset & { }; export type SymbolsAsset = Asset & { name: string; + hash?: string | null | ""; }; export type VfsAsset = Asset & { virtualPath: string; diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonBuilderHelper.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonBuilderHelper.cs index 4c0b1d30260858..de63f0f7f0383b 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonBuilderHelper.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonBuilderHelper.cs @@ -247,6 +247,7 @@ public string TransformResourcesToAssets(BootJsonData config, bool bundlerFriend assets.wasmSymbols = resources.wasmSymbols?.Select(a => new SymbolsAsset() { name = a.Key, + hash = a.Value, cache = GetCacheControl(a.Key, resources) }).ToList(); diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonData.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonData.cs index 7fd50b76f0e224..dd26c45807d2da 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonData.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/BootJsonData.cs @@ -374,6 +374,7 @@ public class JsAsset public class SymbolsAsset { public string name { get; set; } + public string hash { get; set; } public string cache { get; set; } }