diff --git a/Archive/ArchiveTool.cs b/Archive/ArchiveTool.cs index 60212e8..7c07d4d 100644 --- a/Archive/ArchiveTool.cs +++ b/Archive/ArchiveTool.cs @@ -163,6 +163,16 @@ public static int PrintSummary(FileInfo filename, OutputFormat format) uncompressedDataSize += block.UncompressedSize; } + // The data section can be larger than the sum of the blocks, because of the alignment + // padding that separates chunk-based blocks (the BlockPaddingBetweenChunks flag). + long paddingSize = 0; + if (blocks.Length > 0) + { + var lastBlock = blocks[blocks.Length - 1]; + long dataSectionSize = lastBlock.FileOffset + lastBlock.CompressedSize - ArchiveDetector.GetDataOffset(header); + paddingSize = dataSectionSize - dataSize; + } + // Determine the compression algorithm by finding the first block that uses compression. // Individual blocks may be stored uncompressed even when compression is enabled, because // compression is skipped when it provides no size reduction. So the first compressed block @@ -193,6 +203,7 @@ public static int PrintSummary(FileInfo filename, OutputFormat format) unityVersion = header.UnityVersion, fileSize = header.Size, dataSize = dataSize, + blockPaddingSize = paddingSize, uncompressedDataSize = uncompressedDataSize, compressionRatio = Math.Round(compressionRatio, 2), compression = compression, @@ -208,6 +219,8 @@ public static int PrintSummary(FileInfo filename, OutputFormat format) Console.WriteLine($"{"Unity Version",-30} {header.UnityVersion}"); Console.WriteLine($"{"File Size",-30} {header.Size:N0} bytes"); Console.WriteLine($"{"Data Size",-30} {dataSize:N0} bytes"); + if (paddingSize > 0) + Console.WriteLine($"{"Block Padding Size",-30} {paddingSize:N0} bytes"); Console.WriteLine($"{"Uncompressed Data Size",-30} {uncompressedDataSize:N0} bytes"); Console.WriteLine($"{"Compression Ratio",-30} {compressionRatio:F2}x"); Console.WriteLine($"{"Compression",-30} {compression}"); @@ -263,10 +276,11 @@ static string FormatCompressionType(int compressionType) static readonly (uint bit, string name)[] KnownArchiveFlags = { - (0x40, "BlocksAndDirectoryInfoCombined"), - (0x80, "BlocksInfoAtTheEnd"), - (0x100, "OldWebPluginCompatibility"), - (0x200, "BlockInfoNeedPaddingAtStart"), + (ArchiveFlags.BlocksAndDirectoryInfoCombined, "BlocksAndDirectoryInfoCombined"), + (ArchiveFlags.BlocksInfoAtTheEnd, "BlocksInfoAtTheEnd"), + (ArchiveFlags.OldWebPluginCompatibility, "OldWebPluginCompatibility"), + (ArchiveFlags.BlockInfoNeedPaddingAtStart, "BlockInfoNeedPaddingAtStart"), + (ArchiveFlags.BlockPaddingBetweenChunks, "BlockPaddingBetweenChunks"), }; static string[] GetArchiveFlagNames(uint flagBits) diff --git a/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md b/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md index 977dfac..fc46e57 100644 --- a/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md +++ b/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md @@ -6,7 +6,8 @@ The LeadingEdge build scripts regenerate this folder directly, so to update it, ## Layout -* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle. +* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle. LZMA compressed, so each archive has a single data block. +* `AssetBundlesLz4/` - the same bundle layout built with chunk-based (LZ4) compression, giving archives with multiple data blocks separated by alignment padding (archive format version 9). * `ContentDirectory/` - the Content Directory build: content (`.cf`) files, `.resource` files and the build manifest. * `BuildReport-AssetBundles/LastBuild.buildreport` - the AssetBundle build report. * `BuildReport-ContentDirectory/` - the Content Directory build report folder, including `ContentLayout.json`. diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 new file mode 100644 index 0000000..c57ca2d Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest new file mode 100644 index 0000000..6cff95f --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest @@ -0,0 +1,21 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 1054287113 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: ef4548464eae98b6ab651c92a869edc9 + TypeTreeHash: + serializedVersion: 2 + Hash: 9a2ca7bdbd1871f7131daf57de908e0c + IncrementalBuildHash: + serializedVersion: 2 + Hash: 818e83165880f042a3252a52b336d06a +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: [] +Assets: +- Assets/Audio/6.mp3 +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4 b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4 new file mode 100644 index 0000000..51ef32d Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4 differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest new file mode 100644 index 0000000..4ba47c4 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest @@ -0,0 +1,33 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 2935431312 +HashAppended: 0 +AssetBundleManifest: + AssetBundleInfos: + Info_0: + Name: assetbundleroot + Dependencies: + Dependency_0: directaudioclipreference + Dependency_1: singleaudioclipdirectreference + Dependency_2: serializationdemo + Info_1: + Name: directaudioclipreference + Dependencies: + Dependency_0: 6 + Dependency_1: a + Info_2: + Name: singleaudioclipdirectreference + Dependencies: + Dependency_0: a + Info_3: + Name: serializationdemo + Dependencies: {} + Info_4: + Name: 6 + Dependencies: {} + Info_5: + Name: a + Dependencies: {} + Info_6: + Name: scenes + Dependencies: {} diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a new file mode 100644 index 0000000..de585bd Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a.manifest new file mode 100644 index 0000000..a6ccc50 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a.manifest @@ -0,0 +1,21 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 1988020392 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 6175fa89f0b4cfd7cc433b0b143ab713 + TypeTreeHash: + serializedVersion: 2 + Hash: 9a2ca7bdbd1871f7131daf57de908e0c + IncrementalBuildHash: + serializedVersion: 2 + Hash: ac9603f3b25ca1814eb4841392a9ff7e +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: [] +Assets: +- Assets/Audio/a.mp3 +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot new file mode 100644 index 0000000..97ba4b5 Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot.manifest new file mode 100644 index 0000000..c108ccd --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot.manifest @@ -0,0 +1,34 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 4203871746 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 8ae6ceb771f9c15f2dde5117d3f67798 + TypeTreeHash: + serializedVersion: 2 + Hash: e4c92e7ce30487f41e62ec475c399971 + IncrementalBuildHash: + serializedVersion: 2 + Hash: 00f65fc945d785da9075745396600fe2 +HashAppended: 0 +ClassTypes: +- Class: 114 + Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3} +- Class: 114 + Script: {fileID: 11500000, guid: 8623c5efbb626994da80931050f0aba0, type: 3} +- Class: 114 + Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: Assembly-CSharp + ClassName: SerializationDemo/SerializedData +- AssemblyName: UnityEngine.CoreModule + ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2 +Assets: +- Assets/ScriptableObjects/AssetBundleRoot.asset +Dependencies: +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference new file mode 100644 index 0000000..ae08636 Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference.manifest new file mode 100644 index 0000000..aa804ca --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference.manifest @@ -0,0 +1,29 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 3469506813 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 5fd4865f81b71364e957dc7644afd47c + TypeTreeHash: + serializedVersion: 2 + Hash: 4c88c857f41d6968a1090f557f31c5ed + IncrementalBuildHash: + serializedVersion: 2 + Hash: 015f1d03897964d24d88747ea37642b1 +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +- Class: 114 + Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: UnityEngine.CoreModule + ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2 +Assets: +- Assets/ScriptableObjects/DirectAudioClipReference.asset +Dependencies: +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes new file mode 100644 index 0000000..6d70b4e Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes.manifest new file mode 100644 index 0000000..6491976 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes.manifest @@ -0,0 +1,42 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 357273879 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 531bc5793378ad5f958e829f6098afdf + TypeTreeHash: + serializedVersion: 2 + Hash: 62ed4bb4a9e4f0abaa38cfea31ee7bca + IncrementalBuildHash: + serializedVersion: 2 + Hash: 259ca50b232fd1006951e68b47ff4a66 +HashAppended: 0 +ClassTypes: +- Class: 1 + Script: {instanceID: 0} +- Class: 4 + Script: {instanceID: 0} +- Class: 21 + Script: {instanceID: 0} +- Class: 28 + Script: {instanceID: 0} +- Class: 48 + Script: {instanceID: 0} +- Class: 89 + Script: {instanceID: 0} +- Class: 104 + Script: {instanceID: 0} +- Class: 157 + Script: {instanceID: 0} +- Class: 196 + Script: {instanceID: 0} +- Class: 212 + Script: {instanceID: 0} +- Class: 213 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: [] +Assets: +- Assets/Scenes/Scene1.unity +- Assets/Scenes/Scene2.unity +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo new file mode 100644 index 0000000..5a9b97e Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo.manifest new file mode 100644 index 0000000..28226b9 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo.manifest @@ -0,0 +1,25 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 3418023709 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 8787e29fd3c31264a27ae0c18c07b815 + TypeTreeHash: + serializedVersion: 2 + Hash: d0726b8e1199eb6125e2245ce15343f9 + IncrementalBuildHash: + serializedVersion: 2 + Hash: 1393a4a257b5b48e78eead885e48b68d +HashAppended: 0 +ClassTypes: +- Class: 114 + Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: Assembly-CSharp + ClassName: SerializationDemo/SerializedData +Assets: +- Assets/ScriptableObjects/SerializationDemo.asset +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference new file mode 100644 index 0000000..75a364b Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference.manifest new file mode 100644 index 0000000..8e73254 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference.manifest @@ -0,0 +1,28 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0a6 +CRC: 1388156890 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: f3732a953b4cba83b6943cfa13225c56 + TypeTreeHash: + serializedVersion: 2 + Hash: 4c88c857f41d6968a1090f557f31c5ed + IncrementalBuildHash: + serializedVersion: 2 + Hash: 57215df3b8d2b5c11e9501a26ababda4 +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +- Class: 114 + Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: UnityEngine.CoreModule + ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2 +Assets: +- Assets/ScriptableObjects/SingleAudioClipDirectReference.asset +Dependencies: +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a diff --git a/UnityBinaryFormat/ArchiveDetector.cs b/UnityBinaryFormat/ArchiveDetector.cs index 9deb80f..1998887 100644 --- a/UnityBinaryFormat/ArchiveDetector.cs +++ b/UnityBinaryFormat/ArchiveDetector.cs @@ -16,8 +16,10 @@ namespace UnityDataTools.BinaryFormat; /// - Data: One or more blocks of file content. Each block has its own compression type /// recorded in its per-block flags. The metadata section is required to interpret the data. /// A single file can span multiple blocks, and a single block can contain data for multiple files. -/// The blocks account for every byte of the data (there are no offsets stored - no overlapping or -/// gaps can be expressed). However the files could have padding between them. +/// The blocks account for every byte of the uncompressed data (there are no offsets stored - no +/// overlapping or gaps can be expressed). However the files could have padding between them, and +/// when the BlockPaddingBetweenChunks flag is set the on-disk blocks are separated by alignment +/// padding that belongs to no block. /// /// The metadata can appear directly after the header (default layout) or at the end of the /// file after the data (indicated by the BlocksInfoAtTheEnd flag). @@ -47,6 +49,25 @@ public class ArchiveHeaderInfo /// Archive flag bits (bits 6+ of Flags), with compression bits masked out. /// public uint ArchiveFlagBits => Flags & ~0x3Fu; + + /// + /// True when each chunk-based (non-streamed) block is followed by padding up to the next + /// 16-byte boundary of the data section. Introduced with archive format version 9. + /// + public bool HasBlockPaddingBetweenChunks => (Flags & ArchiveFlags.BlockPaddingBetweenChunks) != 0; +} + +/// +/// Archive header flag bits (bits 6 and up of the header Flags field). Bits 0-5 hold the +/// metadata CompressionType instead. +/// +public static class ArchiveFlags +{ + public const uint BlocksAndDirectoryInfoCombined = 0x40; + public const uint BlocksInfoAtTheEnd = 0x80; + public const uint OldWebPluginCompatibility = 0x100; + public const uint BlockInfoNeedPaddingAtStart = 0x200; + public const uint BlockPaddingBetweenChunks = 0x400; } public class ArchiveStorageBlock @@ -276,10 +297,7 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea metadata = null; errorMessage = null; - const uint flagBlocksAndDirectoryInfoCombined = 0x40; - const uint flagBlocksInfoAtTheEnd = 0x80; - - if ((header.ArchiveFlagBits & flagBlocksAndDirectoryInfoCombined) == 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlocksAndDirectoryInfoCombined) == 0) { errorMessage = "This archive does not use the combined BlocksInfo+DirectoryInfo layout. Only the combined layout is supported."; return false; @@ -291,7 +309,7 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea // Calculate where the metadata section starts. long metadataOffset; - if ((header.ArchiveFlagBits & flagBlocksInfoAtTheEnd) != 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlocksInfoAtTheEnd) != 0) metadataOffset = (long)(header.Size - header.CompressedMetadataSize); else metadataOffset = GetHeaderSize(header); @@ -337,14 +355,22 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea var blocksInfo = ParseBlocksInfo(reader); var directoryInfo = ParseDirectoryInfo(reader); - // Populate calculated offsets on each block. - long fileOffset = GetDataOffset(header); + // Populate calculated offsets on each block. Block positions are not stored, so they are + // accumulated from the block sizes. When BlockPaddingBetweenChunks is set, each chunk-based + // block is followed by padding that belongs to no block, so the running offset is aligned + // after each such block. The alignment is relative to the start of the data section, which + // is itself 16-byte aligned in every version that can set this flag, so the blocks are also + // aligned within the file - that is what keeps chunk positions stable for binary patching. + long dataSectionStart = GetDataOffset(header); + long blockOffset = 0; long dataOffset = 0; foreach (var block in blocksInfo.Blocks) { - block.FileOffset = fileOffset; + block.FileOffset = dataSectionStart + blockOffset; block.DataOffset = dataOffset; - fileOffset += block.CompressedSize; + blockOffset += block.CompressedSize; + if (header.HasBlockPaddingBetweenChunks && !block.IsStreamed) + blockOffset = AlignTo16(blockOffset); dataOffset += block.UncompressedSize; } @@ -371,14 +397,11 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea /// public static long GetDataOffset(ArchiveHeaderInfo header) { - const uint flagBlocksInfoAtTheEnd = 0x80; - const uint flagBlockInfoNeedPaddingAtStart = 0x200; - long offset = GetHeaderSize(header); - if ((header.ArchiveFlagBits & flagBlocksInfoAtTheEnd) == 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlocksInfoAtTheEnd) == 0) { - if ((header.ArchiveFlagBits & flagBlockInfoNeedPaddingAtStart) != 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlockInfoNeedPaddingAtStart) != 0) offset += AlignTo16(header.CompressedMetadataSize); else offset += header.CompressedMetadataSize; @@ -430,10 +453,8 @@ static void ValidateMetadata(ArchiveBlocksInfo blocksInfo, ArchiveDirectoryInfo static int GetHeaderSize(ArchiveHeaderInfo header) { - const uint flagOldWebPluginCompatibility = 0x100; - int size; - if ((header.ArchiveFlagBits & flagOldWebPluginCompatibility) != 0) + if ((header.ArchiveFlagBits & ArchiveFlags.OldWebPluginCompatibility) != 0) size = 10; // Legacy web plugin signature portion else size = header.Signature.Length + 1; @@ -452,7 +473,7 @@ static int GetHeaderSize(ArchiveHeaderInfo header) return size; } - static long AlignTo16(uint value) + static long AlignTo16(long value) { return (value + 15) & ~15L; } diff --git a/UnityDataTool.Tests/ArchiveTests.cs b/UnityDataTool.Tests/ArchiveTests.cs index b21d7de..5d1eff7 100644 --- a/UnityDataTool.Tests/ArchiveTests.cs +++ b/UnityDataTool.Tests/ArchiveTests.cs @@ -17,12 +17,17 @@ public class ArchiveTests private string m_TestDataFolder; private string m_ArchivePath; + // An LZ4 (chunk-based) archive in format version 9, where the BlockPaddingBetweenChunks flag + // means the blocks are separated by alignment padding on disk. + private string m_PaddedArchivePath; + [OneTimeSetUp] public void OneTimeSetup() { m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "test_folder"); m_TestDataFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data"); m_ArchivePath = Path.Combine(m_TestDataFolder, "AssetBundles", "2023.1.0a16", "scenes"); + m_PaddedArchivePath = Path.Combine(m_TestDataFolder, "LeadingEdgeBuilds", "AssetBundlesLz4", "scenes"); Directory.CreateDirectory(m_TestOutputFolder); Directory.SetCurrentDirectory(m_TestOutputFolder); } @@ -389,4 +394,109 @@ public async Task ArchiveExtract_WithFilter_ExtractsOnlyMatchingFiles() Assert.IsFalse(File.Exists(Path.Combine(m_TestOutputFolder, "archive", file)), $"File should not have been extracted: {file}"); } } + + [Test] + public async Task ArchiveHeader_BlockPaddingBetweenChunks_ReportsVersion9AndFlag() + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "header", m_PaddedArchivePath, "-f", "Json" })); + + var json = JsonDocument.Parse(sw.ToString()).RootElement; + + Assert.AreEqual(9u, json.GetProperty("version").GetUInt32()); + + var flags = json.GetProperty("flags").EnumerateArray().Select(f => f.GetString()).ToArray(); + Assert.That(flags, Does.Contain("BlockPaddingBetweenChunks")); + Assert.That(flags, Does.Not.Contain("0x400"), "The padding flag bit should be recognized, not reported as a raw hex value."); + } + finally + { + Console.SetOut(currentOut); + } + } + + // The block offsets are not stored in the archive, they are accumulated from the block sizes. + // With BlockPaddingBetweenChunks each chunk-based block is followed by padding belonging to no + // block, so the accumulated offset has to be aligned after every such block. This verifies the + // computed offsets against the actual file: the gaps between the blocks must be alignment + // padding (all zero bytes) and never larger than the alignment. + [Test] + public async Task ArchiveBlocks_BlockPaddingBetweenChunks_OffsetsSkipThePadding() + { + const int alignment = 16; + + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "blocks", m_PaddedArchivePath, "-f", "Json" })); + + var blocks = JsonDocument.Parse(sw.ToString()).RootElement.GetProperty("blocks").EnumerateArray().ToArray(); + Assert.Greater(blocks.Length, 1, "The test archive is expected to have several chunks."); + + var archiveBytes = File.ReadAllBytes(m_PaddedArchivePath); + var dataSectionStart = blocks[0].GetProperty("fileOffset").GetInt64(); + + for (int i = 1; i < blocks.Length; i++) + { + var previousEnd = blocks[i - 1].GetProperty("fileOffset").GetInt64() + + blocks[i - 1].GetProperty("compressedSize").GetInt64(); + var offset = blocks[i].GetProperty("fileOffset").GetInt64(); + + Assert.AreEqual(0, (offset - dataSectionStart) % alignment, + $"Block {i} does not start on an alignment boundary of the data section."); + Assert.That(offset - previousEnd, Is.InRange(0, alignment - 1), + $"The gap before block {i} is not a plausible amount of alignment padding."); + + for (var p = previousEnd; p < offset; p++) + Assert.AreEqual(0, archiveBytes[p], $"Padding byte at offset {p} is not zero."); + } + } + finally + { + Console.SetOut(currentOut); + } + } + + [Test] + public async Task ArchiveInfo_BlockPaddingBetweenChunks_ReportsPaddingSize() + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "info", m_PaddedArchivePath, "-f", "Json" })); + + var json = JsonDocument.Parse(sw.ToString()).RootElement; + + Assert.AreEqual("Lz4HC", json.GetProperty("compression").GetString()); + Assert.Greater(json.GetProperty("blockPaddingSize").GetInt64(), 0); + } + finally + { + Console.SetOut(currentOut); + } + } + + // The native library computes the block offsets independently of the C# parser, so successful + // extraction of content spanning several padded chunks confirms the layout is understood. + [Test] + public async Task ArchiveExtract_BlockPaddingBetweenChunks_FilesExtractedSuccessfully() + { + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "extract", m_PaddedArchivePath })); + + // BuildPlayer-Scene1.sharedAssets is large enough to span several chunks. + var extractedFile = new FileInfo(Path.Combine(m_TestOutputFolder, "archive", "BuildPlayer-Scene1.sharedAssets")); + Assert.IsTrue(extractedFile.Exists, "Expected file not found: BuildPlayer-Scene1.sharedAssets"); + Assert.AreEqual(760788, extractedFile.Length); + } } diff --git a/UnityProjects/LeadingEdge/AGENTS.md b/UnityProjects/LeadingEdge/AGENTS.md index def6bca..7c73503 100644 --- a/UnityProjects/LeadingEdge/AGENTS.md +++ b/UnityProjects/LeadingEdge/AGENTS.md @@ -15,7 +15,7 @@ Both builds start from a root ScriptableObject whose serialized dictionary maps ## Editor scripts (`Assets/Editor`, `ContentDirectory` menu) * `GenerateAssets.cs` - creates the ScriptableObject assets in `Assets/ScriptableObjects`, populating the serialized dictionaries before saving so the entries are serialized into the assets. -* `BuildAssetBundles.cs` - runs the AssetBundle build and copies its build report. +* `BuildAssetBundles.cs` - runs the AssetBundle build and copies its build report. It has a second entry point, `BuildLz4`, that repeats the build with chunk-based (LZ4) compression into a separate folder. * `BuildContentDirectory.cs` - runs the Content Directory build and copies its build report folder. Both build scripts write directly into `TestCommon/Data/LeadingEdgeBuilds` using paths relative to the project root. @@ -27,3 +27,5 @@ The root asset is `ContentDirectoryRoot.asset`. It directly references the `Load ## AssetBundle Build The root asset is `AssetBundleRoot.asset`. AssetBundles do not support `Loadable`, so this build uses the direct-reference variants of the assets instead. Each asset is placed in its own bundle (named after the asset) - a highly granular layout that guarantees no content is duplicated across bundles. The two scenes are placed together in a `scenes` bundle, since AssetBundles require scenes and assets in separate bundles. + +The default build uses LZMA, which produces a single streamed data block per archive. The `Build AssetBundles (LZ4)` menu item repeats the same bundle layout with chunk-based compression, which produces archives with many small blocks - the layout that exercises the alignment padding between chunks added in archive format version 9. diff --git a/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs b/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs index 68a023b..ea4f529 100644 --- a/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs +++ b/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs @@ -8,6 +8,10 @@ // Assets/Audio. This highly granular layout ensures no asset is duplicated across bundles - shared assets // (e.g. a.mp3) live in a single bundle that the others depend on. A scene bundle holds the two scenes (AssetBundles // require scenes and assets in separate bundles). The build report is copied alongside the output. +// +// Two variants are produced into separate folders: the default LZMA build (a single streamed data block) and an +// LZ4 chunk-based build (many small blocks), which is the layout that exercises the alignment padding between +// chunks added in archive format version 9. public static class BuildAssetBundles { const string AudioFolder = "Assets/Audio"; @@ -16,6 +20,7 @@ public static class BuildAssetBundles const string TestDataFolder = "../../TestCommon/Data/LeadingEdgeBuilds"; const string OutputFolder = TestDataFolder + "/AssetBundles"; const string BuildReportFolder = TestDataFolder + "/BuildReport-AssetBundles"; + const string Lz4OutputFolder = TestDataFolder + "/AssetBundlesLz4"; static readonly string[] DirectAssets = { @@ -28,7 +33,19 @@ public static class BuildAssetBundles [MenuItem("ContentDirectory/Build AssetBundles")] public static void Build() { - Directory.CreateDirectory(OutputFolder); + Build(BuildAssetBundleOptions.None, OutputFolder, BuildReportFolder); + } + + [MenuItem("ContentDirectory/Build AssetBundles (LZ4)")] + public static void BuildLz4() + { + Build(BuildAssetBundleOptions.ChunkBasedCompression, Lz4OutputFolder, null); + } + + // buildReportFolder may be null to skip copying the report. + static void Build(BuildAssetBundleOptions options, string outputFolder, string buildReportFolder) + { + Directory.CreateDirectory(outputFolder); var bundles = new List(); @@ -59,9 +76,9 @@ public static void Build() var parameters = new BuildAssetBundlesParameters { - outputPath = OutputFolder, + outputPath = outputFolder, bundleDefinitions = bundles.ToArray(), - options = BuildAssetBundleOptions.None, + options = options, targetPlatform = EditorUserBuildSettings.activeBuildTarget }; @@ -72,9 +89,12 @@ public static void Build() return; } - Directory.CreateDirectory(BuildReportFolder); - File.Copy("Library/LastBuild.buildreport", $"{BuildReportFolder}/LastBuild.buildreport", true); + if (buildReportFolder != null) + { + Directory.CreateDirectory(buildReportFolder); + File.Copy("Library/LastBuild.buildreport", $"{buildReportFolder}/LastBuild.buildreport", true); + } - Debug.Log($"BuildAssetBundles: built {manifest.GetAllAssetBundles().Length} bundles into {OutputFolder}."); + Debug.Log($"BuildAssetBundles: built {manifest.GetAllAssetBundles().Length} bundles into {outputFolder}."); } }