Optimization of layer reading/saving

src/Perpetuum/IO/FileSystem.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.IO;
+using System.Security.Cryptography;
 
 namespace Perpetuum.IO
 {
@@ -37,6 +38,53 @@ public void WriteAllBytes(string path, byte[] bytes)
             File.WriteAllBytes(CreatePath(path),bytes);
         }
 
+        public byte[] WriteAllBytesAndMD5(string path, ReadOnlySpan<byte> bytes, int size)
+        {
+            // Allocate the resulting array to return. MD5 hash is always 16 bytes.
+            // This is the only managed allocation in the entire method.
+            byte[] finalHash = new byte[16];
+
+            // Initialize incremental hashing without creating heavy managed objects
+            using (var incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.MD5))
+            // Open the file handle directly via the OS to bypass FileStream overhead
+            using (var handle = File.OpenHandle(CreatePath(path), FileMode.Create, FileAccess.Write, FileShare.None))
+            {
+                // Allocate a local buffer on the thread's stack (isolated from other threads).
+                // x4096 bytes is an optimal size aligned with most OS disk sector sizes.
+                Span<byte> stackBuffer = stackalloc byte[4096 * size];
+
+                int position = 0;
+                int remaining = bytes.Length;
+
+                while (remaining > 0)
+                {
+                    int chunkSize = Math.Min(remaining, stackBuffer.Length);
+
+                    // ATOMIC SNAPSHOT: Copy data from the shared memory into the protected stack buffer.
+                    // Even if other threads modify sharedBytes concurrently, stackBuffer 
+                    // captures a stable point-in-time snapshot of this specific chunk.
+                    bytes.Slice(position, chunkSize).CopyTo(stackBuffer);
+
+                    // Work strictly with the isolated snapshot
+                    var currentChunk = stackBuffer.Slice(0, chunkSize);
+
+                    // Write the exact captured snapshot to the disk
+                    RandomAccess.Write(handle, currentChunk, position);
+
+                    // Feed the exact same captured snapshot into the hash engine
+                    incrementalHash.AppendData(currentChunk);
+
+                    position += chunkSize;
+                    remaining -= chunkSize;
+                }
+
+                // Finalize the hash computation directly into the allocated return array
+                incrementalHash.GetCurrentHash(finalHash);
+            }
+
+            return finalHash;
+        }
+
         public void WriteAllLines(string path, IEnumerable<string> lines)
         {
             File.WriteAllLines(CreatePath(path), lines);
@@ -82,5 +130,19 @@ public override string ToString()
         {
             return $"Root: {_root}";
         }
+
+        public byte[] MD5SUM(string path)
+        {
+            // We use the hash algorithm MD5
+            using (var md5 = MD5.Create())
+            {
+                // Using a file stream for reading to calculate the hash
+                using (var stream = File.OpenRead(CreatePath(path)))
+                {
+                    // The method itself will read the entire stream to the end
+                    return md5.ComputeHash(stream);
+                }
+            }
+        }
     }
-}
+}

src/Perpetuum/IO/FileSystemExtensions.cs

@@ -1,19 +1,96 @@
+using System;
 using System.IO;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 namespace Perpetuum.IO
 {
     public static class FileSystemExtensions
     {
         public static T[] ReadLayer<T>(this IFileSystem fileSystem, string filename) where T : struct
         {
-            return fileSystem.ReadAllBytes(CreateLayerPath(filename)).ToArray<T>();
+            var path = fileSystem.CreatePath(CreateLayerPath(filename));
+
+            // Open the stream and read data directly into a T[] to avoid an extra intermediate byte[]
+            using var stream = File.OpenRead(path);
+
+            var sizeOfT = Marshal.SizeOf<T>();
+            if (sizeOfT <= 0)
+                return Array.Empty<T>();
+
+            long length = stream.Length;
+            if (length == 0)
+                return Array.Empty<T>();
+
+            int count = (int)(length / sizeOfT);
+
+            // If the file size is not a multiple of sizeof(T), fall back to the previous byte[] reading logic
+            if (length % sizeOfT != 0)
+            {
+                var bytes = fileSystem.ReadAllBytes(CreateLayerPath(filename));
+                return bytes.ToArray<T>();
+            }
+
+            var result = new T[count];
+
+            // For blittable types we can read directly into the byte representation of the T[]
+            if (!RuntimeHelpers.IsReferenceOrContainsReferences<T>())
+            {
+                var span = MemoryMarshal.AsBytes(result.AsSpan());
+
+                int read = 0;
+                while (read < span.Length)
+                {
+                    int n = stream.Read(span.Slice(read));
+                    if (n == 0)
+                        break;
+                    read += n;
+                }
+
+                if (read != span.Length)
+                {
+                    // Read did not complete as expected; fall back to the safe byte[] path
+                    var bytes = fileSystem.ReadAllBytes(CreateLayerPath(filename));
+                    return bytes.ToArray<T>();
+                }
+
+                return result;
+            }
+
+            // For non-blittable structs (with reference fields) read into a byte buffer and convert
+            var buffer = fileSystem.ReadAllBytes(CreateLayerPath(filename));
+            return buffer.ToArray<T>();
         }
 
         public static byte[] ReadLayerAsByteArray(this IFileSystem fileSystem, string filename)
         {
             return fileSystem.ReadAllBytes(CreateLayerPath(filename));
         }
 
+        /// <summary>
+        /// Returns a hash for the layer file.
+        /// </summary>
+        /// <param name="fileSystem"></param>
+        /// <param name="filename">Layer filename</param>
+        /// <returns>16 bytes of hash</returns>
+        public static byte[] MD5(this IFileSystem fileSystem, string filename)
+        {
+            return fileSystem.MD5SUM(CreateLayerPath(filename));
+        }
+
+        /// <summary>
+        /// Writes all bytes to a file on disk, calculating the hash in parallel.
+        /// </summary>
+        /// <param name="fileSystem"></param>
+        /// <param name="filename">Layer filename</param>
+        /// <param name="bytes">Layer data as bytes</param>
+        /// <param name="size">Size fo saved type</param>
+        /// <returns>16 bytes of hash</returns>
+        public static byte[] WriteLayerAndMD5(this IFileSystem fileSystem, string filename, ReadOnlySpan<byte> bytes, int size)
+        {
+            return fileSystem.WriteAllBytesAndMD5(CreateLayerPath(filename), bytes, size);
+        }
+
         private static string CreateLayerPath(string filename)
         {
             return Path.Combine("layers", filename);

src/Perpetuum/IO/IFileSystem.cs

@@ -10,6 +10,15 @@ public interface IFileSystem
         string[] ReadAllLines(string path);
 
         void WriteAllBytes(string path, byte[] bytes);
+
+        /// <summary>
+        /// Writes all bytes to a file on disk, calculating the hash in parallel.
+        /// </summary>
+        /// <param name="path">Path within the game's root directory</param>
+        /// <param name="bytes">Bytes to write</param>
+        /// <param name="size">Size fo saved type</param>
+        /// <returns>MD5 hash</returns>
+        public byte[] WriteAllBytesAndMD5(string path, ReadOnlySpan<byte> bytes, int size);
         void WriteAllLines(string path,IEnumerable<string> lines);
 
         void AppendAllText(string path, string text);
@@ -21,6 +30,13 @@ public interface IFileSystem
 
         string CreatePath(string path);
 
+        /// <summary>
+        /// Calculates the hash using the MD5 algorithm
+        /// </summary>
+        /// <param name="path">Path within the game's root directory</param>
+        /// <returns>16 bytes of hash</returns>
+        byte[] MD5SUM(string path);
+
         IEnumerable<string> GetFiles(string path, string mask);
     }
-}
+}

src/Perpetuum/Zones/ZoneExtensions.cs

@@ -8,7 +8,7 @@
 using Perpetuum.Zones.RemoteControl;
 using Perpetuum.Zones.Terrains;
 using System.Drawing;
-using System.Security.Cryptography;
+using System.Runtime.InteropServices;
 
 namespace Perpetuum.Zones
 {
@@ -46,14 +46,14 @@ public T[] LoadLayerData<T>(IZone zone, string name) where T : struct
         public void SaveLayerToDisk<T>(IZone zone, ILayer<T> layer) where T : struct
         {
             string baseFilename = zone.CreateTerrainDataFilename(layer.LayerType.ToString().ToLower(), "");
-
-            using MD5 md5 = MD5.Create();
             string tmpFn = baseFilename + "tmp" + DateTime.Now.Ticks + ".bin";
-            byte[] layerData = layer.RawData.ToByteArray();
-            _fileSystem.WriteLayer(tmpFn, layerData);
+
+            ReadOnlySpan<byte> layerData = MemoryMarshal.AsBytes(layer.RawData.AsSpan());
+            var hash = _fileSystem.WriteLayerAndMD5(tmpFn, layerData, layer.SizeInBytes);
 
-            if (!md5.ComputeHash(layerData).SequenceEqual(md5.ComputeHash(_fileSystem.ReadLayerAsByteArray(tmpFn))))
+            if (!hash.SequenceEqual(_fileSystem.MD5(tmpFn)))
             {
+                Logger.Error("Layer not saved. (" + baseFilename + ")");
                 return;
             }