From fca8e45fe1eff02d5c8ab323a5eccd6f1b74d56e Mon Sep 17 00:00:00 2001 From: Angel Date: Fri, 4 Sep 2026 13:57:59 +0200 Subject: [PATCH 01/16] Read array attributes in bulk (cherry picked from commit 83c1d29b61fa624c48f47eded05b0f87efb46cd5) --- Datamodel.NET/Arrays.cs | 10 ++++++++++ Datamodel.NET/Codecs/Binary.cs | 30 +++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Datamodel.NET/Arrays.cs b/Datamodel.NET/Arrays.cs index b2dd2bf..7db05f4 100644 --- a/Datamodel.NET/Arrays.cs +++ b/Datamodel.NET/Arrays.cs @@ -57,6 +57,16 @@ internal Array(int capacity) public void AddRange(IEnumerable items) => Inner.AddRange(items); + /// + /// Appends default items and returns them for the caller to fill, so that a codec can read value types in bulk. + /// + internal Span AppendUninitialized(int count) + { + var start = Inner.Count; + System.Runtime.InteropServices.CollectionsMarshal.SetCount(Inner, start + count); + return System.Runtime.InteropServices.CollectionsMarshal.AsSpan(Inner).Slice(start, count); + } + public void RemoveAt(int index) => Inner.RemoveAt(index); public virtual T this[int index] diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index bb5415d..ac3bf23 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -505,14 +505,42 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in var count = reader.ReadInt32(); var array = CodecUtilities.MakeList(type, count); + // value types whose memory layout matches the stream are copied into the list in one read, instead of one boxed item at a time + if (BitConverter.IsLittleEndian && count > 0) + { + switch (array) + { + case IntArray ints: ReadItems(ints, count, reader); return array; + case FloatArray floats: ReadItems(floats, count, reader); return array; + case BoolArray bools: ReadItems(bools, count, reader); return array; + case Vector2Array vectors: ReadItems(vectors, count, reader); return array; + case Vector3Array vectors: ReadItems(vectors, count, reader); return array; + case Vector4Array vectors: ReadItems(vectors, count, reader); return array; + case QuaternionArray quaternions: ReadItems(quaternions, count, reader); return array; + case MatrixArray matrices: ReadItems(matrices, count, reader); return array; + case ColorArray colors: ReadItems(colors, count, reader); return array; + case ByteArray bytes: ReadItems(bytes, count, reader); return array; + case UInt64Array ulongs: ReadItems(ulongs, count, reader); return array; + } + } + var typeId = TypeMap[type.TypeHandle]; - foreach (var x in Enumerable.Range(0, count)) + for (var i = 0; i < count; i++) array.Add(ReadValue(dm, typeId, true, reader)); return array; } } + /// + /// Reads items straight into the list's storage. Only for types stored in the stream exactly as in memory. + /// + static void ReadItems(Array array, int count, BinaryReader reader) where T : unmanaged + { + var bytes = System.Runtime.InteropServices.MemoryMarshal.AsBytes(array.AppendUninitialized(count)); + reader.BaseStream.ReadExactly(bytes); + } + void SkipAttribute(BinaryReader reader) { var (type, isArray) = IdToType(reader.ReadByte()); From 0c7301c21b2f81ff568f4fc0b7db836854eccf02 Mon Sep 17 00:00:00 2001 From: Angel Date: Fri, 4 Sep 2026 13:57:59 +0200 Subject: [PATCH 02/16] Store elements in a dictionary and a list (cherry picked from commit bf8e1668f312bd8296aeb8fcfc2b1ba20ded2f53) --- Datamodel.NET/Datamodel.ElementList.cs | 180 ++++++++++--------------- Datamodel.NET/Element.cs | 11 +- 2 files changed, 71 insertions(+), 120 deletions(-) diff --git a/Datamodel.NET/Datamodel.ElementList.cs b/Datamodel.NET/Datamodel.ElementList.cs index ee612e2..47eef91 100644 --- a/Datamodel.NET/Datamodel.ElementList.cs +++ b/Datamodel.NET/Datamodel.ElementList.cs @@ -4,7 +4,6 @@ using System.Collections.Specialized; using System.Diagnostics; using System.Linq; -using System.Threading; namespace Datamodel { @@ -17,7 +16,7 @@ public partial class Datamodel [DebuggerTypeProxy(typeof(DebugView))] public class ElementList : IEnumerable, INotifyCollectionChanged, IDisposable { - internal ReaderWriterLockSlim ChangeLock = new(LockRecursionPolicy.SupportsRecursion); + internal readonly object ChangeLock = new(); internal class DebugView { @@ -29,10 +28,12 @@ public DebugView(ElementList item) private readonly ElementList Item; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - public Element[] Elements => Item.store.Values.Cast().ToArray(); + public Element[] Elements => [.. Item.order]; } - private readonly OrderedDictionary store = []; + // elements by ID, and in the order they were added, which codecs rely on + private readonly Dictionary byId = []; + private readonly List order = []; private readonly Datamodel Owner; internal ElementList(Datamodel owner) @@ -40,13 +41,16 @@ internal ElementList(Datamodel owner) Owner = owner; } + /// + /// Adds an Element owned by this list's Datamodel. The first Element added becomes the . + /// internal void Add(Element item) { - ChangeLock.EnterUpgradeableReadLock(); - try + bool first; + + lock (ChangeLock) { - Element? existing = (Element?)store[item.ID]; - if (existing != null && !existing.Stub) + if (byId.TryGetValue(item.ID, out var existing) && !existing.Stub) { throw new ElementIdException($"Element ID {item.ID} already in use in this Datamodel."); } @@ -55,27 +59,29 @@ internal void Add(Element item) if (item.Owner != this.Owner) throw new ElementOwnershipException("Cannot add an element from a different Datamodel. Use ImportElement() to create a local copy instead."); - ChangeLock.EnterWriteLock(); - try - { - if (existing != null) - store.Remove(existing.ID); + if (existing != null) + RemoveFromStore(existing); - store.Add(item.ID, item); - } - finally - { - ChangeLock.ExitWriteLock(); - } - } - finally - { - ChangeLock.ExitUpgradeableReadLock(); + byId.Add(item.ID, item); + order.Add(item); + first = order.Count == 1; } + if (first) + Owner.Root = item; + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item)); } + /// + /// Removes an Element from both stores. The caller holds . + /// + void RemoveFromStore(Element item) + { + byId.Remove(item.ID); + order.Remove(item); + } + /// /// Returns the at the specified index. /// @@ -86,12 +92,8 @@ public Element? this[int index] { get { - ChangeLock.EnterReadLock(); - try - { - return (Element?)store[index]; - } - finally { ChangeLock.ExitReadLock(); } + lock (ChangeLock) + return order[index]; } } @@ -104,12 +106,8 @@ public Element? this[Guid id] { get { - ChangeLock.EnterReadLock(); - try - { - return (Element?)store[id]; - } - finally { ChangeLock.ExitReadLock(); } + lock (ChangeLock) + return byId.TryGetValue(id, out var element) ? element : null; } } @@ -120,12 +118,8 @@ public int Count { get { - ChangeLock.EnterReadLock(); - try - { - return store.Count; - } - finally { ChangeLock.ExitReadLock(); } + lock (ChangeLock) + return order.Count; } } @@ -160,47 +154,36 @@ public bool Remove(Element item, RemoveMode mode) { ArgumentNullException.ThrowIfNull(item); - ChangeLock.EnterUpgradeableReadLock(); - try + lock (ChangeLock) { - if (store.Contains(item.ID)) + if (!byId.ContainsKey(item.ID)) + return false; + + RemoveFromStore(item); + Element? replacement = (mode == RemoveMode.MakeStubs) ? new Element(Owner, item.ID) : null; + + foreach (var elem in order) { - ChangeLock.EnterWriteLock(); - try + lock (elem.SyncRoot) { - store.Remove(item.ID); - Element? replacement = (mode == RemoveMode.MakeStubs) ? new Element(Owner, item.ID) : null; - - foreach (Element elem in store.Values) + foreach (var attr in elem.Where(a => a.Value == item).ToArray()) { - lock (elem.SyncRoot) - { - foreach (var attr in elem.Where(a => a.Value == item).ToArray()) - { - elem[attr.Key] = replacement; - } - - foreach (var array in elem.Select(a => a.Value).OfType>()) - for (int i = 0; i < array.Count; i++) - if (array[i] == item) - array[i] = replacement; - } + elem[attr.Key] = replacement; } - if (Owner.Root == item) Owner.Root = replacement; - item.Owner = null; - } - finally - { - ChangeLock.ExitWriteLock(); + foreach (var array in elem.Select(a => a.Value).OfType>()) + for (int i = 0; i < array.Count; i++) + if (array[i] == item) + array[i] = replacement; } - - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item)); - return true; } - else return false; + if (Owner.Root == item) Owner.Root = replacement; + + item.Owner = null; } - finally { ChangeLock.ExitUpgradeableReadLock(); } + + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item)); + return true; } /// @@ -208,21 +191,16 @@ public bool Remove(Element item, RemoveMode mode) /// internal void RemoveUnreferenced(Element item) { - ChangeLock.EnterWriteLock(); - try + lock (ChangeLock) { - if (!store.Contains(item.ID)) + if (!byId.ContainsKey(item.ID)) { return; } - store.Remove(item.ID); + RemoveFromStore(item); item.Owner = null; } - finally - { - ChangeLock.ExitWriteLock(); - } CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item)); } @@ -232,36 +210,20 @@ internal void RemoveUnreferenced(Element item) /// public void Trim() { - ChangeLock.EnterUpgradeableReadLock(); - try + lock (ChangeLock) { var used = new HashSet(); WalkElemTree(Owner.Root, used); - if (used.Count == Count) return; + if (used.Count == order.Count) return; - ChangeLock.EnterWriteLock(); - try + var removed = order.Where(elem => !used.Contains(elem)).ToArray(); + foreach (var elem in removed) { - var removed = this.Except(used).ToArray(); - foreach (var elem in removed) - { - if (elem != null) - { - store.Remove(elem.ID); - elem.Owner = null; - } - } - - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed)); + RemoveFromStore(elem); + elem.Owner = null; } - finally - { - ChangeLock.ExitWriteLock(); - } - } - finally - { - ChangeLock.ExitUpgradeableReadLock(); + + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed)); } } @@ -300,15 +262,14 @@ protected void WalkElemTree(Element? elem, HashSet found) #region Interfaces System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { - return store.GetEnumerator(); + return GetEnumerator(); } /// - /// Returns an Enumerator that iterates through the Elements in collection. + /// Returns an Enumerator that iterates through the Elements in collection, in the order they were added. /// public IEnumerator GetEnumerator() { - foreach (Element elem in store.Values) - yield return elem; + return order.GetEnumerator(); } /// /// Raised when an is added, removed, or replaced. @@ -318,7 +279,6 @@ public IEnumerator GetEnumerator() public void Dispose() { - ChangeLock.Dispose(); } } } diff --git a/Datamodel.NET/Element.cs b/Datamodel.NET/Element.cs index c0bd41a..6f965a9 100644 --- a/Datamodel.NET/Element.cs +++ b/Datamodel.NET/Element.cs @@ -148,16 +148,7 @@ internal set { if (value != null && base.Owner != null && base.Owner.AllElements.Contains(this)) throw new InvalidOperationException("Element already has an owner."); base.Owner = value; - if (value != null) - { - value.AllElements.ChangeLock.EnterWriteLock(); - try - { - value.AllElements.Add(this); - if (value.AllElements.Count == 1) value.Root = this; - } - finally { value.AllElements.ChangeLock.ExitWriteLock(); } - } + value?.AllElements.Add(this); } } From 7c7edbd4e22e9b6bf3df4b9b68ef54119fa1b005 Mon Sep 17 00:00:00 2001 From: Angel Date: Fri, 4 Sep 2026 13:57:59 +0200 Subject: [PATCH 03/16] Generate element IDs on first use (cherry picked from commit a4cabc71090b52cf2380213c13b0d8318f4ba751) --- Datamodel.NET/Element.cs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Datamodel.NET/Element.cs b/Datamodel.NET/Element.cs index 6f965a9..4a06c51 100644 --- a/Datamodel.NET/Element.cs +++ b/Datamodel.NET/Element.cs @@ -67,7 +67,7 @@ public Element(Datamodel owner, Guid id) public Element() : base(null) { - ID = Guid.NewGuid(); + // the ID is generated on first use, so that a codec constructing the Element and assigning the ID from the file does not pay for a random one // For subclasses get the actual classname if (GetType() != typeof(Element)) @@ -95,7 +95,26 @@ public Element() /// . Assign it before the Element joins a or any other /// hash-based collection, because changing it afterwards strands the Element in its old bucket. /// - public Guid ID { get; set; } + public Guid ID + { + get + { + if (!idAssigned) + { + id = Guid.NewGuid(); + idAssigned = true; + } + + return id; + } + set + { + id = value; + idAssigned = true; + } + } + Guid id; + bool idAssigned; /// /// Gets or sets the name of this Element. From 0290370787fe2f6f0680aa84d94f0c44235e2f68 Mon Sep 17 00:00:00 2001 From: Angel Date: Fri, 4 Sep 2026 13:57:59 +0200 Subject: [PATCH 04/16] Skip the type table for class properties (cherry picked from commit 4a7f2414d58364195e385d4ad68dd7940c69c033) --- Datamodel.NET/AttributeList.cs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 877db71..1b0d475 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -276,20 +276,16 @@ public virtual object? this[string name] set { ArgumentNullException.ThrowIfNull(name); - if (value != null && !Datamodel.IsDatamodelType(value.GetType())) - throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); - - if (Owner != null && this == Owner.PrefixAttributes && value?.GetType() == typeof(Element)) - throw new AttributeTypeException("Elements are not supported as prefix attributes."); - var binding = Schema.GetProperty(name); + // a value that fits a class property is a valid attribute type by construction, so it skips the type table + var binding = Schema.Properties.Count > 0 ? Schema.GetProperty(name) : null; if (binding != null) { if (binding.CanWrite) { - // null is fine, it will just set the value to null - if (value != null && !binding.PropertyType.IsInstanceOfType(value)) + // null is fine, it will just set the value to null; an exact type match is the common case and avoids the runtime cast check + if (value != null && binding.PropertyType != value.GetType() && !binding.PropertyType.IsInstanceOfType(value)) { value = ConvertScalar(value, binding.PropertyType) ?? throw new InvalidDataException($"class property '{Schema.ElementType.Name}.{binding.PropertyName}' with type '{binding.PropertyType}' can not hold a value of type '{value.GetType()}' (attribute '{name}'), this is likely a mismatch between the real class and the class from the datamodel"); @@ -324,6 +320,12 @@ public virtual object? this[string name] return; } + if (value != null && !Datamodel.IsDatamodelType(value.GetType())) + throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); + + if (Owner != null && this == Owner.PrefixAttributes && value?.GetType() == typeof(Element)) + throw new AttributeTypeException("Elements are not supported as prefix attributes."); + Attribute? old_attr; Attribute? new_attr; int old_index = -1; @@ -337,7 +339,7 @@ public virtual object? this[string name] old_index = IndexOf(old_attr.Name); Inner.Remove(old_attr); } - Insert(old_index == -1 ? Count : old_index, new Attribute(name, this, value), notify: false); + Insert(old_index == -1 ? Inner.Count : old_index, new_attr, notify: false); } NotifyCollectionChangedEventArgs change_args; From d56f2d459e4bb6e5a5fddf4a121e170db052ce1d Mon Sep 17 00:00:00 2001 From: Angel Date: Fri, 4 Sep 2026 14:20:08 +0200 Subject: [PATCH 05/16] Store arrays in shared chunks (cherry picked from commit e67b6e09160542c7b6721bd483253daae80099dc) --- Datamodel.NET/Arrays.cs | 256 +++++++++++++++++++++---- Datamodel.NET/Codecs/Binary.cs | 70 ++++--- Datamodel.NET/Datamodel.ElementList.cs | 2 +- 3 files changed, 258 insertions(+), 70 deletions(-) diff --git a/Datamodel.NET/Arrays.cs b/Datamodel.NET/Arrays.cs index 7db05f4..19be146 100644 --- a/Datamodel.NET/Arrays.cs +++ b/Datamodel.NET/Arrays.cs @@ -1,13 +1,18 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; namespace Datamodel { + /// + /// A typed attribute array. Items are stored contiguously, either in a private buffer or, for arrays read from a file, + /// in a slice of a chunk shared with the other arrays of that file; the first change that needs more room moves the array to a private buffer. + /// [DebuggerTypeProxy(typeof(Array<>.DebugView))] - [DebuggerDisplay("Count = {Inner.Count}")] + [DebuggerDisplay("Count = {Count}")] public abstract class Array : IList, IList { internal class DebugView(Array arr) @@ -15,10 +20,14 @@ internal class DebugView(Array arr) readonly Array Arr = arr; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - public T[] Items { get { return [.. Arr.Inner]; } } + public T[] Items { get { return Arr.AsSpan().ToArray(); } } } - protected List Inner; + T[] buffer; + int offset; + int count; + int capacity; + bool shared; public virtual AttributeList? Owner { @@ -34,61 +43,135 @@ internal set internal Array() { - Inner = []; + buffer = []; } internal Array(IEnumerable enumerable) { - if (enumerable != null) - Inner = [.. enumerable]; - else - Inner = []; + buffer = enumerable is null ? [] : [.. enumerable]; + count = capacity = buffer.Length; } internal Array(int capacity) { - Inner = new List(capacity); + buffer = capacity > 0 ? new T[capacity] : []; + this.capacity = capacity; } - public int IndexOf(T item) => Inner.IndexOf(item); + /// + /// Creates an array over a slice of a chunk shared with other arrays. The slice belongs to this array alone, but cannot grow in place. + /// + internal Array(T[] buffer, int offset, int count) + { + this.buffer = buffer; + this.offset = offset; + this.count = count; + capacity = count; + shared = true; + } - public void Insert(int index, T item) => Insert_Internal(index, item); - protected virtual void Insert_Internal(int index, T item) => Inner.Insert(index, item); + /// + /// Gets the items as a span. The span is invalidated by any change to the array. + /// + public ReadOnlySpan AsSpan() => new(buffer, offset, count); - public void AddRange(IEnumerable items) => Inner.AddRange(items); + /// + /// The items, writable. Invalidated by any change to the array. + /// + protected Span Items => new(buffer, offset, count); /// - /// Appends default items and returns them for the caller to fill, so that a codec can read value types in bulk. + /// Moves the items to a private buffer with room for at least items. /// - internal Span AppendUninitialized(int count) + void Grow(int minimum) { - var start = Inner.Count; - System.Runtime.InteropServices.CollectionsMarshal.SetCount(Inner, start + count); - return System.Runtime.InteropServices.CollectionsMarshal.AsSpan(Inner).Slice(start, count); + var newCapacity = Math.Max(minimum, Math.Max(capacity * 2, 4)); + var newBuffer = new T[newCapacity]; + Items.CopyTo(newBuffer); + buffer = newBuffer; + offset = 0; + capacity = newCapacity; + shared = false; } - public void RemoveAt(int index) => Inner.RemoveAt(index); + public int IndexOf(T item) + { + var index = System.Array.IndexOf(buffer, item, offset, count); + return index < 0 ? -1 : index - offset; + } + + public void Insert(int index, T item) => Insert_Internal(index, item); + + protected virtual void Insert_Internal(int index, T item) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)index, (uint)count, nameof(index)); + + if (count == capacity) + Grow(count + 1); + + if (index < count) + System.Array.Copy(buffer, offset + index, buffer, offset + index + 1, count - index); + + buffer[offset + index] = item; + count++; + } + + public void AddRange(IEnumerable items) + { + ArgumentNullException.ThrowIfNull(items); + + if (items is ICollection collection) + { + if (count + collection.Count > capacity) + Grow(count + collection.Count); + + collection.CopyTo(buffer, offset + count); + count += collection.Count; + return; + } + + foreach (var item in items) + Add(item); + } + + public void RemoveAt(int index) + { + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual((uint)index, (uint)count, nameof(index)); + + count--; + if (index < count) + System.Array.Copy(buffer, offset + index + 1, buffer, offset + index, count - index); + + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + buffer[offset + count] = default!; + } public virtual T this[int index] { - get => Inner[index]; - set => Inner[index] = value; + get => Items[index]; + set => Items[index] = value; } - public void Add(T item) => Insert(Inner.Count, item); + public void Add(T item) => Insert(count, item); - public void Clear() => Inner.Clear(); + public void Clear() + { + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + Items.Clear(); - public bool Contains(T item) => Inner.Contains(item); + count = 0; + } + + public bool Contains(T item) => IndexOf(item) >= 0; public void CopyTo(T[] array, int offset) { CopyTo_Internal(array, offset); } - protected virtual void CopyTo_Internal(T[] array, int offset) => Inner.CopyTo(array, offset); + protected virtual void CopyTo_Internal(T[] array, int offset) => Items.CopyTo(array.AsSpan(offset)); - public int Count => Inner.Count; + public int Count => count; bool ICollection.IsReadOnly { get { return false; } } @@ -98,7 +181,7 @@ public void CopyTo(T[] array, int offset) public bool IsSynchronized => false; - public object SyncRoot => Inner; + public object SyncRoot => this; object? IList.this[int index] { @@ -106,10 +189,23 @@ public void CopyTo(T[] array, int offset) set => this[index] = value is null ? throw new InvalidOperationException("Trying to set a null object") : (T)value; } - public bool Remove(T item) => Inner.Remove(item); + public bool Remove(T item) + { + var index = IndexOf(item); + if (index < 0) + return false; + + RemoveAt(index); + return true; + } + + public IEnumerator GetEnumerator() + { + for (var i = 0; i < count; i++) + yield return buffer[offset + i]; + } - public IEnumerator GetEnumerator() => Inner.GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => Inner.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); #region IList int IList.Add(object? value) @@ -161,6 +257,55 @@ void ICollection.CopyTo(Array array, int index) #endregion IList } + /// + /// Hands out slices of large shared arrays to the arrays read from one file, so that the collector sees a few large objects + /// that it never moves, instead of hundreds of thousands of small ones that it copies through every generation. + /// + sealed class ArrayChunks + { + /// Chunk size in bytes. Above the large object threshold, so that chunks are never compacted. + const int ChunkBytes = 1 << 20; + + /// Arrays at least this large get their own allocation, which lands on the large object heap anyway. + const int LargeObjectBytes = 85_000; + + sealed class Chunk + { + public T[] Buffer = []; + public int Used; + } + + readonly Dictionary chunks = []; + + /// + /// Returns storage for items. The contents are not zeroed. + /// + public (T[] Buffer, int Offset) Rent(int count) where T : unmanaged + { + var itemSize = Unsafe.SizeOf(); + + if ((long)count * itemSize >= LargeObjectBytes) + return (GC.AllocateUninitializedArray(count), 0); + + if (!chunks.TryGetValue(typeof(T), out var untyped)) + { + untyped = new Chunk(); + chunks[typeof(T)] = untyped; + } + + var chunk = (Chunk)untyped; + if (chunk.Buffer.Length - chunk.Used < count) + { + chunk.Buffer = GC.AllocateUninitializedArray(Math.Max(ChunkBytes / itemSize, count)); + chunk.Used = 0; + } + + var offset = chunk.Used; + chunk.Used += count; + return (chunk.Buffer, offset); + } + } + public class ElementArray : Array { public ElementArray() { } @@ -174,9 +319,9 @@ public ElementArray(int capacity) { } /// - /// Gets the values in the list without attempting destubbing. + /// Gets the items without attempting destubbing. /// - internal IEnumerable RawList { get { foreach (var elem in Inner) yield return elem; } } + internal ReadOnlySpan RawItems => AsSpan(); public override AttributeList? Owner { @@ -187,9 +332,10 @@ internal set if (OwnerDatamodel != null) { - for (int i = 0; i < Count; i++) + var items = Items; + for (int i = 0; i < items.Length; i++) { - var elem = Inner[i]; + var elem = items[i]; if (elem == null) continue; if (elem.Owner == null) @@ -198,7 +344,7 @@ internal set if (importedElement is not null) { - Inner[i] = importedElement; + items[i] = importedElement; } } else if (elem.Owner != OwnerDatamodel) @@ -234,12 +380,13 @@ public override Element this[int index] { get { - var elem = Inner[index]; + var items = Items; + var elem = items[index]; if (elem != null && elem.Stub && elem.Owner != null) { try { - elem = Inner[index] = elem.Owner.OnStubRequest(elem.ID)!; + elem = items[index] = elem.Owner.OnStubRequest(elem.ID)!; } catch (Exception err) { @@ -267,6 +414,9 @@ public IntArray(IEnumerable enumerable) public IntArray(int capacity) : base(capacity) { } + internal IntArray(int[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class FloatArray : Array @@ -278,6 +428,9 @@ public FloatArray(IEnumerable enumerable) public FloatArray(int capacity) : base(capacity) { } + internal FloatArray(float[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class BoolArray : Array @@ -289,6 +442,9 @@ public BoolArray(IEnumerable enumerable) public BoolArray(int capacity) : base(capacity) { } + internal BoolArray(bool[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class StringArray : Array @@ -333,6 +489,9 @@ public ColorArray(IEnumerable enumerable) public ColorArray(int capacity) : base(capacity) { } + internal ColorArray(Color[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class Vector2Array : Array @@ -344,6 +503,9 @@ public Vector2Array(IEnumerable enumerable) public Vector2Array(int capacity) : base(capacity) { } + internal Vector2Array(Vector2[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class Vector3Array : Array @@ -355,6 +517,9 @@ public Vector3Array(IEnumerable enumerable) public Vector3Array(int capacity) : base(capacity) { } + internal Vector3Array(Vector3[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class Vector4Array : Array @@ -366,6 +531,9 @@ public Vector4Array(IEnumerable enumerable) public Vector4Array(int capacity) : base(capacity) { } + internal Vector4Array(Vector4[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class QuaternionArray : Array @@ -377,6 +545,9 @@ public QuaternionArray(IEnumerable enumerable) public QuaternionArray(int capacity) : base(capacity) { } + internal QuaternionArray(Quaternion[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class MatrixArray : Array @@ -388,6 +559,9 @@ public MatrixArray(IEnumerable enumerable) public MatrixArray(int capacity) : base(capacity) { } + internal MatrixArray(Matrix4x4[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class ByteArray : Array @@ -399,6 +573,9 @@ public ByteArray(IEnumerable enumerable) public ByteArray(int capacity) : base(capacity) { } + internal ByteArray(byte[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } [CLSCompliant(false)] @@ -411,5 +588,8 @@ public UInt64Array(IEnumerable enumerable) public UInt64Array(int capacity) : base(capacity) { } + internal UInt64Array(ulong[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } } diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index ac3bf23..796a714 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -500,45 +500,53 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in if (!isArray) return ReadValue(dm, TypeMap[type.TypeHandle], EncodingVersion < 4 || prefix, reader); - else - { - var count = reader.ReadInt32(); - var array = CodecUtilities.MakeList(type, count); - // value types whose memory layout matches the stream are copied into the list in one read, instead of one boxed item at a time - if (BitConverter.IsLittleEndian && count > 0) + return ReadArray(dm, type, reader.ReadInt32(), reader); + } + + /// + /// Storage shared by the value type arrays of this stream, see . + /// + readonly ArrayChunks Chunks = new(); + + /// + /// Reads an array attribute. Value types whose memory layout matches the stream are copied in one read into a slice of a shared chunk, + /// instead of one boxed item at a time into a list of their own. + /// + System.Collections.IList ReadArray(Datamodel dm, Type type, int count, BinaryReader reader) + { + var typeId = TypeMap[type.TypeHandle]; + + if (BitConverter.IsLittleEndian && count > 0) + { + switch (typeId) { - switch (array) - { - case IntArray ints: ReadItems(ints, count, reader); return array; - case FloatArray floats: ReadItems(floats, count, reader); return array; - case BoolArray bools: ReadItems(bools, count, reader); return array; - case Vector2Array vectors: ReadItems(vectors, count, reader); return array; - case Vector3Array vectors: ReadItems(vectors, count, reader); return array; - case Vector4Array vectors: ReadItems(vectors, count, reader); return array; - case QuaternionArray quaternions: ReadItems(quaternions, count, reader); return array; - case MatrixArray matrices: ReadItems(matrices, count, reader); return array; - case ColorArray colors: ReadItems(colors, count, reader); return array; - case ByteArray bytes: ReadItems(bytes, count, reader); return array; - case UInt64Array ulongs: ReadItems(ulongs, count, reader); return array; - } + case 1: { var (buffer, offset) = ReadChunk(count, reader); return new IntArray(buffer, offset, count); } + case 2: { var (buffer, offset) = ReadChunk(count, reader); return new FloatArray(buffer, offset, count); } + case 3: { var (buffer, offset) = ReadChunk(count, reader); return new BoolArray(buffer, offset, count); } + case 7: { var (buffer, offset) = ReadChunk(count, reader); return new ColorArray(buffer, offset, count); } + case 8: { var (buffer, offset) = ReadChunk(count, reader); return new Vector2Array(buffer, offset, count); } + case 9: { var (buffer, offset) = ReadChunk(count, reader); return new Vector3Array(buffer, offset, count); } + case 11: { var (buffer, offset) = ReadChunk(count, reader); return new Vector4Array(buffer, offset, count); } + case 12: { var (buffer, offset) = ReadChunk(count, reader); return new QuaternionArray(buffer, offset, count); } + case 13: { var (buffer, offset) = ReadChunk(count, reader); return new MatrixArray(buffer, offset, count); } + case 14: { var (buffer, offset) = ReadChunk(count, reader); return new ByteArray(buffer, offset, count); } + case 15: { var (buffer, offset) = ReadChunk(count, reader); return new UInt64Array(buffer, offset, count); } } + } - var typeId = TypeMap[type.TypeHandle]; - for (var i = 0; i < count; i++) - array.Add(ReadValue(dm, typeId, true, reader)); + var array = CodecUtilities.MakeList(type, count); + for (var i = 0; i < count; i++) + array.Add(ReadValue(dm, typeId, true, reader)); - return array; - } + return array; } - /// - /// Reads items straight into the list's storage. Only for types stored in the stream exactly as in memory. - /// - static void ReadItems(Array array, int count, BinaryReader reader) where T : unmanaged + (T[] Buffer, int Offset) ReadChunk(int count, BinaryReader reader) where T : unmanaged { - var bytes = System.Runtime.InteropServices.MemoryMarshal.AsBytes(array.AppendUninitialized(count)); - reader.BaseStream.ReadExactly(bytes); + var (buffer, offset) = Chunks.Rent(count); + reader.BaseStream.ReadExactly(System.Runtime.InteropServices.MemoryMarshal.AsBytes(buffer.AsSpan(offset, count))); + return (buffer, offset); } void SkipAttribute(BinaryReader reader) diff --git a/Datamodel.NET/Datamodel.ElementList.cs b/Datamodel.NET/Datamodel.ElementList.cs index 47eef91..a4bdcac 100644 --- a/Datamodel.NET/Datamodel.ElementList.cs +++ b/Datamodel.NET/Datamodel.ElementList.cs @@ -248,7 +248,7 @@ protected void WalkElemTree(Element? elem, HashSet found) } if (value is ElementArray elem_array) { - foreach (var item in elem_array.RawList) + foreach (var item in elem_array.RawItems) { if (item != null && found.Add(item)) { From d76af7f5cf4541725f4b9296590715fe853a3bf7 Mon Sep 17 00:00:00 2001 From: Angel Date: Fri, 4 Sep 2026 14:20:08 +0200 Subject: [PATCH 06/16] Store attributes inline in slots (cherry picked from commit c5e60b959c719b483c428deee1cf00d2dd5c2a8d) --- Datamodel.NET/Attribute.cs | 226 -------- Datamodel.NET/AttributeList.cs | 767 ++++++++++++++++++------- Datamodel.NET/Codecs/Binary.cs | 37 +- Datamodel.NET/Datamodel.ElementList.cs | 2 +- Datamodel.NET/Datamodel.cs | 6 +- Datamodel.NET/Element.cs | 3 +- Tests/Tests.cs | 2 +- 7 files changed, 591 insertions(+), 452 deletions(-) delete mode 100644 Datamodel.NET/Attribute.cs diff --git a/Datamodel.NET/Attribute.cs b/Datamodel.NET/Attribute.cs deleted file mode 100644 index 628811d..0000000 --- a/Datamodel.NET/Attribute.cs +++ /dev/null @@ -1,226 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Numerics; - -using AttrKVP = System.Collections.Generic.KeyValuePair; - -namespace Datamodel -{ - /// - /// A name/value pair associated with an . - /// - class Attribute - { - /// - /// Creates a new Attribute with a specified name and value. - /// - /// The name of the Attribute, which must be unique to its owner. - /// The value of the Attribute, which must be of a supported Datamodel type. - public Attribute(string name, AttributeList owner, object? value) - { - ArgumentNullException.ThrowIfNull(name); - - Name = name; - _Owner = owner; - Value = value; - } - - /// - /// Creates a new Attribute with deferred loading. - /// - /// The name of the Attribute, which must be unique to its owner. - /// The AttributeList which owns this Attribute. - /// The location in the encoded DMX stream at which this Attribute's value can be found. - public Attribute(string name, AttributeList owner, long defer_offset) - : this(name, owner, null) - { - ArgumentNullException.ThrowIfNull(owner); - - Offset = defer_offset; - } - - #region Properties - /// - /// Gets the name of this Attribute. - /// - public string Name { get; private set; } - - /// - /// Gets the Type of this Attribute's Value. - /// - public Type ValueType { get; private set; } = typeof(Element); - - /// - /// Gets or sets the OverrideType of this Attributes. - /// - public AttributeList.OverrideType? OverrideType - { - get - { - return _OverrideType; - } - set - { - switch (value) - { - case null: - break; - case AttributeList.OverrideType.Angle: - if (ValueType != typeof(Vector3)) - throw new AttributeTypeException("OverrideType.Angle can only be applied to Vector3 attributes"); - break; - case AttributeList.OverrideType.Binary: - if (ValueType != typeof(byte[])) - throw new AttributeTypeException("OverrideType.Binary can only be applied to byte[] attributes"); - break; - default: - throw new NotImplementedException(); - } - _OverrideType = value; - } - } - AttributeList.OverrideType? _OverrideType; - - /// - /// Gets the which this Attribute is a member of. - /// - public AttributeList? Owner - { - get { return _Owner; } - internal set - { - if (_Owner == value) return; - - if (Deferred && _Owner != null) DeferredLoad(); - _Owner = value; - } - } - AttributeList? _Owner; - - Datamodel? OwnerDatamodel { get { return Owner?.Owner; } } - - /// - /// Gets whether the value of this Attribute has yet to be decoded. - /// - public bool Deferred { get { return Offset != 0; } } - - /// - /// Loads the value of this Attribute from the encoded source Datamodel. - /// - /// Thrown when the Attribute has already been loaded. - /// Thrown when the deferred load fails. - public void DeferredLoad() - { - if (Offset == 0) throw new InvalidOperationException("Attribute already loaded."); - - if (OwnerDatamodel == null || OwnerDatamodel.Codec == null) - throw new CodecException("Trying to load a deferred Attribute, but could not find codec."); - - try - { - lock (OwnerDatamodel.Codec) - { - _Value = OwnerDatamodel.Codec.DeferredDecodeAttribute(OwnerDatamodel, Offset); - } - } - catch (Exception err) - { - throw new CodecException($"Deferred loading of attribute \"{Name}\" on element {((Element?)Owner)?.ID} using {OwnerDatamodel.Codec} codec threw an exception.", err); - } - Offset = 0; - - if (_Value is ElementArray elem_array) - elem_array.Owner = Owner; - } - - /// - /// Gets or sets the value held by this Attribute. - /// - /// Thrown when deferred value loading fails. - /// Thrown when Element destubbing fails. - public object? Value - { - get - { - if (Offset > 0) - DeferredLoad(); - - if (OwnerDatamodel != null) - { - // expand stubs - if (_Value is Element elem && elem.Stub) - { - try { _Value = OwnerDatamodel.OnStubRequest(elem.ID) ?? _Value; } - catch (Exception err) { throw new DestubException(this, err); } - } - } - - return _Value; - } - set - { - ValueType = value == null ? typeof(Element) : value.GetType(); - - if (!Datamodel.IsDatamodelType(ValueType)) - throw new AttributeTypeException(String.Format("{0} is not a valid Datamodel attribute type. (If this is an array, it must implement IList).", ValueType.FullName)); - - if (value is Element elem) - { - if (elem.Owner == null) - elem.Owner = OwnerDatamodel; - else if (elem.Owner != OwnerDatamodel) - throw new ElementOwnershipException(); - } - - if (value is IEnumerable elem_enumerable) - { - if (elem_enumerable is not ElementArray) - throw new InvalidOperationException("Element array objects must derive from Datamodel.ElementArray"); - - var elem_array = (ElementArray)value; - if (elem_array.Owner == null) - elem_array.Owner = Owner; - else if (elem_array.Owner != Owner) - throw new InvalidOperationException("ElementArray is already owned by a different Datamodel."); - - foreach (var arr_elem in elem_array) - { - if (arr_elem == null) continue; - else if (arr_elem.Owner == null) - arr_elem.Owner = OwnerDatamodel; - - // todo: remove ownership from values - // this is being printed on a debuggerdisplay output for some reason - // else if (arr_elem.Owner != OwnerDatamodel) - // throw new ElementOwnershipException("One or more Elements in the assigned collection are from a different Datamodel. Use ImportElement() to copy them to this one before assigning."); - } - } - - _Value = value; - Offset = 0; - } - } - object? _Value = null; - - /// - /// Gets the Attribute's Value without attempting deferred loading or destubbing. - /// - public object? RawValue { get { return _Value; } } - - #endregion - - long Offset; - - public override string ToString() - { - var type = Value != null ? Value.GetType() : typeof(Element); - var inner_type = Datamodel.GetArrayInnerType(type); - return String.Format("{0} <{1}>", Name, inner_type != null ? inner_type.FullName + "[]" : type.FullName); - } - - public AttrKVP ToKeyValuePair() - { - return new AttrKVP(Name, Value); - } - } -} diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 1b0d475..ff2a5c4 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -1,79 +1,102 @@ -using System; +using System; using System.Collections; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Specialized; -using System.Diagnostics; using System.ComponentModel; +using System.Diagnostics; +using System.IO; using System.Linq; using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using AttrKVP = System.Collections.Generic.KeyValuePair; -using System.IO; namespace Datamodel { /// - /// A thread-safe collection of s. + /// What an holds inline. covers everything stored as an object: strings, binary blobs, matrices, elements, arrays and null. + /// + enum AttributeKind : byte + { + Reference, + Int, + Float, + Bool, + Byte, + UInt64, + Time, + Color, + Vector2, + Vector3, + Vector4, + Quaternion, + QAngle, + } + + /// + /// Sixteen bytes that hold any scalar or vector attribute value without boxing it. + /// + [StructLayout(LayoutKind.Explicit, Size = 16)] + struct InlineValue + { + [FieldOffset(0)] public int Int; + [FieldOffset(0)] public float Float; + [FieldOffset(0)] public bool Bool; + [FieldOffset(0)] public byte Byte; + [FieldOffset(0)] public ulong UInt64; + [FieldOffset(0)] public long Ticks; + [FieldOffset(0)] public Color Color; + [FieldOffset(0)] public Vector2 Vector2; + [FieldOffset(0)] public Vector3 Vector3; + [FieldOffset(0)] public Vector4 Vector4; + [FieldOffset(0)] public Quaternion Quaternion; + [FieldOffset(0)] public QAngle QAngle; + } + + /// + /// One attribute of an : its name and its value, stored inline for value types and as a reference otherwise. + /// Modelled on Valve's fixed-size CDmAttribute, so that a plain element costs one slot per attribute and no further objects. + /// + struct AttributeSlot + { + public string Name; + public object? Reference; + public InlineValue Inline; + /// When not zero, the value has not been read from the stream yet and starts at this position. + public long Offset; + public AttributeKind Kind; + public AttributeList.OverrideType? Override; + } + + /// + /// A thread-safe collection of attributes. /// [DebuggerTypeProxy(typeof(DebugView))] [DebuggerDisplay("Count = {Count}")] public class AttributeList : IDictionary, IDictionary { - internal OrderedDictionary Inner; - protected object Attribute_ChangeLock = new(); + AttributeSlot[]? slots; + int count; /// - /// Gets the properties of this class that are stored as attributes. Empty unless a schema is registered for the class. + /// The object locked while the list is changed. The list itself, which is also its . /// - public ElementSchema Schema { get; } - - private IEnumerable GetPropertyBasedAttributes(bool useSerializationName) - { - foreach (var binding in Schema.Properties) - { - var name = useSerializationName ? binding.AttributeName : binding.PropertyName; - yield return new Attribute(name, this, binding.GetValue(this)); - } - } + protected object Attribute_ChangeLock; /// - /// Converts between the bool, int and float attribute types the way Valve's datamodel does when a value is assigned - /// to an attribute of another of those types. Returns null for any other combination. + /// Gets the properties of this class that are stored as attributes. Empty unless a schema is registered for the class. /// - private static object? ConvertScalar(object value, Type targetType) - { - if (targetType == typeof(int)) - { - return value switch - { - bool b => b ? 1 : 0, - float f => (int)f, - _ => null, - }; - } + public ElementSchema Schema { get; } - if (targetType == typeof(float)) - { - return value switch - { - bool b => b ? 1f : 0f, - int i => (float)i, - _ => null, - }; - } + public AttributeList(Datamodel? owner) + { + Attribute_ChangeLock = this; - if (targetType == typeof(bool)) - { - return value switch - { - int i => i != 0, - float f => f != 0f, - _ => null, - }; - } + var type = GetType(); + Schema = type == typeof(AttributeList) || type == typeof(Element) ? ElementSchema.Empty : ElementSchema.For(type); - return null; + Owner = owner; } internal class DebugView @@ -87,23 +110,18 @@ public DebugView(AttributeList item) [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] public DebugAttribute[] Attributes - => Item.GetPropertyBasedAttributes(useSerializationName: false).Select(attr => new DebugAttribute(attr)) - .Concat(Item.Inner.Values.Cast().Select(attr => new DebugAttribute(attr))) + => Item.Schema.Properties.Select(binding => new DebugAttribute(binding.PropertyName, binding.GetValue(Item))) + .Concat(Item.Select(attr => new DebugAttribute(attr.Key, attr.Value))) .ToArray(); - [DebuggerDisplay("{Value}", Name = "{Attr.Name,nq}", Type = "{Attr.ValueType.FullName,nq}")] - public class DebugAttribute + [DebuggerDisplay("{Value}", Name = "{Name,nq}")] + public class DebugAttribute(string name, object? value) { - public DebugAttribute(Attribute attr) - { - Attr = attr; - } - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - readonly Attribute Attr; + public string Name { get; } = name; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - object? Value { get { return Attr.Value; } } + public object? Value { get; } = value; } } @@ -122,19 +140,242 @@ public enum OverrideType Binary, } - public AttributeList(Datamodel? owner) + /// + /// Gets the that this AttributeList is owned by. + /// + public virtual Datamodel? Owner { get; internal set; } + + #region Slots + + /// + /// Returns the index of the attribute with the given name, or -1. Names of attributes read from a file are usually the same string instance as the query, which the first comparison catches. + /// + int Find(string name) { - var type = GetType(); - Schema = type == typeof(AttributeList) || type == typeof(Element) ? ElementSchema.Empty : ElementSchema.For(type); + var slots = this.slots; + for (var i = 0; i < count; i++) + { + var candidate = slots![i].Name; + if (ReferenceEquals(candidate, name) || candidate == name) + return i; + } - Inner = []; - Owner = owner; + return -1; } /// - /// Gets the that this AttributeList is owned by. + /// Adds an empty slot with the given name at the end. The caller holds the lock. /// - public virtual Datamodel? Owner { get; internal set; } + ref AttributeSlot Append(string name) + { + if (slots == null || count == slots.Length) + System.Array.Resize(ref slots, Math.Max(4, count * 2)); + + ref var slot = ref slots[count++]; + slot = default; + slot.Name = name; + return ref slot; + } + + /// + /// Adds an empty slot with the given name at the given index. The caller holds the lock. + /// + ref AttributeSlot InsertAt(int index, string name) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)index, (uint)count, nameof(index)); + + Append(name); + if (index < count - 1) + { + System.Array.Copy(slots!, index, slots!, index + 1, count - 1 - index); + slots![index] = default; + slots[index].Name = name; + } + + return ref slots![index]; + } + + void RemoveSlot(int index) + { + count--; + if (index < count) + System.Array.Copy(slots!, index + 1, slots!, index, count - index); + + slots![count] = default; + } + + static AttributeKind KindOf() where T : unmanaged + { + if (typeof(T) == typeof(int)) return AttributeKind.Int; + if (typeof(T) == typeof(float)) return AttributeKind.Float; + if (typeof(T) == typeof(bool)) return AttributeKind.Bool; + if (typeof(T) == typeof(byte)) return AttributeKind.Byte; + if (typeof(T) == typeof(ulong)) return AttributeKind.UInt64; + if (typeof(T) == typeof(TimeSpan)) return AttributeKind.Time; + if (typeof(T) == typeof(Color)) return AttributeKind.Color; + if (typeof(T) == typeof(Vector2)) return AttributeKind.Vector2; + if (typeof(T) == typeof(Vector3)) return AttributeKind.Vector3; + if (typeof(T) == typeof(Vector4)) return AttributeKind.Vector4; + if (typeof(T) == typeof(Quaternion)) return AttributeKind.Quaternion; + if (typeof(T) == typeof(QAngle)) return AttributeKind.QAngle; + return AttributeKind.Reference; + } + + static void WriteInline(ref AttributeSlot slot, AttributeKind kind, T value) where T : unmanaged + { + slot.Kind = kind; + slot.Reference = null; + slot.Offset = 0; + slot.Inline = default; + Unsafe.As(ref slot.Inline) = value; + } + + /// + /// Stores a boxed value in a slot, taking ownership of elements and element arrays the way Valve's datamodel does. + /// + void Store(ref AttributeSlot slot, object? value) + { + slot.Offset = 0; + + switch (value) + { + case null: + slot.Kind = AttributeKind.Reference; + slot.Reference = null; + return; + case int v: WriteInline(ref slot, AttributeKind.Int, v); return; + case float v: WriteInline(ref slot, AttributeKind.Float, v); return; + case bool v: WriteInline(ref slot, AttributeKind.Bool, v); return; + case byte v: WriteInline(ref slot, AttributeKind.Byte, v); return; + case ulong v: WriteInline(ref slot, AttributeKind.UInt64, v); return; + case TimeSpan v: WriteInline(ref slot, AttributeKind.Time, v); return; + case Color v: WriteInline(ref slot, AttributeKind.Color, v); return; + case Vector2 v: WriteInline(ref slot, AttributeKind.Vector2, v); return; + case Vector3 v: WriteInline(ref slot, AttributeKind.Vector3, v); return; + case Vector4 v: WriteInline(ref slot, AttributeKind.Vector4, v); return; + case Quaternion v: WriteInline(ref slot, AttributeKind.Quaternion, v); return; + case QAngle v: WriteInline(ref slot, AttributeKind.QAngle, v); return; + case Element elem: + if (elem.Owner == null) + elem.Owner = Owner; + else if (elem.Owner != Owner) + throw new ElementOwnershipException(); + break; + case ElementArray array: + if (array.Owner == null) + array.Owner = this; + else if (array.Owner != this) + throw new InvalidOperationException("ElementArray is already owned by a different Datamodel."); + break; + case IEnumerable: + throw new InvalidOperationException("Element array objects must derive from Datamodel.ElementArray"); + case string or byte[] or Matrix4x4: + break; + default: + if (!Datamodel.IsDatamodelType(value.GetType())) + throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); + break; + } + + slot.Kind = AttributeKind.Reference; + slot.Reference = value; + } + + /// + /// The value as an object, without loading a deferred value or expanding a stub. + /// + static object? RawValue(in AttributeSlot slot) + { + return slot.Kind switch + { + AttributeKind.Reference => slot.Reference, + AttributeKind.Int => slot.Inline.Int, + AttributeKind.Float => slot.Inline.Float, + AttributeKind.Bool => slot.Inline.Bool, + AttributeKind.Byte => slot.Inline.Byte, + AttributeKind.UInt64 => slot.Inline.UInt64, + AttributeKind.Time => TimeSpan.FromTicks(slot.Inline.Ticks), + AttributeKind.Color => slot.Inline.Color, + AttributeKind.Vector2 => slot.Inline.Vector2, + AttributeKind.Vector3 => slot.Inline.Vector3, + AttributeKind.Vector4 => slot.Inline.Vector4, + AttributeKind.Quaternion => slot.Inline.Quaternion, + AttributeKind.QAngle => slot.Inline.QAngle, + _ => throw new InvalidOperationException("Unknown attribute kind."), + }; + } + + /// + /// The value as an object, loading it from the stream if it is deferred and expanding a stub element. + /// + /// Thrown when deferred value loading fails. + /// Thrown when Element destubbing fails. + object? GetValue(int index) + { + if (slots![index].Offset != 0) + LoadDeferred(index); + + ref var slot = ref slots[index]; + + if (slot.Kind == AttributeKind.Reference && slot.Reference is Element { Stub: true } stub && Owner != null) + { + try { slot.Reference = Owner.OnStubRequest(stub.ID) ?? stub; } + catch (Exception err) { throw new DestubException(this, slot.Name, err); } + } + + return RawValue(in slot); + } + + void LoadDeferred(int index) + { + var codec = Owner?.Codec ?? throw new CodecException("Trying to load a deferred Attribute, but could not find codec."); + var offset = slots![index].Offset; + var name = slots[index].Name; + object? value; + + try + { + lock (codec) + { + value = codec.DeferredDecodeAttribute(Owner, offset); + } + } + catch (Exception err) + { + throw new CodecException($"Deferred loading of attribute \"{name}\" on element {(this as Element)?.ID} using {codec} codec threw an exception.", err); + } + + Store(ref slots[index], value); + } + + /// + /// Registers an attribute whose value is read from the stream on first access. + /// + internal void SetDeferred(string name, long offset) + { + lock (Attribute_ChangeLock) + { + var index = Find(name); + ref var slot = ref (index < 0 ? ref Append(name) : ref slots![index]); + slot.Kind = AttributeKind.Reference; + slot.Reference = null; + slot.Override = null; + slot.Offset = offset; + } + } + + /// + /// The reference held by every attribute, without loading deferred values. Value types are skipped, since only elements and arrays matter to callers. + /// + internal IEnumerable EnumerateReferences() + { + for (var i = 0; i < count; i++) + yield return slots![i].Reference; + } + + bool HasListeners => CollectionChanged != null || PropertyChanged != null; + + #endregion /// /// Adds a new attribute to this AttributeList. @@ -147,105 +388,162 @@ public void Add(string key, object? value) } /// - /// Gets the given atttribute's "override type". This applies when multiple Datamodel types map to the same CLR type. + /// Sets a value type attribute without boxing it. Any other type is stored through the indexer. /// - /// The name of the attribute. - /// The attribute's Datamodel type, if different from its CLR type. - /// Thrown when the given attribute is not present in the list. - public OverrideType? GetOverrideType(string key) + public void Set(string name, T value) where T : unmanaged { - var attrib = Inner[key]; + ArgumentNullException.ThrowIfNull(name); - if (attrib is null) + var kind = KindOf(); + if (kind == AttributeKind.Reference || (Schema.Properties.Count > 0 && Schema.GetProperty(name) != null)) { - return null; + this[name] = value; + return; } - return ((Attribute)attrib).OverrideType; + if (this is Element { Stub: true }) + throw new InvalidOperationException("Cannot set attributes on a stub element."); + + lock (Attribute_ChangeLock) + { + var index = Find(name); + if (index < 0) + { + WriteInline(ref Append(name), kind, value); + + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, new AttrKVP(name, value), count - 1)); + } + else + { + ref var slot = ref slots![index]; + var old = HasListeners ? RawValue(in slot) : null; + slot.Override = null; + WriteInline(ref slot, kind, value); + + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, new AttrKVP(name, value), new AttrKVP(name, old), index)); + } + } } /// - /// Sets the given attribute's "override type". This applies when multiple Datamodel types map to the same CLR type. + /// Gets the given atttribute's "override type". This applies when multiple Datamodel types map to the same CLR type. /// /// The name of the attribute. - /// The Datamodel type which the attribute should be stored as when written to DMX, or null. - /// Thrown when the attribute's CLR type does not map to the value given in . - public void SetOverrideType(string key, OverrideType? type) + /// The attribute's Datamodel type, if different from its CLR type. + public OverrideType? GetOverrideType(string key) { - var attrib = Inner[key]; - - if (attrib is not null) + lock (Attribute_ChangeLock) { - ((Attribute)attrib).OverrideType = type; + var index = Find(key); + return index < 0 ? null : slots![index].Override; } - } /// - /// Inserts an Attribute at the given index. + /// Sets the given attribute's "override type". This applies when multiple Datamodel types map to the same CLR type. /// - private void Insert(int index, Attribute item, bool notify = true) + /// The name of the attribute. + /// The Datamodel type which the attribute should be stored as when written to DMX, or null. + /// Thrown when the attribute's CLR type does not map to the value given in . + public void SetOverrideType(string key, OverrideType? type) { lock (Attribute_ChangeLock) { - Inner.Remove(item.Name); - Inner.Insert(index, item.Name, item); - } - item.Owner = this; + var index = Find(key); + if (index < 0) + return; + + ref var slot = ref slots![index]; + switch (type) + { + case null: + break; + case OverrideType.Angle: + if (slot.Kind != AttributeKind.Vector3) + throw new AttributeTypeException("OverrideType.Angle can only be applied to Vector3 attributes"); + break; + case OverrideType.Binary: + if (slot.Reference is not byte[]) + throw new AttributeTypeException("OverrideType.Binary can only be applied to byte[] attributes"); + break; + default: + throw new NotImplementedException(); + } - if (notify) - OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item.ToKeyValuePair(), index)); + slot.Override = type; + } } public bool Remove(string key) { lock (Attribute_ChangeLock) { - var attr = (Attribute?)Inner[key]; - if (attr == null) return false; + var index = Find(key); + if (index < 0) return false; - var index = IndexOf(key); - Inner.Remove(key); - OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, attr.ToKeyValuePair(), index)); + var removed = new AttrKVP(key, RawValue(in slots![index])); + RemoveSlot(index); + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed, index)); return true; } } + /// + /// Gets the value of an attribute without loading it if it is deferred, in which case the value is null. + /// public bool TryGetValue(string key, out object? value) { - Attribute? result; lock (Attribute_ChangeLock) - result = (Attribute?)Inner[key]; - - if (result != null) { - value = result.RawValue; + var index = Find(key); + if (index < 0) + { + value = null; + return false; + } + + ref var slot = ref slots![index]; + value = slot.Offset != 0 ? null : RawValue(in slot); return true; } - else - { - value = null; - return false; - } } public virtual bool ContainsKey(string key) { ArgumentNullException.ThrowIfNull(key); lock (Attribute_ChangeLock) - return Inner[key] != null; + return Find(key) >= 0; } + public ICollection Keys { - get { lock (Attribute_ChangeLock) return Inner.Keys.Cast().ToArray(); } + get + { + lock (Attribute_ChangeLock) + { + var keys = new string[count]; + for (var i = 0; i < count; i++) + keys[i] = slots![i].Name; + return keys; + } + } } + public ICollection Values { - get { lock (Attribute_ChangeLock) return Inner.Values.Cast().Select(attr => attr.Value).ToArray(); } + get + { + var values = new object?[Count]; + for (var i = 0; i < values.Length; i++) + values[i] = GetValue(i); + return values; + } } /// - /// Gets or sets the value of the with the given name. + /// Gets or sets the value of the attribute with the given name. /// /// The name to search for. Cannot be null. /// The value associated with the given name. @@ -253,14 +551,17 @@ public ICollection Values /// Thrown when an attempt is made to get a name that is not present in this AttributeList. /// Thrown when an attempt is made to set the value of the attribute to an Element from a different . /// Thrown when an attempt is made to set a value that is not of a valid Datamodel attribute type. - /// Thrown when the maximum number of Attributes allowed in an AttributeList has been reached. public virtual object? this[string name] { get { ArgumentNullException.ThrowIfNull(name); - var attr = (Attribute?)Inner[name]; - if (attr == null) + + int index; + lock (Attribute_ChangeLock) + index = Find(name); + + if (index < 0) { var binding = Schema.GetProperty(name); if (binding != null) @@ -271,7 +572,7 @@ public virtual object? this[string name] throw new KeyNotFoundException($"{this} does not have an attribute called \"{name}\""); } - return attr.Value; + return GetValue(index); } set { @@ -282,74 +583,114 @@ public virtual object? this[string name] if (binding != null) { - if (binding.CanWrite) - { - // null is fine, it will just set the value to null; an exact type match is the common case and avoids the runtime cast check - if (value != null && binding.PropertyType != value.GetType() && !binding.PropertyType.IsInstanceOfType(value)) - { - value = ConvertScalar(value, binding.PropertyType) - ?? throw new InvalidDataException($"class property '{Schema.ElementType.Name}.{binding.PropertyName}' with type '{binding.PropertyType}' can not hold a value of type '{value.GetType()}' (attribute '{name}'), this is likely a mismatch between the real class and the class from the datamodel"); - } - - binding.SetValue(this, value); - } - else - { - // a read-only array property takes the items of an incoming array of the same type, so that a file can fill it once - var existingArray = binding.GetValue(this) as IList; - var incomingArray = value as IList; - - if (existingArray is not null && incomingArray is not null && existingArray.GetType() == incomingArray.GetType()) - { - if (existingArray.Count == 0) - { - foreach (var item in incomingArray) - existingArray.Add(item); - } - else - { - throw new InvalidOperationException($"Attribute '{name}' modifies property {Schema.ElementType.Name}.{binding.PropertyName}, which is read-only and already has items."); - } - } - else - { - throw new InvalidDataException($"Property '{Schema.ElementType.Name}.{binding.PropertyName}' of deserialisation class must be writeable, make sure it has a setter"); - } - } - + SetProperty(binding, name, value); return; } - if (value != null && !Datamodel.IsDatamodelType(value.GetType())) - throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); - if (Owner != null && this == Owner.PrefixAttributes && value?.GetType() == typeof(Element)) throw new AttributeTypeException("Elements are not supported as prefix attributes."); - Attribute? old_attr; - Attribute? new_attr; - int old_index = -1; lock (Attribute_ChangeLock) { - old_attr = (Attribute?)Inner[name]; - new_attr = new Attribute(name, this, value); + var index = Find(name); + if (index < 0) + { + Store(ref Append(name), value); - if (old_attr != null) + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, new AttrKVP(name, value), count - 1)); + } + else { - old_index = IndexOf(old_attr.Name); - Inner.Remove(old_attr); + ref var slot = ref slots![index]; + var old = HasListeners ? RawValue(in slot) : null; + slot.Override = null; + Store(ref slot, value); + + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, new AttrKVP(name, value), new AttrKVP(name, old), index)); } - Insert(old_index == -1 ? Inner.Count : old_index, new_attr, notify: false); + } + } + } + + /// + /// Assigns a value to the class property that stores the attribute. + /// + void SetProperty(PropertyBinding binding, string name, object? value) + { + if (binding.CanWrite) + { + // null is fine, it will just set the value to null; an exact type match is the common case and avoids the runtime cast check + if (value != null && binding.PropertyType != value.GetType() && !binding.PropertyType.IsInstanceOfType(value)) + { + value = ConvertScalar(value, binding.PropertyType) + ?? throw new InvalidDataException($"class property '{Schema.ElementType.Name}.{binding.PropertyName}' with type '{binding.PropertyType}' can not hold a value of type '{value.GetType()}' (attribute '{name}'), this is likely a mismatch between the real class and the class from the datamodel"); } - NotifyCollectionChangedEventArgs change_args; - if (old_attr != null) - change_args = new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, new_attr.ToKeyValuePair(), old_attr.ToKeyValuePair(), old_index); + binding.SetValue(this, value); + return; + } + + // a read-only array property takes the items of an incoming array of the same type, so that a file can fill it once + var existingArray = binding.GetValue(this) as IList; + var incomingArray = value as IList; + + if (existingArray is not null && incomingArray is not null && existingArray.GetType() == incomingArray.GetType()) + { + if (existingArray.Count == 0) + { + foreach (var item in incomingArray) + existingArray.Add(item); + } else - change_args = new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, new_attr.ToKeyValuePair(), Count); + { + throw new InvalidOperationException($"Attribute '{name}' modifies property {Schema.ElementType.Name}.{binding.PropertyName}, which is read-only and already has items."); + } + } + else + { + throw new InvalidDataException($"Property '{Schema.ElementType.Name}.{binding.PropertyName}' of deserialisation class must be writeable, make sure it has a setter"); + } + } + + /// + /// Converts between the bool, int and float attribute types the way Valve's datamodel does when a value is assigned + /// to an attribute of another of those types. Returns null for any other combination. + /// + private static object? ConvertScalar(object value, Type targetType) + { + if (targetType == typeof(int)) + { + return value switch + { + bool b => b ? 1 : 0, + float f => (int)f, + _ => null, + }; + } + + if (targetType == typeof(float)) + { + return value switch + { + bool b => b ? 1f : 0f, + int i => (float)i, + _ => null, + }; + } - OnCollectionChanged(change_args); + if (targetType == typeof(bool)) + { + return value switch + { + int i => i != 0, + float f => f != 0f, + _ => null, + }; } + + return null; } /// @@ -359,19 +700,17 @@ public AttrKVP this[int index] { get { - var attr = (Attribute?)Inner[index]; - - if (attr is null) - { - throw new InvalidOperationException($"attribute at index {index} doesn't exist"); - } - - return attr.ToKeyValuePair(); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual((uint)index, (uint)count, nameof(index)); + return new AttrKVP(slots![index].Name, GetValue(index)); } set { - RemoveAt(index); - Insert(index, new Attribute(value.Key, this, value.Value)); + lock (Attribute_ChangeLock) + { + RemoveAt(index); + Store(ref InsertAt(index, value.Key), value.Value); + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, value, index)); + } } } @@ -380,32 +719,20 @@ public AttrKVP this[int index] /// public void RemoveAt(int index) { - Attribute? attr; + AttrKVP removed; lock (Attribute_ChangeLock) { - attr = (Attribute?)Inner[index]; - - if (attr is not null) - { - attr.Owner = null; - Inner.RemoveAt(index); - } + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual((uint)index, (uint)count, nameof(index)); + removed = new AttrKVP(slots![index].Name, RawValue(in slots[index])); + RemoveSlot(index); } - OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, attr, index)); + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed, index)); } public int IndexOf(string key) { lock (Attribute_ChangeLock) - { - int i = 0; - foreach (string name in Inner.Keys) - { - if (name == key) return i; - i++; - } - } - return -1; + return Find(key); } /// @@ -414,7 +741,11 @@ public int IndexOf(string key) public void Clear() { lock (Attribute_ChangeLock) - Inner.Clear(); + { + if (slots != null) + System.Array.Clear(slots, 0, count); + count = 0; + } OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } @@ -423,7 +754,7 @@ public int Count get { lock (Attribute_ChangeLock) - return Inner.Count; + return count; } } @@ -440,8 +771,8 @@ public int Count /// public IEnumerable GetAllAttributesForSerialization() { - foreach (var attr in GetPropertyBasedAttributes(useSerializationName: true)) - yield return attr.ToKeyValuePair(); + foreach (var binding in Schema.Properties) + yield return new AttrKVP(binding.AttributeName, binding.GetValue(this)); foreach (var attr in this) yield return attr; @@ -449,8 +780,11 @@ public IEnumerable GetAllAttributesForSerialization() public IEnumerator GetEnumerator() { - foreach (var attr in Inner.Values.Cast().ToArray()) - yield return attr.ToKeyValuePair(); + var pairs = new AttrKVP[Count]; + for (var i = 0; i < pairs.Length; i++) + pairs[i] = new AttrKVP(slots![i].Name, GetValue(i)); + + return ((IEnumerable)pairs).GetEnumerator(); } #region Interfaces @@ -472,13 +806,12 @@ protected virtual void OnPropertyChanged([System.Runtime.CompilerServices.Caller } /// - /// Raised when an is added, removed, or replaced. + /// Raised when an attribute is added, removed, or replaced. /// + /// Only raised while a handler is attached to this event or to . public event NotifyCollectionChangedEventHandler? CollectionChanged; protected virtual void OnCollectionChanged(NotifyCollectionChangedEventArgs e) { - Debug.Assert(!(e.NewItems != null && e.NewItems.OfType().Any()) && !(e.OldItems != null && e.OldItems.OfType().Any())); - switch (e.Action) { case NotifyCollectionChangedAction.Add: @@ -533,10 +866,9 @@ bool ICollection.Remove(AttrKVP item) { lock (Attribute_ChangeLock) { - var attr = (Attribute?)Inner[item.Key]; - if (attr == null || attr.Value != item.Value) return false; - Remove(attr.Name); - return true; + var index = Find(item.Key); + if (index < 0 || !Equals(GetValue(index), item.Value)) return false; + return Remove(item.Key); } } @@ -547,12 +879,11 @@ void ICollection.CopyTo(AttrKVP[] array, int arrayIndex) void ICollection.CopyTo(Array array, int index) { - lock (Attribute_ChangeLock) - foreach (Attribute attr in Inner.Values) - { - array.SetValue(attr.ToKeyValuePair(), index); - index++; - } + foreach (var pair in this) + { + array.SetValue(pair, index); + index++; + } } void ICollection.Add(AttrKVP item) @@ -564,8 +895,8 @@ bool ICollection.Contains(AttrKVP item) { lock (Attribute_ChangeLock) { - var attr = (Attribute?)Inner[item.Key]; - return attr != null && attr.Value == item.Value; + var index = Find(item.Key); + return index >= 0 && Equals(GetValue(index), item.Value); } } diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index 796a714..64e54c6 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -458,7 +458,7 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in } else { - elem.Add(name, DecodeAttribute(dm, false, Reader)); + DecodeAttributeInto(dm, elem, name, Reader); } } } @@ -504,6 +504,41 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in return ReadArray(dm, type, reader.ReadInt32(), reader); } + /// + /// Reads an attribute of an element straight into the element, so that value types are stored inline without being boxed. + /// + void DecodeAttributeInto(Datamodel dm, AttributeList target, string name, BinaryReader reader) + { + var (type, isArray) = IdToType(reader.ReadByte()); + + if (isArray) + { + target[name] = ReadArray(dm, type, reader.ReadInt32(), reader); + return; + } + + switch (TypeMap[type.TypeHandle]) + { + case 0: target[name] = ReadElement(dm, reader); break; + case 1: target.Set(name, reader.ReadInt32()); break; + case 2: target.Set(name, reader.ReadSingle()); break; + case 3: target.Set(name, reader.ReadBoolean()); break; + case 4: target[name] = EncodingVersion < 4 ? ReadString_Raw(reader) : StringDict!.ReadString(reader); break; + case 5: target[name] = reader.ReadBytes(reader.ReadInt32()); break; + case 6: target.Set(name, TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond))); break; + case 7: target.Set(name, ReadColor(reader)); break; + case 8: target.Set(name, ReadVector2(reader)); break; + case 9: target.Set(name, ReadVector3(reader)); break; + case 10: target.Set(name, ReadQAngle(reader)); break; + case 11: target.Set(name, ReadVector4(reader)); break; + case 12: target.Set(name, ReadQuaternion(reader)); break; + case 13: target[name] = ReadMatrix4x4(reader); break; + case 14: target.Set(name, reader.ReadByte()); break; + case 15: target.Set(name, reader.ReadUInt64()); break; + default: throw new ArgumentException("Cannot read value of type"); + } + } + /// /// Storage shared by the value type arrays of this stream, see . /// diff --git a/Datamodel.NET/Datamodel.ElementList.cs b/Datamodel.NET/Datamodel.ElementList.cs index a4bdcac..b7bcd1b 100644 --- a/Datamodel.NET/Datamodel.ElementList.cs +++ b/Datamodel.NET/Datamodel.ElementList.cs @@ -235,7 +235,7 @@ protected void WalkElemTree(Element? elem, HashSet found) } found.Add(elem); - foreach (var value in elem.Inner.Values.Cast().Select(a => a.RawValue)) + foreach (var value in elem.EnumerateReferences()) { if (value is Element value_elem) { diff --git a/Datamodel.NET/Datamodel.cs b/Datamodel.NET/Datamodel.cs index 46d96d8..145d7e2 100644 --- a/Datamodel.NET/Datamodel.cs +++ b/Datamodel.NET/Datamodel.cs @@ -968,11 +968,11 @@ protected CodecException(SerializationInfo info, StreamingContext context) [Serializable] public class DestubException : Exception { - internal DestubException(Attribute attr, Exception innerException) + internal DestubException(AttributeList owner, string attributeName, Exception innerException) : base("An exception occured while destubbing the value of an attribute.", innerException) { - Data.Add("Element", ((Element?)attr.Owner)?.ID); - Data.Add("Attribute", attr.Name); + Data.Add("Element", (owner as Element)?.ID); + Data.Add("Attribute", attributeName); } internal DestubException(ElementArray array, int index, Exception innerException) diff --git a/Datamodel.NET/Element.cs b/Datamodel.NET/Element.cs index 4a06c51..8a61ebe 100644 --- a/Datamodel.NET/Element.cs +++ b/Datamodel.NET/Element.cs @@ -352,8 +352,7 @@ int IEqualityComparer.GetHashCode(object obj) /// The location of the attribute's value in the Datamodel's source stream. internal void Add(string key, long offset) { - lock (Attribute_ChangeLock) - Inner[key] = new Attribute(key, this, offset); + SetDeferred(key, offset); } public override bool ContainsKey(string key) diff --git a/Tests/Tests.cs b/Tests/Tests.cs index d4e8db7..5253305 100644 --- a/Tests/Tests.cs +++ b/Tests/Tests.cs @@ -182,7 +182,7 @@ protected static async Task Populate(Datamodel.Datamodel dm, string encoding_nam var name = value.GetType().Name; dm.Root[name] = value; - await Assert.That(dm.Root[name]).IsSameReferenceAs(value); + await Assert.That(dm.Root[name]).IsEqualTo(value); // value types are stored inline, so the box read back is a new one name += " array"; var list = value.GetType().MakeListType().GetConstructor(Type.EmptyTypes).Invoke(null) as IList; From a335d06c38a25ddc157cc0f04f0ac3e67ff95570 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 13:41:50 +0200 Subject: [PATCH 07/16] Measure the live heap of the typed model alone --- Benchmarks/Program.cs | 67 ++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/Benchmarks/Program.cs b/Benchmarks/Program.cs index ab0ef84..6414883 100644 --- a/Benchmarks/Program.cs +++ b/Benchmarks/Program.cs @@ -1,14 +1,15 @@ using System.Diagnostics; +using System.Runtime.CompilerServices; using Datamodel.Codecs; using Tests.VMAP; using DM = Datamodel.Datamodel; -// Measures loading a vmap as plain elements and as the typed classes of Tests/ValveMap.cs, and saving the typed model. +// Measures loading a vmap as plain elements and as the typed classes of Tests.VMAP, and saving the typed model. // // Benchmarks [--iterations N] ... // // A directory contributes every .vmap inside it, largest last. Each figure is the best of N iterations (default 1). -// "typed alloc" is the managed memory allocated by the typed load, "live heap" the managed heap that survives it. +// "typed alloc" is the managed memory allocated by the typed load, "live heap" the managed heap the typed model keeps, without the file bytes. // dots as decimal separators whatever the machine locale, so that tables can be pasted anywhere System.Globalization.CultureInfo.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture; @@ -44,41 +45,50 @@ { var name = Path.GetFileName(path); var size = new FileInfo(path).Length; - var read = Time(() => File.ReadAllBytes(path), out var bytes); - - var untyped = double.MaxValue; - var typed = double.MaxValue; - var save = double.MaxValue; - long allocated = 0, heap = 0, elements = 0; + var best = new Sample(double.MaxValue, double.MaxValue, double.MaxValue, 0, 0, 0); for (var i = 0; i < iterations; i++) { - Collect(); - untyped = Math.Min(untyped, Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var plain)); - elements = plain.AllElements.Count; - plain.Dispose(); - - Collect(); - var before = GC.GetTotalAllocatedBytes(true); - typed = Math.Min(typed, Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var map)); - allocated = GC.GetTotalAllocatedBytes(true) - before; - heap = GC.GetTotalMemory(true); - - if (map.Root is not CMapRootElement) - { - throw new InvalidOperationException($"{name}: the root was not loaded as {nameof(CMapRootElement)}"); - } - - save = Math.Min(save, Time(() => { map.Save(Stream.Null, "binary", 9); return 0; }, out _)); - map.Dispose(); + var sample = Measure(bytes, name); + best = new Sample(Math.Min(best.Untyped, sample.Untyped), Math.Min(best.Typed, sample.Typed), Math.Min(best.Save, sample.Save), sample.Allocated, sample.Heap, sample.Elements); } - Console.WriteLine($"{name,-26} {size / 1048576.0,7:F1}MB {elements,9} | {Duration(read),10} {Duration(untyped),13} {Duration(typed),11} {typed / untyped,12:F2}x | {allocated / 1048576.0,9:F0}MB {heap / 1048576.0,7:F0}MB | {Duration(save),11}"); + Console.WriteLine($"{name,-26} {size / 1048576.0,7:F1}MB {best.Elements,9} | {Duration(read),10} {Duration(best.Untyped),13} {Duration(best.Typed),11} {best.Typed / best.Untyped,12:F2}x | {best.Allocated / 1048576.0,9:F0}MB {best.Heap / 1048576.0,7:F0}MB | {Duration(best.Save),11}"); } return 0; +/// +/// One load-save round in a frame of its own, so that every model of the round is garbage once it returns. +/// The main loop keeps temporaries alive across iterations, which would count the previous model into the next heap figure. +/// +[MethodImpl(MethodImplOptions.NoInlining)] +static Sample Measure(byte[] bytes, string name) +{ + Collect(); + var untyped = Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var plain); + var elements = plain.AllElements.Count; + plain.Dispose(); + plain = null!; + + Collect(); + var before = GC.GetTotalAllocatedBytes(true); + var typed = Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var map); + var allocated = GC.GetTotalAllocatedBytes(true) - before; + var heap = GC.GetTotalMemory(true) - bytes.Length; + + if (map.Root is not CMapRootElement) + { + throw new InvalidOperationException($"{name}: the root was not loaded as {nameof(CMapRootElement)}"); + } + + var save = Time(() => { map.Save(Stream.Null, "binary", 9); return 0; }, out _); + map.Dispose(); + + return new Sample(untyped, typed, save, allocated, heap, elements); +} + /// Milliseconds up to a tenth of a second, seconds with two decimals above. static string Duration(double milliseconds) { @@ -98,3 +108,6 @@ static void Collect() GC.WaitForPendingFinalizers(); GC.Collect(); } + +/// Times in milliseconds, memory in bytes. +record struct Sample(double Untyped, double Typed, double Save, long Allocated, long Heap, long Elements); From d65a50c03a3a65c7492c26992caf4cffb6ff96d0 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 13:42:58 +0200 Subject: [PATCH 08/16] Set class properties through typed bindings --- Datamodel.NET/AttributeList.cs | 12 ++++- Datamodel.NET/ElementSchema.cs | 90 ++++++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index ff2a5c4..9dff31b 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -394,8 +394,18 @@ public void Set(string name, T value) where T : unmanaged { ArgumentNullException.ThrowIfNull(name); + if (Schema.Properties.Count > 0 && Schema.GetProperty(name) is PropertyBinding binding) + { + // the generated binding of a class property takes the value as it is, so nothing is boxed on the way in + if (binding.CanWrite && binding is PropertyBinding typed) + typed.Set(this, value); + else + SetProperty(binding, name, value); + return; + } + var kind = KindOf(); - if (kind == AttributeKind.Reference || (Schema.Properties.Count > 0 && Schema.GetProperty(name) != null)) + if (kind == AttributeKind.Reference) { this[name] = value; return; diff --git a/Datamodel.NET/ElementSchema.cs b/Datamodel.NET/ElementSchema.cs index 8fe2b1d..ed6544e 100644 --- a/Datamodel.NET/ElementSchema.cs +++ b/Datamodel.NET/ElementSchema.cs @@ -11,9 +11,9 @@ namespace Datamodel /// Instances are emitted by the ElementFactory that the KeyValues2.ElementFactoryGenerator generates into the assembly declaring the class, /// so no reflection is needed to move values between properties and attributes. /// - public sealed class PropertyBinding + public class PropertyBinding { - readonly Func getter; + readonly Func? getter; readonly Action? setter; /// The name of the property in the class. @@ -22,21 +22,28 @@ public sealed class PropertyBinding /// Reads the property of the given element. /// Writes the property of the given element, or null when the property has no setter. public PropertyBinding(string propertyName, string attributeName, Type propertyType, Func getter, Action? setter) + : this(propertyName, attributeName, propertyType, setter != null) + { + ArgumentNullException.ThrowIfNull(getter); + + this.getter = getter; + this.setter = setter; + } + + private protected PropertyBinding(string propertyName, string attributeName, Type propertyType, bool canWrite) { ArgumentNullException.ThrowIfNull(propertyName); ArgumentNullException.ThrowIfNull(attributeName); ArgumentNullException.ThrowIfNull(propertyType); - ArgumentNullException.ThrowIfNull(getter); PropertyName = propertyName; AttributeName = attributeName; PropertyType = propertyType; - this.getter = getter; - this.setter = setter; + CanWrite = canWrite; } /// - /// Creates a binding from typed accessors, so that generated code needs no casts. + /// Creates a binding from typed accessors, so that generated code needs no casts and values move without boxing. /// /// The class declaring the property. /// The type of the property. @@ -44,17 +51,12 @@ public PropertyBinding(string propertyName, string attributeName, Type propertyT /// The name of the attribute in the file. /// Reads the property. /// Writes the property, or null when it has no setter. - public static PropertyBinding Create(string propertyName, string attributeName, Func getter, Action? setter) + public static PropertyBinding Create(string propertyName, string attributeName, Func getter, Action? setter) where TElement : AttributeList { ArgumentNullException.ThrowIfNull(getter); - return new PropertyBinding( - propertyName, - attributeName, - typeof(TValue), - element => getter((TElement)element), - setter == null ? null : (element, value) => setter((TElement)element, (TValue)value!)); + return new PropertyBinding(propertyName, attributeName, getter, setter); } /// @@ -75,18 +77,18 @@ public static PropertyBinding Create(string propertyName, stri /// /// Gets whether the property can be assigned. /// - public bool CanWrite => setter != null; + public bool CanWrite { get; } /// /// Reads the property of the given element. /// - public object? GetValue(AttributeList owner) => getter(owner); + public virtual object? GetValue(AttributeList owner) => getter!(owner); /// /// Writes the property of the given element. /// /// Thrown when the property has no setter. - public void SetValue(AttributeList owner, object? value) + public virtual void SetValue(AttributeList owner, object? value) { if (setter == null) { @@ -99,6 +101,62 @@ public void SetValue(AttributeList owner, object? value) public override string ToString() => $"{PropertyName} <{PropertyType.Name}> as \"{AttributeName}\""; } + /// + /// A whose value type is known, so that codecs and the attribute indexer move values without boxing them. + /// + /// The type of the property. + public abstract class PropertyBinding : PropertyBinding + { + private protected PropertyBinding(string propertyName, string attributeName, bool canWrite) + : base(propertyName, attributeName, typeof(TValue), canWrite) + { + } + + /// + /// Reads the property of the given element. + /// + public abstract TValue Get(AttributeList owner); + + /// + /// Writes the property of the given element. + /// + /// Thrown when the property has no setter. + public abstract void Set(AttributeList owner, TValue value); + + public override object? GetValue(AttributeList owner) => Get(owner); + + public override void SetValue(AttributeList owner, object? value) => Set(owner, (TValue)value!); + } + + /// + /// The binding makes: the generated accessors of one property, called without any cast of the value. + /// + sealed class PropertyBinding : PropertyBinding + where TElement : AttributeList + { + readonly Func getter; + readonly Action? setter; + + public PropertyBinding(string propertyName, string attributeName, Func getter, Action? setter) + : base(propertyName, attributeName, setter != null) + { + this.getter = getter; + this.setter = setter; + } + + public override TValue Get(AttributeList owner) => getter((TElement)owner); + + public override void Set(AttributeList owner, TValue value) + { + if (setter == null) + { + throw new InvalidOperationException($"Property '{PropertyName}' is read-only."); + } + + setter((TElement)owner, value); + } + } + /// /// Describes how an subclass maps onto a file: its class name and the properties that are stored as attributes. /// From f4b2a0f2d0f88de1f206755d432014fcd7725db4 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 13:52:00 +0200 Subject: [PATCH 09/16] Size the element tables from the file --- Datamodel.NET/Codecs/Binary.cs | 19 ++++++++++++++----- Datamodel.NET/Datamodel.ElementList.cs | 10 ++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index 64e54c6..c7537dd 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -147,7 +147,9 @@ public StringDictionary(Binary codec, BinaryReader reader) Dummy = EncodingVersion == 1; if (!Dummy) { - foreach (var i in Enumerable.Range(0, LengthSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32())) + var count = LengthSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32(); + Strings.Capacity = count; + for (var i = 0; i < count; i++) AddString(Codec.ReadString_Raw(reader)); } } @@ -422,13 +424,20 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in StringDict = new StringDictionary(this, Reader); var num_elements = Reader.ReadInt32(); + // the file states how many elements follow, so the tables that hold them are sized once + ElementIndex.Capacity = num_elements; + dm.AllElements.EnsureCapacity(num_elements); + // read index - foreach (var i in Enumerable.Range(0, num_elements)) + Span id_bits = stackalloc byte[16]; + for (var i = 0; i < num_elements; i++) { var type = StringDict.ReadString(Reader); var name = EncodingVersion >= 4 ? StringDict.ReadString(Reader) : ReadString_Raw(Reader); - var id_bits = Reader.ReadBytes(16); - var id = new Guid(BitConverter.IsLittleEndian ? id_bits : id_bits.Reverse().ToArray()); + Reader.BaseStream.ReadExactly(id_bits); + if (!BitConverter.IsLittleEndian) + id_bits.Reverse(); + var id = new Guid(id_bits); if (!CodecUtilities.TryConstructCustomElement(resolver, dm, type, name, id, out var elem)) { @@ -448,7 +457,7 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in var num_attrs = Reader.ReadInt32(); - foreach (var i in Enumerable.Range(0, num_attrs)) + for (var i = 0; i < num_attrs; i++) { var name = StringDict.ReadString(Reader); if (defer_mode == DeferredMode.Automatic) diff --git a/Datamodel.NET/Datamodel.ElementList.cs b/Datamodel.NET/Datamodel.ElementList.cs index b7bcd1b..e63a88b 100644 --- a/Datamodel.NET/Datamodel.ElementList.cs +++ b/Datamodel.NET/Datamodel.ElementList.cs @@ -41,6 +41,16 @@ internal ElementList(Datamodel owner) Owner = owner; } + /// Makes room for the given number of elements, so that a codec that knows the count adds them without growing the tables. + internal void EnsureCapacity(int count) + { + lock (ChangeLock) + { + byId.EnsureCapacity(count); + order.EnsureCapacity(count); + } + } + /// /// Adds an Element owned by this list's Datamodel. The first Element added becomes the . /// From 408e2fac525d1039c8b4b5757b171db4bd2aced5 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 13:52:55 +0200 Subject: [PATCH 10/16] Shrink attribute slots to 40 bytes --- Datamodel.NET/AttributeList.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 9dff31b..0fbf5ee 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -20,6 +20,8 @@ namespace Datamodel enum AttributeKind : byte { Reference, + /// The value has not been read from the stream yet; holds the position it starts at. + Deferred, Int, Float, Bool, @@ -63,8 +65,6 @@ struct AttributeSlot public string Name; public object? Reference; public InlineValue Inline; - /// When not zero, the value has not been read from the stream yet and starts at this position. - public long Offset; public AttributeKind Kind; public AttributeList.OverrideType? Override; } @@ -225,7 +225,6 @@ static void WriteInline(ref AttributeSlot slot, AttributeKind kind, T value) { slot.Kind = kind; slot.Reference = null; - slot.Offset = 0; slot.Inline = default; Unsafe.As(ref slot.Inline) = value; } @@ -235,8 +234,6 @@ static void WriteInline(ref AttributeSlot slot, AttributeKind kind, T value) /// void Store(ref AttributeSlot slot, object? value) { - slot.Offset = 0; - switch (value) { case null: @@ -289,6 +286,7 @@ void Store(ref AttributeSlot slot, object? value) return slot.Kind switch { AttributeKind.Reference => slot.Reference, + AttributeKind.Deferred => null, AttributeKind.Int => slot.Inline.Int, AttributeKind.Float => slot.Inline.Float, AttributeKind.Bool => slot.Inline.Bool, @@ -312,7 +310,7 @@ void Store(ref AttributeSlot slot, object? value) /// Thrown when Element destubbing fails. object? GetValue(int index) { - if (slots![index].Offset != 0) + if (slots![index].Kind == AttributeKind.Deferred) LoadDeferred(index); ref var slot = ref slots[index]; @@ -329,7 +327,7 @@ void Store(ref AttributeSlot slot, object? value) void LoadDeferred(int index) { var codec = Owner?.Codec ?? throw new CodecException("Trying to load a deferred Attribute, but could not find codec."); - var offset = slots![index].Offset; + var offset = slots![index].Inline.Ticks; var name = slots[index].Name; object? value; @@ -357,10 +355,11 @@ internal void SetDeferred(string name, long offset) { var index = Find(name); ref var slot = ref (index < 0 ? ref Append(name) : ref slots![index]); - slot.Kind = AttributeKind.Reference; + slot.Kind = AttributeKind.Deferred; slot.Reference = null; slot.Override = null; - slot.Offset = offset; + slot.Inline = default; + slot.Inline.Ticks = offset; } } @@ -515,7 +514,7 @@ public bool TryGetValue(string key, out object? value) } ref var slot = ref slots![index]; - value = slot.Offset != 0 ? null : RawValue(in slot); + value = RawValue(in slot); return true; } } From e63ea7b6e0dea08cce9a9d857afd1ce173681c62 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 14:02:18 +0200 Subject: [PATCH 11/16] Write elements from their slots and arrays in one piece --- Datamodel.NET/AttributeList.cs | 131 ++++++- Datamodel.NET/Codecs/Binary.cs | 600 ++++++++++++++++++--------------- Datamodel.NET/ElementSchema.cs | 28 ++ 3 files changed, 481 insertions(+), 278 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 0fbf5ee..76aa1bd 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; @@ -69,6 +70,17 @@ struct AttributeSlot public AttributeList.OverrideType? Override; } + /// + /// Receives the attributes of an in the form their slots hold them, so that a codec writes them without boxing. See . + /// + interface IAttributeVisitor + { + /// Called once before the attributes, with how many follow. + void Begin(int count); + + void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference); + } + /// /// A thread-safe collection of attributes. /// @@ -310,18 +322,8 @@ void Store(ref AttributeSlot slot, object? value) /// Thrown when Element destubbing fails. object? GetValue(int index) { - if (slots![index].Kind == AttributeKind.Deferred) - LoadDeferred(index); - - ref var slot = ref slots[index]; - - if (slot.Kind == AttributeKind.Reference && slot.Reference is Element { Stub: true } stub && Owner != null) - { - try { slot.Reference = Owner.OnStubRequest(stub.ID) ?? stub; } - catch (Exception err) { throw new DestubException(this, slot.Name, err); } - } - - return RawValue(in slot); + Resolve(index); + return RawValue(in slots![index]); } void LoadDeferred(int index) @@ -796,6 +798,111 @@ public IEnumerator GetEnumerator() return ((IEnumerable)pairs).GetEnumerator(); } + /// + /// Loads the value of a deferred slot and expands a stub, so that the slot holds its final value. + /// + void Resolve(int index) + { + if (slots![index].Kind == AttributeKind.Deferred) + LoadDeferred(index); + + ref var slot = ref slots[index]; + if (slot.Kind == AttributeKind.Reference && slot.Reference is Element { Stub: true } stub && Owner != null) + { + try { slot.Reference = Owner.OnStubRequest(stub.ID) ?? stub; } + catch (Exception err) { throw new DestubException(this, slot.Name, err); } + } + } + + /// + /// Passes every attribute a codec writes to the visitor in the form its slot holds it: class properties first, in declaration order, then the plain attributes in the order they were added. + /// Deferred values are loaded and stubs expanded first, as does, but no value is boxed and the lock is released before the visitor runs. + /// + internal void VisitAttributes(ref TVisitor visitor) where TVisitor : struct, IAttributeVisitor + { + var properties = Schema.Properties; + AttributeSlot[] copy; + int copied; + + lock (Attribute_ChangeLock) + { + copied = count; + copy = ArrayPool.Shared.Rent(copied); + for (var i = 0; i < copied; i++) + { + Resolve(i); + copy[i] = slots![i]; + } + } + + try + { + visitor.Begin(properties.Count + copied); + + foreach (var binding in properties) + { + binding.Read(this, out var kind, out var inline, out var reference); + visitor.Visit(binding.AttributeName, kind, in inline, reference); + } + + for (var i = 0; i < copied; i++) + { + ref var slot = ref copy[i]; + visitor.Visit(slot.Name, slot.Kind, in slot.Inline, slot.Reference); + } + } + finally + { + System.Array.Clear(copy, 0, copied); + ArrayPool.Shared.Return(copy); + } + } + + /// + /// Splits a boxed value into the form a slot stores it in. Anything that is not a scalar or vector kind stays a reference, whether or not it is a valid attribute value. + /// + internal static void Classify(object? value, out AttributeKind kind, out InlineValue inline, out object? reference) + { + inline = default; + reference = null; + switch (value) + { + case int v: kind = AttributeKind.Int; inline.Int = v; return; + case float v: kind = AttributeKind.Float; inline.Float = v; return; + case bool v: kind = AttributeKind.Bool; inline.Bool = v; return; + case byte v: kind = AttributeKind.Byte; inline.Byte = v; return; + case ulong v: kind = AttributeKind.UInt64; inline.UInt64 = v; return; + case TimeSpan v: kind = AttributeKind.Time; inline.Ticks = v.Ticks; return; + case Color v: kind = AttributeKind.Color; inline.Color = v; return; + case Vector2 v: kind = AttributeKind.Vector2; inline.Vector2 = v; return; + case Vector3 v: kind = AttributeKind.Vector3; inline.Vector3 = v; return; + case Vector4 v: kind = AttributeKind.Vector4; inline.Vector4 = v; return; + case Quaternion v: kind = AttributeKind.Quaternion; inline.Quaternion = v; return; + case QAngle v: kind = AttributeKind.QAngle; inline.QAngle = v; return; + default: kind = AttributeKind.Reference; reference = value; return; + } + } + + /// + /// The kind a slot stores values of the given type as: inline for the scalar and vector types, a reference for everything else. + /// + internal static AttributeKind KindOf(Type type) + { + if (type == typeof(int)) return AttributeKind.Int; + if (type == typeof(float)) return AttributeKind.Float; + if (type == typeof(bool)) return AttributeKind.Bool; + if (type == typeof(byte)) return AttributeKind.Byte; + if (type == typeof(ulong)) return AttributeKind.UInt64; + if (type == typeof(TimeSpan)) return AttributeKind.Time; + if (type == typeof(Color)) return AttributeKind.Color; + if (type == typeof(Vector2)) return AttributeKind.Vector2; + if (type == typeof(Vector3)) return AttributeKind.Vector3; + if (type == typeof(Vector4)) return AttributeKind.Vector4; + if (type == typeof(Quaternion)) return AttributeKind.Quaternion; + if (type == typeof(QAngle)) return AttributeKind.QAngle; + return AttributeKind.Reference; + } + #region Interfaces IEnumerator IEnumerable.GetEnumerator() diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index c7537dd..45dbe0f 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; @@ -6,6 +7,7 @@ using System.IO; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; namespace Datamodel.Codecs @@ -150,88 +152,35 @@ public StringDictionary(Binary codec, BinaryReader reader) var count = LengthSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32(); Strings.Capacity = count; for (var i = 0; i < count; i++) - AddString(Codec.ReadString_Raw(reader)); + Strings.Add(Codec.ReadString_Raw(reader)); } } /// - /// Constructs a new from a object. + /// Constructs an empty dictionary for writing. The encoder adds every string it meets, in the order it meets them. /// - public StringDictionary(int encoding_version, BinaryWriter writer, Datamodel dm, SerializationContext context) + public StringDictionary(int encoding_version) { EncodingVersion = encoding_version; - Context = context; - Dummy = EncodingVersion == 1; if (!Dummy) - { Indices = []; - Scraped = []; - - ScrapeElement(dm.Root); - - // the prefix attributes are also written as a regular element in version 9 - if (EncodingVersion >= 9 && dm.PrefixAttributes.Count > 0) - { - AddString(string.Empty); - AddString(PrefixElementClass); - foreach (var attr in dm.PrefixAttributes) - { - AddString(attr.Key); - if (attr.Value is string stringValue) - AddString(stringValue); - } - } - } - } - - private readonly HashSet Scraped = []; - private readonly SerializationContext? Context; - - void ScrapeElement(Element? elem) - { - if (elem == null || elem.Stub || Scraped.Contains(elem)) return; - Scraped.Add(elem); - - AddString(elem.Name); - AddString(elem.ClassName); - foreach (var attr in Context!.Attributes[elem]) - { - AddString(attr.Key); - switch (attr.Value) - { - case string stringValue: - AddString(stringValue); - break; - case Element elementValue: - ScrapeElement(elementValue); - break; - case IList elementListValue: - foreach (var array_elem in elementListValue) - ScrapeElement(array_elem); - break; - } - } } /// - /// Add non-nullable string. + /// Adds a string to the table unless it is there already. Nothing is added for a version that writes every string in place. /// - /// - void AddString(string value) + public void AddString(string? value) { - value ??= string.Empty; - if (Indices == null) - { - Strings.Add(value); return; - } + value ??= string.Empty; if (Indices.TryAdd(value, Strings.Count)) Strings.Add(value); } + int GetIndex(string value) { value ??= string.Empty; @@ -661,28 +610,36 @@ void SkipAttribute(BinaryReader reader) reader.BaseStream.Seek(length * count, SeekOrigin.Current); } - readonly struct Encoder + /// + /// Writes a datamodel the way Valve's CDmSerializerBinary does: one pass from the root gathers the strings and fixes the order of the elements, + /// a second pass writes the bodies. Attributes are read in the form their slots hold them, so no value is boxed, and an array of plain values is written in one piece. + /// + sealed class Encoder { - readonly Dictionary ElementIndices; - readonly List ElementOrder; readonly BinaryWriter Writer; readonly StringDictionary StringDict; readonly Datamodel Datamodel; - readonly SerializationContext Context; - readonly int EncodingVersion; + /// The bodies in the order their index entries are written: the root, the prefix attributes when the version stores them as an element, then every element in the order it is first reached. + readonly List Order = []; + readonly Dictionary Indices = []; + + /// Type ids of the inline kinds, filled in as they are met, since a version may not support every kind. + readonly byte[] KindIds = new byte[16]; + readonly Dictionary ArrayIds = []; + readonly byte ElementId, StringId, BinaryId, MatrixId; + public Encoder(BinaryWriter writer, Datamodel dm, int version) { EncodingVersion = version; Writer = writer; Datamodel = dm; - - Context = new SerializationContext(); - StringDict = new StringDictionary(version, writer, dm, Context); - ElementIndices = []; - ElementOrder = []; - + StringDict = new StringDictionary(version); + ElementId = TypeToId(typeof(Element), version); + StringId = TypeToId(typeof(string), version); + BinaryId = TypeToId(typeof(byte[]), version); + MatrixId = TypeToId(typeof(Matrix4x4), version); } public void Encode() @@ -694,107 +651,105 @@ public void Encode() WritePrefixAttributes(); } - StringDict.WriteSelf(Writer); - var hasPrefixElement = EncodingVersion >= 9 && Datamodel.PrefixAttributes.Count > 0; - var elementCount = CountChildren(Datamodel.Root, []) + (hasPrefixElement ? 1 : 0); - Writer.Write(elementCount); - var root = Datamodel.Root; if (root != null && !root.Stub) { - WriteIndexEntry(root, root.ClassName, root.Name, root.ID); + Indices[root] = 0; + Order.Add(root); // the prefix attributes are also stored as an unreferenced element right after the root if (hasPrefixElement) - WriteIndexEntry(Datamodel.PrefixAttributes, PrefixElementClass, string.Empty, Datamodel.PrefixElementId); + Order.Add(Datamodel.PrefixAttributes); - WriteIndexChildren(root); - } + Gather(root); - foreach (var body in ElementOrder) - WriteBody(body); - } - - int CountChildren(Element? elem, HashSet counter) - { - if (elem is null) - { - return 0; + if (hasPrefixElement) + { + StringDict.AddString(string.Empty); + StringDict.AddString(PrefixElementClass); + foreach (var attr in Datamodel.PrefixAttributes) + { + StringDict.AddString(attr.Key); + if (attr.Value is string stringValue) + StringDict.AddString(stringValue); + } + } } - if (elem.Stub) return 0; - int num_elems = 1; - counter.Add(elem); - foreach (var attr in Context.Attributes[elem]) - { - if (attr.Value == null) continue; + StringDict.WriteSelf(Writer); + Writer.Write(Order.Count); - if (attr.Value is Element child_elem && !counter.Contains(child_elem)) - { - num_elems += CountChildren(child_elem, counter); - } - else if (attr.Value is IEnumerable child_array) - { - foreach (var array_elem in child_array.Where(c => c != null && !counter.Contains(c))) - num_elems += CountChildren(array_elem, counter); - } + Span id = stackalloc byte[16]; + foreach (var body in Order) + { + var (className, name, elementId) = body is Element elem ? (elem.ClassName, elem.Name, elem.ID) : (PrefixElementClass, string.Empty, Datamodel.PrefixElementId); + StringDict.WriteString(className, Writer); + if (EncodingVersion >= 4) StringDict.WriteString(name, Writer); + else Writer.Write(name); + elementId.TryWriteBytes(id); + Writer.Write(id); } - return num_elems; + foreach (var body in Order) + WriteBody(body); } - void WriteIndex(Element? elem) + /// + /// Adds the strings of an element to the table and reaches the elements it refers to, depth first in attribute order, which is the order of the index. + /// + void Gather(Element elem) { - if (elem is null || elem.Stub || ElementIndices.ContainsKey(elem)) return; + StringDict.AddString(elem.Name); + StringDict.AddString(elem.ClassName); - WriteIndexEntry(elem, elem.ClassName, elem.Name, elem.ID); - WriteIndexChildren(elem); + var visitor = new GatherVisitor(this); + elem.VisitAttributes(ref visitor); } - void WriteIndexEntry(AttributeList body, string className, string name, Guid id) + void Reach(Element? child) { - if (body is Element elem) - ElementIndices[elem] = ElementOrder.Count; - ElementOrder.Add(body); + if (child == null || child.Stub || Indices.ContainsKey(child)) + return; - StringDict.WriteString(className, Writer); - if (EncodingVersion >= 4) StringDict.WriteString(name, Writer); - else Writer.Write(name); - Writer.Write(id.ToByteArray()); + Indices[child] = Order.Count; + Order.Add(child); + Gather(child); } - void WriteIndexChildren(Element elem) + readonly struct GatherVisitor(Encoder encoder) : IAttributeVisitor { - foreach (var attr in Context.Attributes[elem]) + public void Begin(int count) { - var child_elem = attr.Value as Element; - if (child_elem != null) - { - if (!ElementIndices.ContainsKey(child_elem)) - WriteIndex(child_elem); - } - else + } + + public void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference) + { + encoder.StringDict.AddString(name); + + switch (reference) { - var elem_list = attr.Value as IList; - if (elem_list != null) - { - var elem_indices = ElementIndices; // workaround for .Net 4 lambda limitation in structs - foreach (var item in elem_list.Where(e => e != null && !elem_indices.ContainsKey(e))) - WriteIndex(item); - } + case string stringValue: + encoder.StringDict.AddString(stringValue); + break; + case Element child: + encoder.Reach(child); + break; + case ElementArray children: + foreach (var child in children.AsSpan()) + encoder.Reach(child); + break; + case IList children: + foreach (var child in children) + encoder.Reach(child); + break; } } } - /// - /// Prefix attributes are stored as a list of prefix elements, each a list of name/typed value pairs. - /// Only the first prefix element is read back, so everything is written into a single one. - /// void WritePrefixAttributes() { var prefixAttributes = Datamodel.PrefixAttributes.Where(attr => attr.Value != null).ToArray(); - if (prefixAttributes.Length == 0) { Writer.Write(0); @@ -803,184 +758,297 @@ void WritePrefixAttributes() Writer.Write(1); Writer.Write(prefixAttributes.Length); - foreach (var attr in prefixAttributes) { Writer.Write(attr.Key); - WriteTypedValue(attr.Value, raw_string: true); + AttributeList.Classify(attr.Value, out var kind, out var inline, out var reference); + WriteValue(kind, in inline, reference, rawStrings: true); } } - void WriteBody(AttributeList elem) + void WriteBody(AttributeList body) + { + var visitor = new WriteVisitor(this); + body.VisitAttributes(ref visitor); + } + + readonly struct WriteVisitor(Encoder encoder) : IAttributeVisitor { - var attributesIterated = elem is Element element ? Context.Attributes[element] : elem.GetAllAttributesForSerialization().ToArray(); - Writer.Write(attributesIterated.Length); - foreach (var attr in attributesIterated) + public void Begin(int count) { - StringDict.WriteString(attr.Key, Writer); - WriteTypedValue(attr.Value, raw_string: false); + encoder.Writer.Write(count); + } + + public void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference) + { + encoder.StringDict.WriteString(name, encoder.Writer); + encoder.WriteValue(kind, in inline, reference, rawStrings: false); } } /// - /// Writes the type id of a value followed by the value itself, or by the item count and items for arrays. + /// Writes the type id of a value and the value itself. /// - void WriteTypedValue(object? value, bool raw_string) + /// Whether a string is written in place rather than as an index into the table, as the prefix attributes and array items are. + void WriteValue(AttributeKind kind, in InlineValue inline, object? reference, bool rawStrings) { - var attr_type = value == null ? typeof(Element) : value.GetType(); - var attr_type_id = TypeToId(attr_type, EncodingVersion); - Writer.Write(attr_type_id); - - if (value == null || value is byte[] || !Datamodel.IsDatamodelArrayType(attr_type)) + if (kind != AttributeKind.Reference) { - WriteAttribute(value, raw_string); + Writer.Write(IdOf(kind)); + WriteInline(kind, in inline); return; } - var array = (System.Collections.IList)value; - Writer.Write(array.Count); - foreach (var item in array) - WriteAttribute(item, true); + switch (reference) + { + case null: + Writer.Write(ElementId); + Writer.Write(-1); + return; + case Element elem: + Writer.Write(ElementId); + WriteElement(elem); + return; + case string stringValue: + Writer.Write(StringId); + WriteString(stringValue, rawStrings); + return; + case byte[] binary: + Writer.Write(BinaryId); + Writer.Write(binary.Length); + Writer.Write(binary); + return; + case Matrix4x4 matrix: + Writer.Write(MatrixId); + WriteMatrix(in matrix); + return; + case IList array: + WriteArray(array); + return; + default: + throw new InvalidOperationException("Unrecognised output Type."); + } } - /// Whether the value is an array item or a prefix attribute, in which case strings are written inline rather than through the dictionary. - void WriteAttribute(object? value, bool in_array) + /// + /// Writes an array with its type id. The items of an array whose memory layout matches the stream are written in one piece. + /// + void WriteArray(IList array) { - if (value == null) + Writer.Write(IdOf(array.GetType())); + Writer.Write(array.Count); + + switch (array) { - Writer.Write(-1); - return; + case ElementArray elements: + foreach (var elem in elements.AsSpan()) + { + if (elem == null) + Writer.Write(-1); + else + WriteElement(elem); + } + return; + case StringArray strings: + foreach (var stringValue in strings.AsSpan()) + Writer.Write(stringValue); + return; + case BinaryArray binaries: + foreach (var binary in binaries.AsSpan()) + { + if (binary == null) + { + Writer.Write(-1); + continue; + } + + Writer.Write(binary.Length); + Writer.Write(binary); + } + return; + case TimeSpanArray times: + foreach (var time in times.AsSpan()) + Writer.Write(ToTicks(time)); + return; + case IntArray a: WriteItems(a.AsSpan()); return; + case FloatArray a: WriteItems(a.AsSpan()); return; + case BoolArray a: WriteItems(a.AsSpan()); return; + case ColorArray a: WriteItems(a.AsSpan()); return; + case Vector2Array a: WriteItems(a.AsSpan()); return; + case Vector3Array a: WriteItems(a.AsSpan()); return; + case Vector4Array a: WriteItems(a.AsSpan()); return; + case QuaternionArray a: WriteItems(a.AsSpan()); return; + case MatrixArray a: WriteItems(a.AsSpan()); return; + case ByteArray a: WriteItems(a.AsSpan()); return; + case UInt64Array a: WriteItems(a.AsSpan()); return; } - if (value is Element child_elem) + foreach (var item in array) { - if (child_elem.Stub) + AttributeList.Classify(item, out var kind, out var inline, out var reference); + if (kind != AttributeKind.Reference) + WriteInline(kind, in inline); + else if (reference == null) + Writer.Write(-1); + else if (reference is Element elem) + WriteElement(elem); + else if (reference is string stringValue) + Writer.Write(stringValue); + else if (reference is byte[] binary) { - Writer.Write(-2); - Writer.Write(child_elem.ID.ToString().ToArray()); // yes, ToString()! - Writer.Write((byte)0); + Writer.Write(binary.Length); + Writer.Write(binary); } + else if (reference is Matrix4x4 matrix) + WriteMatrix(in matrix); else - Writer.Write(ElementIndices[child_elem]); - return; - } - - if (value is string string_value) - { - if (EncodingVersion < 4 || in_array) - Writer.Write(string_value); - else - StringDict.WriteString(string_value, Writer); - return; + throw new InvalidOperationException("Unrecognised output Type."); } + } - if (value is bool bool_value) + /// + /// Writes items whose layout in memory is their layout in the stream: the scalars, the vectors and the four by four matrix, all little-endian floats and integers. + /// + void WriteItems(ReadOnlySpan items) where T : unmanaged + { + if (BitConverter.IsLittleEndian) { - Writer.Write(bool_value == true ? (byte)1 : (byte)0); + Writer.Write(MemoryMarshal.AsBytes(items)); return; } - if (value is byte[] binary_value) + foreach (var item in items) { - Writer.Write(binary_value.Length); - Writer.Write(binary_value); - return; + AttributeList.Classify(item, out var kind, out var inline, out var reference); + if (kind != AttributeKind.Reference) + WriteInline(kind, in inline); + else + WriteMatrix((Matrix4x4)reference!); } + } - if (value is TimeSpan time_span) + void WriteInline(AttributeKind kind, in InlineValue inline) + { + switch (kind) { - Writer.Write((int)(time_span.Ticks / (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond))); - return; + case AttributeKind.Int: Writer.Write(inline.Int); return; + case AttributeKind.Float: Writer.Write(inline.Float); return; + case AttributeKind.Bool: Writer.Write(inline.Bool ? (byte)1 : (byte)0); return; + case AttributeKind.Byte: Writer.Write(inline.Byte); return; + case AttributeKind.UInt64: Writer.Write(inline.UInt64); return; + case AttributeKind.Time: Writer.Write(ToTicks(TimeSpan.FromTicks(inline.Ticks))); return; + case AttributeKind.Color: + Writer.Write(inline.Color.R); + Writer.Write(inline.Color.G); + Writer.Write(inline.Color.B); + Writer.Write(inline.Color.A); + return; + case AttributeKind.Vector2: + Writer.Write(inline.Vector2.X); + Writer.Write(inline.Vector2.Y); + return; + case AttributeKind.Vector3: + Writer.Write(inline.Vector3.X); + Writer.Write(inline.Vector3.Y); + Writer.Write(inline.Vector3.Z); + return; + case AttributeKind.QAngle: + Writer.Write(inline.QAngle.Pitch); + Writer.Write(inline.QAngle.Yaw); + Writer.Write(inline.QAngle.Roll); + return; + case AttributeKind.Vector4: + Writer.Write(inline.Vector4.X); + Writer.Write(inline.Vector4.Y); + Writer.Write(inline.Vector4.Z); + Writer.Write(inline.Vector4.W); + return; + case AttributeKind.Quaternion: + Writer.Write(inline.Quaternion.X); + Writer.Write(inline.Quaternion.Y); + Writer.Write(inline.Quaternion.Z); + Writer.Write(inline.Quaternion.W); + return; + default: + throw new InvalidOperationException("Unrecognised output Type."); } + } - if (value is Color colour_value) - { - Writer.Write(colour_value.ToBytes()); - return; - } + void WriteMatrix(in Matrix4x4 matrix) + { + Writer.Write(matrix.M11); + Writer.Write(matrix.M12); + Writer.Write(matrix.M13); + Writer.Write(matrix.M14); + Writer.Write(matrix.M21); + Writer.Write(matrix.M22); + Writer.Write(matrix.M23); + Writer.Write(matrix.M24); + Writer.Write(matrix.M31); + Writer.Write(matrix.M32); + Writer.Write(matrix.M33); + Writer.Write(matrix.M34); + Writer.Write(matrix.M41); + Writer.Write(matrix.M42); + Writer.Write(matrix.M43); + Writer.Write(matrix.M44); + } - if (value is Vector2 vector2) - { - Writer.Write(vector2.X); - Writer.Write(vector2.Y); - return; - } - if (value is Vector3 vector3) - { - Writer.Write(vector3.X); - Writer.Write(vector3.Y); - Writer.Write(vector3.Z); - return; - } - if (value is QAngle qangle) - { - Writer.Write(qangle.Pitch); - Writer.Write(qangle.Yaw); - Writer.Write(qangle.Roll); - return; - } - if (value is Vector4 vector4) - { - Writer.Write(vector4.X); - Writer.Write(vector4.Y); - Writer.Write(vector4.Z); - Writer.Write(vector4.W); - return; - } - if (value is Quaternion quaternion) + void WriteElement(Element elem) + { + if (elem.Stub) { - Writer.Write(quaternion.X); - Writer.Write(quaternion.Y); - Writer.Write(quaternion.Z); - Writer.Write(quaternion.W); - return; + Writer.Write(-2); + Writer.Write(elem.ID.ToString().ToCharArray()); // yes, ToString()! + Writer.Write((byte)0); } - if (value is Matrix4x4 matrix) + else { - Writer.Write(matrix.M11); - Writer.Write(matrix.M12); - Writer.Write(matrix.M13); - Writer.Write(matrix.M14); - Writer.Write(matrix.M21); - Writer.Write(matrix.M22); - Writer.Write(matrix.M23); - Writer.Write(matrix.M24); - Writer.Write(matrix.M31); - Writer.Write(matrix.M32); - Writer.Write(matrix.M33); - Writer.Write(matrix.M34); - Writer.Write(matrix.M41); - Writer.Write(matrix.M42); - Writer.Write(matrix.M43); - Writer.Write(matrix.M44); - return; + Writer.Write(Indices[elem]); } + } - if (value is int intValue) - { - Writer.Write(intValue); - return; - } - if (value is float floatValue) - { - Writer.Write(floatValue); - return; - } + void WriteString(string value, bool raw) + { + if (EncodingVersion < 4 || raw) + Writer.Write(value); + else + StringDict.WriteString(value, Writer); + } - if (value is byte byteValue) - { - Writer.Write(byteValue); - return; - } + static int ToTicks(TimeSpan time) => (int)(time.Ticks / (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)); - if (value is ulong ulongValue) - { - Writer.Write(ulongValue); - return; - } + byte IdOf(AttributeKind kind) + { + ref var id = ref KindIds[(int)kind]; + if (id == 0) + id = TypeToId(TypeOf(kind), EncodingVersion); + return id; + } - throw new InvalidOperationException("Unrecognised output Type."); + byte IdOf(Type arrayType) + { + if (!ArrayIds.TryGetValue(arrayType, out var id)) + ArrayIds[arrayType] = id = TypeToId(arrayType, EncodingVersion); + return id; } + + static Type TypeOf(AttributeKind kind) => kind switch + { + AttributeKind.Int => typeof(int), + AttributeKind.Float => typeof(float), + AttributeKind.Bool => typeof(bool), + AttributeKind.Byte => typeof(byte), + AttributeKind.UInt64 => typeof(ulong), + AttributeKind.Time => typeof(TimeSpan), + AttributeKind.Color => typeof(Color), + AttributeKind.Vector2 => typeof(Vector2), + AttributeKind.Vector3 => typeof(Vector3), + AttributeKind.Vector4 => typeof(Vector4), + AttributeKind.Quaternion => typeof(Quaternion), + AttributeKind.QAngle => typeof(QAngle), + _ => throw new InvalidOperationException("Unrecognised output Type."), + }; } class DmxBinaryWriter : BinaryWriter diff --git a/Datamodel.NET/ElementSchema.cs b/Datamodel.NET/ElementSchema.cs index ed6544e..7272674 100644 --- a/Datamodel.NET/ElementSchema.cs +++ b/Datamodel.NET/ElementSchema.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace Datamodel { @@ -98,6 +99,14 @@ public virtual void SetValue(AttributeList owner, object? value) setter(owner, value); } + /// + /// Reads the property in the form an attribute slot stores it, so that a codec writes it without boxing when the binding is typed. + /// + internal virtual void Read(AttributeList owner, out AttributeKind kind, out InlineValue inline, out object? reference) + { + AttributeList.Classify(GetValue(owner), out kind, out inline, out reference); + } + public override string ToString() => $"{PropertyName} <{PropertyType.Name}> as \"{AttributeName}\""; } @@ -146,6 +155,25 @@ public PropertyBinding(string propertyName, string attributeName, Func getter((TElement)owner); + /// The kind a slot stores values of as, decided once per type. + static readonly AttributeKind ValueKind = AttributeList.KindOf(typeof(TValue)); + + internal override void Read(AttributeList owner, out AttributeKind kind, out InlineValue inline, out object? reference) + { + var value = getter((TElement)owner); + kind = ValueKind; + inline = default; + if (ValueKind == AttributeKind.Reference) + { + reference = value; + } + else + { + reference = null; + Unsafe.As(ref inline) = value; + } + } + public override void Set(AttributeList owner, TValue value) { if (setter == null) From 3c755d4296ae2d9f9519bb72ccf0cff3db97a323 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 14:07:28 +0200 Subject: [PATCH 12/16] Buffer the output and look attribute names up by identity --- Datamodel.NET/Codecs/Binary.cs | 171 +++++++++++++++++++++++++++------ 1 file changed, 139 insertions(+), 32 deletions(-) diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index 45dbe0f..21dc751 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers.Binary; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -133,6 +134,12 @@ class StringDictionary /// Fast string index lookup. readonly Dictionary? Indices; + /// + /// Indices of the attribute and class name instances met so far. Names of class properties and of attributes read from a file are shared instances, + /// so this finds them without hashing their characters, the way Valve looks attribute names up as symbols. + /// + readonly Dictionary NameIndices = new(ReferenceEqualityComparer.Instance); + public bool Dummy; // binary 4 uses int for dictionary length, but short for dictionary indices. Whoops! @@ -187,25 +194,53 @@ int GetIndex(string value) return Indices!.TryGetValue(value, out var index) ? index : -1; } + /// + /// Adds an attribute or class name to the table unless it is there already. + /// + public void AddName(string name) + { + if (Indices == null || NameIndices.ContainsKey(name)) + return; + + AddString(name); + NameIndices[name] = Indices[name]; + } + + public void WriteName(string name, OutputBuffer writer) + { + if (Dummy) + { + writer.Write(name); + return; + } + + if (!NameIndices.TryGetValue(name, out var index)) + NameIndices[name] = index = GetIndex(name); + + WriteIndex(index, writer); + } + public string ReadString(BinaryReader reader) { if (Dummy) return Codec!.ReadString_Raw(reader); return Strings[IndiceSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32()]; } - public void WriteString(string value, BinaryWriter writer) + public void WriteString(string value, OutputBuffer writer) { if (Dummy) writer.Write(value); else - { - var index = GetIndex(value); - if (IndiceSize == sizeof(short)) writer.Write((short)index); - else writer.Write(index); - } + WriteIndex(GetIndex(value), writer); } - public void WriteSelf(BinaryWriter writer) + void WriteIndex(int index, OutputBuffer writer) + { + if (IndiceSize == sizeof(short)) writer.Write((short)index); + else writer.Write(index); + } + + public void WriteSelf(OutputBuffer writer) { if (Dummy) return; @@ -222,9 +257,9 @@ public void WriteSelf(BinaryWriter writer) public void Encode(Datamodel dm, string encoding, int encoding_version, Stream stream) { - using var writer = new DmxBinaryWriter(stream); - var encoder = new Encoder(writer, dm, encoding_version); - encoder.Encode(); + var output = new OutputBuffer(stream); + new Encoder(output, dm, encoding_version).Encode(); + output.Flush(); } private static readonly Dictionary TypeMap = new Dictionary @@ -616,7 +651,7 @@ void SkipAttribute(BinaryReader reader) /// sealed class Encoder { - readonly BinaryWriter Writer; + readonly OutputBuffer Writer; readonly StringDictionary StringDict; readonly Datamodel Datamodel; readonly int EncodingVersion; @@ -630,7 +665,7 @@ sealed class Encoder readonly Dictionary ArrayIds = []; readonly byte ElementId, StringId, BinaryId, MatrixId; - public Encoder(BinaryWriter writer, Datamodel dm, int version) + public Encoder(OutputBuffer writer, Datamodel dm, int version) { EncodingVersion = version; Writer = writer; @@ -684,7 +719,7 @@ public void Encode() foreach (var body in Order) { var (className, name, elementId) = body is Element elem ? (elem.ClassName, elem.Name, elem.ID) : (PrefixElementClass, string.Empty, Datamodel.PrefixElementId); - StringDict.WriteString(className, Writer); + StringDict.WriteName(className, Writer); if (EncodingVersion >= 4) StringDict.WriteString(name, Writer); else Writer.Write(name); elementId.TryWriteBytes(id); @@ -701,7 +736,7 @@ public void Encode() void Gather(Element elem) { StringDict.AddString(elem.Name); - StringDict.AddString(elem.ClassName); + StringDict.AddName(elem.ClassName); var visitor = new GatherVisitor(this); elem.VisitAttributes(ref visitor); @@ -725,7 +760,7 @@ public void Begin(int count) public void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference) { - encoder.StringDict.AddString(name); + encoder.StringDict.AddName(name); switch (reference) { @@ -781,7 +816,7 @@ public void Begin(int count) public void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference) { - encoder.StringDict.WriteString(name, encoder.Writer); + encoder.StringDict.WriteName(name, encoder.Writer); encoder.WriteValue(kind, in inline, reference, rawStrings: false); } } @@ -999,8 +1034,7 @@ void WriteElement(Element elem) if (elem.Stub) { Writer.Write(-2); - Writer.Write(elem.ID.ToString().ToCharArray()); // yes, ToString()! - Writer.Write((byte)0); + Writer.Write(elem.ID.ToString()); // yes, ToString()! } else { @@ -1051,27 +1085,100 @@ byte IdOf(Type arrayType) }; } - class DmxBinaryWriter : BinaryWriter + /// + /// Collects the output and hands it to the stream in large pieces, the way Valve's CUtlBuffer does, so that writing a value is a few stores rather than a call into the stream. + /// + sealed class OutputBuffer(Stream stream) { - public DmxBinaryWriter(Stream output) - : base(output, Datamodel.TextEncoding) - { } + readonly byte[] buffer = new byte[1 << 16]; + int used; + + public void Flush() + { + if (used > 0) + { + stream.Write(buffer, 0, used); + used = 0; + } + } + + void Reserve(int size) + { + if (buffer.Length - used < size) + Flush(); + } + + public void Write(byte value) + { + Reserve(1); + buffer[used++] = value; + } + + public void Write(short value) + { + Reserve(2); + BinaryPrimitives.WriteInt16LittleEndian(buffer.AsSpan(used), value); + used += 2; + } + + public void Write(int value) + { + Reserve(4); + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(used), value); + used += 4; + } + + public void Write(float value) + { + Reserve(4); + BinaryPrimitives.WriteSingleLittleEndian(buffer.AsSpan(used), value); + used += 4; + } + + public void Write(ulong value) + { + Reserve(8); + BinaryPrimitives.WriteUInt64LittleEndian(buffer.AsSpan(used), value); + used += 8; + } + + public void Write(ReadOnlySpan bytes) + { + if (bytes.Length > buffer.Length - used) + { + Flush(); + if (bytes.Length > buffer.Length) + { + stream.Write(bytes); + return; + } + } + + bytes.CopyTo(buffer.AsSpan(used)); + used += bytes.Length; + } /// - /// Writes a null-terminated string to the underlying stream using . + /// Writes a string in followed by a zero byte. /// - /// - [System.Security.SecuritySafeCritical] - public override void Write(string value) + public void Write(string? value) { if (value != null) - base.Write(Datamodel.TextEncoding.GetBytes(value)); - base.Write((byte)0); - } + { + var encoding = Datamodel.TextEncoding; + var room = encoding.GetMaxByteCount(value.Length); + if (room > buffer.Length) + { + Write(encoding.GetBytes(value)); + } + else + { + Reserve(room); + used += encoding.GetBytes(value, buffer.AsSpan(used)); + } + } - protected override void Dispose(bool disposing) - { - return; // don't mess with the base stream! + Write((byte)0); } } } From 069c7bdefb851080d5e388c0ed1e8aab0d35492f Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 14:08:28 +0200 Subject: [PATCH 13/16] Size plain elements from the attribute count --- Datamodel.NET/AttributeList.cs | 12 ++++++++++++ Datamodel.NET/Codecs/Binary.cs | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 76aa1bd..9c24d46 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -178,6 +178,18 @@ int Find(string name) /// /// Adds an empty slot with the given name at the end. The caller holds the lock. /// + /// + /// Makes room for the given number of attributes, so that a codec that knows the count adds them without growing the slots. + /// + internal void EnsureCapacity(int capacity) + { + lock (Attribute_ChangeLock) + { + if (slots == null || slots.Length < capacity) + System.Array.Resize(ref slots, capacity); + } + } + ref AttributeSlot Append(string name) { if (slots == null || count == slots.Length) diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index 21dc751..96bcc2d 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -441,6 +441,10 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in var num_attrs = Reader.ReadInt32(); + // a plain element gets exactly the slots it needs; a class keeps most attributes in its properties, so its few slots grow as they come + if (elem.Schema.Properties.Count == 0) + elem.EnsureCapacity(num_attrs); + for (var i = 0; i < num_attrs; i++) { var name = StringDict.ReadString(Reader); From 9933eca5eacc69750378a635d47d1e79741efba7 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 8 Sep 2026 15:07:10 +0200 Subject: [PATCH 14/16] Name the attribute types once, for slots and the binary encoding --- Datamodel.NET/AttributeList.cs | 202 +++++++++------- Datamodel.NET/Codecs/Binary.cs | 428 ++++++++++++++++----------------- Datamodel.NET/ElementSchema.cs | 21 +- 3 files changed, 336 insertions(+), 315 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 9c24d46..256f344 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -16,25 +16,31 @@ namespace Datamodel { /// - /// What an holds inline. covers everything stored as an object: strings, binary blobs, matrices, elements, arrays and null. + /// The types an attribute value can have, named as Valve's DmAttributeType_t names them: what a slot stores, what a class property is read as + /// and what the binary encoding writes. Values of the scalar and vector types live in the slot itself, see ; the rest are held as references. /// - enum AttributeKind : byte + enum AttributeType : byte { - Reference, - /// The value has not been read from the stream yet; holds the position it starts at. - Deferred, + Element, Int, Float, Bool, - Byte, - UInt64, + String, + Binary, Time, Color, Vector2, Vector3, Vector4, - Quaternion, QAngle, + Quaternion, + Matrix, + UInt64, + Byte, + /// An array of any of the above. + Array, + /// The value has not been read from the stream yet; holds the position it starts at. + Deferred, } /// @@ -66,7 +72,7 @@ struct AttributeSlot public string Name; public object? Reference; public InlineValue Inline; - public AttributeKind Kind; + public AttributeType Kind; public AttributeList.OverrideType? Override; } @@ -78,7 +84,7 @@ interface IAttributeVisitor /// Called once before the attributes, with how many follow. void Begin(int count); - void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference); + void Visit(string name, AttributeType kind, in InlineValue inline, object? reference); } /// @@ -228,24 +234,29 @@ void RemoveSlot(int index) slots![count] = default; } - static AttributeKind KindOf() where T : unmanaged + /// Whether values of the type are stored in the slot itself rather than as a reference. + internal static bool IsInline(AttributeType type) => type is AttributeType.Int or AttributeType.Float or AttributeType.Bool or AttributeType.Byte or AttributeType.UInt64 + or AttributeType.Time or AttributeType.Color or AttributeType.Vector2 or AttributeType.Vector3 or AttributeType.Vector4 or AttributeType.Quaternion or AttributeType.QAngle; + + /// The type a slot stores values of as, or null when it is not one of the types stored inline. + static AttributeType? KindOf() where T : unmanaged { - if (typeof(T) == typeof(int)) return AttributeKind.Int; - if (typeof(T) == typeof(float)) return AttributeKind.Float; - if (typeof(T) == typeof(bool)) return AttributeKind.Bool; - if (typeof(T) == typeof(byte)) return AttributeKind.Byte; - if (typeof(T) == typeof(ulong)) return AttributeKind.UInt64; - if (typeof(T) == typeof(TimeSpan)) return AttributeKind.Time; - if (typeof(T) == typeof(Color)) return AttributeKind.Color; - if (typeof(T) == typeof(Vector2)) return AttributeKind.Vector2; - if (typeof(T) == typeof(Vector3)) return AttributeKind.Vector3; - if (typeof(T) == typeof(Vector4)) return AttributeKind.Vector4; - if (typeof(T) == typeof(Quaternion)) return AttributeKind.Quaternion; - if (typeof(T) == typeof(QAngle)) return AttributeKind.QAngle; - return AttributeKind.Reference; + if (typeof(T) == typeof(int)) return AttributeType.Int; + if (typeof(T) == typeof(float)) return AttributeType.Float; + if (typeof(T) == typeof(bool)) return AttributeType.Bool; + if (typeof(T) == typeof(byte)) return AttributeType.Byte; + if (typeof(T) == typeof(ulong)) return AttributeType.UInt64; + if (typeof(T) == typeof(TimeSpan)) return AttributeType.Time; + if (typeof(T) == typeof(Color)) return AttributeType.Color; + if (typeof(T) == typeof(Vector2)) return AttributeType.Vector2; + if (typeof(T) == typeof(Vector3)) return AttributeType.Vector3; + if (typeof(T) == typeof(Vector4)) return AttributeType.Vector4; + if (typeof(T) == typeof(Quaternion)) return AttributeType.Quaternion; + if (typeof(T) == typeof(QAngle)) return AttributeType.QAngle; + return null; } - static void WriteInline(ref AttributeSlot slot, AttributeKind kind, T value) where T : unmanaged + static void WriteInline(ref AttributeSlot slot, AttributeType kind, T value) where T : unmanaged { slot.Kind = kind; slot.Reference = null; @@ -261,44 +272,53 @@ void Store(ref AttributeSlot slot, object? value) switch (value) { case null: - slot.Kind = AttributeKind.Reference; + slot.Kind = AttributeType.Element; slot.Reference = null; return; - case int v: WriteInline(ref slot, AttributeKind.Int, v); return; - case float v: WriteInline(ref slot, AttributeKind.Float, v); return; - case bool v: WriteInline(ref slot, AttributeKind.Bool, v); return; - case byte v: WriteInline(ref slot, AttributeKind.Byte, v); return; - case ulong v: WriteInline(ref slot, AttributeKind.UInt64, v); return; - case TimeSpan v: WriteInline(ref slot, AttributeKind.Time, v); return; - case Color v: WriteInline(ref slot, AttributeKind.Color, v); return; - case Vector2 v: WriteInline(ref slot, AttributeKind.Vector2, v); return; - case Vector3 v: WriteInline(ref slot, AttributeKind.Vector3, v); return; - case Vector4 v: WriteInline(ref slot, AttributeKind.Vector4, v); return; - case Quaternion v: WriteInline(ref slot, AttributeKind.Quaternion, v); return; - case QAngle v: WriteInline(ref slot, AttributeKind.QAngle, v); return; + case int v: WriteInline(ref slot, AttributeType.Int, v); return; + case float v: WriteInline(ref slot, AttributeType.Float, v); return; + case bool v: WriteInline(ref slot, AttributeType.Bool, v); return; + case byte v: WriteInline(ref slot, AttributeType.Byte, v); return; + case ulong v: WriteInline(ref slot, AttributeType.UInt64, v); return; + case TimeSpan v: WriteInline(ref slot, AttributeType.Time, v); return; + case Color v: WriteInline(ref slot, AttributeType.Color, v); return; + case Vector2 v: WriteInline(ref slot, AttributeType.Vector2, v); return; + case Vector3 v: WriteInline(ref slot, AttributeType.Vector3, v); return; + case Vector4 v: WriteInline(ref slot, AttributeType.Vector4, v); return; + case Quaternion v: WriteInline(ref slot, AttributeType.Quaternion, v); return; + case QAngle v: WriteInline(ref slot, AttributeType.QAngle, v); return; case Element elem: if (elem.Owner == null) elem.Owner = Owner; else if (elem.Owner != Owner) throw new ElementOwnershipException(); + slot.Kind = AttributeType.Element; break; case ElementArray array: if (array.Owner == null) array.Owner = this; else if (array.Owner != this) throw new InvalidOperationException("ElementArray is already owned by a different Datamodel."); + slot.Kind = AttributeType.Array; break; case IEnumerable: throw new InvalidOperationException("Element array objects must derive from Datamodel.ElementArray"); - case string or byte[] or Matrix4x4: + case string: + slot.Kind = AttributeType.String; + break; + case byte[]: + slot.Kind = AttributeType.Binary; + break; + case Matrix4x4: + slot.Kind = AttributeType.Matrix; break; default: if (!Datamodel.IsDatamodelType(value.GetType())) throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); + slot.Kind = AttributeType.Array; break; } - slot.Kind = AttributeKind.Reference; slot.Reference = value; } @@ -309,20 +329,20 @@ void Store(ref AttributeSlot slot, object? value) { return slot.Kind switch { - AttributeKind.Reference => slot.Reference, - AttributeKind.Deferred => null, - AttributeKind.Int => slot.Inline.Int, - AttributeKind.Float => slot.Inline.Float, - AttributeKind.Bool => slot.Inline.Bool, - AttributeKind.Byte => slot.Inline.Byte, - AttributeKind.UInt64 => slot.Inline.UInt64, - AttributeKind.Time => TimeSpan.FromTicks(slot.Inline.Ticks), - AttributeKind.Color => slot.Inline.Color, - AttributeKind.Vector2 => slot.Inline.Vector2, - AttributeKind.Vector3 => slot.Inline.Vector3, - AttributeKind.Vector4 => slot.Inline.Vector4, - AttributeKind.Quaternion => slot.Inline.Quaternion, - AttributeKind.QAngle => slot.Inline.QAngle, + AttributeType.Element or AttributeType.String or AttributeType.Binary or AttributeType.Matrix or AttributeType.Array => slot.Reference, + AttributeType.Deferred => null, + AttributeType.Int => slot.Inline.Int, + AttributeType.Float => slot.Inline.Float, + AttributeType.Bool => slot.Inline.Bool, + AttributeType.Byte => slot.Inline.Byte, + AttributeType.UInt64 => slot.Inline.UInt64, + AttributeType.Time => TimeSpan.FromTicks(slot.Inline.Ticks), + AttributeType.Color => slot.Inline.Color, + AttributeType.Vector2 => slot.Inline.Vector2, + AttributeType.Vector3 => slot.Inline.Vector3, + AttributeType.Vector4 => slot.Inline.Vector4, + AttributeType.Quaternion => slot.Inline.Quaternion, + AttributeType.QAngle => slot.Inline.QAngle, _ => throw new InvalidOperationException("Unknown attribute kind."), }; } @@ -369,7 +389,7 @@ internal void SetDeferred(string name, long offset) { var index = Find(name); ref var slot = ref (index < 0 ? ref Append(name) : ref slots![index]); - slot.Kind = AttributeKind.Deferred; + slot.Kind = AttributeType.Deferred; slot.Reference = null; slot.Override = null; slot.Inline = default; @@ -417,8 +437,7 @@ public void Set(string name, T value) where T : unmanaged return; } - var kind = KindOf(); - if (kind == AttributeKind.Reference) + if (KindOf() is not AttributeType kind) { this[name] = value; return; @@ -484,7 +503,7 @@ public void SetOverrideType(string key, OverrideType? type) case null: break; case OverrideType.Angle: - if (slot.Kind != AttributeKind.Vector3) + if (slot.Kind != AttributeType.Vector3) throw new AttributeTypeException("OverrideType.Angle can only be applied to Vector3 attributes"); break; case OverrideType.Binary: @@ -815,11 +834,11 @@ public IEnumerator GetEnumerator() /// void Resolve(int index) { - if (slots![index].Kind == AttributeKind.Deferred) + if (slots![index].Kind == AttributeType.Deferred) LoadDeferred(index); ref var slot = ref slots[index]; - if (slot.Kind == AttributeKind.Reference && slot.Reference is Element { Stub: true } stub && Owner != null) + if (slot.Kind == AttributeType.Element && slot.Reference is Element { Stub: true } stub && Owner != null) { try { slot.Reference = Owner.OnStubRequest(stub.ID) ?? stub; } catch (Exception err) { throw new DestubException(this, slot.Name, err); } @@ -871,48 +890,53 @@ internal void VisitAttributes(ref TVisitor visitor) where TVisitor : s } /// - /// Splits a boxed value into the form a slot stores it in. Anything that is not a scalar or vector kind stays a reference, whether or not it is a valid attribute value. + /// Splits a boxed value into the form a slot stores it in. Anything that is not a scalar, vector, element, string, blob or matrix counts as an array, whether or not it is a valid attribute value. /// - internal static void Classify(object? value, out AttributeKind kind, out InlineValue inline, out object? reference) + internal static void Classify(object? value, out AttributeType kind, out InlineValue inline, out object? reference) { inline = default; reference = null; switch (value) { - case int v: kind = AttributeKind.Int; inline.Int = v; return; - case float v: kind = AttributeKind.Float; inline.Float = v; return; - case bool v: kind = AttributeKind.Bool; inline.Bool = v; return; - case byte v: kind = AttributeKind.Byte; inline.Byte = v; return; - case ulong v: kind = AttributeKind.UInt64; inline.UInt64 = v; return; - case TimeSpan v: kind = AttributeKind.Time; inline.Ticks = v.Ticks; return; - case Color v: kind = AttributeKind.Color; inline.Color = v; return; - case Vector2 v: kind = AttributeKind.Vector2; inline.Vector2 = v; return; - case Vector3 v: kind = AttributeKind.Vector3; inline.Vector3 = v; return; - case Vector4 v: kind = AttributeKind.Vector4; inline.Vector4 = v; return; - case Quaternion v: kind = AttributeKind.Quaternion; inline.Quaternion = v; return; - case QAngle v: kind = AttributeKind.QAngle; inline.QAngle = v; return; - default: kind = AttributeKind.Reference; reference = value; return; + case int v: kind = AttributeType.Int; inline.Int = v; return; + case float v: kind = AttributeType.Float; inline.Float = v; return; + case bool v: kind = AttributeType.Bool; inline.Bool = v; return; + case byte v: kind = AttributeType.Byte; inline.Byte = v; return; + case ulong v: kind = AttributeType.UInt64; inline.UInt64 = v; return; + case TimeSpan v: kind = AttributeType.Time; inline.Ticks = v.Ticks; return; + case Color v: kind = AttributeType.Color; inline.Color = v; return; + case Vector2 v: kind = AttributeType.Vector2; inline.Vector2 = v; return; + case Vector3 v: kind = AttributeType.Vector3; inline.Vector3 = v; return; + case Vector4 v: kind = AttributeType.Vector4; inline.Vector4 = v; return; + case Quaternion v: kind = AttributeType.Quaternion; inline.Quaternion = v; return; + case QAngle v: kind = AttributeType.QAngle; inline.QAngle = v; return; + case null: kind = AttributeType.Element; return; + case Element: kind = AttributeType.Element; reference = value; return; + case string: kind = AttributeType.String; reference = value; return; + case byte[]: kind = AttributeType.Binary; reference = value; return; + case Matrix4x4: kind = AttributeType.Matrix; reference = value; return; + default: kind = AttributeType.Array; reference = value; return; } } /// /// The kind a slot stores values of the given type as: inline for the scalar and vector types, a reference for everything else. /// - internal static AttributeKind KindOf(Type type) + internal static AttributeType? KindOf(Type type) { - if (type == typeof(int)) return AttributeKind.Int; - if (type == typeof(float)) return AttributeKind.Float; - if (type == typeof(bool)) return AttributeKind.Bool; - if (type == typeof(byte)) return AttributeKind.Byte; - if (type == typeof(ulong)) return AttributeKind.UInt64; - if (type == typeof(TimeSpan)) return AttributeKind.Time; - if (type == typeof(Color)) return AttributeKind.Color; - if (type == typeof(Vector2)) return AttributeKind.Vector2; - if (type == typeof(Vector3)) return AttributeKind.Vector3; - if (type == typeof(Vector4)) return AttributeKind.Vector4; - if (type == typeof(Quaternion)) return AttributeKind.Quaternion; - if (type == typeof(QAngle)) return AttributeKind.QAngle; - return AttributeKind.Reference; + if (type == typeof(int)) return AttributeType.Int; + if (type == typeof(float)) return AttributeType.Float; + if (type == typeof(bool)) return AttributeType.Bool; + if (type == typeof(byte)) return AttributeType.Byte; + if (type == typeof(ulong)) return AttributeType.UInt64; + if (type == typeof(TimeSpan)) return AttributeType.Time; + if (type == typeof(Color)) return AttributeType.Color; + if (type == typeof(Vector2)) return AttributeType.Vector2; + if (type == typeof(Vector3)) return AttributeType.Vector3; + if (type == typeof(Vector4)) return AttributeType.Vector4; + if (type == typeof(Quaternion)) return AttributeType.Quaternion; + if (type == typeof(QAngle)) return AttributeType.QAngle; + return null; } #region Interfaces diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index 96bcc2d..1460c2b 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -15,7 +15,8 @@ namespace Datamodel.Codecs { class Binary : IDeferredAttributeCodec { - static readonly Dictionary SupportedAttributes = []; + /// The types each encoding version supports, in the order of their ids. A null marks an id the version reserves for something the library does not read. + static readonly Dictionary SupportedAttributes = []; BinaryReader? Reader; /// @@ -32,29 +33,63 @@ static Binary() { SupportedAttributes[1] = SupportedAttributes[2] = [ - typeof(Element), typeof(int), typeof(float), typeof(bool), typeof(string), typeof(byte[]), - null /* ObjectID */, typeof(Color), typeof(Vector2), typeof(Vector3), typeof(Vector4), typeof(Vector3) /* angle*/, typeof(Quaternion), typeof(Matrix4x4) + AttributeType.Element, AttributeType.Int, AttributeType.Float, AttributeType.Bool, AttributeType.String, AttributeType.Binary, + null /* ObjectID */, AttributeType.Color, AttributeType.Vector2, AttributeType.Vector3, AttributeType.Vector4, AttributeType.Vector3 /* angle */, AttributeType.Quaternion, AttributeType.Matrix ]; + SupportedAttributes[3] = SupportedAttributes[4] = SupportedAttributes[5] = [ - typeof(Element), typeof(int), typeof(float), typeof(bool), typeof(string), typeof(byte[]), - typeof(TimeSpan), typeof(Color), typeof(Vector2), typeof(Vector3), typeof(Vector4), typeof(Vector3) /* angle*/, typeof(Quaternion), typeof(Matrix4x4) + AttributeType.Element, AttributeType.Int, AttributeType.Float, AttributeType.Bool, AttributeType.String, AttributeType.Binary, + AttributeType.Time, AttributeType.Color, AttributeType.Vector2, AttributeType.Vector3, AttributeType.Vector4, AttributeType.Vector3 /* angle */, AttributeType.Quaternion, AttributeType.Matrix ]; + SupportedAttributes[9] = [ - typeof(Element), typeof(int), typeof(float), typeof(bool), typeof(string), typeof(byte[]), - typeof(TimeSpan), typeof(Color), typeof(Vector2), typeof(Vector3), typeof(Vector4), typeof(QAngle), typeof(Quaternion), typeof(Matrix4x4), - typeof(ulong), typeof(byte) + AttributeType.Element, AttributeType.Int, AttributeType.Float, AttributeType.Bool, AttributeType.String, AttributeType.Binary, + AttributeType.Time, AttributeType.Color, AttributeType.Vector2, AttributeType.Vector3, AttributeType.Vector4, AttributeType.QAngle, AttributeType.Quaternion, AttributeType.Matrix, + AttributeType.UInt64, AttributeType.Byte ]; } + /// The CLR type a value of the given type is stored as. + static Type ClrType(AttributeType type) => type switch + { + AttributeType.Element => typeof(Element), + AttributeType.Int => typeof(int), + AttributeType.Float => typeof(float), + AttributeType.Bool => typeof(bool), + AttributeType.String => typeof(string), + AttributeType.Binary => typeof(byte[]), + AttributeType.Time => typeof(TimeSpan), + AttributeType.Color => typeof(Color), + AttributeType.Vector2 => typeof(Vector2), + AttributeType.Vector3 => typeof(Vector3), + AttributeType.Vector4 => typeof(Vector4), + AttributeType.QAngle => typeof(QAngle), + AttributeType.Quaternion => typeof(Quaternion), + AttributeType.Matrix => typeof(Matrix4x4), + AttributeType.UInt64 => typeof(ulong), + AttributeType.Byte => typeof(byte), + _ => throw new ArgumentOutOfRangeException(nameof(type)), + }; + + /// The id a version writes for a value type. + static byte TypeToId(AttributeType type, int version) + { + var index = System.Array.IndexOf(SupportedAttributes[version], (AttributeType?)type); + if (index < 0) + throw new CodecException($"\"{type}\" is not supported in encoding binary {version}"); + + return (byte)(index + 1); + } + static byte TypeToId(Type type, int version) { // a byte[] is a "binary" blob, distinct from a "uint8_array" (Array) in encoding version 9 bool array = type != typeof(byte[]) && Datamodel.IsDatamodelArrayType(type); var search_type = array ? Datamodel.GetArrayInnerType(type) : type; - if (array && search_type == typeof(byte) && !SupportedAttributes[version].Contains(typeof(byte))) + if (array && search_type == typeof(byte) && !SupportedAttributes[version].Contains(AttributeType.Byte)) { search_type = typeof(byte[]); // Recent version of DMX support both "binary" and "uint8_array" attributes. These are the same thing! array = false; @@ -63,10 +98,10 @@ static byte TypeToId(Type type, int version) byte i = 0; foreach (var list_type in type_list) { - if (list_type == typeof(Element) && type.IsSubclassOf(typeof(Element))) + if (list_type == AttributeType.Element && type.IsSubclassOf(typeof(Element))) break; - if (list_type == search_type) + if (list_type is AttributeType known && ClrType(known) == search_type) break; i++; } @@ -79,7 +114,7 @@ static byte TypeToId(Type type, int version) /// /// Maps a type id of the stream to the attribute type, or to the item type when the id denotes an array. /// - (Type Type, bool IsArray) IdToType(byte id) + (AttributeType Type, bool IsArray) IdToType(byte id) { var type_list = SupportedAttributes[EncodingVersion]; bool array = false; @@ -100,7 +135,7 @@ static byte TypeToId(Type type, int version) } } - if (id >= type_list.Length || type_list[id] is not Type type) + if (id >= type_list.Length || type_list[id] is not AttributeType type) { throw new CodecException(String.Format("Unrecognised attribute type: {0}", id + 1)); } @@ -262,47 +297,27 @@ public void Encode(Datamodel dm, string encoding, int encoding_version, Stream s output.Flush(); } - private static readonly Dictionary TypeMap = new Dictionary - { - { typeof(Element).TypeHandle, 0 }, - { typeof(int).TypeHandle, 1 }, - { typeof(float).TypeHandle, 2 }, - { typeof(bool).TypeHandle, 3 }, - { typeof(string).TypeHandle, 4 }, - { typeof(byte[]).TypeHandle, 5 }, - { typeof(TimeSpan).TypeHandle, 6 }, - { typeof(Color).TypeHandle, 7 }, - { typeof(Vector2).TypeHandle, 8 }, - { typeof(Vector3).TypeHandle, 9 }, - { typeof(QAngle).TypeHandle, 10 }, - { typeof(Vector4).TypeHandle, 11 }, - { typeof(Quaternion).TypeHandle, 12 }, - { typeof(Matrix4x4).TypeHandle, 13 }, - { typeof(byte).TypeHandle, 14 }, - { typeof(UInt64).TypeHandle, 15 } - }; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - object? ReadValue(Datamodel dm, int typeIndex, bool raw_string, BinaryReader reader) + object? ReadValue(Datamodel dm, AttributeType type, bool raw_string, BinaryReader reader) { - return typeIndex switch + return type switch { - 0 => ReadElement(dm, reader), - 1 => reader.ReadInt32(), - 2 => reader.ReadSingle(), - 3 => reader.ReadBoolean(), - 4 => raw_string ? ReadString_Raw(reader) : StringDict!.ReadString(reader), - 5 => reader.ReadBytes(reader.ReadInt32()), - 6 => TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)), - 7 => ReadColor(reader), - 8 => ReadVector2(reader), - 9 => ReadVector3(reader), - 10 => ReadQAngle(reader), - 11 => ReadVector4(reader), - 12 => ReadQuaternion(reader), - 13 => ReadMatrix4x4(reader), - 14 => reader.ReadByte(), - 15 => reader.ReadUInt64(), + AttributeType.Element => ReadElement(dm, reader), + AttributeType.Int => reader.ReadInt32(), + AttributeType.Float => reader.ReadSingle(), + AttributeType.Bool => reader.ReadBoolean(), + AttributeType.String => raw_string ? ReadString_Raw(reader) : StringDict!.ReadString(reader), + AttributeType.Binary => reader.ReadBytes(reader.ReadInt32()), + AttributeType.Time => TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)), + AttributeType.Color => ReadColor(reader), + AttributeType.Vector2 => ReadVector2(reader), + AttributeType.Vector3 => ReadVector3(reader), + AttributeType.QAngle => ReadQAngle(reader), + AttributeType.Vector4 => ReadVector4(reader), + AttributeType.Quaternion => ReadQuaternion(reader), + AttributeType.Matrix => ReadMatrix4x4(reader), + AttributeType.Byte => reader.ReadByte(), + AttributeType.UInt64 => reader.ReadUInt64(), _ => throw new ArgumentException("Cannot read value of type") }; } @@ -496,7 +511,7 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in var (type, isArray) = IdToType(reader.ReadByte()); if (!isArray) - return ReadValue(dm, TypeMap[type.TypeHandle], EncodingVersion < 4 || prefix, reader); + return ReadValue(dm, type, EncodingVersion < 4 || prefix, reader); return ReadArray(dm, type, reader.ReadInt32(), reader); } @@ -514,24 +529,24 @@ void DecodeAttributeInto(Datamodel dm, AttributeList target, string name, Binary return; } - switch (TypeMap[type.TypeHandle]) + switch (type) { - case 0: target[name] = ReadElement(dm, reader); break; - case 1: target.Set(name, reader.ReadInt32()); break; - case 2: target.Set(name, reader.ReadSingle()); break; - case 3: target.Set(name, reader.ReadBoolean()); break; - case 4: target[name] = EncodingVersion < 4 ? ReadString_Raw(reader) : StringDict!.ReadString(reader); break; - case 5: target[name] = reader.ReadBytes(reader.ReadInt32()); break; - case 6: target.Set(name, TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond))); break; - case 7: target.Set(name, ReadColor(reader)); break; - case 8: target.Set(name, ReadVector2(reader)); break; - case 9: target.Set(name, ReadVector3(reader)); break; - case 10: target.Set(name, ReadQAngle(reader)); break; - case 11: target.Set(name, ReadVector4(reader)); break; - case 12: target.Set(name, ReadQuaternion(reader)); break; - case 13: target[name] = ReadMatrix4x4(reader); break; - case 14: target.Set(name, reader.ReadByte()); break; - case 15: target.Set(name, reader.ReadUInt64()); break; + case AttributeType.Element: target[name] = ReadElement(dm, reader); break; + case AttributeType.Int: target.Set(name, reader.ReadInt32()); break; + case AttributeType.Float: target.Set(name, reader.ReadSingle()); break; + case AttributeType.Bool: target.Set(name, reader.ReadBoolean()); break; + case AttributeType.String: target[name] = EncodingVersion < 4 ? ReadString_Raw(reader) : StringDict!.ReadString(reader); break; + case AttributeType.Binary: target[name] = reader.ReadBytes(reader.ReadInt32()); break; + case AttributeType.Time: target.Set(name, TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond))); break; + case AttributeType.Color: target.Set(name, ReadColor(reader)); break; + case AttributeType.Vector2: target.Set(name, ReadVector2(reader)); break; + case AttributeType.Vector3: target.Set(name, ReadVector3(reader)); break; + case AttributeType.QAngle: target.Set(name, ReadQAngle(reader)); break; + case AttributeType.Vector4: target.Set(name, ReadVector4(reader)); break; + case AttributeType.Quaternion: target.Set(name, ReadQuaternion(reader)); break; + case AttributeType.Matrix: target[name] = ReadMatrix4x4(reader); break; + case AttributeType.Byte: target.Set(name, reader.ReadByte()); break; + case AttributeType.UInt64: target.Set(name, reader.ReadUInt64()); break; default: throw new ArgumentException("Cannot read value of type"); } } @@ -545,31 +560,29 @@ void DecodeAttributeInto(Datamodel dm, AttributeList target, string name, Binary /// Reads an array attribute. Value types whose memory layout matches the stream are copied in one read into a slice of a shared chunk, /// instead of one boxed item at a time into a list of their own. /// - System.Collections.IList ReadArray(Datamodel dm, Type type, int count, BinaryReader reader) + System.Collections.IList ReadArray(Datamodel dm, AttributeType type, int count, BinaryReader reader) { - var typeId = TypeMap[type.TypeHandle]; - if (BitConverter.IsLittleEndian && count > 0) { - switch (typeId) + switch (type) { - case 1: { var (buffer, offset) = ReadChunk(count, reader); return new IntArray(buffer, offset, count); } - case 2: { var (buffer, offset) = ReadChunk(count, reader); return new FloatArray(buffer, offset, count); } - case 3: { var (buffer, offset) = ReadChunk(count, reader); return new BoolArray(buffer, offset, count); } - case 7: { var (buffer, offset) = ReadChunk(count, reader); return new ColorArray(buffer, offset, count); } - case 8: { var (buffer, offset) = ReadChunk(count, reader); return new Vector2Array(buffer, offset, count); } - case 9: { var (buffer, offset) = ReadChunk(count, reader); return new Vector3Array(buffer, offset, count); } - case 11: { var (buffer, offset) = ReadChunk(count, reader); return new Vector4Array(buffer, offset, count); } - case 12: { var (buffer, offset) = ReadChunk(count, reader); return new QuaternionArray(buffer, offset, count); } - case 13: { var (buffer, offset) = ReadChunk(count, reader); return new MatrixArray(buffer, offset, count); } - case 14: { var (buffer, offset) = ReadChunk(count, reader); return new ByteArray(buffer, offset, count); } - case 15: { var (buffer, offset) = ReadChunk(count, reader); return new UInt64Array(buffer, offset, count); } + case AttributeType.Int: { var (buffer, offset) = ReadChunk(count, reader); return new IntArray(buffer, offset, count); } + case AttributeType.Float: { var (buffer, offset) = ReadChunk(count, reader); return new FloatArray(buffer, offset, count); } + case AttributeType.Bool: { var (buffer, offset) = ReadChunk(count, reader); return new BoolArray(buffer, offset, count); } + case AttributeType.Color: { var (buffer, offset) = ReadChunk(count, reader); return new ColorArray(buffer, offset, count); } + case AttributeType.Vector2: { var (buffer, offset) = ReadChunk(count, reader); return new Vector2Array(buffer, offset, count); } + case AttributeType.Vector3: { var (buffer, offset) = ReadChunk(count, reader); return new Vector3Array(buffer, offset, count); } + case AttributeType.Vector4: { var (buffer, offset) = ReadChunk(count, reader); return new Vector4Array(buffer, offset, count); } + case AttributeType.Quaternion: { var (buffer, offset) = ReadChunk(count, reader); return new QuaternionArray(buffer, offset, count); } + case AttributeType.Matrix: { var (buffer, offset) = ReadChunk(count, reader); return new MatrixArray(buffer, offset, count); } + case AttributeType.Byte: { var (buffer, offset) = ReadChunk(count, reader); return new ByteArray(buffer, offset, count); } + case AttributeType.UInt64: { var (buffer, offset) = ReadChunk(count, reader); return new UInt64Array(buffer, offset, count); } } } - var array = CodecUtilities.MakeList(type, count); + var array = CodecUtilities.MakeList(ClrType(type), count); for (var i = 0; i < count; i++) - array.Add(ReadValue(dm, typeId, true, reader)); + array.Add(ReadValue(dm, type, true, reader)); return array; } @@ -592,59 +605,60 @@ void SkipAttribute(BinaryReader reader) count = reader.ReadInt32(); } - if (type == typeof(Element)) - { - foreach (int i in Enumerable.Range(0, count)) - if (reader.ReadInt32() == -2) reader.BaseStream.Seek(37, SeekOrigin.Current); // skip GUID + null terminator if a stub - return; - } - int length; - - if (type == typeof(TimeSpan)) - length = sizeof(int); - else if (type == typeof(Color)) - length = 4; - else if (type == typeof(bool)) - length = 1; - else if (type == typeof(byte[])) + switch (type) { - foreach (var i in Enumerable.Range(0, count)) - reader.BaseStream.Seek(reader.ReadInt32(), SeekOrigin.Current); - return; - } - else if (type == typeof(string)) - { - if (!StringDict!.Dummy && !isArray && EncodingVersion >= 4) - length = StringDict.IndiceSize; - else - { - foreach (var i in Enumerable.Range(0, count)) + case AttributeType.Element: + for (var i = 0; i < count; i++) + if (reader.ReadInt32() == -2) reader.BaseStream.Seek(37, SeekOrigin.Current); // skip GUID + null terminator if a stub + return; + case AttributeType.Binary: + for (var i = 0; i < count; i++) + reader.BaseStream.Seek(reader.ReadInt32(), SeekOrigin.Current); + return; + case AttributeType.String: + if (!StringDict!.Dummy && !isArray && EncodingVersion >= 4) + { + length = StringDict.IndiceSize; + break; + } + + for (var i = 0; i < count; i++) { byte b; do { b = reader.ReadByte(); } while (b != 0); } return; - } + case AttributeType.Bool: + case AttributeType.Byte: + length = 1; + break; + case AttributeType.Int: + case AttributeType.Float: + case AttributeType.Time: + case AttributeType.Color: + length = 4; + break; + case AttributeType.Vector2: + length = sizeof(float) * 2; + break; + case AttributeType.Vector3: + case AttributeType.QAngle: + length = sizeof(float) * 3; + break; + case AttributeType.Vector4: + case AttributeType.Quaternion: + length = sizeof(float) * 4; + break; + case AttributeType.UInt64: + length = sizeof(ulong); + break; + case AttributeType.Matrix: + length = sizeof(float) * 4 * 4; + break; + default: + throw new CodecException($"Cannot skip an attribute of type {type}."); } - else if (type == typeof(Vector2)) - length = sizeof(float) * 2; - else if (type == typeof(Vector3)) - length = sizeof(float) * 3; - else if (type == typeof(Vector4) || type == typeof(Quaternion)) - length = sizeof(float) * 4; - else if (type == typeof(Matrix4x4)) - length = sizeof(float) * 4 * 4; - else if (type == typeof(QAngle)) - length = sizeof(float) * 3; - else if (type == typeof(int) || type == typeof(float)) - length = 4; - else if (type == typeof(byte)) - length = sizeof(byte); - else if (type == typeof(ulong)) - length = sizeof(ulong); - else - throw new CodecException($"Cannot skip an attribute of type {type.Name}."); reader.BaseStream.Seek(length * count, SeekOrigin.Current); } @@ -665,9 +679,8 @@ sealed class Encoder readonly Dictionary Indices = []; /// Type ids of the inline kinds, filled in as they are met, since a version may not support every kind. - readonly byte[] KindIds = new byte[16]; + readonly byte[] KindIds = new byte[(int)AttributeType.Deferred + 1]; readonly Dictionary ArrayIds = []; - readonly byte ElementId, StringId, BinaryId, MatrixId; public Encoder(OutputBuffer writer, Datamodel dm, int version) { @@ -675,10 +688,6 @@ public Encoder(OutputBuffer writer, Datamodel dm, int version) Writer = writer; Datamodel = dm; StringDict = new StringDictionary(version); - ElementId = TypeToId(typeof(Element), version); - StringId = TypeToId(typeof(string), version); - BinaryId = TypeToId(typeof(byte[]), version); - MatrixId = TypeToId(typeof(Matrix4x4), version); } public void Encode() @@ -762,7 +771,7 @@ public void Begin(int count) { } - public void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference) + public void Visit(string name, AttributeType kind, in InlineValue inline, object? reference) { encoder.StringDict.AddName(name); @@ -818,7 +827,7 @@ public void Begin(int count) encoder.Writer.Write(count); } - public void Visit(string name, AttributeKind kind, in InlineValue inline, object? reference) + public void Visit(string name, AttributeType kind, in InlineValue inline, object? reference) { encoder.StringDict.WriteName(name, encoder.Writer); encoder.WriteValue(kind, in inline, reference, rawStrings: false); @@ -829,43 +838,40 @@ public void Visit(string name, AttributeKind kind, in InlineValue inline, object /// Writes the type id of a value and the value itself. /// /// Whether a string is written in place rather than as an index into the table, as the prefix attributes and array items are. - void WriteValue(AttributeKind kind, in InlineValue inline, object? reference, bool rawStrings) + void WriteValue(AttributeType kind, in InlineValue inline, object? reference, bool rawStrings) { - if (kind != AttributeKind.Reference) - { - Writer.Write(IdOf(kind)); - WriteInline(kind, in inline); - return; - } - - switch (reference) + switch (kind) { - case null: - Writer.Write(ElementId); - Writer.Write(-1); + case AttributeType.Element: + Writer.Write(IdOf(kind)); + if (reference is Element elem) + WriteElement(elem); + else + Writer.Write(-1); return; - case Element elem: - Writer.Write(ElementId); - WriteElement(elem); + case AttributeType.String: + Writer.Write(IdOf(kind)); + WriteString((string)reference!, rawStrings); return; - case string stringValue: - Writer.Write(StringId); - WriteString(stringValue, rawStrings); - return; - case byte[] binary: - Writer.Write(BinaryId); + case AttributeType.Binary: + Writer.Write(IdOf(kind)); + var binary = (byte[])reference!; Writer.Write(binary.Length); Writer.Write(binary); return; - case Matrix4x4 matrix: - Writer.Write(MatrixId); - WriteMatrix(in matrix); + case AttributeType.Matrix: + Writer.Write(IdOf(kind)); + WriteMatrix((Matrix4x4)reference!); return; - case IList array: - WriteArray(array); + case AttributeType.Array: + WriteArray((IList)reference!); return; + case AttributeType.Deferred: + throw new InvalidOperationException("A deferred attribute was not loaded before being written."); default: - throw new InvalidOperationException("Unrecognised output Type."); + Writer.Write(IdOf(kind)); + WriteInline(kind, in inline); + return; } } @@ -925,23 +931,31 @@ void WriteArray(IList array) foreach (var item in array) { AttributeList.Classify(item, out var kind, out var inline, out var reference); - if (kind != AttributeKind.Reference) - WriteInline(kind, in inline); - else if (reference == null) - Writer.Write(-1); - else if (reference is Element elem) - WriteElement(elem); - else if (reference is string stringValue) - Writer.Write(stringValue); - else if (reference is byte[] binary) + switch (kind) { - Writer.Write(binary.Length); - Writer.Write(binary); + case AttributeType.Element: + if (reference is Element elem) + WriteElement(elem); + else + Writer.Write(-1); + break; + case AttributeType.String: + Writer.Write((string)reference!); + break; + case AttributeType.Binary: + var binary = (byte[])reference!; + Writer.Write(binary.Length); + Writer.Write(binary); + break; + case AttributeType.Matrix: + WriteMatrix((Matrix4x4)reference!); + break; + case AttributeType.Array: + throw new InvalidOperationException("Unrecognised output Type."); + default: + WriteInline(kind, in inline); + break; } - else if (reference is Matrix4x4 matrix) - WriteMatrix(in matrix); - else - throw new InvalidOperationException("Unrecognised output Type."); } } @@ -959,50 +973,50 @@ void WriteItems(ReadOnlySpan items) where T : unmanaged foreach (var item in items) { AttributeList.Classify(item, out var kind, out var inline, out var reference); - if (kind != AttributeKind.Reference) - WriteInline(kind, in inline); - else + if (kind == AttributeType.Matrix) WriteMatrix((Matrix4x4)reference!); + else + WriteInline(kind, in inline); } } - void WriteInline(AttributeKind kind, in InlineValue inline) + void WriteInline(AttributeType kind, in InlineValue inline) { switch (kind) { - case AttributeKind.Int: Writer.Write(inline.Int); return; - case AttributeKind.Float: Writer.Write(inline.Float); return; - case AttributeKind.Bool: Writer.Write(inline.Bool ? (byte)1 : (byte)0); return; - case AttributeKind.Byte: Writer.Write(inline.Byte); return; - case AttributeKind.UInt64: Writer.Write(inline.UInt64); return; - case AttributeKind.Time: Writer.Write(ToTicks(TimeSpan.FromTicks(inline.Ticks))); return; - case AttributeKind.Color: + case AttributeType.Int: Writer.Write(inline.Int); return; + case AttributeType.Float: Writer.Write(inline.Float); return; + case AttributeType.Bool: Writer.Write(inline.Bool ? (byte)1 : (byte)0); return; + case AttributeType.Byte: Writer.Write(inline.Byte); return; + case AttributeType.UInt64: Writer.Write(inline.UInt64); return; + case AttributeType.Time: Writer.Write(ToTicks(TimeSpan.FromTicks(inline.Ticks))); return; + case AttributeType.Color: Writer.Write(inline.Color.R); Writer.Write(inline.Color.G); Writer.Write(inline.Color.B); Writer.Write(inline.Color.A); return; - case AttributeKind.Vector2: + case AttributeType.Vector2: Writer.Write(inline.Vector2.X); Writer.Write(inline.Vector2.Y); return; - case AttributeKind.Vector3: + case AttributeType.Vector3: Writer.Write(inline.Vector3.X); Writer.Write(inline.Vector3.Y); Writer.Write(inline.Vector3.Z); return; - case AttributeKind.QAngle: + case AttributeType.QAngle: Writer.Write(inline.QAngle.Pitch); Writer.Write(inline.QAngle.Yaw); Writer.Write(inline.QAngle.Roll); return; - case AttributeKind.Vector4: + case AttributeType.Vector4: Writer.Write(inline.Vector4.X); Writer.Write(inline.Vector4.Y); Writer.Write(inline.Vector4.Z); Writer.Write(inline.Vector4.W); return; - case AttributeKind.Quaternion: + case AttributeType.Quaternion: Writer.Write(inline.Quaternion.X); Writer.Write(inline.Quaternion.Y); Writer.Write(inline.Quaternion.Z); @@ -1056,11 +1070,11 @@ void WriteString(string value, bool raw) static int ToTicks(TimeSpan time) => (int)(time.Ticks / (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)); - byte IdOf(AttributeKind kind) + byte IdOf(AttributeType kind) { ref var id = ref KindIds[(int)kind]; if (id == 0) - id = TypeToId(TypeOf(kind), EncodingVersion); + id = TypeToId(kind, EncodingVersion); return id; } @@ -1071,22 +1085,6 @@ byte IdOf(Type arrayType) return id; } - static Type TypeOf(AttributeKind kind) => kind switch - { - AttributeKind.Int => typeof(int), - AttributeKind.Float => typeof(float), - AttributeKind.Bool => typeof(bool), - AttributeKind.Byte => typeof(byte), - AttributeKind.UInt64 => typeof(ulong), - AttributeKind.Time => typeof(TimeSpan), - AttributeKind.Color => typeof(Color), - AttributeKind.Vector2 => typeof(Vector2), - AttributeKind.Vector3 => typeof(Vector3), - AttributeKind.Vector4 => typeof(Vector4), - AttributeKind.Quaternion => typeof(Quaternion), - AttributeKind.QAngle => typeof(QAngle), - _ => throw new InvalidOperationException("Unrecognised output Type."), - }; } /// diff --git a/Datamodel.NET/ElementSchema.cs b/Datamodel.NET/ElementSchema.cs index 7272674..d7904be 100644 --- a/Datamodel.NET/ElementSchema.cs +++ b/Datamodel.NET/ElementSchema.cs @@ -102,7 +102,7 @@ public virtual void SetValue(AttributeList owner, object? value) /// /// Reads the property in the form an attribute slot stores it, so that a codec writes it without boxing when the binding is typed. /// - internal virtual void Read(AttributeList owner, out AttributeKind kind, out InlineValue inline, out object? reference) + internal virtual void Read(AttributeList owner, out AttributeType kind, out InlineValue inline, out object? reference) { AttributeList.Classify(GetValue(owner), out kind, out inline, out reference); } @@ -155,23 +155,22 @@ public PropertyBinding(string propertyName, string attributeName, Func getter((TElement)owner); - /// The kind a slot stores values of as, decided once per type. - static readonly AttributeKind ValueKind = AttributeList.KindOf(typeof(TValue)); + /// The type a slot stores values of as when they are stored inline, decided once per type; null for the reference types, which are classified per value. + static readonly AttributeType? ValueKind = AttributeList.KindOf(typeof(TValue)); - internal override void Read(AttributeList owner, out AttributeKind kind, out InlineValue inline, out object? reference) + internal override void Read(AttributeList owner, out AttributeType kind, out InlineValue inline, out object? reference) { var value = getter((TElement)owner); - kind = ValueKind; - inline = default; - if (ValueKind == AttributeKind.Reference) - { - reference = value; - } - else + if (ValueKind is AttributeType inlineKind) { + kind = inlineKind; reference = null; + inline = default; Unsafe.As(ref inline) = value; + return; } + + AttributeList.Classify(value, out kind, out inline, out reference); } public override void Set(AttributeList owner, TValue value) From 5dc983d4de6f5c89a37fc3f6543dce269a934783 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:24:19 +0000 Subject: [PATCH 15/16] Add Offset alias to InlineValue for deferred attribute position Co-authored-by: kristiker <26466974+kristiker@users.noreply.github.com> --- Datamodel.NET/AttributeList.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 256f344..49d791d 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -39,7 +39,7 @@ enum AttributeType : byte Byte, /// An array of any of the above. Array, - /// The value has not been read from the stream yet; holds the position it starts at. + /// The value has not been read from the stream yet; holds the position it starts at. Deferred, } @@ -55,6 +55,8 @@ struct InlineValue [FieldOffset(0)] public byte Byte; [FieldOffset(0)] public ulong UInt64; [FieldOffset(0)] public long Ticks; + /// Alias of , used when the slot is and holds a stream offset rather than a duration. + [FieldOffset(0)] public long Offset; [FieldOffset(0)] public Color Color; [FieldOffset(0)] public Vector2 Vector2; [FieldOffset(0)] public Vector3 Vector3; @@ -361,7 +363,7 @@ void Store(ref AttributeSlot slot, object? value) void LoadDeferred(int index) { var codec = Owner?.Codec ?? throw new CodecException("Trying to load a deferred Attribute, but could not find codec."); - var offset = slots![index].Inline.Ticks; + var offset = slots![index].Inline.Offset; var name = slots[index].Name; object? value; @@ -393,7 +395,7 @@ internal void SetDeferred(string name, long offset) slot.Reference = null; slot.Override = null; slot.Inline = default; - slot.Inline.Ticks = offset; + slot.Inline.Offset = offset; } } From 18c1e892c86b2429cf17e63f4f8be1ec1beb5d38 Mon Sep 17 00:00:00 2001 From: Kristi K <26466974+kristiker@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:40:45 +0200 Subject: [PATCH 16/16] Bump version --- Datamodel.NET/AttributeList.cs | 1 - Datamodel.NET/Datamodel.NET.csproj | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 49d791d..bbf0ba8 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -55,7 +55,6 @@ struct InlineValue [FieldOffset(0)] public byte Byte; [FieldOffset(0)] public ulong UInt64; [FieldOffset(0)] public long Ticks; - /// Alias of , used when the slot is and holds a stream offset rather than a duration. [FieldOffset(0)] public long Offset; [FieldOffset(0)] public Color Color; [FieldOffset(0)] public Vector2 Vector2; diff --git a/Datamodel.NET/Datamodel.NET.csproj b/Datamodel.NET/Datamodel.NET.csproj index efec38c..5616644 100644 --- a/Datamodel.NET/Datamodel.NET.csproj +++ b/Datamodel.NET/Datamodel.NET.csproj @@ -4,7 +4,7 @@ Library Datamodel KeyValues2 - 1.0 + 2.0-beta enable MIT README.md