diff --git a/Benchmarks/Program.cs b/Benchmarks/Program.cs index ab0ef84..6414883 100644 --- a/Benchmarks/Program.cs +++ b/Benchmarks/Program.cs @@ -1,14 +1,15 @@ using System.Diagnostics; +using System.Runtime.CompilerServices; using Datamodel.Codecs; using Tests.VMAP; using DM = Datamodel.Datamodel; -// Measures loading a vmap as plain elements and as the typed classes of Tests/ValveMap.cs, and saving the typed model. +// Measures loading a vmap as plain elements and as the typed classes of Tests.VMAP, and saving the typed model. // // Benchmarks [--iterations N] ... // // A directory contributes every .vmap inside it, largest last. Each figure is the best of N iterations (default 1). -// "typed alloc" is the managed memory allocated by the typed load, "live heap" the managed heap that survives it. +// "typed alloc" is the managed memory allocated by the typed load, "live heap" the managed heap the typed model keeps, without the file bytes. // dots as decimal separators whatever the machine locale, so that tables can be pasted anywhere System.Globalization.CultureInfo.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture; @@ -44,41 +45,50 @@ { var name = Path.GetFileName(path); var size = new FileInfo(path).Length; - var read = Time(() => File.ReadAllBytes(path), out var bytes); - - var untyped = double.MaxValue; - var typed = double.MaxValue; - var save = double.MaxValue; - long allocated = 0, heap = 0, elements = 0; + var best = new Sample(double.MaxValue, double.MaxValue, double.MaxValue, 0, 0, 0); for (var i = 0; i < iterations; i++) { - Collect(); - untyped = Math.Min(untyped, Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var plain)); - elements = plain.AllElements.Count; - plain.Dispose(); - - Collect(); - var before = GC.GetTotalAllocatedBytes(true); - typed = Math.Min(typed, Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var map)); - allocated = GC.GetTotalAllocatedBytes(true) - before; - heap = GC.GetTotalMemory(true); - - if (map.Root is not CMapRootElement) - { - throw new InvalidOperationException($"{name}: the root was not loaded as {nameof(CMapRootElement)}"); - } - - save = Math.Min(save, Time(() => { map.Save(Stream.Null, "binary", 9); return 0; }, out _)); - map.Dispose(); + var sample = Measure(bytes, name); + best = new Sample(Math.Min(best.Untyped, sample.Untyped), Math.Min(best.Typed, sample.Typed), Math.Min(best.Save, sample.Save), sample.Allocated, sample.Heap, sample.Elements); } - Console.WriteLine($"{name,-26} {size / 1048576.0,7:F1}MB {elements,9} | {Duration(read),10} {Duration(untyped),13} {Duration(typed),11} {typed / untyped,12:F2}x | {allocated / 1048576.0,9:F0}MB {heap / 1048576.0,7:F0}MB | {Duration(save),11}"); + Console.WriteLine($"{name,-26} {size / 1048576.0,7:F1}MB {best.Elements,9} | {Duration(read),10} {Duration(best.Untyped),13} {Duration(best.Typed),11} {best.Typed / best.Untyped,12:F2}x | {best.Allocated / 1048576.0,9:F0}MB {best.Heap / 1048576.0,7:F0}MB | {Duration(best.Save),11}"); } return 0; +/// +/// One load-save round in a frame of its own, so that every model of the round is garbage once it returns. +/// The main loop keeps temporaries alive across iterations, which would count the previous model into the next heap figure. +/// +[MethodImpl(MethodImplOptions.NoInlining)] +static Sample Measure(byte[] bytes, string name) +{ + Collect(); + var untyped = Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var plain); + var elements = plain.AllElements.Count; + plain.Dispose(); + plain = null!; + + Collect(); + var before = GC.GetTotalAllocatedBytes(true); + var typed = Time(() => DM.Load(new MemoryStream(bytes, false), DeferredMode.Disabled), out var map); + var allocated = GC.GetTotalAllocatedBytes(true) - before; + var heap = GC.GetTotalMemory(true) - bytes.Length; + + if (map.Root is not CMapRootElement) + { + throw new InvalidOperationException($"{name}: the root was not loaded as {nameof(CMapRootElement)}"); + } + + var save = Time(() => { map.Save(Stream.Null, "binary", 9); return 0; }, out _); + map.Dispose(); + + return new Sample(untyped, typed, save, allocated, heap, elements); +} + /// Milliseconds up to a tenth of a second, seconds with two decimals above. static string Duration(double milliseconds) { @@ -98,3 +108,6 @@ static void Collect() GC.WaitForPendingFinalizers(); GC.Collect(); } + +/// Times in milliseconds, memory in bytes. +record struct Sample(double Untyped, double Typed, double Save, long Allocated, long Heap, long Elements); diff --git a/Datamodel.NET/Arrays.cs b/Datamodel.NET/Arrays.cs index b2dd2bf..19be146 100644 --- a/Datamodel.NET/Arrays.cs +++ b/Datamodel.NET/Arrays.cs @@ -1,13 +1,18 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; namespace Datamodel { + /// + /// A typed attribute array. Items are stored contiguously, either in a private buffer or, for arrays read from a file, + /// in a slice of a chunk shared with the other arrays of that file; the first change that needs more room moves the array to a private buffer. + /// [DebuggerTypeProxy(typeof(Array<>.DebugView))] - [DebuggerDisplay("Count = {Inner.Count}")] + [DebuggerDisplay("Count = {Count}")] public abstract class Array : IList, IList { internal class DebugView(Array arr) @@ -15,10 +20,14 @@ internal class DebugView(Array arr) readonly Array Arr = arr; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - public T[] Items { get { return [.. Arr.Inner]; } } + public T[] Items { get { return Arr.AsSpan().ToArray(); } } } - protected List Inner; + T[] buffer; + int offset; + int count; + int capacity; + bool shared; public virtual AttributeList? Owner { @@ -34,51 +43,135 @@ internal set internal Array() { - Inner = []; + buffer = []; } internal Array(IEnumerable enumerable) { - if (enumerable != null) - Inner = [.. enumerable]; - else - Inner = []; + buffer = enumerable is null ? [] : [.. enumerable]; + count = capacity = buffer.Length; } internal Array(int capacity) { - Inner = new List(capacity); + buffer = capacity > 0 ? new T[capacity] : []; + this.capacity = capacity; } - public int IndexOf(T item) => Inner.IndexOf(item); + /// + /// Creates an array over a slice of a chunk shared with other arrays. The slice belongs to this array alone, but cannot grow in place. + /// + internal Array(T[] buffer, int offset, int count) + { + this.buffer = buffer; + this.offset = offset; + this.count = count; + capacity = count; + shared = true; + } + + /// + /// Gets the items as a span. The span is invalidated by any change to the array. + /// + public ReadOnlySpan AsSpan() => new(buffer, offset, count); + + /// + /// The items, writable. Invalidated by any change to the array. + /// + protected Span Items => new(buffer, offset, count); + + /// + /// Moves the items to a private buffer with room for at least items. + /// + void Grow(int minimum) + { + var newCapacity = Math.Max(minimum, Math.Max(capacity * 2, 4)); + var newBuffer = new T[newCapacity]; + Items.CopyTo(newBuffer); + buffer = newBuffer; + offset = 0; + capacity = newCapacity; + shared = false; + } + + public int IndexOf(T item) + { + var index = System.Array.IndexOf(buffer, item, offset, count); + return index < 0 ? -1 : index - offset; + } public void Insert(int index, T item) => Insert_Internal(index, item); - protected virtual void Insert_Internal(int index, T item) => Inner.Insert(index, item); - public void AddRange(IEnumerable items) => Inner.AddRange(items); + protected virtual void Insert_Internal(int index, T item) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)index, (uint)count, nameof(index)); + + if (count == capacity) + Grow(count + 1); - public void RemoveAt(int index) => Inner.RemoveAt(index); + if (index < count) + System.Array.Copy(buffer, offset + index, buffer, offset + index + 1, count - index); + + buffer[offset + index] = item; + count++; + } + + public void AddRange(IEnumerable items) + { + ArgumentNullException.ThrowIfNull(items); + + if (items is ICollection collection) + { + if (count + collection.Count > capacity) + Grow(count + collection.Count); + + collection.CopyTo(buffer, offset + count); + count += collection.Count; + return; + } + + foreach (var item in items) + Add(item); + } + + public void RemoveAt(int index) + { + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual((uint)index, (uint)count, nameof(index)); + + count--; + if (index < count) + System.Array.Copy(buffer, offset + index + 1, buffer, offset + index, count - index); + + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + buffer[offset + count] = default!; + } public virtual T this[int index] { - get => Inner[index]; - set => Inner[index] = value; + get => Items[index]; + set => Items[index] = value; } - public void Add(T item) => Insert(Inner.Count, item); + public void Add(T item) => Insert(count, item); - public void Clear() => Inner.Clear(); + public void Clear() + { + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + Items.Clear(); + + count = 0; + } - public bool Contains(T item) => Inner.Contains(item); + public bool Contains(T item) => IndexOf(item) >= 0; public void CopyTo(T[] array, int offset) { CopyTo_Internal(array, offset); } - protected virtual void CopyTo_Internal(T[] array, int offset) => Inner.CopyTo(array, offset); + protected virtual void CopyTo_Internal(T[] array, int offset) => Items.CopyTo(array.AsSpan(offset)); - public int Count => Inner.Count; + public int Count => count; bool ICollection.IsReadOnly { get { return false; } } @@ -88,7 +181,7 @@ public void CopyTo(T[] array, int offset) public bool IsSynchronized => false; - public object SyncRoot => Inner; + public object SyncRoot => this; object? IList.this[int index] { @@ -96,10 +189,23 @@ public void CopyTo(T[] array, int offset) set => this[index] = value is null ? throw new InvalidOperationException("Trying to set a null object") : (T)value; } - public bool Remove(T item) => Inner.Remove(item); + public bool Remove(T item) + { + var index = IndexOf(item); + if (index < 0) + return false; + + RemoveAt(index); + return true; + } + + public IEnumerator GetEnumerator() + { + for (var i = 0; i < count; i++) + yield return buffer[offset + i]; + } - public IEnumerator GetEnumerator() => Inner.GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => Inner.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); #region IList int IList.Add(object? value) @@ -151,6 +257,55 @@ void ICollection.CopyTo(Array array, int index) #endregion IList } + /// + /// Hands out slices of large shared arrays to the arrays read from one file, so that the collector sees a few large objects + /// that it never moves, instead of hundreds of thousands of small ones that it copies through every generation. + /// + sealed class ArrayChunks + { + /// Chunk size in bytes. Above the large object threshold, so that chunks are never compacted. + const int ChunkBytes = 1 << 20; + + /// Arrays at least this large get their own allocation, which lands on the large object heap anyway. + const int LargeObjectBytes = 85_000; + + sealed class Chunk + { + public T[] Buffer = []; + public int Used; + } + + readonly Dictionary chunks = []; + + /// + /// Returns storage for items. The contents are not zeroed. + /// + public (T[] Buffer, int Offset) Rent(int count) where T : unmanaged + { + var itemSize = Unsafe.SizeOf(); + + if ((long)count * itemSize >= LargeObjectBytes) + return (GC.AllocateUninitializedArray(count), 0); + + if (!chunks.TryGetValue(typeof(T), out var untyped)) + { + untyped = new Chunk(); + chunks[typeof(T)] = untyped; + } + + var chunk = (Chunk)untyped; + if (chunk.Buffer.Length - chunk.Used < count) + { + chunk.Buffer = GC.AllocateUninitializedArray(Math.Max(ChunkBytes / itemSize, count)); + chunk.Used = 0; + } + + var offset = chunk.Used; + chunk.Used += count; + return (chunk.Buffer, offset); + } + } + public class ElementArray : Array { public ElementArray() { } @@ -164,9 +319,9 @@ public ElementArray(int capacity) { } /// - /// Gets the values in the list without attempting destubbing. + /// Gets the items without attempting destubbing. /// - internal IEnumerable RawList { get { foreach (var elem in Inner) yield return elem; } } + internal ReadOnlySpan RawItems => AsSpan(); public override AttributeList? Owner { @@ -177,9 +332,10 @@ internal set if (OwnerDatamodel != null) { - for (int i = 0; i < Count; i++) + var items = Items; + for (int i = 0; i < items.Length; i++) { - var elem = Inner[i]; + var elem = items[i]; if (elem == null) continue; if (elem.Owner == null) @@ -188,7 +344,7 @@ internal set if (importedElement is not null) { - Inner[i] = importedElement; + items[i] = importedElement; } } else if (elem.Owner != OwnerDatamodel) @@ -224,12 +380,13 @@ public override Element this[int index] { get { - var elem = Inner[index]; + var items = Items; + var elem = items[index]; if (elem != null && elem.Stub && elem.Owner != null) { try { - elem = Inner[index] = elem.Owner.OnStubRequest(elem.ID)!; + elem = items[index] = elem.Owner.OnStubRequest(elem.ID)!; } catch (Exception err) { @@ -257,6 +414,9 @@ public IntArray(IEnumerable enumerable) public IntArray(int capacity) : base(capacity) { } + internal IntArray(int[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class FloatArray : Array @@ -268,6 +428,9 @@ public FloatArray(IEnumerable enumerable) public FloatArray(int capacity) : base(capacity) { } + internal FloatArray(float[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class BoolArray : Array @@ -279,6 +442,9 @@ public BoolArray(IEnumerable enumerable) public BoolArray(int capacity) : base(capacity) { } + internal BoolArray(bool[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class StringArray : Array @@ -323,6 +489,9 @@ public ColorArray(IEnumerable enumerable) public ColorArray(int capacity) : base(capacity) { } + internal ColorArray(Color[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class Vector2Array : Array @@ -334,6 +503,9 @@ public Vector2Array(IEnumerable enumerable) public Vector2Array(int capacity) : base(capacity) { } + internal Vector2Array(Vector2[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class Vector3Array : Array @@ -345,6 +517,9 @@ public Vector3Array(IEnumerable enumerable) public Vector3Array(int capacity) : base(capacity) { } + internal Vector3Array(Vector3[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class Vector4Array : Array @@ -356,6 +531,9 @@ public Vector4Array(IEnumerable enumerable) public Vector4Array(int capacity) : base(capacity) { } + internal Vector4Array(Vector4[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class QuaternionArray : Array @@ -367,6 +545,9 @@ public QuaternionArray(IEnumerable enumerable) public QuaternionArray(int capacity) : base(capacity) { } + internal QuaternionArray(Quaternion[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class MatrixArray : Array @@ -378,6 +559,9 @@ public MatrixArray(IEnumerable enumerable) public MatrixArray(int capacity) : base(capacity) { } + internal MatrixArray(Matrix4x4[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } public class ByteArray : Array @@ -389,6 +573,9 @@ public ByteArray(IEnumerable enumerable) public ByteArray(int capacity) : base(capacity) { } + internal ByteArray(byte[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } [CLSCompliant(false)] @@ -401,5 +588,8 @@ public UInt64Array(IEnumerable enumerable) public UInt64Array(int capacity) : base(capacity) { } + internal UInt64Array(ulong[] buffer, int offset, int count) + : base(buffer, offset, count) + { } } } diff --git a/Datamodel.NET/Attribute.cs b/Datamodel.NET/Attribute.cs deleted file mode 100644 index 628811d..0000000 --- a/Datamodel.NET/Attribute.cs +++ /dev/null @@ -1,226 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Numerics; - -using AttrKVP = System.Collections.Generic.KeyValuePair; - -namespace Datamodel -{ - /// - /// A name/value pair associated with an . - /// - class Attribute - { - /// - /// Creates a new Attribute with a specified name and value. - /// - /// The name of the Attribute, which must be unique to its owner. - /// The value of the Attribute, which must be of a supported Datamodel type. - public Attribute(string name, AttributeList owner, object? value) - { - ArgumentNullException.ThrowIfNull(name); - - Name = name; - _Owner = owner; - Value = value; - } - - /// - /// Creates a new Attribute with deferred loading. - /// - /// The name of the Attribute, which must be unique to its owner. - /// The AttributeList which owns this Attribute. - /// The location in the encoded DMX stream at which this Attribute's value can be found. - public Attribute(string name, AttributeList owner, long defer_offset) - : this(name, owner, null) - { - ArgumentNullException.ThrowIfNull(owner); - - Offset = defer_offset; - } - - #region Properties - /// - /// Gets the name of this Attribute. - /// - public string Name { get; private set; } - - /// - /// Gets the Type of this Attribute's Value. - /// - public Type ValueType { get; private set; } = typeof(Element); - - /// - /// Gets or sets the OverrideType of this Attributes. - /// - public AttributeList.OverrideType? OverrideType - { - get - { - return _OverrideType; - } - set - { - switch (value) - { - case null: - break; - case AttributeList.OverrideType.Angle: - if (ValueType != typeof(Vector3)) - throw new AttributeTypeException("OverrideType.Angle can only be applied to Vector3 attributes"); - break; - case AttributeList.OverrideType.Binary: - if (ValueType != typeof(byte[])) - throw new AttributeTypeException("OverrideType.Binary can only be applied to byte[] attributes"); - break; - default: - throw new NotImplementedException(); - } - _OverrideType = value; - } - } - AttributeList.OverrideType? _OverrideType; - - /// - /// Gets the which this Attribute is a member of. - /// - public AttributeList? Owner - { - get { return _Owner; } - internal set - { - if (_Owner == value) return; - - if (Deferred && _Owner != null) DeferredLoad(); - _Owner = value; - } - } - AttributeList? _Owner; - - Datamodel? OwnerDatamodel { get { return Owner?.Owner; } } - - /// - /// Gets whether the value of this Attribute has yet to be decoded. - /// - public bool Deferred { get { return Offset != 0; } } - - /// - /// Loads the value of this Attribute from the encoded source Datamodel. - /// - /// Thrown when the Attribute has already been loaded. - /// Thrown when the deferred load fails. - public void DeferredLoad() - { - if (Offset == 0) throw new InvalidOperationException("Attribute already loaded."); - - if (OwnerDatamodel == null || OwnerDatamodel.Codec == null) - throw new CodecException("Trying to load a deferred Attribute, but could not find codec."); - - try - { - lock (OwnerDatamodel.Codec) - { - _Value = OwnerDatamodel.Codec.DeferredDecodeAttribute(OwnerDatamodel, Offset); - } - } - catch (Exception err) - { - throw new CodecException($"Deferred loading of attribute \"{Name}\" on element {((Element?)Owner)?.ID} using {OwnerDatamodel.Codec} codec threw an exception.", err); - } - Offset = 0; - - if (_Value is ElementArray elem_array) - elem_array.Owner = Owner; - } - - /// - /// Gets or sets the value held by this Attribute. - /// - /// Thrown when deferred value loading fails. - /// Thrown when Element destubbing fails. - public object? Value - { - get - { - if (Offset > 0) - DeferredLoad(); - - if (OwnerDatamodel != null) - { - // expand stubs - if (_Value is Element elem && elem.Stub) - { - try { _Value = OwnerDatamodel.OnStubRequest(elem.ID) ?? _Value; } - catch (Exception err) { throw new DestubException(this, err); } - } - } - - return _Value; - } - set - { - ValueType = value == null ? typeof(Element) : value.GetType(); - - if (!Datamodel.IsDatamodelType(ValueType)) - throw new AttributeTypeException(String.Format("{0} is not a valid Datamodel attribute type. (If this is an array, it must implement IList).", ValueType.FullName)); - - if (value is Element elem) - { - if (elem.Owner == null) - elem.Owner = OwnerDatamodel; - else if (elem.Owner != OwnerDatamodel) - throw new ElementOwnershipException(); - } - - if (value is IEnumerable elem_enumerable) - { - if (elem_enumerable is not ElementArray) - throw new InvalidOperationException("Element array objects must derive from Datamodel.ElementArray"); - - var elem_array = (ElementArray)value; - if (elem_array.Owner == null) - elem_array.Owner = Owner; - else if (elem_array.Owner != Owner) - throw new InvalidOperationException("ElementArray is already owned by a different Datamodel."); - - foreach (var arr_elem in elem_array) - { - if (arr_elem == null) continue; - else if (arr_elem.Owner == null) - arr_elem.Owner = OwnerDatamodel; - - // todo: remove ownership from values - // this is being printed on a debuggerdisplay output for some reason - // else if (arr_elem.Owner != OwnerDatamodel) - // throw new ElementOwnershipException("One or more Elements in the assigned collection are from a different Datamodel. Use ImportElement() to copy them to this one before assigning."); - } - } - - _Value = value; - Offset = 0; - } - } - object? _Value = null; - - /// - /// Gets the Attribute's Value without attempting deferred loading or destubbing. - /// - public object? RawValue { get { return _Value; } } - - #endregion - - long Offset; - - public override string ToString() - { - var type = Value != null ? Value.GetType() : typeof(Element); - var inner_type = Datamodel.GetArrayInnerType(type); - return String.Format("{0} <{1}>", Name, inner_type != null ? inner_type.FullName + "[]" : type.FullName); - } - - public AttrKVP ToKeyValuePair() - { - return new AttrKVP(Name, Value); - } - } -} diff --git a/Datamodel.NET/AttributeList.cs b/Datamodel.NET/AttributeList.cs index 877db71..bbf0ba8 100644 --- a/Datamodel.NET/AttributeList.cs +++ b/Datamodel.NET/AttributeList.cs @@ -1,79 +1,121 @@ -using System; +using System; +using System.Buffers; using System.Collections; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Specialized; -using System.Diagnostics; using System.ComponentModel; +using System.Diagnostics; +using System.IO; using System.Linq; using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using AttrKVP = System.Collections.Generic.KeyValuePair; -using System.IO; namespace Datamodel { /// - /// A thread-safe collection of s. + /// The types an attribute value can have, named as Valve's DmAttributeType_t names them: what a slot stores, what a class property is read as + /// and what the binary encoding writes. Values of the scalar and vector types live in the slot itself, see ; the rest are held as references. + /// + enum AttributeType : byte + { + Element, + Int, + Float, + Bool, + String, + Binary, + Time, + Color, + Vector2, + Vector3, + Vector4, + QAngle, + Quaternion, + Matrix, + UInt64, + Byte, + /// An array of any of the above. + Array, + /// The value has not been read from the stream yet; holds the position it starts at. + Deferred, + } + + /// + /// Sixteen bytes that hold any scalar or vector attribute value without boxing it. + /// + [StructLayout(LayoutKind.Explicit, Size = 16)] + struct InlineValue + { + [FieldOffset(0)] public int Int; + [FieldOffset(0)] public float Float; + [FieldOffset(0)] public bool Bool; + [FieldOffset(0)] public byte Byte; + [FieldOffset(0)] public ulong UInt64; + [FieldOffset(0)] public long Ticks; + [FieldOffset(0)] public long Offset; + [FieldOffset(0)] public Color Color; + [FieldOffset(0)] public Vector2 Vector2; + [FieldOffset(0)] public Vector3 Vector3; + [FieldOffset(0)] public Vector4 Vector4; + [FieldOffset(0)] public Quaternion Quaternion; + [FieldOffset(0)] public QAngle QAngle; + } + + /// + /// One attribute of an : its name and its value, stored inline for value types and as a reference otherwise. + /// Modelled on Valve's fixed-size CDmAttribute, so that a plain element costs one slot per attribute and no further objects. + /// + struct AttributeSlot + { + public string Name; + public object? Reference; + public InlineValue Inline; + public AttributeType Kind; + public AttributeList.OverrideType? Override; + } + + /// + /// Receives the attributes of an in the form their slots hold them, so that a codec writes them without boxing. See . + /// + interface IAttributeVisitor + { + /// Called once before the attributes, with how many follow. + void Begin(int count); + + void Visit(string name, AttributeType kind, in InlineValue inline, object? reference); + } + + /// + /// A thread-safe collection of attributes. /// [DebuggerTypeProxy(typeof(DebugView))] [DebuggerDisplay("Count = {Count}")] public class AttributeList : IDictionary, IDictionary { - internal OrderedDictionary Inner; - protected object Attribute_ChangeLock = new(); + AttributeSlot[]? slots; + int count; /// - /// Gets the properties of this class that are stored as attributes. Empty unless a schema is registered for the class. + /// The object locked while the list is changed. The list itself, which is also its . /// - public ElementSchema Schema { get; } - - private IEnumerable GetPropertyBasedAttributes(bool useSerializationName) - { - foreach (var binding in Schema.Properties) - { - var name = useSerializationName ? binding.AttributeName : binding.PropertyName; - yield return new Attribute(name, this, binding.GetValue(this)); - } - } + protected object Attribute_ChangeLock; /// - /// Converts between the bool, int and float attribute types the way Valve's datamodel does when a value is assigned - /// to an attribute of another of those types. Returns null for any other combination. + /// Gets the properties of this class that are stored as attributes. Empty unless a schema is registered for the class. /// - private static object? ConvertScalar(object value, Type targetType) - { - if (targetType == typeof(int)) - { - return value switch - { - bool b => b ? 1 : 0, - float f => (int)f, - _ => null, - }; - } + public ElementSchema Schema { get; } - if (targetType == typeof(float)) - { - return value switch - { - bool b => b ? 1f : 0f, - int i => (float)i, - _ => null, - }; - } + public AttributeList(Datamodel? owner) + { + Attribute_ChangeLock = this; - if (targetType == typeof(bool)) - { - return value switch - { - int i => i != 0, - float f => f != 0f, - _ => null, - }; - } + var type = GetType(); + Schema = type == typeof(AttributeList) || type == typeof(Element) ? ElementSchema.Empty : ElementSchema.For(type); - return null; + Owner = owner; } internal class DebugView @@ -87,23 +129,18 @@ public DebugView(AttributeList item) [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] public DebugAttribute[] Attributes - => Item.GetPropertyBasedAttributes(useSerializationName: false).Select(attr => new DebugAttribute(attr)) - .Concat(Item.Inner.Values.Cast().Select(attr => new DebugAttribute(attr))) + => Item.Schema.Properties.Select(binding => new DebugAttribute(binding.PropertyName, binding.GetValue(Item))) + .Concat(Item.Select(attr => new DebugAttribute(attr.Key, attr.Value))) .ToArray(); - [DebuggerDisplay("{Value}", Name = "{Attr.Name,nq}", Type = "{Attr.ValueType.FullName,nq}")] - public class DebugAttribute + [DebuggerDisplay("{Value}", Name = "{Name,nq}")] + public class DebugAttribute(string name, object? value) { - public DebugAttribute(Attribute attr) - { - Attr = attr; - } - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - readonly Attribute Attr; + public string Name { get; } = name; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - object? Value { get { return Attr.Value; } } + public object? Value { get; } = value; } } @@ -122,19 +159,257 @@ public enum OverrideType Binary, } - public AttributeList(Datamodel? owner) + /// + /// Gets the that this AttributeList is owned by. + /// + public virtual Datamodel? Owner { get; internal set; } + + #region Slots + + /// + /// Returns the index of the attribute with the given name, or -1. Names of attributes read from a file are usually the same string instance as the query, which the first comparison catches. + /// + int Find(string name) { - var type = GetType(); - Schema = type == typeof(AttributeList) || type == typeof(Element) ? ElementSchema.Empty : ElementSchema.For(type); + var slots = this.slots; + for (var i = 0; i < count; i++) + { + var candidate = slots![i].Name; + if (ReferenceEquals(candidate, name) || candidate == name) + return i; + } - Inner = []; - Owner = owner; + return -1; } /// - /// Gets the that this AttributeList is owned by. + /// Adds an empty slot with the given name at the end. The caller holds the lock. /// - public virtual Datamodel? Owner { get; internal set; } + /// + /// Makes room for the given number of attributes, so that a codec that knows the count adds them without growing the slots. + /// + internal void EnsureCapacity(int capacity) + { + lock (Attribute_ChangeLock) + { + if (slots == null || slots.Length < capacity) + System.Array.Resize(ref slots, capacity); + } + } + + ref AttributeSlot Append(string name) + { + if (slots == null || count == slots.Length) + System.Array.Resize(ref slots, Math.Max(4, count * 2)); + + ref var slot = ref slots[count++]; + slot = default; + slot.Name = name; + return ref slot; + } + + /// + /// Adds an empty slot with the given name at the given index. The caller holds the lock. + /// + ref AttributeSlot InsertAt(int index, string name) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)index, (uint)count, nameof(index)); + + Append(name); + if (index < count - 1) + { + System.Array.Copy(slots!, index, slots!, index + 1, count - 1 - index); + slots![index] = default; + slots[index].Name = name; + } + + return ref slots![index]; + } + + void RemoveSlot(int index) + { + count--; + if (index < count) + System.Array.Copy(slots!, index + 1, slots!, index, count - index); + + slots![count] = default; + } + + /// Whether values of the type are stored in the slot itself rather than as a reference. + internal static bool IsInline(AttributeType type) => type is AttributeType.Int or AttributeType.Float or AttributeType.Bool or AttributeType.Byte or AttributeType.UInt64 + or AttributeType.Time or AttributeType.Color or AttributeType.Vector2 or AttributeType.Vector3 or AttributeType.Vector4 or AttributeType.Quaternion or AttributeType.QAngle; + + /// The type a slot stores values of as, or null when it is not one of the types stored inline. + static AttributeType? KindOf() where T : unmanaged + { + if (typeof(T) == typeof(int)) return AttributeType.Int; + if (typeof(T) == typeof(float)) return AttributeType.Float; + if (typeof(T) == typeof(bool)) return AttributeType.Bool; + if (typeof(T) == typeof(byte)) return AttributeType.Byte; + if (typeof(T) == typeof(ulong)) return AttributeType.UInt64; + if (typeof(T) == typeof(TimeSpan)) return AttributeType.Time; + if (typeof(T) == typeof(Color)) return AttributeType.Color; + if (typeof(T) == typeof(Vector2)) return AttributeType.Vector2; + if (typeof(T) == typeof(Vector3)) return AttributeType.Vector3; + if (typeof(T) == typeof(Vector4)) return AttributeType.Vector4; + if (typeof(T) == typeof(Quaternion)) return AttributeType.Quaternion; + if (typeof(T) == typeof(QAngle)) return AttributeType.QAngle; + return null; + } + + static void WriteInline(ref AttributeSlot slot, AttributeType kind, T value) where T : unmanaged + { + slot.Kind = kind; + slot.Reference = null; + slot.Inline = default; + Unsafe.As(ref slot.Inline) = value; + } + + /// + /// Stores a boxed value in a slot, taking ownership of elements and element arrays the way Valve's datamodel does. + /// + void Store(ref AttributeSlot slot, object? value) + { + switch (value) + { + case null: + slot.Kind = AttributeType.Element; + slot.Reference = null; + return; + case int v: WriteInline(ref slot, AttributeType.Int, v); return; + case float v: WriteInline(ref slot, AttributeType.Float, v); return; + case bool v: WriteInline(ref slot, AttributeType.Bool, v); return; + case byte v: WriteInline(ref slot, AttributeType.Byte, v); return; + case ulong v: WriteInline(ref slot, AttributeType.UInt64, v); return; + case TimeSpan v: WriteInline(ref slot, AttributeType.Time, v); return; + case Color v: WriteInline(ref slot, AttributeType.Color, v); return; + case Vector2 v: WriteInline(ref slot, AttributeType.Vector2, v); return; + case Vector3 v: WriteInline(ref slot, AttributeType.Vector3, v); return; + case Vector4 v: WriteInline(ref slot, AttributeType.Vector4, v); return; + case Quaternion v: WriteInline(ref slot, AttributeType.Quaternion, v); return; + case QAngle v: WriteInline(ref slot, AttributeType.QAngle, v); return; + case Element elem: + if (elem.Owner == null) + elem.Owner = Owner; + else if (elem.Owner != Owner) + throw new ElementOwnershipException(); + slot.Kind = AttributeType.Element; + break; + case ElementArray array: + if (array.Owner == null) + array.Owner = this; + else if (array.Owner != this) + throw new InvalidOperationException("ElementArray is already owned by a different Datamodel."); + slot.Kind = AttributeType.Array; + break; + case IEnumerable: + throw new InvalidOperationException("Element array objects must derive from Datamodel.ElementArray"); + case string: + slot.Kind = AttributeType.String; + break; + case byte[]: + slot.Kind = AttributeType.Binary; + break; + case Matrix4x4: + slot.Kind = AttributeType.Matrix; + break; + default: + if (!Datamodel.IsDatamodelType(value.GetType())) + throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); + slot.Kind = AttributeType.Array; + break; + } + + slot.Reference = value; + } + + /// + /// The value as an object, without loading a deferred value or expanding a stub. + /// + static object? RawValue(in AttributeSlot slot) + { + return slot.Kind switch + { + AttributeType.Element or AttributeType.String or AttributeType.Binary or AttributeType.Matrix or AttributeType.Array => slot.Reference, + AttributeType.Deferred => null, + AttributeType.Int => slot.Inline.Int, + AttributeType.Float => slot.Inline.Float, + AttributeType.Bool => slot.Inline.Bool, + AttributeType.Byte => slot.Inline.Byte, + AttributeType.UInt64 => slot.Inline.UInt64, + AttributeType.Time => TimeSpan.FromTicks(slot.Inline.Ticks), + AttributeType.Color => slot.Inline.Color, + AttributeType.Vector2 => slot.Inline.Vector2, + AttributeType.Vector3 => slot.Inline.Vector3, + AttributeType.Vector4 => slot.Inline.Vector4, + AttributeType.Quaternion => slot.Inline.Quaternion, + AttributeType.QAngle => slot.Inline.QAngle, + _ => throw new InvalidOperationException("Unknown attribute kind."), + }; + } + + /// + /// The value as an object, loading it from the stream if it is deferred and expanding a stub element. + /// + /// Thrown when deferred value loading fails. + /// Thrown when Element destubbing fails. + object? GetValue(int index) + { + Resolve(index); + return RawValue(in slots![index]); + } + + void LoadDeferred(int index) + { + var codec = Owner?.Codec ?? throw new CodecException("Trying to load a deferred Attribute, but could not find codec."); + var offset = slots![index].Inline.Offset; + var name = slots[index].Name; + object? value; + + try + { + lock (codec) + { + value = codec.DeferredDecodeAttribute(Owner, offset); + } + } + catch (Exception err) + { + throw new CodecException($"Deferred loading of attribute \"{name}\" on element {(this as Element)?.ID} using {codec} codec threw an exception.", err); + } + + Store(ref slots[index], value); + } + + /// + /// Registers an attribute whose value is read from the stream on first access. + /// + internal void SetDeferred(string name, long offset) + { + lock (Attribute_ChangeLock) + { + var index = Find(name); + ref var slot = ref (index < 0 ? ref Append(name) : ref slots![index]); + slot.Kind = AttributeType.Deferred; + slot.Reference = null; + slot.Override = null; + slot.Inline = default; + slot.Inline.Offset = offset; + } + } + + /// + /// The reference held by every attribute, without loading deferred values. Value types are skipped, since only elements and arrays matter to callers. + /// + internal IEnumerable EnumerateReferences() + { + for (var i = 0; i < count; i++) + yield return slots![i].Reference; + } + + bool HasListeners => CollectionChanged != null || PropertyChanged != null; + + #endregion /// /// Adds a new attribute to this AttributeList. @@ -147,105 +422,171 @@ public void Add(string key, object? value) } /// - /// Gets the given atttribute's "override type". This applies when multiple Datamodel types map to the same CLR type. + /// Sets a value type attribute without boxing it. Any other type is stored through the indexer. /// - /// The name of the attribute. - /// The attribute's Datamodel type, if different from its CLR type. - /// Thrown when the given attribute is not present in the list. - public OverrideType? GetOverrideType(string key) + public void Set(string name, T value) where T : unmanaged { - var attrib = Inner[key]; + ArgumentNullException.ThrowIfNull(name); + + if (Schema.Properties.Count > 0 && Schema.GetProperty(name) is PropertyBinding binding) + { + // the generated binding of a class property takes the value as it is, so nothing is boxed on the way in + if (binding.CanWrite && binding is PropertyBinding typed) + typed.Set(this, value); + else + SetProperty(binding, name, value); + return; + } - if (attrib is null) + if (KindOf() is not AttributeType kind) { - return null; + this[name] = value; + return; } - return ((Attribute)attrib).OverrideType; + if (this is Element { Stub: true }) + throw new InvalidOperationException("Cannot set attributes on a stub element."); + + lock (Attribute_ChangeLock) + { + var index = Find(name); + if (index < 0) + { + WriteInline(ref Append(name), kind, value); + + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, new AttrKVP(name, value), count - 1)); + } + else + { + ref var slot = ref slots![index]; + var old = HasListeners ? RawValue(in slot) : null; + slot.Override = null; + WriteInline(ref slot, kind, value); + + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, new AttrKVP(name, value), new AttrKVP(name, old), index)); + } + } } /// - /// Sets the given attribute's "override type". This applies when multiple Datamodel types map to the same CLR type. + /// Gets the given atttribute's "override type". This applies when multiple Datamodel types map to the same CLR type. /// /// The name of the attribute. - /// The Datamodel type which the attribute should be stored as when written to DMX, or null. - /// Thrown when the attribute's CLR type does not map to the value given in . - public void SetOverrideType(string key, OverrideType? type) + /// The attribute's Datamodel type, if different from its CLR type. + public OverrideType? GetOverrideType(string key) { - var attrib = Inner[key]; - - if (attrib is not null) + lock (Attribute_ChangeLock) { - ((Attribute)attrib).OverrideType = type; + var index = Find(key); + return index < 0 ? null : slots![index].Override; } - } /// - /// Inserts an Attribute at the given index. + /// Sets the given attribute's "override type". This applies when multiple Datamodel types map to the same CLR type. /// - private void Insert(int index, Attribute item, bool notify = true) + /// The name of the attribute. + /// The Datamodel type which the attribute should be stored as when written to DMX, or null. + /// Thrown when the attribute's CLR type does not map to the value given in . + public void SetOverrideType(string key, OverrideType? type) { lock (Attribute_ChangeLock) { - Inner.Remove(item.Name); - Inner.Insert(index, item.Name, item); - } - item.Owner = this; + var index = Find(key); + if (index < 0) + return; + + ref var slot = ref slots![index]; + switch (type) + { + case null: + break; + case OverrideType.Angle: + if (slot.Kind != AttributeType.Vector3) + throw new AttributeTypeException("OverrideType.Angle can only be applied to Vector3 attributes"); + break; + case OverrideType.Binary: + if (slot.Reference is not byte[]) + throw new AttributeTypeException("OverrideType.Binary can only be applied to byte[] attributes"); + break; + default: + throw new NotImplementedException(); + } - if (notify) - OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item.ToKeyValuePair(), index)); + slot.Override = type; + } } public bool Remove(string key) { lock (Attribute_ChangeLock) { - var attr = (Attribute?)Inner[key]; - if (attr == null) return false; + var index = Find(key); + if (index < 0) return false; - var index = IndexOf(key); - Inner.Remove(key); - OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, attr.ToKeyValuePair(), index)); + var removed = new AttrKVP(key, RawValue(in slots![index])); + RemoveSlot(index); + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed, index)); return true; } } + /// + /// Gets the value of an attribute without loading it if it is deferred, in which case the value is null. + /// public bool TryGetValue(string key, out object? value) { - Attribute? result; lock (Attribute_ChangeLock) - result = (Attribute?)Inner[key]; - - if (result != null) { - value = result.RawValue; + var index = Find(key); + if (index < 0) + { + value = null; + return false; + } + + ref var slot = ref slots![index]; + value = RawValue(in slot); return true; } - else - { - value = null; - return false; - } } public virtual bool ContainsKey(string key) { ArgumentNullException.ThrowIfNull(key); lock (Attribute_ChangeLock) - return Inner[key] != null; + return Find(key) >= 0; } + public ICollection Keys { - get { lock (Attribute_ChangeLock) return Inner.Keys.Cast().ToArray(); } + get + { + lock (Attribute_ChangeLock) + { + var keys = new string[count]; + for (var i = 0; i < count; i++) + keys[i] = slots![i].Name; + return keys; + } + } } + public ICollection Values { - get { lock (Attribute_ChangeLock) return Inner.Values.Cast().Select(attr => attr.Value).ToArray(); } + get + { + var values = new object?[Count]; + for (var i = 0; i < values.Length; i++) + values[i] = GetValue(i); + return values; + } } /// - /// Gets or sets the value of the with the given name. + /// Gets or sets the value of the attribute with the given name. /// /// The name to search for. Cannot be null. /// The value associated with the given name. @@ -253,14 +594,17 @@ public ICollection Values /// Thrown when an attempt is made to get a name that is not present in this AttributeList. /// Thrown when an attempt is made to set the value of the attribute to an Element from a different . /// Thrown when an attempt is made to set a value that is not of a valid Datamodel attribute type. - /// Thrown when the maximum number of Attributes allowed in an AttributeList has been reached. public virtual object? this[string name] { get { ArgumentNullException.ThrowIfNull(name); - var attr = (Attribute?)Inner[name]; - if (attr == null) + + int index; + lock (Attribute_ChangeLock) + index = Find(name); + + if (index < 0) { var binding = Schema.GetProperty(name); if (binding != null) @@ -271,83 +615,125 @@ public virtual object? this[string name] throw new KeyNotFoundException($"{this} does not have an attribute called \"{name}\""); } - return attr.Value; + return GetValue(index); } set { ArgumentNullException.ThrowIfNull(name); - if (value != null && !Datamodel.IsDatamodelType(value.GetType())) - throw new AttributeTypeException($"{value.GetType().FullName} is not a valid Datamodel attribute type. (If this is an array, it must implement IList)."); + + // a value that fits a class property is a valid attribute type by construction, so it skips the type table + var binding = Schema.Properties.Count > 0 ? Schema.GetProperty(name) : null; + + if (binding != null) + { + SetProperty(binding, name, value); + return; + } if (Owner != null && this == Owner.PrefixAttributes && value?.GetType() == typeof(Element)) throw new AttributeTypeException("Elements are not supported as prefix attributes."); - var binding = Schema.GetProperty(name); - - if (binding != null) + lock (Attribute_ChangeLock) { - if (binding.CanWrite) + var index = Find(name); + if (index < 0) { - // null is fine, it will just set the value to null - if (value != null && !binding.PropertyType.IsInstanceOfType(value)) - { - value = ConvertScalar(value, binding.PropertyType) - ?? throw new InvalidDataException($"class property '{Schema.ElementType.Name}.{binding.PropertyName}' with type '{binding.PropertyType}' can not hold a value of type '{value.GetType()}' (attribute '{name}'), this is likely a mismatch between the real class and the class from the datamodel"); - } - - binding.SetValue(this, value); + Store(ref Append(name), value); + + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, new AttrKVP(name, value), count - 1)); } else { - // a read-only array property takes the items of an incoming array of the same type, so that a file can fill it once - var existingArray = binding.GetValue(this) as IList; - var incomingArray = value as IList; - - if (existingArray is not null && incomingArray is not null && existingArray.GetType() == incomingArray.GetType()) - { - if (existingArray.Count == 0) - { - foreach (var item in incomingArray) - existingArray.Add(item); - } - else - { - throw new InvalidOperationException($"Attribute '{name}' modifies property {Schema.ElementType.Name}.{binding.PropertyName}, which is read-only and already has items."); - } - } - else - { - throw new InvalidDataException($"Property '{Schema.ElementType.Name}.{binding.PropertyName}' of deserialisation class must be writeable, make sure it has a setter"); - } - } + ref var slot = ref slots![index]; + var old = HasListeners ? RawValue(in slot) : null; + slot.Override = null; + Store(ref slot, value); - return; + if (HasListeners) + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, new AttrKVP(name, value), new AttrKVP(name, old), index)); + } } + } + } - Attribute? old_attr; - Attribute? new_attr; - int old_index = -1; - lock (Attribute_ChangeLock) + /// + /// Assigns a value to the class property that stores the attribute. + /// + void SetProperty(PropertyBinding binding, string name, object? value) + { + if (binding.CanWrite) + { + // null is fine, it will just set the value to null; an exact type match is the common case and avoids the runtime cast check + if (value != null && binding.PropertyType != value.GetType() && !binding.PropertyType.IsInstanceOfType(value)) { - old_attr = (Attribute?)Inner[name]; - new_attr = new Attribute(name, this, value); - - if (old_attr != null) - { - old_index = IndexOf(old_attr.Name); - Inner.Remove(old_attr); - } - Insert(old_index == -1 ? Count : old_index, new Attribute(name, this, value), notify: false); + value = ConvertScalar(value, binding.PropertyType) + ?? throw new InvalidDataException($"class property '{Schema.ElementType.Name}.{binding.PropertyName}' with type '{binding.PropertyType}' can not hold a value of type '{value.GetType()}' (attribute '{name}'), this is likely a mismatch between the real class and the class from the datamodel"); } - NotifyCollectionChangedEventArgs change_args; - if (old_attr != null) - change_args = new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, new_attr.ToKeyValuePair(), old_attr.ToKeyValuePair(), old_index); + binding.SetValue(this, value); + return; + } + + // a read-only array property takes the items of an incoming array of the same type, so that a file can fill it once + var existingArray = binding.GetValue(this) as IList; + var incomingArray = value as IList; + + if (existingArray is not null && incomingArray is not null && existingArray.GetType() == incomingArray.GetType()) + { + if (existingArray.Count == 0) + { + foreach (var item in incomingArray) + existingArray.Add(item); + } else - change_args = new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, new_attr.ToKeyValuePair(), Count); + { + throw new InvalidOperationException($"Attribute '{name}' modifies property {Schema.ElementType.Name}.{binding.PropertyName}, which is read-only and already has items."); + } + } + else + { + throw new InvalidDataException($"Property '{Schema.ElementType.Name}.{binding.PropertyName}' of deserialisation class must be writeable, make sure it has a setter"); + } + } - OnCollectionChanged(change_args); + /// + /// Converts between the bool, int and float attribute types the way Valve's datamodel does when a value is assigned + /// to an attribute of another of those types. Returns null for any other combination. + /// + private static object? ConvertScalar(object value, Type targetType) + { + if (targetType == typeof(int)) + { + return value switch + { + bool b => b ? 1 : 0, + float f => (int)f, + _ => null, + }; + } + + if (targetType == typeof(float)) + { + return value switch + { + bool b => b ? 1f : 0f, + int i => (float)i, + _ => null, + }; + } + + if (targetType == typeof(bool)) + { + return value switch + { + int i => i != 0, + float f => f != 0f, + _ => null, + }; } + + return null; } /// @@ -357,19 +743,17 @@ public AttrKVP this[int index] { get { - var attr = (Attribute?)Inner[index]; - - if (attr is null) - { - throw new InvalidOperationException($"attribute at index {index} doesn't exist"); - } - - return attr.ToKeyValuePair(); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual((uint)index, (uint)count, nameof(index)); + return new AttrKVP(slots![index].Name, GetValue(index)); } set { - RemoveAt(index); - Insert(index, new Attribute(value.Key, this, value.Value)); + lock (Attribute_ChangeLock) + { + RemoveAt(index); + Store(ref InsertAt(index, value.Key), value.Value); + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, value, index)); + } } } @@ -378,32 +762,20 @@ public AttrKVP this[int index] /// public void RemoveAt(int index) { - Attribute? attr; + AttrKVP removed; lock (Attribute_ChangeLock) { - attr = (Attribute?)Inner[index]; - - if (attr is not null) - { - attr.Owner = null; - Inner.RemoveAt(index); - } + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual((uint)index, (uint)count, nameof(index)); + removed = new AttrKVP(slots![index].Name, RawValue(in slots[index])); + RemoveSlot(index); } - OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, attr, index)); + OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed, index)); } public int IndexOf(string key) { lock (Attribute_ChangeLock) - { - int i = 0; - foreach (string name in Inner.Keys) - { - if (name == key) return i; - i++; - } - } - return -1; + return Find(key); } /// @@ -412,7 +784,11 @@ public int IndexOf(string key) public void Clear() { lock (Attribute_ChangeLock) - Inner.Clear(); + { + if (slots != null) + System.Array.Clear(slots, 0, count); + count = 0; + } OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } @@ -421,7 +797,7 @@ public int Count get { lock (Attribute_ChangeLock) - return Inner.Count; + return count; } } @@ -438,8 +814,8 @@ public int Count /// public IEnumerable GetAllAttributesForSerialization() { - foreach (var attr in GetPropertyBasedAttributes(useSerializationName: true)) - yield return attr.ToKeyValuePair(); + foreach (var binding in Schema.Properties) + yield return new AttrKVP(binding.AttributeName, binding.GetValue(this)); foreach (var attr in this) yield return attr; @@ -447,8 +823,121 @@ public IEnumerable GetAllAttributesForSerialization() public IEnumerator GetEnumerator() { - foreach (var attr in Inner.Values.Cast().ToArray()) - yield return attr.ToKeyValuePair(); + var pairs = new AttrKVP[Count]; + for (var i = 0; i < pairs.Length; i++) + pairs[i] = new AttrKVP(slots![i].Name, GetValue(i)); + + return ((IEnumerable)pairs).GetEnumerator(); + } + + /// + /// Loads the value of a deferred slot and expands a stub, so that the slot holds its final value. + /// + void Resolve(int index) + { + if (slots![index].Kind == AttributeType.Deferred) + LoadDeferred(index); + + ref var slot = ref slots[index]; + if (slot.Kind == AttributeType.Element && slot.Reference is Element { Stub: true } stub && Owner != null) + { + try { slot.Reference = Owner.OnStubRequest(stub.ID) ?? stub; } + catch (Exception err) { throw new DestubException(this, slot.Name, err); } + } + } + + /// + /// Passes every attribute a codec writes to the visitor in the form its slot holds it: class properties first, in declaration order, then the plain attributes in the order they were added. + /// Deferred values are loaded and stubs expanded first, as does, but no value is boxed and the lock is released before the visitor runs. + /// + internal void VisitAttributes(ref TVisitor visitor) where TVisitor : struct, IAttributeVisitor + { + var properties = Schema.Properties; + AttributeSlot[] copy; + int copied; + + lock (Attribute_ChangeLock) + { + copied = count; + copy = ArrayPool.Shared.Rent(copied); + for (var i = 0; i < copied; i++) + { + Resolve(i); + copy[i] = slots![i]; + } + } + + try + { + visitor.Begin(properties.Count + copied); + + foreach (var binding in properties) + { + binding.Read(this, out var kind, out var inline, out var reference); + visitor.Visit(binding.AttributeName, kind, in inline, reference); + } + + for (var i = 0; i < copied; i++) + { + ref var slot = ref copy[i]; + visitor.Visit(slot.Name, slot.Kind, in slot.Inline, slot.Reference); + } + } + finally + { + System.Array.Clear(copy, 0, copied); + ArrayPool.Shared.Return(copy); + } + } + + /// + /// Splits a boxed value into the form a slot stores it in. Anything that is not a scalar, vector, element, string, blob or matrix counts as an array, whether or not it is a valid attribute value. + /// + internal static void Classify(object? value, out AttributeType kind, out InlineValue inline, out object? reference) + { + inline = default; + reference = null; + switch (value) + { + case int v: kind = AttributeType.Int; inline.Int = v; return; + case float v: kind = AttributeType.Float; inline.Float = v; return; + case bool v: kind = AttributeType.Bool; inline.Bool = v; return; + case byte v: kind = AttributeType.Byte; inline.Byte = v; return; + case ulong v: kind = AttributeType.UInt64; inline.UInt64 = v; return; + case TimeSpan v: kind = AttributeType.Time; inline.Ticks = v.Ticks; return; + case Color v: kind = AttributeType.Color; inline.Color = v; return; + case Vector2 v: kind = AttributeType.Vector2; inline.Vector2 = v; return; + case Vector3 v: kind = AttributeType.Vector3; inline.Vector3 = v; return; + case Vector4 v: kind = AttributeType.Vector4; inline.Vector4 = v; return; + case Quaternion v: kind = AttributeType.Quaternion; inline.Quaternion = v; return; + case QAngle v: kind = AttributeType.QAngle; inline.QAngle = v; return; + case null: kind = AttributeType.Element; return; + case Element: kind = AttributeType.Element; reference = value; return; + case string: kind = AttributeType.String; reference = value; return; + case byte[]: kind = AttributeType.Binary; reference = value; return; + case Matrix4x4: kind = AttributeType.Matrix; reference = value; return; + default: kind = AttributeType.Array; reference = value; return; + } + } + + /// + /// The kind a slot stores values of the given type as: inline for the scalar and vector types, a reference for everything else. + /// + internal static AttributeType? KindOf(Type type) + { + if (type == typeof(int)) return AttributeType.Int; + if (type == typeof(float)) return AttributeType.Float; + if (type == typeof(bool)) return AttributeType.Bool; + if (type == typeof(byte)) return AttributeType.Byte; + if (type == typeof(ulong)) return AttributeType.UInt64; + if (type == typeof(TimeSpan)) return AttributeType.Time; + if (type == typeof(Color)) return AttributeType.Color; + if (type == typeof(Vector2)) return AttributeType.Vector2; + if (type == typeof(Vector3)) return AttributeType.Vector3; + if (type == typeof(Vector4)) return AttributeType.Vector4; + if (type == typeof(Quaternion)) return AttributeType.Quaternion; + if (type == typeof(QAngle)) return AttributeType.QAngle; + return null; } #region Interfaces @@ -470,13 +959,12 @@ protected virtual void OnPropertyChanged([System.Runtime.CompilerServices.Caller } /// - /// Raised when an is added, removed, or replaced. + /// Raised when an attribute is added, removed, or replaced. /// + /// Only raised while a handler is attached to this event or to . public event NotifyCollectionChangedEventHandler? CollectionChanged; protected virtual void OnCollectionChanged(NotifyCollectionChangedEventArgs e) { - Debug.Assert(!(e.NewItems != null && e.NewItems.OfType().Any()) && !(e.OldItems != null && e.OldItems.OfType().Any())); - switch (e.Action) { case NotifyCollectionChangedAction.Add: @@ -531,10 +1019,9 @@ bool ICollection.Remove(AttrKVP item) { lock (Attribute_ChangeLock) { - var attr = (Attribute?)Inner[item.Key]; - if (attr == null || attr.Value != item.Value) return false; - Remove(attr.Name); - return true; + var index = Find(item.Key); + if (index < 0 || !Equals(GetValue(index), item.Value)) return false; + return Remove(item.Key); } } @@ -545,12 +1032,11 @@ void ICollection.CopyTo(AttrKVP[] array, int arrayIndex) void ICollection.CopyTo(Array array, int index) { - lock (Attribute_ChangeLock) - foreach (Attribute attr in Inner.Values) - { - array.SetValue(attr.ToKeyValuePair(), index); - index++; - } + foreach (var pair in this) + { + array.SetValue(pair, index); + index++; + } } void ICollection.Add(AttrKVP item) @@ -562,8 +1048,8 @@ bool ICollection.Contains(AttrKVP item) { lock (Attribute_ChangeLock) { - var attr = (Attribute?)Inner[item.Key]; - return attr != null && attr.Value == item.Value; + var index = Find(item.Key); + return index >= 0 && Equals(GetValue(index), item.Value); } } diff --git a/Datamodel.NET/Codecs/Binary.cs b/Datamodel.NET/Codecs/Binary.cs index bb5415d..1460c2b 100644 --- a/Datamodel.NET/Codecs/Binary.cs +++ b/Datamodel.NET/Codecs/Binary.cs @@ -1,4 +1,6 @@ using System; +using System.Buffers.Binary; +using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; @@ -6,13 +8,15 @@ using System.IO; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; namespace Datamodel.Codecs { class Binary : IDeferredAttributeCodec { - static readonly Dictionary SupportedAttributes = []; + /// The types each encoding version supports, in the order of their ids. A null marks an id the version reserves for something the library does not read. + static readonly Dictionary SupportedAttributes = []; BinaryReader? Reader; /// @@ -29,29 +33,63 @@ static Binary() { SupportedAttributes[1] = SupportedAttributes[2] = [ - typeof(Element), typeof(int), typeof(float), typeof(bool), typeof(string), typeof(byte[]), - null /* ObjectID */, typeof(Color), typeof(Vector2), typeof(Vector3), typeof(Vector4), typeof(Vector3) /* angle*/, typeof(Quaternion), typeof(Matrix4x4) + AttributeType.Element, AttributeType.Int, AttributeType.Float, AttributeType.Bool, AttributeType.String, AttributeType.Binary, + null /* ObjectID */, AttributeType.Color, AttributeType.Vector2, AttributeType.Vector3, AttributeType.Vector4, AttributeType.Vector3 /* angle */, AttributeType.Quaternion, AttributeType.Matrix ]; + SupportedAttributes[3] = SupportedAttributes[4] = SupportedAttributes[5] = [ - typeof(Element), typeof(int), typeof(float), typeof(bool), typeof(string), typeof(byte[]), - typeof(TimeSpan), typeof(Color), typeof(Vector2), typeof(Vector3), typeof(Vector4), typeof(Vector3) /* angle*/, typeof(Quaternion), typeof(Matrix4x4) + AttributeType.Element, AttributeType.Int, AttributeType.Float, AttributeType.Bool, AttributeType.String, AttributeType.Binary, + AttributeType.Time, AttributeType.Color, AttributeType.Vector2, AttributeType.Vector3, AttributeType.Vector4, AttributeType.Vector3 /* angle */, AttributeType.Quaternion, AttributeType.Matrix ]; + SupportedAttributes[9] = [ - typeof(Element), typeof(int), typeof(float), typeof(bool), typeof(string), typeof(byte[]), - typeof(TimeSpan), typeof(Color), typeof(Vector2), typeof(Vector3), typeof(Vector4), typeof(QAngle), typeof(Quaternion), typeof(Matrix4x4), - typeof(ulong), typeof(byte) + AttributeType.Element, AttributeType.Int, AttributeType.Float, AttributeType.Bool, AttributeType.String, AttributeType.Binary, + AttributeType.Time, AttributeType.Color, AttributeType.Vector2, AttributeType.Vector3, AttributeType.Vector4, AttributeType.QAngle, AttributeType.Quaternion, AttributeType.Matrix, + AttributeType.UInt64, AttributeType.Byte ]; } + /// The CLR type a value of the given type is stored as. + static Type ClrType(AttributeType type) => type switch + { + AttributeType.Element => typeof(Element), + AttributeType.Int => typeof(int), + AttributeType.Float => typeof(float), + AttributeType.Bool => typeof(bool), + AttributeType.String => typeof(string), + AttributeType.Binary => typeof(byte[]), + AttributeType.Time => typeof(TimeSpan), + AttributeType.Color => typeof(Color), + AttributeType.Vector2 => typeof(Vector2), + AttributeType.Vector3 => typeof(Vector3), + AttributeType.Vector4 => typeof(Vector4), + AttributeType.QAngle => typeof(QAngle), + AttributeType.Quaternion => typeof(Quaternion), + AttributeType.Matrix => typeof(Matrix4x4), + AttributeType.UInt64 => typeof(ulong), + AttributeType.Byte => typeof(byte), + _ => throw new ArgumentOutOfRangeException(nameof(type)), + }; + + /// The id a version writes for a value type. + static byte TypeToId(AttributeType type, int version) + { + var index = System.Array.IndexOf(SupportedAttributes[version], (AttributeType?)type); + if (index < 0) + throw new CodecException($"\"{type}\" is not supported in encoding binary {version}"); + + return (byte)(index + 1); + } + static byte TypeToId(Type type, int version) { // a byte[] is a "binary" blob, distinct from a "uint8_array" (Array) in encoding version 9 bool array = type != typeof(byte[]) && Datamodel.IsDatamodelArrayType(type); var search_type = array ? Datamodel.GetArrayInnerType(type) : type; - if (array && search_type == typeof(byte) && !SupportedAttributes[version].Contains(typeof(byte))) + if (array && search_type == typeof(byte) && !SupportedAttributes[version].Contains(AttributeType.Byte)) { search_type = typeof(byte[]); // Recent version of DMX support both "binary" and "uint8_array" attributes. These are the same thing! array = false; @@ -60,10 +98,10 @@ static byte TypeToId(Type type, int version) byte i = 0; foreach (var list_type in type_list) { - if (list_type == typeof(Element) && type.IsSubclassOf(typeof(Element))) + if (list_type == AttributeType.Element && type.IsSubclassOf(typeof(Element))) break; - if (list_type == search_type) + if (list_type is AttributeType known && ClrType(known) == search_type) break; i++; } @@ -76,7 +114,7 @@ static byte TypeToId(Type type, int version) /// /// Maps a type id of the stream to the attribute type, or to the item type when the id denotes an array. /// - (Type Type, bool IsArray) IdToType(byte id) + (AttributeType Type, bool IsArray) IdToType(byte id) { var type_list = SupportedAttributes[EncodingVersion]; bool array = false; @@ -97,7 +135,7 @@ static byte TypeToId(Type type, int version) } } - if (id >= type_list.Length || type_list[id] is not Type type) + if (id >= type_list.Length || type_list[id] is not AttributeType type) { throw new CodecException(String.Format("Unrecognised attribute type: {0}", id + 1)); } @@ -131,6 +169,12 @@ class StringDictionary /// Fast string index lookup. readonly Dictionary? Indices; + /// + /// Indices of the attribute and class name instances met so far. Names of class properties and of attributes read from a file are shared instances, + /// so this finds them without hashing their characters, the way Valve looks attribute names up as symbols. + /// + readonly Dictionary NameIndices = new(ReferenceEqualityComparer.Instance); + public bool Dummy; // binary 4 uses int for dictionary length, but short for dictionary indices. Whoops! @@ -147,93 +191,68 @@ public StringDictionary(Binary codec, BinaryReader reader) Dummy = EncodingVersion == 1; if (!Dummy) { - foreach (var i in Enumerable.Range(0, LengthSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32())) - AddString(Codec.ReadString_Raw(reader)); + var count = LengthSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32(); + Strings.Capacity = count; + for (var i = 0; i < count; i++) + Strings.Add(Codec.ReadString_Raw(reader)); } } /// - /// Constructs a new from a object. + /// Constructs an empty dictionary for writing. The encoder adds every string it meets, in the order it meets them. /// - public StringDictionary(int encoding_version, BinaryWriter writer, Datamodel dm, SerializationContext context) + public StringDictionary(int encoding_version) { EncodingVersion = encoding_version; - Context = context; - Dummy = EncodingVersion == 1; if (!Dummy) - { Indices = []; - Scraped = []; + } - ScrapeElement(dm.Root); + /// + /// Adds a string to the table unless it is there already. Nothing is added for a version that writes every string in place. + /// + public void AddString(string? value) + { + if (Indices == null) + return; - // the prefix attributes are also written as a regular element in version 9 - if (EncodingVersion >= 9 && dm.PrefixAttributes.Count > 0) - { - AddString(string.Empty); - AddString(PrefixElementClass); - foreach (var attr in dm.PrefixAttributes) - { - AddString(attr.Key); - if (attr.Value is string stringValue) - AddString(stringValue); - } - } - } + value ??= string.Empty; + if (Indices.TryAdd(value, Strings.Count)) + Strings.Add(value); } - private readonly HashSet Scraped = []; - private readonly SerializationContext? Context; - void ScrapeElement(Element? elem) + int GetIndex(string value) { - if (elem == null || elem.Stub || Scraped.Contains(elem)) return; - Scraped.Add(elem); - - AddString(elem.Name); - AddString(elem.ClassName); - foreach (var attr in Context!.Attributes[elem]) - { - AddString(attr.Key); - switch (attr.Value) - { - case string stringValue: - AddString(stringValue); - break; - case Element elementValue: - ScrapeElement(elementValue); - break; - case IList elementListValue: - foreach (var array_elem in elementListValue) - ScrapeElement(array_elem); - break; - } - } + value ??= string.Empty; + return Indices!.TryGetValue(value, out var index) ? index : -1; } /// - /// Add non-nullable string. + /// Adds an attribute or class name to the table unless it is there already. /// - /// - void AddString(string value) + public void AddName(string name) { - value ??= string.Empty; + if (Indices == null || NameIndices.ContainsKey(name)) + return; - if (Indices == null) + AddString(name); + NameIndices[name] = Indices[name]; + } + + public void WriteName(string name, OutputBuffer writer) + { + if (Dummy) { - Strings.Add(value); + writer.Write(name); return; } - if (Indices.TryAdd(value, Strings.Count)) - Strings.Add(value); - } + if (!NameIndices.TryGetValue(name, out var index)) + NameIndices[name] = index = GetIndex(name); - int GetIndex(string value) - { - value ??= string.Empty; - return Indices!.TryGetValue(value, out var index) ? index : -1; + WriteIndex(index, writer); } public string ReadString(BinaryReader reader) @@ -242,19 +261,21 @@ public string ReadString(BinaryReader reader) return Strings[IndiceSize == sizeof(short) ? reader.ReadInt16() : reader.ReadInt32()]; } - public void WriteString(string value, BinaryWriter writer) + public void WriteString(string value, OutputBuffer writer) { if (Dummy) writer.Write(value); else - { - var index = GetIndex(value); - if (IndiceSize == sizeof(short)) writer.Write((short)index); - else writer.Write(index); - } + WriteIndex(GetIndex(value), writer); + } + + void WriteIndex(int index, OutputBuffer writer) + { + if (IndiceSize == sizeof(short)) writer.Write((short)index); + else writer.Write(index); } - public void WriteSelf(BinaryWriter writer) + public void WriteSelf(OutputBuffer writer) { if (Dummy) return; @@ -271,52 +292,32 @@ public void WriteSelf(BinaryWriter writer) public void Encode(Datamodel dm, string encoding, int encoding_version, Stream stream) { - using var writer = new DmxBinaryWriter(stream); - var encoder = new Encoder(writer, dm, encoding_version); - encoder.Encode(); + var output = new OutputBuffer(stream); + new Encoder(output, dm, encoding_version).Encode(); + output.Flush(); } - private static readonly Dictionary TypeMap = new Dictionary - { - { typeof(Element).TypeHandle, 0 }, - { typeof(int).TypeHandle, 1 }, - { typeof(float).TypeHandle, 2 }, - { typeof(bool).TypeHandle, 3 }, - { typeof(string).TypeHandle, 4 }, - { typeof(byte[]).TypeHandle, 5 }, - { typeof(TimeSpan).TypeHandle, 6 }, - { typeof(Color).TypeHandle, 7 }, - { typeof(Vector2).TypeHandle, 8 }, - { typeof(Vector3).TypeHandle, 9 }, - { typeof(QAngle).TypeHandle, 10 }, - { typeof(Vector4).TypeHandle, 11 }, - { typeof(Quaternion).TypeHandle, 12 }, - { typeof(Matrix4x4).TypeHandle, 13 }, - { typeof(byte).TypeHandle, 14 }, - { typeof(UInt64).TypeHandle, 15 } - }; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - object? ReadValue(Datamodel dm, int typeIndex, bool raw_string, BinaryReader reader) + object? ReadValue(Datamodel dm, AttributeType type, bool raw_string, BinaryReader reader) { - return typeIndex switch - { - 0 => ReadElement(dm, reader), - 1 => reader.ReadInt32(), - 2 => reader.ReadSingle(), - 3 => reader.ReadBoolean(), - 4 => raw_string ? ReadString_Raw(reader) : StringDict!.ReadString(reader), - 5 => reader.ReadBytes(reader.ReadInt32()), - 6 => TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)), - 7 => ReadColor(reader), - 8 => ReadVector2(reader), - 9 => ReadVector3(reader), - 10 => ReadQAngle(reader), - 11 => ReadVector4(reader), - 12 => ReadQuaternion(reader), - 13 => ReadMatrix4x4(reader), - 14 => reader.ReadByte(), - 15 => reader.ReadUInt64(), + return type switch + { + AttributeType.Element => ReadElement(dm, reader), + AttributeType.Int => reader.ReadInt32(), + AttributeType.Float => reader.ReadSingle(), + AttributeType.Bool => reader.ReadBoolean(), + AttributeType.String => raw_string ? ReadString_Raw(reader) : StringDict!.ReadString(reader), + AttributeType.Binary => reader.ReadBytes(reader.ReadInt32()), + AttributeType.Time => TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)), + AttributeType.Color => ReadColor(reader), + AttributeType.Vector2 => ReadVector2(reader), + AttributeType.Vector3 => ReadVector3(reader), + AttributeType.QAngle => ReadQAngle(reader), + AttributeType.Vector4 => ReadVector4(reader), + AttributeType.Quaternion => ReadQuaternion(reader), + AttributeType.Matrix => ReadMatrix4x4(reader), + AttributeType.Byte => reader.ReadByte(), + AttributeType.UInt64 => reader.ReadUInt64(), _ => throw new ArgumentException("Cannot read value of type") }; } @@ -422,13 +423,20 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in StringDict = new StringDictionary(this, Reader); var num_elements = Reader.ReadInt32(); + // the file states how many elements follow, so the tables that hold them are sized once + ElementIndex.Capacity = num_elements; + dm.AllElements.EnsureCapacity(num_elements); + // read index - foreach (var i in Enumerable.Range(0, num_elements)) + Span id_bits = stackalloc byte[16]; + for (var i = 0; i < num_elements; i++) { var type = StringDict.ReadString(Reader); var name = EncodingVersion >= 4 ? StringDict.ReadString(Reader) : ReadString_Raw(Reader); - var id_bits = Reader.ReadBytes(16); - var id = new Guid(BitConverter.IsLittleEndian ? id_bits : id_bits.Reverse().ToArray()); + Reader.BaseStream.ReadExactly(id_bits); + if (!BitConverter.IsLittleEndian) + id_bits.Reverse(); + var id = new Guid(id_bits); if (!CodecUtilities.TryConstructCustomElement(resolver, dm, type, name, id, out var elem)) { @@ -448,7 +456,11 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in var num_attrs = Reader.ReadInt32(); - foreach (var i in Enumerable.Range(0, num_attrs)) + // a plain element gets exactly the slots it needs; a class keeps most attributes in its properties, so its few slots grow as they come + if (elem.Schema.Properties.Count == 0) + elem.EnsureCapacity(num_attrs); + + for (var i = 0; i < num_attrs; i++) { var name = StringDict.ReadString(Reader); if (defer_mode == DeferredMode.Automatic) @@ -458,7 +470,7 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in } else { - elem.Add(name, DecodeAttribute(dm, false, Reader)); + DecodeAttributeInto(dm, elem, name, Reader); } } } @@ -499,18 +511,87 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in var (type, isArray) = IdToType(reader.ReadByte()); if (!isArray) - return ReadValue(dm, TypeMap[type.TypeHandle], EncodingVersion < 4 || prefix, reader); - else + return ReadValue(dm, type, EncodingVersion < 4 || prefix, reader); + + return ReadArray(dm, type, reader.ReadInt32(), reader); + } + + /// + /// Reads an attribute of an element straight into the element, so that value types are stored inline without being boxed. + /// + void DecodeAttributeInto(Datamodel dm, AttributeList target, string name, BinaryReader reader) + { + var (type, isArray) = IdToType(reader.ReadByte()); + + if (isArray) { - var count = reader.ReadInt32(); - var array = CodecUtilities.MakeList(type, count); + target[name] = ReadArray(dm, type, reader.ReadInt32(), reader); + return; + } - var typeId = TypeMap[type.TypeHandle]; - foreach (var x in Enumerable.Range(0, count)) - array.Add(ReadValue(dm, typeId, true, reader)); + switch (type) + { + case AttributeType.Element: target[name] = ReadElement(dm, reader); break; + case AttributeType.Int: target.Set(name, reader.ReadInt32()); break; + case AttributeType.Float: target.Set(name, reader.ReadSingle()); break; + case AttributeType.Bool: target.Set(name, reader.ReadBoolean()); break; + case AttributeType.String: target[name] = EncodingVersion < 4 ? ReadString_Raw(reader) : StringDict!.ReadString(reader); break; + case AttributeType.Binary: target[name] = reader.ReadBytes(reader.ReadInt32()); break; + case AttributeType.Time: target.Set(name, TimeSpan.FromTicks(reader.ReadInt32() * (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond))); break; + case AttributeType.Color: target.Set(name, ReadColor(reader)); break; + case AttributeType.Vector2: target.Set(name, ReadVector2(reader)); break; + case AttributeType.Vector3: target.Set(name, ReadVector3(reader)); break; + case AttributeType.QAngle: target.Set(name, ReadQAngle(reader)); break; + case AttributeType.Vector4: target.Set(name, ReadVector4(reader)); break; + case AttributeType.Quaternion: target.Set(name, ReadQuaternion(reader)); break; + case AttributeType.Matrix: target[name] = ReadMatrix4x4(reader); break; + case AttributeType.Byte: target.Set(name, reader.ReadByte()); break; + case AttributeType.UInt64: target.Set(name, reader.ReadUInt64()); break; + default: throw new ArgumentException("Cannot read value of type"); + } + } - return array; + /// + /// Storage shared by the value type arrays of this stream, see . + /// + readonly ArrayChunks Chunks = new(); + + /// + /// Reads an array attribute. Value types whose memory layout matches the stream are copied in one read into a slice of a shared chunk, + /// instead of one boxed item at a time into a list of their own. + /// + System.Collections.IList ReadArray(Datamodel dm, AttributeType type, int count, BinaryReader reader) + { + if (BitConverter.IsLittleEndian && count > 0) + { + switch (type) + { + case AttributeType.Int: { var (buffer, offset) = ReadChunk(count, reader); return new IntArray(buffer, offset, count); } + case AttributeType.Float: { var (buffer, offset) = ReadChunk(count, reader); return new FloatArray(buffer, offset, count); } + case AttributeType.Bool: { var (buffer, offset) = ReadChunk(count, reader); return new BoolArray(buffer, offset, count); } + case AttributeType.Color: { var (buffer, offset) = ReadChunk(count, reader); return new ColorArray(buffer, offset, count); } + case AttributeType.Vector2: { var (buffer, offset) = ReadChunk(count, reader); return new Vector2Array(buffer, offset, count); } + case AttributeType.Vector3: { var (buffer, offset) = ReadChunk(count, reader); return new Vector3Array(buffer, offset, count); } + case AttributeType.Vector4: { var (buffer, offset) = ReadChunk(count, reader); return new Vector4Array(buffer, offset, count); } + case AttributeType.Quaternion: { var (buffer, offset) = ReadChunk(count, reader); return new QuaternionArray(buffer, offset, count); } + case AttributeType.Matrix: { var (buffer, offset) = ReadChunk(count, reader); return new MatrixArray(buffer, offset, count); } + case AttributeType.Byte: { var (buffer, offset) = ReadChunk(count, reader); return new ByteArray(buffer, offset, count); } + case AttributeType.UInt64: { var (buffer, offset) = ReadChunk(count, reader); return new UInt64Array(buffer, offset, count); } + } } + + var array = CodecUtilities.MakeList(ClrType(type), count); + for (var i = 0; i < count; i++) + array.Add(ReadValue(dm, type, true, reader)); + + return array; + } + + (T[] Buffer, int Offset) ReadChunk(int count, BinaryReader reader) where T : unmanaged + { + var (buffer, offset) = Chunks.Rent(count); + reader.BaseStream.ReadExactly(System.Runtime.InteropServices.MemoryMarshal.AsBytes(buffer.AsSpan(offset, count))); + return (buffer, offset); } void SkipAttribute(BinaryReader reader) @@ -524,85 +605,89 @@ void SkipAttribute(BinaryReader reader) count = reader.ReadInt32(); } - if (type == typeof(Element)) - { - foreach (int i in Enumerable.Range(0, count)) - if (reader.ReadInt32() == -2) reader.BaseStream.Seek(37, SeekOrigin.Current); // skip GUID + null terminator if a stub - return; - } - int length; - - if (type == typeof(TimeSpan)) - length = sizeof(int); - else if (type == typeof(Color)) - length = 4; - else if (type == typeof(bool)) - length = 1; - else if (type == typeof(byte[])) - { - foreach (var i in Enumerable.Range(0, count)) - reader.BaseStream.Seek(reader.ReadInt32(), SeekOrigin.Current); - return; - } - else if (type == typeof(string)) + switch (type) { - if (!StringDict!.Dummy && !isArray && EncodingVersion >= 4) - length = StringDict.IndiceSize; - else - { - foreach (var i in Enumerable.Range(0, count)) + case AttributeType.Element: + for (var i = 0; i < count; i++) + if (reader.ReadInt32() == -2) reader.BaseStream.Seek(37, SeekOrigin.Current); // skip GUID + null terminator if a stub + return; + case AttributeType.Binary: + for (var i = 0; i < count; i++) + reader.BaseStream.Seek(reader.ReadInt32(), SeekOrigin.Current); + return; + case AttributeType.String: + if (!StringDict!.Dummy && !isArray && EncodingVersion >= 4) + { + length = StringDict.IndiceSize; + break; + } + + for (var i = 0; i < count; i++) { byte b; do { b = reader.ReadByte(); } while (b != 0); } return; - } + case AttributeType.Bool: + case AttributeType.Byte: + length = 1; + break; + case AttributeType.Int: + case AttributeType.Float: + case AttributeType.Time: + case AttributeType.Color: + length = 4; + break; + case AttributeType.Vector2: + length = sizeof(float) * 2; + break; + case AttributeType.Vector3: + case AttributeType.QAngle: + length = sizeof(float) * 3; + break; + case AttributeType.Vector4: + case AttributeType.Quaternion: + length = sizeof(float) * 4; + break; + case AttributeType.UInt64: + length = sizeof(ulong); + break; + case AttributeType.Matrix: + length = sizeof(float) * 4 * 4; + break; + default: + throw new CodecException($"Cannot skip an attribute of type {type}."); } - else if (type == typeof(Vector2)) - length = sizeof(float) * 2; - else if (type == typeof(Vector3)) - length = sizeof(float) * 3; - else if (type == typeof(Vector4) || type == typeof(Quaternion)) - length = sizeof(float) * 4; - else if (type == typeof(Matrix4x4)) - length = sizeof(float) * 4 * 4; - else if (type == typeof(QAngle)) - length = sizeof(float) * 3; - else if (type == typeof(int) || type == typeof(float)) - length = 4; - else if (type == typeof(byte)) - length = sizeof(byte); - else if (type == typeof(ulong)) - length = sizeof(ulong); - else - throw new CodecException($"Cannot skip an attribute of type {type.Name}."); reader.BaseStream.Seek(length * count, SeekOrigin.Current); } - readonly struct Encoder + /// + /// Writes a datamodel the way Valve's CDmSerializerBinary does: one pass from the root gathers the strings and fixes the order of the elements, + /// a second pass writes the bodies. Attributes are read in the form their slots hold them, so no value is boxed, and an array of plain values is written in one piece. + /// + sealed class Encoder { - readonly Dictionary ElementIndices; - readonly List ElementOrder; - readonly BinaryWriter Writer; + readonly OutputBuffer Writer; readonly StringDictionary StringDict; readonly Datamodel Datamodel; - readonly SerializationContext Context; - readonly int EncodingVersion; - public Encoder(BinaryWriter writer, Datamodel dm, int version) + /// The bodies in the order their index entries are written: the root, the prefix attributes when the version stores them as an element, then every element in the order it is first reached. + readonly List Order = []; + readonly Dictionary Indices = []; + + /// Type ids of the inline kinds, filled in as they are met, since a version may not support every kind. + readonly byte[] KindIds = new byte[(int)AttributeType.Deferred + 1]; + readonly Dictionary ArrayIds = []; + + public Encoder(OutputBuffer writer, Datamodel dm, int version) { EncodingVersion = version; Writer = writer; Datamodel = dm; - - Context = new SerializationContext(); - StringDict = new StringDictionary(version, writer, dm, Context); - ElementIndices = []; - ElementOrder = []; - + StringDict = new StringDictionary(version); } public void Encode() @@ -614,107 +699,105 @@ public void Encode() WritePrefixAttributes(); } - StringDict.WriteSelf(Writer); - var hasPrefixElement = EncodingVersion >= 9 && Datamodel.PrefixAttributes.Count > 0; - var elementCount = CountChildren(Datamodel.Root, []) + (hasPrefixElement ? 1 : 0); - Writer.Write(elementCount); - var root = Datamodel.Root; if (root != null && !root.Stub) { - WriteIndexEntry(root, root.ClassName, root.Name, root.ID); + Indices[root] = 0; + Order.Add(root); // the prefix attributes are also stored as an unreferenced element right after the root if (hasPrefixElement) - WriteIndexEntry(Datamodel.PrefixAttributes, PrefixElementClass, string.Empty, Datamodel.PrefixElementId); + Order.Add(Datamodel.PrefixAttributes); - WriteIndexChildren(root); - } + Gather(root); - foreach (var body in ElementOrder) - WriteBody(body); - } - - int CountChildren(Element? elem, HashSet counter) - { - if (elem is null) - { - return 0; + if (hasPrefixElement) + { + StringDict.AddString(string.Empty); + StringDict.AddString(PrefixElementClass); + foreach (var attr in Datamodel.PrefixAttributes) + { + StringDict.AddString(attr.Key); + if (attr.Value is string stringValue) + StringDict.AddString(stringValue); + } + } } - if (elem.Stub) return 0; - int num_elems = 1; - counter.Add(elem); - foreach (var attr in Context.Attributes[elem]) - { - if (attr.Value == null) continue; + StringDict.WriteSelf(Writer); + Writer.Write(Order.Count); - if (attr.Value is Element child_elem && !counter.Contains(child_elem)) - { - num_elems += CountChildren(child_elem, counter); - } - else if (attr.Value is IEnumerable child_array) - { - foreach (var array_elem in child_array.Where(c => c != null && !counter.Contains(c))) - num_elems += CountChildren(array_elem, counter); - } + Span id = stackalloc byte[16]; + foreach (var body in Order) + { + var (className, name, elementId) = body is Element elem ? (elem.ClassName, elem.Name, elem.ID) : (PrefixElementClass, string.Empty, Datamodel.PrefixElementId); + StringDict.WriteName(className, Writer); + if (EncodingVersion >= 4) StringDict.WriteString(name, Writer); + else Writer.Write(name); + elementId.TryWriteBytes(id); + Writer.Write(id); } - return num_elems; + foreach (var body in Order) + WriteBody(body); } - void WriteIndex(Element? elem) + /// + /// Adds the strings of an element to the table and reaches the elements it refers to, depth first in attribute order, which is the order of the index. + /// + void Gather(Element elem) { - if (elem is null || elem.Stub || ElementIndices.ContainsKey(elem)) return; + StringDict.AddString(elem.Name); + StringDict.AddName(elem.ClassName); - WriteIndexEntry(elem, elem.ClassName, elem.Name, elem.ID); - WriteIndexChildren(elem); + var visitor = new GatherVisitor(this); + elem.VisitAttributes(ref visitor); } - void WriteIndexEntry(AttributeList body, string className, string name, Guid id) + void Reach(Element? child) { - if (body is Element elem) - ElementIndices[elem] = ElementOrder.Count; - ElementOrder.Add(body); + if (child == null || child.Stub || Indices.ContainsKey(child)) + return; - StringDict.WriteString(className, Writer); - if (EncodingVersion >= 4) StringDict.WriteString(name, Writer); - else Writer.Write(name); - Writer.Write(id.ToByteArray()); + Indices[child] = Order.Count; + Order.Add(child); + Gather(child); } - void WriteIndexChildren(Element elem) + readonly struct GatherVisitor(Encoder encoder) : IAttributeVisitor { - foreach (var attr in Context.Attributes[elem]) + public void Begin(int count) { - var child_elem = attr.Value as Element; - if (child_elem != null) - { - if (!ElementIndices.ContainsKey(child_elem)) - WriteIndex(child_elem); - } - else + } + + public void Visit(string name, AttributeType kind, in InlineValue inline, object? reference) + { + encoder.StringDict.AddName(name); + + switch (reference) { - var elem_list = attr.Value as IList; - if (elem_list != null) - { - var elem_indices = ElementIndices; // workaround for .Net 4 lambda limitation in structs - foreach (var item in elem_list.Where(e => e != null && !elem_indices.ContainsKey(e))) - WriteIndex(item); - } + case string stringValue: + encoder.StringDict.AddString(stringValue); + break; + case Element child: + encoder.Reach(child); + break; + case ElementArray children: + foreach (var child in children.AsSpan()) + encoder.Reach(child); + break; + case IList children: + foreach (var child in children) + encoder.Reach(child); + break; } } } - /// - /// Prefix attributes are stored as a list of prefix elements, each a list of name/typed value pairs. - /// Only the first prefix element is read back, so everything is written into a single one. - /// void WritePrefixAttributes() { var prefixAttributes = Datamodel.PrefixAttributes.Where(attr => attr.Value != null).ToArray(); - if (prefixAttributes.Length == 0) { Writer.Write(0); @@ -723,207 +806,381 @@ void WritePrefixAttributes() Writer.Write(1); Writer.Write(prefixAttributes.Length); - foreach (var attr in prefixAttributes) { Writer.Write(attr.Key); - WriteTypedValue(attr.Value, raw_string: true); + AttributeList.Classify(attr.Value, out var kind, out var inline, out var reference); + WriteValue(kind, in inline, reference, rawStrings: true); } } - void WriteBody(AttributeList elem) + void WriteBody(AttributeList body) { - var attributesIterated = elem is Element element ? Context.Attributes[element] : elem.GetAllAttributesForSerialization().ToArray(); - Writer.Write(attributesIterated.Length); - foreach (var attr in attributesIterated) + var visitor = new WriteVisitor(this); + body.VisitAttributes(ref visitor); + } + + readonly struct WriteVisitor(Encoder encoder) : IAttributeVisitor + { + public void Begin(int count) + { + encoder.Writer.Write(count); + } + + public void Visit(string name, AttributeType kind, in InlineValue inline, object? reference) { - StringDict.WriteString(attr.Key, Writer); - WriteTypedValue(attr.Value, raw_string: false); + encoder.StringDict.WriteName(name, encoder.Writer); + encoder.WriteValue(kind, in inline, reference, rawStrings: false); } } /// - /// Writes the type id of a value followed by the value itself, or by the item count and items for arrays. + /// Writes the type id of a value and the value itself. /// - void WriteTypedValue(object? value, bool raw_string) + /// Whether a string is written in place rather than as an index into the table, as the prefix attributes and array items are. + void WriteValue(AttributeType kind, in InlineValue inline, object? reference, bool rawStrings) { - var attr_type = value == null ? typeof(Element) : value.GetType(); - var attr_type_id = TypeToId(attr_type, EncodingVersion); - Writer.Write(attr_type_id); - - if (value == null || value is byte[] || !Datamodel.IsDatamodelArrayType(attr_type)) + switch (kind) { - WriteAttribute(value, raw_string); - return; + case AttributeType.Element: + Writer.Write(IdOf(kind)); + if (reference is Element elem) + WriteElement(elem); + else + Writer.Write(-1); + return; + case AttributeType.String: + Writer.Write(IdOf(kind)); + WriteString((string)reference!, rawStrings); + return; + case AttributeType.Binary: + Writer.Write(IdOf(kind)); + var binary = (byte[])reference!; + Writer.Write(binary.Length); + Writer.Write(binary); + return; + case AttributeType.Matrix: + Writer.Write(IdOf(kind)); + WriteMatrix((Matrix4x4)reference!); + return; + case AttributeType.Array: + WriteArray((IList)reference!); + return; + case AttributeType.Deferred: + throw new InvalidOperationException("A deferred attribute was not loaded before being written."); + default: + Writer.Write(IdOf(kind)); + WriteInline(kind, in inline); + return; } - - var array = (System.Collections.IList)value; - Writer.Write(array.Count); - foreach (var item in array) - WriteAttribute(item, true); } - /// Whether the value is an array item or a prefix attribute, in which case strings are written inline rather than through the dictionary. - void WriteAttribute(object? value, bool in_array) + /// + /// Writes an array with its type id. The items of an array whose memory layout matches the stream are written in one piece. + /// + void WriteArray(IList array) { - if (value == null) + Writer.Write(IdOf(array.GetType())); + Writer.Write(array.Count); + + switch (array) { - Writer.Write(-1); - return; + case ElementArray elements: + foreach (var elem in elements.AsSpan()) + { + if (elem == null) + Writer.Write(-1); + else + WriteElement(elem); + } + return; + case StringArray strings: + foreach (var stringValue in strings.AsSpan()) + Writer.Write(stringValue); + return; + case BinaryArray binaries: + foreach (var binary in binaries.AsSpan()) + { + if (binary == null) + { + Writer.Write(-1); + continue; + } + + Writer.Write(binary.Length); + Writer.Write(binary); + } + return; + case TimeSpanArray times: + foreach (var time in times.AsSpan()) + Writer.Write(ToTicks(time)); + return; + case IntArray a: WriteItems(a.AsSpan()); return; + case FloatArray a: WriteItems(a.AsSpan()); return; + case BoolArray a: WriteItems(a.AsSpan()); return; + case ColorArray a: WriteItems(a.AsSpan()); return; + case Vector2Array a: WriteItems(a.AsSpan()); return; + case Vector3Array a: WriteItems(a.AsSpan()); return; + case Vector4Array a: WriteItems(a.AsSpan()); return; + case QuaternionArray a: WriteItems(a.AsSpan()); return; + case MatrixArray a: WriteItems(a.AsSpan()); return; + case ByteArray a: WriteItems(a.AsSpan()); return; + case UInt64Array a: WriteItems(a.AsSpan()); return; } - if (value is Element child_elem) + foreach (var item in array) { - if (child_elem.Stub) + AttributeList.Classify(item, out var kind, out var inline, out var reference); + switch (kind) { - Writer.Write(-2); - Writer.Write(child_elem.ID.ToString().ToArray()); // yes, ToString()! - Writer.Write((byte)0); + case AttributeType.Element: + if (reference is Element elem) + WriteElement(elem); + else + Writer.Write(-1); + break; + case AttributeType.String: + Writer.Write((string)reference!); + break; + case AttributeType.Binary: + var binary = (byte[])reference!; + Writer.Write(binary.Length); + Writer.Write(binary); + break; + case AttributeType.Matrix: + WriteMatrix((Matrix4x4)reference!); + break; + case AttributeType.Array: + throw new InvalidOperationException("Unrecognised output Type."); + default: + WriteInline(kind, in inline); + break; } - else - Writer.Write(ElementIndices[child_elem]); - return; - } - - if (value is string string_value) - { - if (EncodingVersion < 4 || in_array) - Writer.Write(string_value); - else - StringDict.WriteString(string_value, Writer); - return; } + } - if (value is bool bool_value) + /// + /// Writes items whose layout in memory is their layout in the stream: the scalars, the vectors and the four by four matrix, all little-endian floats and integers. + /// + void WriteItems(ReadOnlySpan items) where T : unmanaged + { + if (BitConverter.IsLittleEndian) { - Writer.Write(bool_value == true ? (byte)1 : (byte)0); + Writer.Write(MemoryMarshal.AsBytes(items)); return; } - if (value is byte[] binary_value) + foreach (var item in items) { - Writer.Write(binary_value.Length); - Writer.Write(binary_value); - return; + AttributeList.Classify(item, out var kind, out var inline, out var reference); + if (kind == AttributeType.Matrix) + WriteMatrix((Matrix4x4)reference!); + else + WriteInline(kind, in inline); } + } - if (value is TimeSpan time_span) + void WriteInline(AttributeType kind, in InlineValue inline) + { + switch (kind) { - Writer.Write((int)(time_span.Ticks / (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond))); - return; + case AttributeType.Int: Writer.Write(inline.Int); return; + case AttributeType.Float: Writer.Write(inline.Float); return; + case AttributeType.Bool: Writer.Write(inline.Bool ? (byte)1 : (byte)0); return; + case AttributeType.Byte: Writer.Write(inline.Byte); return; + case AttributeType.UInt64: Writer.Write(inline.UInt64); return; + case AttributeType.Time: Writer.Write(ToTicks(TimeSpan.FromTicks(inline.Ticks))); return; + case AttributeType.Color: + Writer.Write(inline.Color.R); + Writer.Write(inline.Color.G); + Writer.Write(inline.Color.B); + Writer.Write(inline.Color.A); + return; + case AttributeType.Vector2: + Writer.Write(inline.Vector2.X); + Writer.Write(inline.Vector2.Y); + return; + case AttributeType.Vector3: + Writer.Write(inline.Vector3.X); + Writer.Write(inline.Vector3.Y); + Writer.Write(inline.Vector3.Z); + return; + case AttributeType.QAngle: + Writer.Write(inline.QAngle.Pitch); + Writer.Write(inline.QAngle.Yaw); + Writer.Write(inline.QAngle.Roll); + return; + case AttributeType.Vector4: + Writer.Write(inline.Vector4.X); + Writer.Write(inline.Vector4.Y); + Writer.Write(inline.Vector4.Z); + Writer.Write(inline.Vector4.W); + return; + case AttributeType.Quaternion: + Writer.Write(inline.Quaternion.X); + Writer.Write(inline.Quaternion.Y); + Writer.Write(inline.Quaternion.Z); + Writer.Write(inline.Quaternion.W); + return; + default: + throw new InvalidOperationException("Unrecognised output Type."); } + } - if (value is Color colour_value) - { - Writer.Write(colour_value.ToBytes()); - return; - } + void WriteMatrix(in Matrix4x4 matrix) + { + Writer.Write(matrix.M11); + Writer.Write(matrix.M12); + Writer.Write(matrix.M13); + Writer.Write(matrix.M14); + Writer.Write(matrix.M21); + Writer.Write(matrix.M22); + Writer.Write(matrix.M23); + Writer.Write(matrix.M24); + Writer.Write(matrix.M31); + Writer.Write(matrix.M32); + Writer.Write(matrix.M33); + Writer.Write(matrix.M34); + Writer.Write(matrix.M41); + Writer.Write(matrix.M42); + Writer.Write(matrix.M43); + Writer.Write(matrix.M44); + } - if (value is Vector2 vector2) - { - Writer.Write(vector2.X); - Writer.Write(vector2.Y); - return; - } - if (value is Vector3 vector3) - { - Writer.Write(vector3.X); - Writer.Write(vector3.Y); - Writer.Write(vector3.Z); - return; - } - if (value is QAngle qangle) - { - Writer.Write(qangle.Pitch); - Writer.Write(qangle.Yaw); - Writer.Write(qangle.Roll); - return; - } - if (value is Vector4 vector4) + void WriteElement(Element elem) + { + if (elem.Stub) { - Writer.Write(vector4.X); - Writer.Write(vector4.Y); - Writer.Write(vector4.Z); - Writer.Write(vector4.W); - return; + Writer.Write(-2); + Writer.Write(elem.ID.ToString()); // yes, ToString()! } - if (value is Quaternion quaternion) - { - Writer.Write(quaternion.X); - Writer.Write(quaternion.Y); - Writer.Write(quaternion.Z); - Writer.Write(quaternion.W); - return; - } - if (value is Matrix4x4 matrix) + else { - Writer.Write(matrix.M11); - Writer.Write(matrix.M12); - Writer.Write(matrix.M13); - Writer.Write(matrix.M14); - Writer.Write(matrix.M21); - Writer.Write(matrix.M22); - Writer.Write(matrix.M23); - Writer.Write(matrix.M24); - Writer.Write(matrix.M31); - Writer.Write(matrix.M32); - Writer.Write(matrix.M33); - Writer.Write(matrix.M34); - Writer.Write(matrix.M41); - Writer.Write(matrix.M42); - Writer.Write(matrix.M43); - Writer.Write(matrix.M44); - return; + Writer.Write(Indices[elem]); } + } - if (value is int intValue) - { - Writer.Write(intValue); - return; - } - if (value is float floatValue) - { - Writer.Write(floatValue); - return; - } + void WriteString(string value, bool raw) + { + if (EncodingVersion < 4 || raw) + Writer.Write(value); + else + StringDict.WriteString(value, Writer); + } + + static int ToTicks(TimeSpan time) => (int)(time.Ticks / (TimeSpan.TicksPerSecond / DatamodelTicksPerSecond)); + + byte IdOf(AttributeType kind) + { + ref var id = ref KindIds[(int)kind]; + if (id == 0) + id = TypeToId(kind, EncodingVersion); + return id; + } + + byte IdOf(Type arrayType) + { + if (!ArrayIds.TryGetValue(arrayType, out var id)) + ArrayIds[arrayType] = id = TypeToId(arrayType, EncodingVersion); + return id; + } - if (value is byte byteValue) + } + + /// + /// Collects the output and hands it to the stream in large pieces, the way Valve's CUtlBuffer does, so that writing a value is a few stores rather than a call into the stream. + /// + sealed class OutputBuffer(Stream stream) + { + readonly byte[] buffer = new byte[1 << 16]; + int used; + + public void Flush() + { + if (used > 0) { - Writer.Write(byteValue); - return; + stream.Write(buffer, 0, used); + used = 0; } + } + + void Reserve(int size) + { + if (buffer.Length - used < size) + Flush(); + } + + public void Write(byte value) + { + Reserve(1); + buffer[used++] = value; + } + + public void Write(short value) + { + Reserve(2); + BinaryPrimitives.WriteInt16LittleEndian(buffer.AsSpan(used), value); + used += 2; + } + + public void Write(int value) + { + Reserve(4); + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(used), value); + used += 4; + } + + public void Write(float value) + { + Reserve(4); + BinaryPrimitives.WriteSingleLittleEndian(buffer.AsSpan(used), value); + used += 4; + } + + public void Write(ulong value) + { + Reserve(8); + BinaryPrimitives.WriteUInt64LittleEndian(buffer.AsSpan(used), value); + used += 8; + } - if (value is ulong ulongValue) + public void Write(ReadOnlySpan bytes) + { + if (bytes.Length > buffer.Length - used) { - Writer.Write(ulongValue); - return; + Flush(); + if (bytes.Length > buffer.Length) + { + stream.Write(bytes); + return; + } } - throw new InvalidOperationException("Unrecognised output Type."); + bytes.CopyTo(buffer.AsSpan(used)); + used += bytes.Length; } - } - - class DmxBinaryWriter : BinaryWriter - { - public DmxBinaryWriter(Stream output) - : base(output, Datamodel.TextEncoding) - { } /// - /// Writes a null-terminated string to the underlying stream using . + /// Writes a string in followed by a zero byte. /// - /// - [System.Security.SecuritySafeCritical] - public override void Write(string value) + public void Write(string? value) { if (value != null) - base.Write(Datamodel.TextEncoding.GetBytes(value)); - base.Write((byte)0); - } + { + var encoding = Datamodel.TextEncoding; + var room = encoding.GetMaxByteCount(value.Length); + if (room > buffer.Length) + { + Write(encoding.GetBytes(value)); + } + else + { + Reserve(room); + used += encoding.GetBytes(value, buffer.AsSpan(used)); + } + } - protected override void Dispose(bool disposing) - { - return; // don't mess with the base stream! + Write((byte)0); } } } diff --git a/Datamodel.NET/Datamodel.ElementList.cs b/Datamodel.NET/Datamodel.ElementList.cs index ee612e2..e63a88b 100644 --- a/Datamodel.NET/Datamodel.ElementList.cs +++ b/Datamodel.NET/Datamodel.ElementList.cs @@ -4,7 +4,6 @@ using System.Collections.Specialized; using System.Diagnostics; using System.Linq; -using System.Threading; namespace Datamodel { @@ -17,7 +16,7 @@ public partial class Datamodel [DebuggerTypeProxy(typeof(DebugView))] public class ElementList : IEnumerable, INotifyCollectionChanged, IDisposable { - internal ReaderWriterLockSlim ChangeLock = new(LockRecursionPolicy.SupportsRecursion); + internal readonly object ChangeLock = new(); internal class DebugView { @@ -29,10 +28,12 @@ public DebugView(ElementList item) private readonly ElementList Item; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - public Element[] Elements => Item.store.Values.Cast().ToArray(); + public Element[] Elements => [.. Item.order]; } - private readonly OrderedDictionary store = []; + // elements by ID, and in the order they were added, which codecs rely on + private readonly Dictionary byId = []; + private readonly List order = []; private readonly Datamodel Owner; internal ElementList(Datamodel owner) @@ -40,13 +41,26 @@ internal ElementList(Datamodel owner) Owner = owner; } + /// Makes room for the given number of elements, so that a codec that knows the count adds them without growing the tables. + internal void EnsureCapacity(int count) + { + lock (ChangeLock) + { + byId.EnsureCapacity(count); + order.EnsureCapacity(count); + } + } + + /// + /// Adds an Element owned by this list's Datamodel. The first Element added becomes the . + /// internal void Add(Element item) { - ChangeLock.EnterUpgradeableReadLock(); - try + bool first; + + lock (ChangeLock) { - Element? existing = (Element?)store[item.ID]; - if (existing != null && !existing.Stub) + if (byId.TryGetValue(item.ID, out var existing) && !existing.Stub) { throw new ElementIdException($"Element ID {item.ID} already in use in this Datamodel."); } @@ -55,27 +69,29 @@ internal void Add(Element item) if (item.Owner != this.Owner) throw new ElementOwnershipException("Cannot add an element from a different Datamodel. Use ImportElement() to create a local copy instead."); - ChangeLock.EnterWriteLock(); - try - { - if (existing != null) - store.Remove(existing.ID); + if (existing != null) + RemoveFromStore(existing); - store.Add(item.ID, item); - } - finally - { - ChangeLock.ExitWriteLock(); - } - } - finally - { - ChangeLock.ExitUpgradeableReadLock(); + byId.Add(item.ID, item); + order.Add(item); + first = order.Count == 1; } + if (first) + Owner.Root = item; + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item)); } + /// + /// Removes an Element from both stores. The caller holds . + /// + void RemoveFromStore(Element item) + { + byId.Remove(item.ID); + order.Remove(item); + } + /// /// Returns the at the specified index. /// @@ -86,12 +102,8 @@ public Element? this[int index] { get { - ChangeLock.EnterReadLock(); - try - { - return (Element?)store[index]; - } - finally { ChangeLock.ExitReadLock(); } + lock (ChangeLock) + return order[index]; } } @@ -104,12 +116,8 @@ public Element? this[Guid id] { get { - ChangeLock.EnterReadLock(); - try - { - return (Element?)store[id]; - } - finally { ChangeLock.ExitReadLock(); } + lock (ChangeLock) + return byId.TryGetValue(id, out var element) ? element : null; } } @@ -120,12 +128,8 @@ public int Count { get { - ChangeLock.EnterReadLock(); - try - { - return store.Count; - } - finally { ChangeLock.ExitReadLock(); } + lock (ChangeLock) + return order.Count; } } @@ -160,47 +164,36 @@ public bool Remove(Element item, RemoveMode mode) { ArgumentNullException.ThrowIfNull(item); - ChangeLock.EnterUpgradeableReadLock(); - try + lock (ChangeLock) { - if (store.Contains(item.ID)) + if (!byId.ContainsKey(item.ID)) + return false; + + RemoveFromStore(item); + Element? replacement = (mode == RemoveMode.MakeStubs) ? new Element(Owner, item.ID) : null; + + foreach (var elem in order) { - ChangeLock.EnterWriteLock(); - try + lock (elem.SyncRoot) { - store.Remove(item.ID); - Element? replacement = (mode == RemoveMode.MakeStubs) ? new Element(Owner, item.ID) : null; - - foreach (Element elem in store.Values) + foreach (var attr in elem.Where(a => a.Value == item).ToArray()) { - lock (elem.SyncRoot) - { - foreach (var attr in elem.Where(a => a.Value == item).ToArray()) - { - elem[attr.Key] = replacement; - } - - foreach (var array in elem.Select(a => a.Value).OfType>()) - for (int i = 0; i < array.Count; i++) - if (array[i] == item) - array[i] = replacement; - } + elem[attr.Key] = replacement; } - if (Owner.Root == item) Owner.Root = replacement; - item.Owner = null; - } - finally - { - ChangeLock.ExitWriteLock(); + foreach (var array in elem.Select(a => a.Value).OfType>()) + for (int i = 0; i < array.Count; i++) + if (array[i] == item) + array[i] = replacement; } - - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item)); - return true; } - else return false; + if (Owner.Root == item) Owner.Root = replacement; + + item.Owner = null; } - finally { ChangeLock.ExitUpgradeableReadLock(); } + + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item)); + return true; } /// @@ -208,21 +201,16 @@ public bool Remove(Element item, RemoveMode mode) /// internal void RemoveUnreferenced(Element item) { - ChangeLock.EnterWriteLock(); - try + lock (ChangeLock) { - if (!store.Contains(item.ID)) + if (!byId.ContainsKey(item.ID)) { return; } - store.Remove(item.ID); + RemoveFromStore(item); item.Owner = null; } - finally - { - ChangeLock.ExitWriteLock(); - } CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item)); } @@ -232,36 +220,20 @@ internal void RemoveUnreferenced(Element item) /// public void Trim() { - ChangeLock.EnterUpgradeableReadLock(); - try + lock (ChangeLock) { var used = new HashSet(); WalkElemTree(Owner.Root, used); - if (used.Count == Count) return; - - ChangeLock.EnterWriteLock(); - try - { - var removed = this.Except(used).ToArray(); - foreach (var elem in removed) - { - if (elem != null) - { - store.Remove(elem.ID); - elem.Owner = null; - } - } + if (used.Count == order.Count) return; - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed)); - } - finally + var removed = order.Where(elem => !used.Contains(elem)).ToArray(); + foreach (var elem in removed) { - ChangeLock.ExitWriteLock(); + RemoveFromStore(elem); + elem.Owner = null; } - } - finally - { - ChangeLock.ExitUpgradeableReadLock(); + + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, removed)); } } @@ -273,7 +245,7 @@ protected void WalkElemTree(Element? elem, HashSet found) } found.Add(elem); - foreach (var value in elem.Inner.Values.Cast().Select(a => a.RawValue)) + foreach (var value in elem.EnumerateReferences()) { if (value is Element value_elem) { @@ -286,7 +258,7 @@ protected void WalkElemTree(Element? elem, HashSet found) } if (value is ElementArray elem_array) { - foreach (var item in elem_array.RawList) + foreach (var item in elem_array.RawItems) { if (item != null && found.Add(item)) { @@ -300,15 +272,14 @@ protected void WalkElemTree(Element? elem, HashSet found) #region Interfaces System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { - return store.GetEnumerator(); + return GetEnumerator(); } /// - /// Returns an Enumerator that iterates through the Elements in collection. + /// Returns an Enumerator that iterates through the Elements in collection, in the order they were added. /// public IEnumerator GetEnumerator() { - foreach (Element elem in store.Values) - yield return elem; + return order.GetEnumerator(); } /// /// Raised when an is added, removed, or replaced. @@ -318,7 +289,6 @@ public IEnumerator GetEnumerator() public void Dispose() { - ChangeLock.Dispose(); } } } diff --git a/Datamodel.NET/Datamodel.NET.csproj b/Datamodel.NET/Datamodel.NET.csproj index efec38c..5616644 100644 --- a/Datamodel.NET/Datamodel.NET.csproj +++ b/Datamodel.NET/Datamodel.NET.csproj @@ -4,7 +4,7 @@ Library Datamodel KeyValues2 - 1.0 + 2.0-beta enable MIT README.md diff --git a/Datamodel.NET/Datamodel.cs b/Datamodel.NET/Datamodel.cs index 46d96d8..145d7e2 100644 --- a/Datamodel.NET/Datamodel.cs +++ b/Datamodel.NET/Datamodel.cs @@ -968,11 +968,11 @@ protected CodecException(SerializationInfo info, StreamingContext context) [Serializable] public class DestubException : Exception { - internal DestubException(Attribute attr, Exception innerException) + internal DestubException(AttributeList owner, string attributeName, Exception innerException) : base("An exception occured while destubbing the value of an attribute.", innerException) { - Data.Add("Element", ((Element?)attr.Owner)?.ID); - Data.Add("Attribute", attr.Name); + Data.Add("Element", (owner as Element)?.ID); + Data.Add("Attribute", attributeName); } internal DestubException(ElementArray array, int index, Exception innerException) diff --git a/Datamodel.NET/Element.cs b/Datamodel.NET/Element.cs index c0bd41a..8a61ebe 100644 --- a/Datamodel.NET/Element.cs +++ b/Datamodel.NET/Element.cs @@ -67,7 +67,7 @@ public Element(Datamodel owner, Guid id) public Element() : base(null) { - ID = Guid.NewGuid(); + // the ID is generated on first use, so that a codec constructing the Element and assigning the ID from the file does not pay for a random one // For subclasses get the actual classname if (GetType() != typeof(Element)) @@ -95,7 +95,26 @@ public Element() /// . Assign it before the Element joins a or any other /// hash-based collection, because changing it afterwards strands the Element in its old bucket. /// - public Guid ID { get; set; } + public Guid ID + { + get + { + if (!idAssigned) + { + id = Guid.NewGuid(); + idAssigned = true; + } + + return id; + } + set + { + id = value; + idAssigned = true; + } + } + Guid id; + bool idAssigned; /// /// Gets or sets the name of this Element. @@ -148,16 +167,7 @@ internal set { if (value != null && base.Owner != null && base.Owner.AllElements.Contains(this)) throw new InvalidOperationException("Element already has an owner."); base.Owner = value; - if (value != null) - { - value.AllElements.ChangeLock.EnterWriteLock(); - try - { - value.AllElements.Add(this); - if (value.AllElements.Count == 1) value.Root = this; - } - finally { value.AllElements.ChangeLock.ExitWriteLock(); } - } + value?.AllElements.Add(this); } } @@ -342,8 +352,7 @@ int IEqualityComparer.GetHashCode(object obj) /// The location of the attribute's value in the Datamodel's source stream. internal void Add(string key, long offset) { - lock (Attribute_ChangeLock) - Inner[key] = new Attribute(key, this, offset); + SetDeferred(key, offset); } public override bool ContainsKey(string key) diff --git a/Datamodel.NET/ElementSchema.cs b/Datamodel.NET/ElementSchema.cs index 8fe2b1d..d7904be 100644 --- a/Datamodel.NET/ElementSchema.cs +++ b/Datamodel.NET/ElementSchema.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace Datamodel { @@ -11,9 +12,9 @@ namespace Datamodel /// Instances are emitted by the ElementFactory that the KeyValues2.ElementFactoryGenerator generates into the assembly declaring the class, /// so no reflection is needed to move values between properties and attributes. /// - public sealed class PropertyBinding + public class PropertyBinding { - readonly Func getter; + readonly Func? getter; readonly Action? setter; /// The name of the property in the class. @@ -22,21 +23,28 @@ public sealed class PropertyBinding /// Reads the property of the given element. /// Writes the property of the given element, or null when the property has no setter. public PropertyBinding(string propertyName, string attributeName, Type propertyType, Func getter, Action? setter) + : this(propertyName, attributeName, propertyType, setter != null) + { + ArgumentNullException.ThrowIfNull(getter); + + this.getter = getter; + this.setter = setter; + } + + private protected PropertyBinding(string propertyName, string attributeName, Type propertyType, bool canWrite) { ArgumentNullException.ThrowIfNull(propertyName); ArgumentNullException.ThrowIfNull(attributeName); ArgumentNullException.ThrowIfNull(propertyType); - ArgumentNullException.ThrowIfNull(getter); PropertyName = propertyName; AttributeName = attributeName; PropertyType = propertyType; - this.getter = getter; - this.setter = setter; + CanWrite = canWrite; } /// - /// Creates a binding from typed accessors, so that generated code needs no casts. + /// Creates a binding from typed accessors, so that generated code needs no casts and values move without boxing. /// /// The class declaring the property. /// The type of the property. @@ -44,17 +52,12 @@ public PropertyBinding(string propertyName, string attributeName, Type propertyT /// The name of the attribute in the file. /// Reads the property. /// Writes the property, or null when it has no setter. - public static PropertyBinding Create(string propertyName, string attributeName, Func getter, Action? setter) + public static PropertyBinding Create(string propertyName, string attributeName, Func getter, Action? setter) where TElement : AttributeList { ArgumentNullException.ThrowIfNull(getter); - return new PropertyBinding( - propertyName, - attributeName, - typeof(TValue), - element => getter((TElement)element), - setter == null ? null : (element, value) => setter((TElement)element, (TValue)value!)); + return new PropertyBinding(propertyName, attributeName, getter, setter); } /// @@ -75,18 +78,18 @@ public static PropertyBinding Create(string propertyName, stri /// /// Gets whether the property can be assigned. /// - public bool CanWrite => setter != null; + public bool CanWrite { get; } /// /// Reads the property of the given element. /// - public object? GetValue(AttributeList owner) => getter(owner); + public virtual object? GetValue(AttributeList owner) => getter!(owner); /// /// Writes the property of the given element. /// /// Thrown when the property has no setter. - public void SetValue(AttributeList owner, object? value) + public virtual void SetValue(AttributeList owner, object? value) { if (setter == null) { @@ -96,9 +99,91 @@ public void SetValue(AttributeList owner, object? value) setter(owner, value); } + /// + /// Reads the property in the form an attribute slot stores it, so that a codec writes it without boxing when the binding is typed. + /// + internal virtual void Read(AttributeList owner, out AttributeType kind, out InlineValue inline, out object? reference) + { + AttributeList.Classify(GetValue(owner), out kind, out inline, out reference); + } + public override string ToString() => $"{PropertyName} <{PropertyType.Name}> as \"{AttributeName}\""; } + /// + /// A whose value type is known, so that codecs and the attribute indexer move values without boxing them. + /// + /// The type of the property. + public abstract class PropertyBinding : PropertyBinding + { + private protected PropertyBinding(string propertyName, string attributeName, bool canWrite) + : base(propertyName, attributeName, typeof(TValue), canWrite) + { + } + + /// + /// Reads the property of the given element. + /// + public abstract TValue Get(AttributeList owner); + + /// + /// Writes the property of the given element. + /// + /// Thrown when the property has no setter. + public abstract void Set(AttributeList owner, TValue value); + + public override object? GetValue(AttributeList owner) => Get(owner); + + public override void SetValue(AttributeList owner, object? value) => Set(owner, (TValue)value!); + } + + /// + /// The binding makes: the generated accessors of one property, called without any cast of the value. + /// + sealed class PropertyBinding : PropertyBinding + where TElement : AttributeList + { + readonly Func getter; + readonly Action? setter; + + public PropertyBinding(string propertyName, string attributeName, Func getter, Action? setter) + : base(propertyName, attributeName, setter != null) + { + this.getter = getter; + this.setter = setter; + } + + public override TValue Get(AttributeList owner) => getter((TElement)owner); + + /// The type a slot stores values of as when they are stored inline, decided once per type; null for the reference types, which are classified per value. + static readonly AttributeType? ValueKind = AttributeList.KindOf(typeof(TValue)); + + internal override void Read(AttributeList owner, out AttributeType kind, out InlineValue inline, out object? reference) + { + var value = getter((TElement)owner); + if (ValueKind is AttributeType inlineKind) + { + kind = inlineKind; + reference = null; + inline = default; + Unsafe.As(ref inline) = value; + return; + } + + AttributeList.Classify(value, out kind, out inline, out reference); + } + + public override void Set(AttributeList owner, TValue value) + { + if (setter == null) + { + throw new InvalidOperationException($"Property '{PropertyName}' is read-only."); + } + + setter((TElement)owner, value); + } + } + /// /// Describes how an subclass maps onto a file: its class name and the properties that are stored as attributes. /// diff --git a/Tests/Tests.cs b/Tests/Tests.cs index d4e8db7..5253305 100644 --- a/Tests/Tests.cs +++ b/Tests/Tests.cs @@ -182,7 +182,7 @@ protected static async Task Populate(Datamodel.Datamodel dm, string encoding_nam var name = value.GetType().Name; dm.Root[name] = value; - await Assert.That(dm.Root[name]).IsSameReferenceAs(value); + await Assert.That(dm.Root[name]).IsEqualTo(value); // value types are stored inline, so the box read back is a new one name += " array"; var list = value.GetType().MakeListType().GetConstructor(Type.EmptyTypes).Invoke(null) as IList;