From c555ef1fc13cd52022f27270730a195212869bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=82ngelo=20Tadeucci?= Date: Sun, 25 Oct 2026 15:24:05 -0300 Subject: [PATCH 1/2] Fix CRLF asset metadata, shared flat index, and culture-dependent float parsing AssetIndex.ParseNtFile now trims a trailing carriage return. Every .nt file in a repacked asset-web-metadata.m2d ends lines with CRLF, the regex rejected all of them, and test setup in TestUtils failed for 213 of 255 tests. RuntimeClassLookup patches the shared FlatTypeIndex with TryAdd. A second XBlockParser over the same index threw a duplicate key exception on ProxyNifAsset. Float parsing and formatting no longer depend on the current culture. The generated Vector3 and array accessors, FlatProperty, AiParser and EntityModel use the invariant culture. M2dReader no longer sets the caller's CurrentCulture to en-US, which was the only thing keeping pt-BR from reading "1.5" as 15. Adds data-free tests for the CRLF parse and the three culture paths, and a data test that builds two XBlockParsers over one index. Bumps the package to 2.4.30. Co-Authored-By: Claude Opus 5.5 --- Maple2.File.Generator/XmlArrayGenerator.cs | 11 ++- Maple2.File.Generator/XmlVector3Generator.cs | 8 +-- Maple2.File.IO/M2dReader.cs | 6 +- Maple2.File.Parser/AiParser.cs | 5 +- Maple2.File.Parser/Flat/Convert/AssetIndex.cs | 6 +- .../Flat/Convert/EntityModel.cs | 3 +- Maple2.File.Parser/Flat/FlatProperty.cs | 9 +-- .../MapXBlock/RuntimeClassLookup.cs | 22 +++--- Maple2.File.Parser/Maple2.File.Parser.csproj | 6 +- Maple2.File.Tests/DataFreeParsingTest.cs | 69 +++++++++++++++++++ Maple2.File.Tests/XBlockParserTest.cs | 11 +++ 11 files changed, 126 insertions(+), 30 deletions(-) create mode 100644 Maple2.File.Tests/DataFreeParsingTest.cs diff --git a/Maple2.File.Generator/XmlArrayGenerator.cs b/Maple2.File.Generator/XmlArrayGenerator.cs index 0fc2865e..4d24f188 100644 --- a/Maple2.File.Generator/XmlArrayGenerator.cs +++ b/Maple2.File.Generator/XmlArrayGenerator.cs @@ -66,9 +66,18 @@ public string _{xmlAttributeName} {{ private void AddSerializer(StringBuilder source, IFieldSymbol field, char delimiter) { string fieldName = $"this.{field.FieldName()}"; + var arrayType = field.Type as IArrayTypeSymbol; + if (arrayType != null && IsFloatingPoint(arrayType.ElementType)) { + source.Append(@$" return {fieldName} != null ? string.Join('{delimiter}', global::System.Linq.Enumerable.Select({fieldName}, v => v.ToString(global::System.Globalization.CultureInfo.InvariantCulture))) : string.Empty;"); + return; + } source.Append(@$" return {fieldName} != null ? string.Join('{delimiter}', {fieldName}) : string.Empty;"); } + private static bool IsFloatingPoint(ITypeSymbol type) { + return type.SpecialType is SpecialType.System_Single or SpecialType.System_Double or SpecialType.System_Decimal; + } + private void AddDeserializer(GeneratorExecutionContext context, StringBuilder source, IFieldSymbol field, char delimiter, bool keepEmpty) { var arrayType = field.Type as IArrayTypeSymbol; @@ -101,7 +110,7 @@ private void AddDeserializer(GeneratorExecutionContext context, StringBuilder so if (arrayType.ElementType.ToString() == "bool") { source.AppendLine($@"{fieldName}[i] = bool.TryParse(val, out bool result) ? result : (val != ""0"");"); } else { - source.AppendLine($"{fieldName}[i] = {arrayType.ElementType}.Parse(val);"); + source.AppendLine($"{fieldName}[i] = {arrayType.ElementType}.Parse(val, global::System.Globalization.CultureInfo.InvariantCulture);"); } } else { source.AppendLine($"{fieldName}[i] = val;"); diff --git a/Maple2.File.Generator/XmlVector3Generator.cs b/Maple2.File.Generator/XmlVector3Generator.cs index e94cd28b..7a5e3472 100644 --- a/Maple2.File.Generator/XmlVector3Generator.cs +++ b/Maple2.File.Generator/XmlVector3Generator.cs @@ -50,12 +50,12 @@ private string ProcessField(GeneratorExecutionContext context, IFieldSymbol fiel source.Append($@" [XmlAttribute(""{xmlAttributeName}"")] public string _{xmlAttributeName} {{ - get => $""{{{fieldName}.X}},{{{fieldName}.Y}},{{{fieldName}.Z}}""; + get => global::System.FormattableString.Invariant($""{{{fieldName}.X}},{{{fieldName}.Y}},{{{fieldName}.Z}}""); set {{ string[] split = value.Split(new[] {{',', ' '}}, StringSplitOptions.RemoveEmptyEntries); - float x = split.Length > 0 ? float.Parse(split[0]) : 0; - float y = split.Length > 1 ? float.Parse(split[1]) : 0; - float z = split.Length > 2 ? float.Parse(split[2]) : 0; + float x = split.Length > 0 ? float.Parse(split[0], global::System.Globalization.CultureInfo.InvariantCulture) : 0; + float y = split.Length > 1 ? float.Parse(split[1], global::System.Globalization.CultureInfo.InvariantCulture) : 0; + float z = split.Length > 2 ? float.Parse(split[2], global::System.Globalization.CultureInfo.InvariantCulture) : 0; {fieldName} = new System.Numerics.Vector3(x, y, z); }} }}"); diff --git a/Maple2.File.IO/M2dReader.cs b/Maple2.File.IO/M2dReader.cs index 41eb5a50..4988dee0 100644 --- a/Maple2.File.IO/M2dReader.cs +++ b/Maple2.File.IO/M2dReader.cs @@ -1,5 +1,4 @@ -using System.Globalization; -using System.Text; +using System.Text; using System.Xml; using Maple2.File.IO.Crypto; using Maple2.File.IO.Crypto.Common; @@ -12,9 +11,6 @@ public class M2dReader : IDisposable { public readonly IReadOnlyList Files; public M2dReader(string path) { - // Force Globalization to en-US because we use periods instead of commas for decimals - CultureInfo.CurrentCulture = new("en-US"); - m2dFile = System.IO.File.OpenRead(path); // Create an index from the header file diff --git a/Maple2.File.Parser/AiParser.cs b/Maple2.File.Parser/AiParser.cs index cee615d3..ec5a8443 100644 --- a/Maple2.File.Parser/AiParser.cs +++ b/Maple2.File.Parser/AiParser.cs @@ -1,5 +1,6 @@ using Maple2.File.Parser.Xml.AI; using System.Diagnostics; +using System.Globalization; using System.Numerics; using System.Reflection; using Maple2.File.IO; @@ -211,14 +212,14 @@ private static T ParseAttributes(XmlNode xmlNode) where T : Entry { field.SetValue(result, outLong); break; case "Single": - if (!float.TryParse(attrib.Value, out float outFloat)) { + if (!float.TryParse(attrib.Value, NumberStyles.Float, CultureInfo.InvariantCulture, out float outFloat)) { throw new InvalidDataException($"invalid float '{attrib.Value}' for {typeof(T).Name} member {field.Name}"); } field.SetValue(result, outFloat); break; case "Vector3": - float[] floatValues = Array.ConvertAll(attrib.Value.Split(",", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries), float.Parse); + float[] floatValues = Array.ConvertAll(attrib.Value.Split(",", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries), value => float.Parse(value, CultureInfo.InvariantCulture)); if (floatValues.Length != 3) { throw new InvalidDataException($"invalid Vector3 '{attrib.Value}' for {typeof(T).Name} member {field.Name}"); diff --git a/Maple2.File.Parser/Flat/Convert/AssetIndex.cs b/Maple2.File.Parser/Flat/Convert/AssetIndex.cs index 9a3584c3..75fc4c50 100644 --- a/Maple2.File.Parser/Flat/Convert/AssetIndex.cs +++ b/Maple2.File.Parser/Flat/Convert/AssetIndex.cs @@ -157,9 +157,11 @@ public static AssetIndex Deserialize(BinaryReader reader) { return (name, path, string.Join(':', tags)); } - private static Dictionary ParseNtFile(string data) { + internal static Dictionary ParseNtFile(string data) { var result = new Dictionary(); - foreach (string line in data.Split("\n")) { + foreach (string rawLine in data.Split("\n")) { + // Repacked archives can carry CRLF line endings, which the trailing ".$" would reject. + string line = rawLine.TrimEnd('\r'); if (string.IsNullOrWhiteSpace(line)) { continue; } diff --git a/Maple2.File.Parser/Flat/Convert/EntityModel.cs b/Maple2.File.Parser/Flat/Convert/EntityModel.cs index a52da9e9..e185bce4 100644 --- a/Maple2.File.Parser/Flat/Convert/EntityModel.cs +++ b/Maple2.File.Parser/Flat/Convert/EntityModel.cs @@ -1,4 +1,5 @@ using System.Drawing; +using System.Globalization; using System.Numerics; using System.Xml; using System.Xml.Schema; @@ -177,7 +178,7 @@ private Action GetTypeWriter(string type, object value) { case long: case float: case double: - writer.WriteAttributeString("Value", value.ToString()); + writer.WriteAttributeString("Value", System.Convert.ToString(value, CultureInfo.InvariantCulture)); break; case Vector3 point3: writer.WriteStartElement("X"); diff --git a/Maple2.File.Parser/Flat/FlatProperty.cs b/Maple2.File.Parser/Flat/FlatProperty.cs index 2c6300fa..a6ce079b 100644 --- a/Maple2.File.Parser/Flat/FlatProperty.cs +++ b/Maple2.File.Parser/Flat/FlatProperty.cs @@ -1,5 +1,6 @@ using System.Collections; using System.Drawing; +using System.Globalization; using System.Numerics; using System.Text.RegularExpressions; @@ -25,9 +26,9 @@ public static object ParseType(string type, string value) { case "SInt32": return int.Parse(value); case "Float32": - return float.Parse(value); + return float.Parse(value, CultureInfo.InvariantCulture); case "Float64": - return double.Parse(value); + return double.Parse(value, CultureInfo.InvariantCulture); case "Point3": float[] point3 = FloatList(value); return new Vector3(point3[0], point3[1], point3[2]); @@ -91,7 +92,7 @@ private static float[] FloatList(string value) { string[] split = value.Split(", "); float[] result = new float[split.Length]; for (int i = 0; i < split.Length; i++) { - result[i] = float.Parse(split[i]); + result[i] = float.Parse(split[i], CultureInfo.InvariantCulture); } return result; @@ -108,7 +109,7 @@ public string ValueString() { } public string ValueCodeString() { - string value = Value.ToString(); + string value = System.Convert.ToString(Value, CultureInfo.InvariantCulture); if (Value is float) { value = Regex.Replace(value, "(\\d+\\.\\d+)", "$1f"); } diff --git a/Maple2.File.Parser/MapXBlock/RuntimeClassLookup.cs b/Maple2.File.Parser/MapXBlock/RuntimeClassLookup.cs index 88ff6074..3f89bb5d 100644 --- a/Maple2.File.Parser/MapXBlock/RuntimeClassLookup.cs +++ b/Maple2.File.Parser/MapXBlock/RuntimeClassLookup.cs @@ -35,19 +35,21 @@ public RuntimeClassLookup(FlatTypeIndex index) : base(index) { var npcCount = new FlatProperty { Name = "NpcCount", Type = "UInt32", Value = (uint) 0 }; var reactableSequenceName = new FlatProperty { Name = "reactableSequenceName", Type = "String", Value = "" }; + // TryAdd because the index is shared: a second lookup over the same index, or an index + // serialized after a lookup patched it, already carries these properties. FlatType ms2CubeProp = index.GetType("MS2CubeProp"); - ms2CubeProp.Properties.Add(proxyNifAsset.Name, proxyNifAsset); - ms2CubeProp.Properties.Add(interval.Name, interval); - ms2CubeProp.Properties.Add(friendly.Name, friendly); + ms2CubeProp.Properties.TryAdd(proxyNifAsset.Name, proxyNifAsset); + ms2CubeProp.Properties.TryAdd(interval.Name, interval); + ms2CubeProp.Properties.TryAdd(friendly.Name, friendly); FlatType eventSpawnPointItem = index.GetType("EventSpawnPointItem"); - eventSpawnPointItem.Properties.Add(dropID.Name, dropID); + eventSpawnPointItem.Properties.TryAdd(dropID.Name, dropID); FlatType ms2Actor = index.GetType("MS2Actor"); - ms2Actor.Properties.Add(npcList.Name, npcList); - ms2Actor.Properties.Add(spawnPointID.Name, spawnPointID); - ms2Actor.Properties.Add(proxyNifAsset.Name, proxyNifAsset); - ms2Actor.Properties.Add(spawnRadius.Name, spawnRadius); - ms2Actor.Properties.Add(npcCount.Name, npcCount); - ms2Actor.Properties.Add(reactableSequenceName.Name, reactableSequenceName); + ms2Actor.Properties.TryAdd(npcList.Name, npcList); + ms2Actor.Properties.TryAdd(spawnPointID.Name, spawnPointID); + ms2Actor.Properties.TryAdd(proxyNifAsset.Name, proxyNifAsset); + ms2Actor.Properties.TryAdd(spawnRadius.Name, spawnRadius); + ms2Actor.Properties.TryAdd(npcCount.Name, npcCount); + ms2Actor.Properties.TryAdd(reactableSequenceName.Name, reactableSequenceName); // Some MS2InteractActor entities double as NPC spawn points (e.g. // 11000119_MS2InteractActor_Moonrabbit on 80000022_bonus) and carry the SpawnPointNPC // property set, which the flat definition does not declare. diff --git a/Maple2.File.Parser/Maple2.File.Parser.csproj b/Maple2.File.Parser/Maple2.File.Parser.csproj index 2a37cf9f..80fb8ec1 100644 --- a/Maple2.File.Parser/Maple2.File.Parser.csproj +++ b/Maple2.File.Parser/Maple2.File.Parser.csproj @@ -13,7 +13,7 @@ MapleStory2, File, Parser, m2d, xml true - 2.4.29 + 2.4.30 net8.0 README.md enable @@ -47,4 +47,8 @@ + + + + diff --git a/Maple2.File.Tests/DataFreeParsingTest.cs b/Maple2.File.Tests/DataFreeParsingTest.cs new file mode 100644 index 00000000..23e02b29 --- /dev/null +++ b/Maple2.File.Tests/DataFreeParsingTest.cs @@ -0,0 +1,69 @@ +using System.Globalization; +using System.Numerics; +using Maple2.File.Parser.Flat; +using Maple2.File.Parser.Flat.Convert; +using Maple2.File.Parser.Xml.AdditionalEffect; +using Maple2.File.Parser.Xml.Item; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Maple2.File.Tests; + +// These tests need no game data, so they run without MS2_DATA_FOLDER. +[TestClass] +public class DataFreeParsingTest { + // pt-BR reads "." as a thousands separator, so a culture-sensitive parse turns "1.5" into 15. + private static readonly CultureInfo CommaDecimalCulture = new("pt-BR"); + + private static void WithCulture(CultureInfo culture, Action action) { + CultureInfo previous = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = culture; + try { + action(); + } finally { + CultureInfo.CurrentCulture = previous; + } + } + + [TestMethod] + public void TestParseNtFileAcceptsLfAndCrlf() { + const string lf = " \"application\".\n" + + " \"application\".\n"; + string crlf = lf.Replace("\n", "\r\n"); + + foreach (string data in new[] { lf, crlf }) { + Dictionary result = AssetIndex.ParseNtFile(data); + Assert.AreEqual(2, result.Count); + Assert.AreEqual("application", result["urn:uuid:ba8762a8-e692-43f2-a0ba-0a1bfb1c42a9"]); + } + } + + [TestMethod] + public void TestFlatPropertyIgnoresCurrentCulture() { + WithCulture(CommaDecimalCulture, () => { + Assert.AreEqual(1.5f, FlatProperty.ParseType("Float32", "1.5")); + Assert.AreEqual(1.5d, FlatProperty.ParseType("Float64", "1.5")); + Assert.AreEqual(new Vector3(1.5f, -2.25f, 3f), FlatProperty.ParseType("Point3", "1.5, -2.25, 3")); + + var property = new FlatProperty { Name = "Scale", Type = "Float32", Value = 1.5f }; + Assert.AreEqual("1.5f", property.ValueCodeString()); + }); + } + + [TestMethod] + public void TestGeneratedVector3IgnoresCurrentCulture() { + WithCulture(CommaDecimalCulture, () => { + var install = new Install { _physXdimension = "1.5,-2.25,3" }; + Assert.AreEqual(new Vector3(1.5f, -2.25f, 3f), install.physXdimension); + Assert.AreEqual("1.5,-2.25,3", install._physXdimension); + }); + } + + [TestMethod] + public void TestGeneratedFloatArrayIgnoresCurrentCulture() { + WithCulture(CommaDecimalCulture, () => { + var property = new InvokeEffectProperty { _values = "0.5,1.25,-3" }; + CollectionAssert.AreEqual(new[] { 0.5f, 1.25f, -3f }, property.values); + Assert.AreEqual("0.5,1.25,-3", property._values); + }); + } +} diff --git a/Maple2.File.Tests/XBlockParserTest.cs b/Maple2.File.Tests/XBlockParserTest.cs index 516a2b49..6f42f85b 100644 --- a/Maple2.File.Tests/XBlockParserTest.cs +++ b/Maple2.File.Tests/XBlockParserTest.cs @@ -35,6 +35,17 @@ public void TestXBlockParser() { }); } + [TestMethod] + public void TestXBlockParsersShareIndex() { + var index = new FlatTypeIndex(TestUtils.ExportedReader); + _ = new XBlockParser(TestUtils.ExportedReader, index); + var second = new XBlockParser(TestUtils.ExportedReader, index); + + int count = 0; + second.ParseMap("02000070_in", entities => count = entities.Count()); + Assert.IsTrue(count > 0); + } + [Ignore] public void TestXBlockParserAll() { var sw = new Stopwatch(); From 8349fabfa1d72cb561ef5a3c0306487da022c7c0 Mon Sep 17 00:00:00 2001 From: AngeloTadeucci <15664821+AngeloTadeucci@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:25:46 +0000 Subject: [PATCH 2/2] style: auto-format whitespace --- Maple2.File.Tests/DataFreeParsingTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Maple2.File.Tests/DataFreeParsingTest.cs b/Maple2.File.Tests/DataFreeParsingTest.cs index 23e02b29..f1921781 100644 --- a/Maple2.File.Tests/DataFreeParsingTest.cs +++ b/Maple2.File.Tests/DataFreeParsingTest.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; using System.Numerics; using Maple2.File.Parser.Flat; using Maple2.File.Parser.Flat.Convert;