From 943b0d380754b960fc235c5a7642d128998a00d6 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 23 Sep 2026 11:58:09 +0530 Subject: [PATCH 1/4] Surface JSON decoder position in reader errors The file, string, and URL readers replaced the JSONDecodeError with a fixed message, so the CLI could only say the input was invalid. Append the decoder message so users see the line and column. Fixes #436 --- json2xml/utils.py | 7 ++++--- lat.md/behavior.md | 2 +- lat.md/tests.md | 4 ++++ tests/test_cli.py | 15 +++++++++++++++ tests/test_utils.py | 24 ++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/json2xml/utils.py b/json2xml/utils.py index ed91ab1..8844e74 100644 --- a/json2xml/utils.py +++ b/json2xml/utils.py @@ -70,7 +70,8 @@ def readfromjson(filename: str) -> JSONValue: except OSError as error: raise JSONReadError("Could not read JSON file") from error except ValueError as error: - raise JSONReadError("Invalid JSON File") from error + # Keep the decoder message so callers can show where parsing stopped. + raise JSONReadError(f"Invalid JSON File: {error}") from error # @lat: [[behavior#URL security boundaries]] @@ -351,7 +352,7 @@ def readfromurl( try: return json.loads(response_data.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise URLReadError("URL did not return valid JSON") from error + raise URLReadError(f"URL did not return valid JSON: {error}") from error def readfromstring(jsondata: object) -> JSONValue: @@ -361,4 +362,4 @@ def readfromstring(jsondata: object) -> JSONValue: try: return json.loads(jsondata) except ValueError as error: - raise StringReadError("Input is not a proper JSON string") from error + raise StringReadError(f"Input is not a proper JSON string: {error}") from error diff --git a/lat.md/behavior.md b/lat.md/behavior.md index 6e65aac..06f97f2 100644 --- a/lat.md/behavior.md +++ b/lat.md/behavior.md @@ -6,7 +6,7 @@ This file captures the observable conversion and input rules that matter more th The input helpers convert files, strings, URLs, and stdin into Python data structures while surfacing source-specific errors to callers. -[[json2xml/utils.py#readfromjson]] wraps file and JSON decoding failures in `JSONReadError`. [[json2xml/utils.py#readfromstring]] rejects non-string inputs and malformed JSON with `StringReadError`. +[[json2xml/utils.py#readfromjson]] wraps file and JSON decoding failures in `JSONReadError`. [[json2xml/utils.py#readfromstring]] rejects non-string inputs and malformed JSON with `StringReadError`. Decoding failures keep the decoder's line and column in the message so the CLI can show where the input breaks. [[json2xml/utils.py#readfromurl]] lazily initializes the HTTP client, performs a bounded GET request, and raises `URLReadError` for hostname encoding, network, status, size, decoding, and JSON failures. diff --git a/lat.md/tests.md b/lat.md/tests.md index 96cd1ab..8216938 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -42,6 +42,10 @@ These tests verify the concrete reader helpers against realistic source behavior A file that cannot be opened reports a read failure, while a file whose content does not parse reports invalid JSON, so callers can tell the two apart from the message. +### Invalid JSON errors include the decoder position + +Malformed file, string, and URL input carry the decoder's line and column in the wrapped error, so the CLI can say where the document breaks instead of only that it is invalid. + ### URL reader uses real HTTP and wraps failures URL input should read valid JSON over HTTP and wrap status, network, and decoding failures in `URLReadError`. diff --git a/tests/test_cli.py b/tests/test_cli.py index fcbd06f..43d79de 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -404,6 +404,21 @@ def test_invalid_json_file(self) -> None: assert "Could not parse JSON file" in result.stderr assert str(json_file) in result.stderr + # @lat: [[tests#Input readers#Invalid JSON errors include the decoder position]] + def test_invalid_json_file_reports_position(self) -> None: + """The CLI shows where the decoder stopped, not only that it failed.""" + json_file = Path(__file__).parent.parent / "examples" / "wrongjson.json" + + result = subprocess.run( + [sys.executable, "-m", "json2xml.cli", str(json_file)], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "Could not parse JSON file" in result.stderr + assert "line 372 column 3" in result.stderr + def test_output_file_permission_error(self) -> None: """Test error handling when output file cannot be written.""" with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/test_utils.py b/tests/test_utils.py index b781cfc..c45c2e3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -128,6 +128,20 @@ def test_readfromjson_invalid_json_content(self) -> None: os.unlink(temp_filename) + def test_readfromjson_reports_decoder_position(self) -> None: + """The wrapped error carries the decoder's line and column.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write('{"a": 1 "b": 2}') + temp_filename = f.name + + try: + with pytest.raises(JSONReadError, match="line 1 column 9"): + readfromjson(temp_filename) + finally: + import os + + os.unlink(temp_filename) + # @lat: [[tests#Input readers#File reader distinguishes unreadable files from invalid JSON]] def test_readfromjson_file_not_found(self) -> None: """Test reading a non-existent file.""" @@ -186,6 +200,11 @@ def test_readfromurl_invalid_json_response(self, json_server: str) -> None: with pytest.raises(URLReadError, match="URL did not return valid JSON"): readfromurl(f"{json_server}/invalid.json", allow_private_networks=True) + def test_readfromurl_reports_decoder_position(self, json_server: str) -> None: + """The wrapped error carries the decoder's message.""" + with pytest.raises(URLReadError, match=r"URL did not return valid JSON: .+line 1"): + readfromurl(f"{json_server}/invalid.json", allow_private_networks=True) + def test_readfromurl_network_error(self) -> None: """Test network failures are wrapped as URLReadError.""" with socket.socket() as unused_socket: @@ -715,6 +734,11 @@ def test_readfromstring_complex_object(self) -> None: } assert result == expected + def test_readfromstring_reports_decoder_position(self) -> None: + """The wrapped error carries the decoder's line and column.""" + with pytest.raises(StringReadError, match="line 1 column 9"): + readfromstring('{"a": 1 "b": 2}') + def test_readfromstring_invalid_type_int(self) -> None: """Test reading with integer input.""" with pytest.raises(StringReadError, match="Input is not a proper JSON string"): From e13e26241991727b49b357be5b249989d040abf7 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 23 Sep 2026 11:59:29 +0530 Subject: [PATCH 2/4] Accept a UTF-8 byte order mark in JSON input Files, URL bodies, and strings starting with a BOM were rejected as invalid JSON. Windows tools such as PowerShell and Notepad write the BOM by default, so decode with utf-8-sig and drop a leading U+FEFF from string input. Fixes #437 --- json2xml/utils.py | 8 +++++--- lat.md/behavior.md | 2 +- lat.md/tests.md | 4 ++++ tests/test_cli.py | 15 +++++++++++++++ tests/test_utils.py | 20 ++++++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/json2xml/utils.py b/json2xml/utils.py index 8844e74..73eb683 100644 --- a/json2xml/utils.py +++ b/json2xml/utils.py @@ -65,7 +65,8 @@ def _effective_port(parsed: SplitResult) -> int: def readfromjson(filename: str) -> JSONValue: """Read JSON data from a file.""" try: - with open(filename, encoding="utf-8") as jsondata: + # utf-8-sig drops the BOM that Windows tools write by default. + with open(filename, encoding="utf-8-sig") as jsondata: return json.load(jsondata) except OSError as error: raise JSONReadError("Could not read JSON file") from error @@ -350,7 +351,7 @@ def readfromurl( response.close() try: - return json.loads(response_data.decode("utf-8")) + return json.loads(response_data.decode("utf-8-sig")) except (UnicodeDecodeError, json.JSONDecodeError) as error: raise URLReadError(f"URL did not return valid JSON: {error}") from error @@ -360,6 +361,7 @@ def readfromstring(jsondata: object) -> JSONValue: if not isinstance(jsondata, str): raise StringReadError("Input is not a proper JSON string") try: - return json.loads(jsondata) + # Text sources such as stdin deliver a BOM as a leading U+FEFF. + return json.loads(jsondata.removeprefix("\ufeff")) except ValueError as error: raise StringReadError(f"Input is not a proper JSON string: {error}") from error diff --git a/lat.md/behavior.md b/lat.md/behavior.md index 06f97f2..22acf56 100644 --- a/lat.md/behavior.md +++ b/lat.md/behavior.md @@ -6,7 +6,7 @@ This file captures the observable conversion and input rules that matter more th The input helpers convert files, strings, URLs, and stdin into Python data structures while surfacing source-specific errors to callers. -[[json2xml/utils.py#readfromjson]] wraps file and JSON decoding failures in `JSONReadError`. [[json2xml/utils.py#readfromstring]] rejects non-string inputs and malformed JSON with `StringReadError`. Decoding failures keep the decoder's line and column in the message so the CLI can show where the input breaks. +[[json2xml/utils.py#readfromjson]] wraps file and JSON decoding failures in `JSONReadError`. [[json2xml/utils.py#readfromstring]] rejects non-string inputs and malformed JSON with `StringReadError`. Decoding failures keep the decoder's line and column in the message so the CLI can show where the input breaks. All three readers ignore a leading UTF-8 byte order mark, which Windows tools write by default. [[json2xml/utils.py#readfromurl]] lazily initializes the HTTP client, performs a bounded GET request, and raises `URLReadError` for hostname encoding, network, status, size, decoding, and JSON failures. diff --git a/lat.md/tests.md b/lat.md/tests.md index 8216938..d673af8 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -46,6 +46,10 @@ A file that cannot be opened reports a read failure, while a file whose content Malformed file, string, and URL input carry the decoder's line and column in the wrapped error, so the CLI can say where the document breaks instead of only that it is invalid. +### Readers accept a UTF-8 byte order mark + +File, URL, and string input that starts with a UTF-8 byte order mark parses normally, because Windows tools write that prefix by default and it carries no data. + ### URL reader uses real HTTP and wraps failures URL input should read valid JSON over HTTP and wrap status, network, and decoding failures in `URLReadError`. diff --git a/tests/test_cli.py b/tests/test_cli.py index 43d79de..53d89f3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -419,6 +419,21 @@ def test_invalid_json_file_reports_position(self) -> None: assert "Could not parse JSON file" in result.stderr assert "line 372 column 3" in result.stderr + def test_json_file_with_utf8_bom(self) -> None: + """A BOM-prefixed file converts instead of being reported as invalid.""" + with tempfile.TemporaryDirectory() as tmpdir: + json_file = Path(tmpdir) / "bom.json" + json_file.write_bytes(b'\xef\xbb\xbf{"bom": true}') + + result = subprocess.run( + [sys.executable, "-m", "json2xml.cli", str(json_file)], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert 'true' in result.stdout + def test_output_file_permission_error(self) -> None: """Test error handling when output file cannot be written.""" with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/test_utils.py b/tests/test_utils.py index c45c2e3..f75fd4c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,6 +7,7 @@ import threading import zlib from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, cast from unittest.mock import Mock, patch @@ -34,6 +35,7 @@ class JsonTestHandler(BaseHTTPRequestHandler): "/data.json": (200, b'{"key": "value", "number": 42}'), "/api": (200, b'{"result": "success"}'), "/invalid.json": (200, b"invalid json content"), + "/bom.json": (200, b'\xef\xbb\xbf{"bom": true}'), "/error.json": (500, b'{"error": true}'), "/api.json": (200, b'{"api": "response", "status": "ok"}'), } @@ -142,6 +144,14 @@ def test_readfromjson_reports_decoder_position(self) -> None: os.unlink(temp_filename) + # @lat: [[tests#Input readers#Readers accept a UTF-8 byte order mark]] + def test_readfromjson_accepts_utf8_bom(self, tmp_path: Path) -> None: + """A leading UTF-8 BOM, as written by Windows tools, is not an error.""" + bom_file = tmp_path / "bom.json" + bom_file.write_bytes(b'\xef\xbb\xbf{"bom": true}') + + assert readfromjson(str(bom_file)) == {"bom": True} + # @lat: [[tests#Input readers#File reader distinguishes unreadable files from invalid JSON]] def test_readfromjson_file_not_found(self) -> None: """Test reading a non-existent file.""" @@ -205,6 +215,12 @@ def test_readfromurl_reports_decoder_position(self, json_server: str) -> None: with pytest.raises(URLReadError, match=r"URL did not return valid JSON: .+line 1"): readfromurl(f"{json_server}/invalid.json", allow_private_networks=True) + def test_readfromurl_accepts_utf8_bom(self, json_server: str) -> None: + """A BOM-prefixed response body decodes like any other UTF-8 body.""" + result = readfromurl(f"{json_server}/bom.json", allow_private_networks=True) + + assert result == {"bom": True} + def test_readfromurl_network_error(self) -> None: """Test network failures are wrapped as URLReadError.""" with socket.socket() as unused_socket: @@ -739,6 +755,10 @@ def test_readfromstring_reports_decoder_position(self) -> None: with pytest.raises(StringReadError, match="line 1 column 9"): readfromstring('{"a": 1 "b": 2}') + def test_readfromstring_accepts_utf8_bom(self) -> None: + """A leading U+FEFF, as stdin delivers a BOM, is dropped before parsing.""" + assert readfromstring('\ufeff{"bom": true}') == {"bom": True} + def test_readfromstring_invalid_type_int(self) -> None: """Test reading with integer input.""" with pytest.raises(StringReadError, match="Input is not a proper JSON string"): From 4602c3482e8600db1bc93ff2ed28772454605c0d Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 23 Sep 2026 12:05:07 +0530 Subject: [PATCH 3/4] Assert decoder position without pinning wording PyPy reports the unterminated object at line 1 for the sample file while CPython reports the missing comma at line 372. The CLI contract is that a position is shown, so match the pattern, not the text. Refs #436 --- tests/test_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 53d89f3..3ab5dd0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import io +import re import subprocess import sys import tempfile @@ -417,7 +418,8 @@ def test_invalid_json_file_reports_position(self) -> None: assert result.returncode == 1 assert "Could not parse JSON file" in result.stderr - assert "line 372 column 3" in result.stderr + # CPython and PyPy report different positions for the same file. + assert re.search(r"line \d+ column \d+", result.stderr) def test_json_file_with_utf8_bom(self) -> None: """A BOM-prefixed file converts instead of being reported as invalid.""" From 5601b1c0d8cc5ab7001ffc7a389f232dd4bbdaa4 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 23 Sep 2026 12:11:53 +0530 Subject: [PATCH 4/4] Fix ty 0.0.83 diagnostics in tests ty 0.0.83 marks Number.__hash__ abstract and enforces dict and list invariance, so the CustomNumber helpers could not be instantiated and nested payload literals were not assignable to JSONValue. Implement __hash__ and declare the payloads as JSONValue. Also make the omit-list test compare the result instead of asserting a literal. Fixes #439 --- tests/test_dict2xml.py | 6 ++++-- tests/test_dicttoxml_unit.py | 3 +++ tests/test_json2xml.py | 25 +++++++++++++------------ tests/test_missing_coverage.py | 3 +++ 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/test_dict2xml.py b/tests/test_dict2xml.py index 74e2560..f1e7d97 100644 --- a/tests/test_dict2xml.py +++ b/tests/test_dict2xml.py @@ -190,8 +190,10 @@ def test_dict2xml_with_flat(self) -> None: def test_dict2xml_omit_list(self) -> None: """Test dicttoxml with list omission.""" obj = {"list": [{"bike": "blue"}, {"wheel": "black"}]} - dicttoxml.dicttoxml(obj, root=False, attr_type=False, item_wrap=False) - assert b"blueblack" + result = dicttoxml.dicttoxml( + obj, root=False, attr_type=False, item_wrap=False + ) + assert result == b"blueblack" def test_dict2xml_with_val_and_custom_attr(self) -> None: """Test dicttoxml with @val and custom attributes.""" diff --git a/tests/test_dicttoxml_unit.py b/tests/test_dicttoxml_unit.py index 3b4ec3b..8dcf10f 100644 --- a/tests/test_dicttoxml_unit.py +++ b/tests/test_dicttoxml_unit.py @@ -13,6 +13,9 @@ class CustomNumber(numbers.Number): + def __hash__(self) -> int: + return 7 + def __complex__(self) -> complex: return complex(7) diff --git a/tests/test_json2xml.py b/tests/test_json2xml.py index 3fc4147..edbb832 100644 --- a/tests/test_json2xml.py +++ b/tests/test_json2xml.py @@ -12,6 +12,7 @@ from json2xml import json2xml from json2xml.json2xml import _positive_limit +from json2xml.types import JSONValue from json2xml.utils import ( InvalidDataError, JSONReadError, @@ -215,7 +216,7 @@ def test_attrs(self) -> None: assert "dict" == old_dict["all"]["empty_dict"]["@type"] def test_dicttoxml_bug(self) -> None: - input_dict = { + input_dict: JSONValue = { "response": { "results": { "user": [ @@ -342,7 +343,7 @@ def test_exact_compact_output_limit_is_enforced_after_backend_render( def test_pretty_output_limit_counts_indentation(self) -> None: """Pretty-only whitespace is included in the exact output byte budget.""" - data = {"outer": {"inner": {"leaf": "value"}}} + data: JSONValue = {"outer": {"inner": {"leaf": "value"}}} compact_size = len(json2xml.Json2xml(data).to_xml() or b"") with pytest.raises(InvalidDataError, match="XML output size limit exceeded"): @@ -430,7 +431,7 @@ def test_read_boolean_data_from_json2(self) -> None: assert dict_from_xml["all"]["string_array"]["item"][2]["#text"] == "c" def test_dict_attr_crash(self) -> None: - data = { + data: JSONValue = { "product": { "@attrs": {"attr_name": "attr_value", "a": "b"}, "@val": [], @@ -461,7 +462,7 @@ def test_encoding_without_pretty_print(self) -> None: # @lat: [[tests#Conversion behavior#XPath format adds functions namespace]] def test_xpath_format_basic(self) -> None: """Test XPath 3.1 json-to-xml format with basic types.""" - data = {"name": "John", "age": 30, "active": True} + data: JSONValue = {"name": "John", "age": 30, "active": True} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b'xmlns="http://www.w3.org/2005/xpath-functions"' in xmldata @@ -471,7 +472,7 @@ def test_xpath_format_basic(self) -> None: def test_xpath_format_nested_dict(self) -> None: """Test XPath 3.1 format with nested dictionaries.""" - data = {"person": {"name": "Alice", "age": 25}} + data: JSONValue = {"person": {"name": "Alice", "age": 25}} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b'' in xmldata @@ -480,7 +481,7 @@ def test_xpath_format_nested_dict(self) -> None: def test_xpath_format_array(self) -> None: """Test XPath 3.1 format with arrays.""" - data = {"numbers": [1, 2, 3]} + data: JSONValue = {"numbers": [1, 2, 3]} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b'' in xmldata @@ -490,14 +491,14 @@ def test_xpath_format_array(self) -> None: def test_xpath_format_null(self) -> None: """Test XPath 3.1 format with null values.""" - data = {"value": None} + data: JSONValue = {"value": None} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b'' in xmldata def test_xpath_format_mixed_array(self) -> None: """Test XPath 3.1 format with mixed type arrays.""" - data = {"items": ["text", 42, True, None]} + data: JSONValue = {"items": ["text", 42, True, None]} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b'' in xmldata @@ -508,7 +509,7 @@ def test_xpath_format_mixed_array(self) -> None: def test_xpath_format_complex_nested(self) -> None: """Test XPath 3.1 format with complex nested structures.""" - data = { + data: JSONValue = { "content": [ {"id": 70805774, "value": "1001", "position": [1004.0, 288.0]}, ] @@ -523,7 +524,7 @@ def test_xpath_format_complex_nested(self) -> None: def test_xpath_format_escaping(self) -> None: """Test XPath 3.1 format properly escapes special characters.""" - data = {"text": ""} + data: JSONValue = {"text": ""} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b"<script>" in xmldata @@ -531,7 +532,7 @@ def test_xpath_format_escaping(self) -> None: def test_xpath_format_with_pretty_print(self) -> None: """Test XPath 3.1 format works with pretty printing.""" - data = {"name": "Test"} + data: JSONValue = {"name": "Test"} xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=True).to_xml() assert isinstance(xmldata, str) assert 'xmlns="http://www.w3.org/2005/xpath-functions"' in xmldata @@ -539,7 +540,7 @@ def test_xpath_format_with_pretty_print(self) -> None: def test_xpath_format_root_array(self) -> None: """Test XPath 3.1 format with root-level array.""" - data = [1, 2, 3] + data: JSONValue = [1, 2, 3] xmldata = json2xml.Json2xml(data, xpath_format=True, pretty=False).to_xml() assert isinstance(xmldata, bytes) assert b'' in xmldata diff --git a/tests/test_missing_coverage.py b/tests/test_missing_coverage.py index 935dfd9..737fd5b 100644 --- a/tests/test_missing_coverage.py +++ b/tests/test_missing_coverage.py @@ -203,6 +203,9 @@ def __init__(self, value: float) -> None: def __str__(self) -> str: return str(self.value) + def __hash__(self) -> int: + return hash(self.value) + result = get_xml_type(CustomNumber(42)) assert result == "number"