From 01da008df4fb4bb88151237e094bc8648e1f358d Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Wed, 17 Jun 2026 16:13:57 +0000 Subject: [PATCH 1/3] Fix DuckDB bulk_insert failing when quoted CSV cells fall outside sniffer sample (#62) DuckDB's read_csv sniffer only examines the first ~20 480 rows; if none are quoted, it sets quote=(empty) and then errors on any later quoted cell with a column-count mismatch. Passing quote='"' explicitly bypasses auto-detection. Adds a regression test that inserts 25 000 plain rows followed by one row whose value contains a comma (triggering csv.writer quoting). Co-Authored-By: Claude Sonnet 4.6 --- src/xml2db/dialect/duckdb.py | 2 +- tests/test_bulk_insert.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/xml2db/dialect/duckdb.py b/src/xml2db/dialect/duckdb.py index 249b63b..aa54de4 100644 --- a/src/xml2db/dialect/duckdb.py +++ b/src/xml2db/dialect/duckdb.py @@ -158,7 +158,7 @@ def bulk_insert(self, conn: Any, table: Any, records: list) -> None: sql = text( f"INSERT INTO {full_name} ({insert_cols}) " f"SELECT {select_exprs} " - f"FROM read_csv('{safe_path}', header=true, nullstr='', all_varchar=true)" + f"FROM read_csv('{safe_path}', header=true, nullstr='', all_varchar=true, quote='\"')" ) conn.execute(sql) finally: diff --git a/tests/test_bulk_insert.py b/tests/test_bulk_insert.py index d4fbd63..5bcd939 100644 --- a/tests/test_bulk_insert.py +++ b/tests/test_bulk_insert.py @@ -168,6 +168,27 @@ def test_duckdb_bulk_insert_scalar_column_default(duckdb_engine): assert rows[1]["flag"] is False +def test_duckdb_bulk_insert_quoted_csv_field_after_large_unquoted_sample(duckdb_engine): + """Regression: DuckDB's CSV sniffer uses only the first ~20k rows as a sample. + + If all sampled rows are unquoted, the sniffer sets quote=(empty), causing a + column-count error when it later hits a row whose cell value contains a comma + (making csv.writer emit a quoted field). Explicitly passing quote='"' to + read_csv bypasses auto-detection and must always be present. + """ + table = _make_table(duckdb_engine, "quoted_field_test") + # 'vals' value that contains a comma — document.py's 'join' transform can produce + # strings like '"val,ue",other' which csv.writer then wraps in outer quotes, + # yielding a quoted CSV cell. + problematic_value = '"val,ue",other_value' + records = [ + {"id": i, "label": "simple"} for i in range(25_000) # exceeds sniffer sample + ] + [{"id": 25_000, "label": problematic_value}] + rows = _roundtrip(duckdb_engine, table, records) + assert len(rows) == 25_001 + assert rows[-1]["label"] == problematic_value + + def test_duckdb_bulk_insert_empty(duckdb_engine): table = _make_table(duckdb_engine, "empty_test") dialect = DuckDBDialect() From ee17fb1105a94bb286d67798b0aa854803a3a9cc Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Wed, 17 Jun 2026 16:21:34 +0000 Subject: [PATCH 2/3] Bump version to 0.13.3 Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 40b0918..b6253bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xml2db" -version = "0.13.2" +version = "0.13.3" authors = [ { name="Commission de régulation de l'énergie", email="opensource@cre.fr" }, ] From 5f32e0e95d6cc131564f1f36813b9b99ef3244c5 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Wed, 17 Jun 2026 16:32:27 +0000 Subject: [PATCH 3/3] Add explicit escape='\"' to read_csv alongside quote='\"' Makes the RFC 4180 doubling behaviour explicit rather than relying on DuckDB defaulting escape to the same char as quote. Co-Authored-By: Claude Sonnet 4.6 --- src/xml2db/dialect/duckdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xml2db/dialect/duckdb.py b/src/xml2db/dialect/duckdb.py index aa54de4..af750b0 100644 --- a/src/xml2db/dialect/duckdb.py +++ b/src/xml2db/dialect/duckdb.py @@ -158,7 +158,7 @@ def bulk_insert(self, conn: Any, table: Any, records: list) -> None: sql = text( f"INSERT INTO {full_name} ({insert_cols}) " f"SELECT {select_exprs} " - f"FROM read_csv('{safe_path}', header=true, nullstr='', all_varchar=true, quote='\"')" + f"FROM read_csv('{safe_path}', header=true, nullstr='', all_varchar=true, quote='\"', escape='\"')" ) conn.execute(sql) finally: