From c6019474d90fb0fdac85936f83cf35a928abd8a9 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 10:05:48 +0000 Subject: [PATCH 1/9] fix: correct four API-level bugs requiring a minor version bump - DataModel._build_model: table-name validation iterated over self.model_config (which has no "tables" key after _validate_config) instead of self.tables_config, so the guard was silently dead and unknown table names in model_config were never caught - XMLConverter.parse_xml: skip_validation defaulted to False (validate) while Document.parse_xml and DataModel.parse_xml both default to True (skip); aligns the low-level default with the documented behaviour and with how the method is always called by its callers - DatabaseDialect.db_identifier: remove the dead temp_prefix parameter which was never passed by any caller and whose implementation (max_len += 14) was the inverse of what its docstring described - DataModelColumn: rename the model_config constructor argument and attribute to config to reflect that it holds the table-level config dict, not the top-level model config Co-Authored-By: Claude Sonnet 4.6 --- src/xml2db/dialect/base.py | 5 +---- src/xml2db/model.py | 2 +- src/xml2db/table/column.py | 8 ++++---- src/xml2db/xml_converter.py | 5 +++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/xml2db/dialect/base.py b/src/xml2db/dialect/base.py index 27f5f94..afcc9a7 100644 --- a/src/xml2db/dialect/base.py +++ b/src/xml2db/dialect/base.py @@ -47,7 +47,7 @@ def __init__(self, **kwargs): # Identifier handling # ------------------------------------------------------------------ - def db_identifier(self, logical_name: str, temp_prefix: bool = False) -> str: + def db_identifier(self, logical_name: str) -> str: """Return the physical database identifier for a logical name. Names longer than :attr:`MAX_IDENTIFIER_LENGTH` are truncated using a @@ -57,15 +57,12 @@ def db_identifier(self, logical_name: str, temp_prefix: bool = False) -> str: Args: logical_name: The full logical name used inside the Python model (e.g. ``"very_long_table_name_derived_from_xsd"``). - temp_prefix: Should we save 14 more characters for the temp prefix? Returns: A string that is safe to use as a database identifier for this backend. Guaranteed to be stable across calls with the same input. """ max_len = self.MAX_IDENTIFIER_LENGTH - if temp_prefix: - max_len += 14 if len(logical_name) <= max_len: return logical_name diff --git a/src/xml2db/model.py b/src/xml2db/model.py index 05e70cf..79818f1 100644 --- a/src/xml2db/model.py +++ b/src/xml2db/model.py @@ -224,7 +224,7 @@ def _build_model(self): # compute a text representation of the original data model and store it self.source_tree = str(root_table) # check user-provided configuration for tables - for tb_config in self.model_config.get("tables", {}): + for tb_config in self.tables_config: if tb_config not in self.names_types_map: raise DataModelConfigError( f"Table '{tb_config}' provided in config does not exist" diff --git a/src/xml2db/table/column.py b/src/xml2db/table/column.py index 9579360..f4bd491 100644 --- a/src/xml2db/table/column.py +++ b/src/xml2db/table/column.py @@ -25,7 +25,7 @@ class DataModelColumn: is_content: is the column used to store the content value of a mixed complex type? allow_empty: is nullable ? ngroup: a string key used to handle nested sequences (``str(hash(parent_node))``) or ``None`` - model_config: the table-level config dict; may contain a ``fields`` entry with a custom column type + config: the table-level config dict; may contain a ``fields`` entry with a custom column type data_model: the DataModel object it belongs to Attributes: @@ -47,7 +47,7 @@ def __init__( is_content: bool, allow_empty: bool, ngroup: Union[str, None], - model_config: dict[str, Any], + config: dict[str, Any], data_model: "DataModel", ): """Constructor method""" @@ -62,7 +62,7 @@ def __init__( self.is_content = is_content self.allow_empty = allow_empty self.ngroup = ngroup - self.model_config = model_config + self.config = config self.data_model = data_model self.other_table = None # just to avoid a linting warning @@ -101,7 +101,7 @@ def get_sqlalchemy_column(self, temp: bool = False) -> Iterable[Column]: temp: temp table or target table ? """ # use type specified in config if exists - column_type = self.model_config.get("fields", {}).get(self.name, {}).get( + column_type = self.config.get("fields", {}).get(self.name, {}).get( "type" ) or self.data_model.dialect.column_type(self, temp) db_col = self.data_model.dialect.db_identifier(self.name) diff --git a/src/xml2db/xml_converter.py b/src/xml2db/xml_converter.py index a16f401..67a137f 100644 --- a/src/xml2db/xml_converter.py +++ b/src/xml2db/xml_converter.py @@ -46,7 +46,7 @@ def parse_xml( self, xml_file: Union[str, BytesIO], file_path: str = None, - skip_validation: bool = False, + skip_validation: bool = True, recover: bool = False, iterparse: bool = True, ) -> tuple: @@ -57,7 +57,8 @@ def parse_xml( Args: xml_file: An XML file path or file content to be converted file_path: The file path to be printed in logs - skip_validation: Whether we should validate XML against the schema before parsing + skip_validation: Whether to skip XML validation against the schema before parsing + (default ``True``; set to ``False`` to validate) recover: Try to process malformed XML (lxml option) iterparse: Parse XML using iterative parsing, which is a bit slower but uses less memory From b512578d452b6bfd3f728116ad9ff31cce75035b Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 10:07:07 +0000 Subject: [PATCH 2/9] chore: bump version to 0.14.0 Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b6253bd..92f5c64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xml2db" -version = "0.13.3" +version = "0.14.0" authors = [ { name="Commission de régulation de l'énergie", email="opensource@cre.fr" }, ] From 31aa3d0201601dc4bab4ba3f2a3bccf86c50bb9a Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 12:29:42 +0000 Subject: [PATCH 3/9] feat: add 'rename' field config option to set physical DB column name Allows overriding the database column name independently of the internal logical name, so XSD element names that are awkward or conflict with SQL reserved words can be mapped to a cleaner identifier without touching the schema or any other part of the pipeline. Co-Authored-By: Claude Sonnet 4.6 --- docs/configuring.md | 38 ++++++++++++++++++++++++++++++++++++++ src/xml2db/table/column.py | 8 +++----- tests/test_conversions.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/docs/configuring.md b/docs/configuring.md index 86b767b..d7115ec 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -123,6 +123,44 @@ defined as `sqlalchemy` types and will be passed to the `sqlalchemy.Column` cons You can infer `my_table` and `my_column` when visualizing the data model. +### Renaming columns + +The physical database column name for any field can be overridden while keeping the original XML element name as the +internal logical key. This is useful when XSD element names are awkward, conflict with reserved SQL words, or need to +follow a naming convention that differs from the source schema. + +The rename applies to both the target table and the staging table. All internal references (data dict keys, foreign key +lookups, merge statements) continue to use the original logical name, so only the visible DB column name changes. + +Configuration: `"rename":` `"new_column_name"` (no default — omit to keep the original name) + +!!! example + Rename the `orderid` attribute to `order_id` in the `shiporder` table: + ```python + model_config = { + "tables": { + "shiporder": { + "fields": { + "orderid": {"rename": "order_id"} + } + } + } + } + ``` + + Elevated fields (those pulled up from a child table) are renamed using their prefixed name in the parent table: + ```python + model_config = { + "tables": { + "shiporder": { + "fields": { + "orderperson_name": {"rename": "contact_name"} + } + } + } + } + ``` + ### Joining values for simple types By default, XML simple type elements with types in `["string", "date", "dateTime", "NMTOKEN", "time", "base64Binary", "decimal"]` and max diff --git a/src/xml2db/table/column.py b/src/xml2db/table/column.py index f4bd491..877874c 100644 --- a/src/xml2db/table/column.py +++ b/src/xml2db/table/column.py @@ -100,9 +100,7 @@ def get_sqlalchemy_column(self, temp: bool = False) -> Iterable[Column]: Args: temp: temp table or target table ? """ - # use type specified in config if exists - column_type = self.config.get("fields", {}).get(self.name, {}).get( - "type" - ) or self.data_model.dialect.column_type(self, temp) - db_col = self.data_model.dialect.db_identifier(self.name) + field_config = self.config.get("fields", {}).get(self.name, {}) + column_type = field_config.get("type") or self.data_model.dialect.column_type(self, temp) + db_col = self.data_model.dialect.db_identifier(field_config.get("rename", self.name)) yield Column(db_col, column_type, key=self.name) diff --git a/tests/test_conversions.py b/tests/test_conversions.py index b279cfa..4712d70 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -117,6 +117,34 @@ def test_document_tree_to_xml(test_config): assert xml == ref_xml +def test_field_rename(): + """Test that 'rename' in field config sets the physical DB column name without affecting internal logic""" + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "tables": { + "shiporder": { + "fields": { + "orderid": {"rename": "order_id"}, + } + } + } + }, + ) + shiporder_table = model.tables["shipordertype"] + + # physical column name in the DB uses the renamed value + assert shiporder_table.table.c["orderid"].name == "order_id" + # temp table should also use the renamed physical name + assert shiporder_table.temp_table.c["orderid"].name == "order_id" + + # parsing still works and data dict uses the original logical name + doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order1.xml"))) + records = doc.data["shipordertype"]["records"] + assert len(records) > 0 + assert "orderid" in records[0] + + @pytest.mark.parametrize( "test_config", [ From 7d57c096326f9ed587612dedf7a2af38b1c42ca5 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 12:35:37 +0000 Subject: [PATCH 4/9] test: add rename coverage to orders model version 0 roundtrip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames orderid → order_id in the shiporder table of the version 0 sample config to exercise the rename feature through the full DB roundtrip test, and regenerates the three DDL snapshots accordingly. ERD and tree snapshots are unaffected (they display logical names). Co-Authored-By: Claude Sonnet 4.6 --- tests/sample_models/models.py | 7 ++++++- tests/sample_models/orders/orders_ddl_mssql_version0.sql | 2 +- tests/sample_models/orders/orders_ddl_mysql_version0.sql | 2 +- .../orders/orders_ddl_postgresql_version0.sql | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index 50cd865..b1f6955 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -23,7 +23,12 @@ def wrapped(): { "config": { "tables": { - "shiporder": {"fields": {"orderperson": {"transform": False}}}, + "shiporder": { + "fields": { + "orderperson": {"transform": False}, + "orderid": {"rename": "order_id"}, + } + }, "item": None, }, "record_hash_column_name": "record_hash", diff --git a/tests/sample_models/orders/orders_ddl_mssql_version0.sql b/tests/sample_models/orders/orders_ddl_mssql_version0.sql index a4c4121..e7c4a05 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version0.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version0.sql @@ -76,7 +76,7 @@ CREATE TABLE item_product_features_stringfeature ( CREATE TABLE shiporder ( pk_shiporder INTEGER NOT NULL IDENTITY, - orderid VARCHAR(1000) NULL, + order_id VARCHAR(1000) NULL, processed_at DATETIMEOFFSET NULL, fk_orderperson INTEGER NULL, shipto_fk_orderperson INTEGER NULL, diff --git a/tests/sample_models/orders/orders_ddl_mysql_version0.sql b/tests/sample_models/orders/orders_ddl_mysql_version0.sql index 7b7b79f..6da9028 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version0.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version0.sql @@ -76,7 +76,7 @@ CREATE TABLE item_product_features_stringfeature ( CREATE TABLE shiporder ( pk_shiporder INTEGER NOT NULL AUTO_INCREMENT, - orderid VARCHAR(255), + order_id VARCHAR(255), processed_at DATETIME, fk_orderperson INTEGER, shipto_fk_orderperson INTEGER, diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version0.sql b/tests/sample_models/orders/orders_ddl_postgresql_version0.sql index ee1c43d..fdbd56c 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version0.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version0.sql @@ -76,7 +76,7 @@ CREATE TABLE item_product_features_stringfeature ( CREATE TABLE shiporder ( pk_shiporder SERIAL NOT NULL, - orderid VARCHAR(1000), + order_id VARCHAR(1000), processed_at TIMESTAMP WITH TIME ZONE, fk_orderperson INTEGER, shipto_fk_orderperson INTEGER, From 4e95d588df7f84d27032e9949d4b084355796771 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 12:42:52 +0000 Subject: [PATCH 5/9] test: add rename of elevated field to orders model version 1 roundtrip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames product_name → prd_name in the item table of version 1, where item.reuse=False and product is elevated by default (no field-level transform config). Exercises rename on a post-elevation prefixed column name through the full DB roundtrip test. Co-Authored-By: Claude Sonnet 4.6 --- tests/sample_models/models.py | 5 ++++- tests/sample_models/orders/orders_ddl_mssql_version1.sql | 2 +- tests/sample_models/orders/orders_ddl_mysql_version1.sql | 2 +- .../sample_models/orders/orders_ddl_postgresql_version1.sql | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index b1f6955..9a9f695 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -44,7 +44,10 @@ def wrapped(): "config": { "row_numbers": True, "tables": { - "item": {"reuse": False}, + "item": { + "reuse": False, + "fields": {"product_name": {"rename": "prd_name"}}, + }, "shiporder": {"fields": {"orderperson": {"transform": False}}}, "companyId": {"choice_transform": False}, }, diff --git a/tests/sample_models/orders/orders_ddl_mssql_version1.sql b/tests/sample_models/orders/orders_ddl_mssql_version1.sql index 7ec3ed2..51f3837 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version1.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version1.sql @@ -81,7 +81,7 @@ CREATE TABLE item ( temp_pk_item INTEGER NULL, fk_parent_shiporder INTEGER NULL, xml2db_row_number INTEGER NOT NULL, - product_name VARCHAR(1000) NULL, + prd_name VARCHAR(1000) NULL, product_version VARCHAR(1000) NULL, note VARCHAR(1000) NULL, quantity INTEGER NULL, diff --git a/tests/sample_models/orders/orders_ddl_mysql_version1.sql b/tests/sample_models/orders/orders_ddl_mysql_version1.sql index 07865a6..8a381d3 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version1.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version1.sql @@ -81,7 +81,7 @@ CREATE TABLE item ( temp_pk_item INTEGER, fk_parent_shiporder INTEGER, xml2db_row_number INTEGER NOT NULL, - product_name VARCHAR(255), + prd_name VARCHAR(255), product_version VARCHAR(255), note VARCHAR(255), quantity INTEGER, diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version1.sql b/tests/sample_models/orders/orders_ddl_postgresql_version1.sql index f7f8fc2..03dce19 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version1.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version1.sql @@ -81,7 +81,7 @@ CREATE TABLE item ( temp_pk_item INTEGER, fk_parent_shiporder INTEGER, xml2db_row_number INTEGER NOT NULL, - product_name VARCHAR(1000), + prd_name VARCHAR(1000), product_version VARCHAR(1000), note VARCHAR(1000), quantity INTEGER, From f22d1235fa1cff29560035213e17fcf7cd0ff285 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 13:00:43 +0000 Subject: [PATCH 6/9] feat: add 'transform: skip' to exclude fields from the data model Columns and relations marked with {"transform": "skip"} are entirely removed from the table schema (no DB column created), omitted from fields_transforms (XML converter ignores them naturally), and absent from parsed records. Skipped relations are also removed from relations_1/relations_n so no FK columns or relation tables are built; child tables with no other parents are pruned from the model. Three tests cover a skipped scalar column, a skipped rel1 (including child-table pruning), and a DB insert with a skipped field. Co-Authored-By: Claude Sonnet 4.6 --- docs/configuring.md | 30 +++++++++ src/xml2db/table/transformed_table.py | 26 +++++--- tests/test_conversions.py | 90 +++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 7 deletions(-) diff --git a/docs/configuring.md b/docs/configuring.md index d7115ec..2f547b4 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -186,6 +186,36 @@ automatically applied `join`, as it would require a complex process of adding a } ``` +### Skipping fields + +Any field — column or relation — can be excluded from the data model entirely by setting its transform to `"skip"`. +The field will be absent from the target table schema and all data for it will be silently dropped during XML +parsing. This is useful for PII columns, large binary blobs, or fields that are irrelevant for analysis. + +For a skipped relation, the child table is also pruned from the model unless it is referenced by another relation +elsewhere in the schema. + +Configuration: `"transform": "skip"` + +!!! warning + Skipped fields are not recoverable — data for them is never stored. Round-trip XML reconstruction will omit + any skipped field even when it was present in the source document. + +!!! example + Skip an optional scalar column and an optional relation: + ```python + model_config = { + "tables": { + "item": { + "fields": { + "note": {"transform": "skip"}, + "delivery": {"transform": "skip"}, + } + } + } + } + ``` + ### Elevate children to upper level A mandatory child (min occurrences = 1, i.e. `[1, 1]`) is always elevated to its parent by default, as long as it diff --git a/src/xml2db/table/transformed_table.py b/src/xml2db/table/transformed_table.py index 031d858..65382dd 100644 --- a/src/xml2db/table/transformed_table.py +++ b/src/xml2db/table/transformed_table.py @@ -140,6 +140,8 @@ def _get_field_transform( if "transform" in field_config: if field_config["transform"] is False: return None + if field_config["transform"] == "skip": + return "skip" if self._can_transform_field( field_type, field_name, field_config["transform"] ): @@ -292,12 +294,16 @@ def simplify_table(self) -> Tuple[dict, dict]: fields_transforms = {} for field_type, field_name, field in self.fields: if field_type == "col": - if self._get_field_transform("col", field_name) == "join": - fields_transforms[(self.type_name, field_name)] = ( - None, - "join", - ) - out_fields.append(("col", field_name, field)) + transform = self._get_field_transform("col", field_name) + if transform == "skip": + del self.columns[field_name] + else: + if transform == "join": + fields_transforms[(self.type_name, field_name)] = ( + None, + "join", + ) + out_fields.append(("col", field_name, field)) else: # simplify child table @@ -310,7 +316,13 @@ def simplify_table(self) -> Tuple[dict, dict]: # check if children can be "elevated" to the upper level transform = self._get_field_transform(field_type, field_name) - if transform is not None: + if transform == "skip": + if field_type == "rel1": + del self.relations_1[field_name] + elif field_type == "reln": + del self.relations_n[field_name] + # don't add to out_fields, don't set keep_table, don't add to fields_transforms + elif transform is not None: if field_type == "rel1": elevated_fields = self._elevate_relation_1( field_name, transform diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 4712d70..3178c88 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -145,6 +145,96 @@ def test_field_rename(): assert "orderid" in records[0] +def test_field_skip_column(): + """Test that transform='skip' removes a scalar column from the schema and parsed records""" + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "tables": { + "item": { + "fields": { + "note": {"transform": "skip"}, + } + } + } + }, + ) + item_table = model.tables["itemtype"] + + # column must be absent from both the DataModelTable and the SQLAlchemy table + assert "note" not in item_table.columns + assert "note" not in item_table.table.c + # neighbouring columns are unaffected + assert "quantity" in item_table.table.c + + # parsing works; 'note' is absent from records even for files that contain + doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order1.xml"))) + records = doc.data["itemtype"]["records"] + assert len(records) > 0 + assert "note" not in records[0] + + +def test_field_skip_relation(): + """Test that transform='skip' on a relation removes its generated columns and prunes the child table""" + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "tables": { + "item": { + "fields": { + "delivery": {"transform": "skip"}, + } + } + } + }, + ) + item_table = model.tables["itemtype"] + + # delivery is a rel1 that would normally be elevated → its FK columns must be absent + assert "delivery_from_fk_orderperson" not in item_table.table.c + assert "delivery_to_fk_orderperson" not in item_table.table.c + # 'delivery' is also absent from relations_1 (no FK column, no merge statements) + assert "delivery" not in item_table.relations_1 + + # deliveryType has no other parents, so it should be pruned from the model + assert "deliveryType" not in model.tables + + # parsing works even for XML files that contain ; delivery data is silently dropped + doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order3.xml"))) + records = doc.data["itemtype"]["records"] + assert len(records) > 0 + assert all("delivery" not in key for key in records[0]) + + +@pytest.mark.dbtest +def test_field_skip_db(conn_string): + """Test that skipped fields do not cause DB errors during insert""" + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + connection_string=conn_string, + db_schema="test_xml2db_skip", + model_config={ + "tables": { + "item": { + "fields": { + "note": {"transform": "skip"}, + } + } + } + }, + ) + model.create_db_schema() + model.drop_all_tables() + model.create_all_tables() + try: + # insert a file that contains — must succeed without errors + xml_path = str(os.path.join(models_path, "orders", "xml", "order1.xml")) + doc = model.parse_xml(xml_path) + doc.insert_into_target_tables() + finally: + model.drop_all_tables() + + @pytest.mark.parametrize( "test_config", [ From a63aa2024b237af48b5c8e47c6bd90255979fba0 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 13:37:19 +0000 Subject: [PATCH 7/9] test: cover transform=skip via sample model equivalent_xml test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds optional 'comment' field to itemtype in orders.xsd, skips it in all three versions' item config, and adds to order1a.xml. The equivalent_xml test now verifies that a file WITH the skipped field and one WITHOUT produce identical flat data — the core contract of skip. Also removes test_field_skip_column and test_field_skip_db (superseded by the DDL snapshot and roundtrip tests), keeping only test_field_skip_relation which tests the unique relation-skip and child-table-pruning behaviour. Co-Authored-By: Claude Sonnet 4.6 --- tests/sample_models/models.py | 15 ++++- .../orders/equivalent_xml/order1a.xml | 1 + tests/sample_models/orders/orders.xsd | 1 + .../orders/orders_source_tree_version0.txt | 1 + .../orders/orders_source_tree_version1.txt | 1 + .../orders/orders_source_tree_version2.txt | 1 + tests/test_conversions.py | 57 ------------------- 7 files changed, 17 insertions(+), 60 deletions(-) diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index 9a9f695..af1e4c6 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -29,7 +29,8 @@ def wrapped(): "orderid": {"rename": "order_id"}, } }, - "item": None, + "item": {"fields": {"comment": {"transform": "skip"}}}, + "orderperson": None, }, "record_hash_column_name": "record_hash", "metadata_columns": [ @@ -46,7 +47,10 @@ def wrapped(): "tables": { "item": { "reuse": False, - "fields": {"product_name": {"rename": "prd_name"}}, + "fields": { + "product_name": {"rename": "prd_name"}, + "comment": {"transform": "skip"}, + }, }, "shiporder": {"fields": {"orderperson": {"transform": False}}}, "companyId": {"choice_transform": False}, @@ -71,7 +75,12 @@ def wrapped(): "config": { "tables": { "shiporder": {"reuse": False}, - "item": {"fields": {"product": {"transform": False}}}, + "item": { + "fields": { + "product": {"transform": False}, + "comment": {"transform": "skip"}, + } + }, }, "metadata_columns": [ { diff --git a/tests/sample_models/orders/equivalent_xml/order1a.xml b/tests/sample_models/orders/equivalent_xml/order1a.xml index 7a6f7c1..82234d8 100644 --- a/tests/sample_models/orders/equivalent_xml/order1a.xml +++ b/tests/sample_models/orders/equivalent_xml/order1a.xml @@ -48,6 +48,7 @@ regular string + this comment should be skipped 8 56.2 EUR diff --git a/tests/sample_models/orders/orders.xsd b/tests/sample_models/orders/orders.xsd index 151fc76..beb0724 100644 --- a/tests/sample_models/orders/orders.xsd +++ b/tests/sample_models/orders/orders.xsd @@ -87,6 +87,7 @@ + diff --git a/tests/sample_models/orders/orders_source_tree_version0.txt b/tests/sample_models/orders/orders_source_tree_version0.txt index aaead3a..5563bcf 100644 --- a/tests/sample_models/orders/orders_source_tree_version0.txt +++ b/tests/sample_models/orders/orders_source_tree_version0.txt @@ -52,6 +52,7 @@ orders: id[1, 1]: string value[1, 1]: string note[0, 1]: string + comment[0, 1]: string quantity[1, 1]: integer price[1, 1]: decimal currency[1, 1]: string diff --git a/tests/sample_models/orders/orders_source_tree_version1.txt b/tests/sample_models/orders/orders_source_tree_version1.txt index aaead3a..5563bcf 100644 --- a/tests/sample_models/orders/orders_source_tree_version1.txt +++ b/tests/sample_models/orders/orders_source_tree_version1.txt @@ -52,6 +52,7 @@ orders: id[1, 1]: string value[1, 1]: string note[0, 1]: string + comment[0, 1]: string quantity[1, 1]: integer price[1, 1]: decimal currency[1, 1]: string diff --git a/tests/sample_models/orders/orders_source_tree_version2.txt b/tests/sample_models/orders/orders_source_tree_version2.txt index aaead3a..5563bcf 100644 --- a/tests/sample_models/orders/orders_source_tree_version2.txt +++ b/tests/sample_models/orders/orders_source_tree_version2.txt @@ -52,6 +52,7 @@ orders: id[1, 1]: string value[1, 1]: string note[0, 1]: string + comment[0, 1]: string quantity[1, 1]: integer price[1, 1]: decimal currency[1, 1]: string diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 3178c88..b880b34 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -145,35 +145,6 @@ def test_field_rename(): assert "orderid" in records[0] -def test_field_skip_column(): - """Test that transform='skip' removes a scalar column from the schema and parsed records""" - model = DataModel( - str(os.path.join(models_path, "orders", "orders.xsd")), - model_config={ - "tables": { - "item": { - "fields": { - "note": {"transform": "skip"}, - } - } - } - }, - ) - item_table = model.tables["itemtype"] - - # column must be absent from both the DataModelTable and the SQLAlchemy table - assert "note" not in item_table.columns - assert "note" not in item_table.table.c - # neighbouring columns are unaffected - assert "quantity" in item_table.table.c - - # parsing works; 'note' is absent from records even for files that contain - doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order1.xml"))) - records = doc.data["itemtype"]["records"] - assert len(records) > 0 - assert "note" not in records[0] - - def test_field_skip_relation(): """Test that transform='skip' on a relation removes its generated columns and prunes the child table""" model = DataModel( @@ -206,34 +177,6 @@ def test_field_skip_relation(): assert all("delivery" not in key for key in records[0]) -@pytest.mark.dbtest -def test_field_skip_db(conn_string): - """Test that skipped fields do not cause DB errors during insert""" - model = DataModel( - str(os.path.join(models_path, "orders", "orders.xsd")), - connection_string=conn_string, - db_schema="test_xml2db_skip", - model_config={ - "tables": { - "item": { - "fields": { - "note": {"transform": "skip"}, - } - } - } - }, - ) - model.create_db_schema() - model.drop_all_tables() - model.create_all_tables() - try: - # insert a file that contains — must succeed without errors - xml_path = str(os.path.join(models_path, "orders", "xml", "order1.xml")) - doc = model.parse_xml(xml_path) - doc.insert_into_target_tables() - finally: - model.drop_all_tables() - @pytest.mark.parametrize( "test_config", From caeb886f759f93511726ab93a526fbeea21f36ef Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 13:54:56 +0000 Subject: [PATCH 8/9] docs: replace em-dashes in configuring.md --- docs/configuring.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuring.md b/docs/configuring.md index 2f547b4..a10f634 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -132,7 +132,7 @@ follow a naming convention that differs from the source schema. The rename applies to both the target table and the staging table. All internal references (data dict keys, foreign key lookups, merge statements) continue to use the original logical name, so only the visible DB column name changes. -Configuration: `"rename":` `"new_column_name"` (no default — omit to keep the original name) +Configuration: `"rename":` `"new_column_name"` (no default; omit to keep the original name) !!! example Rename the `orderid` attribute to `order_id` in the `shiporder` table: @@ -188,7 +188,7 @@ automatically applied `join`, as it would require a complex process of adding a ### Skipping fields -Any field — column or relation — can be excluded from the data model entirely by setting its transform to `"skip"`. +Any field (column or relation) can be excluded from the data model entirely by setting its transform to `"skip"`. The field will be absent from the target table schema and all data for it will be silently dropped during XML parsing. This is useful for PII columns, large binary blobs, or fields that are irrelevant for analysis. @@ -198,7 +198,7 @@ elsewhere in the schema. Configuration: `"transform": "skip"` !!! warning - Skipped fields are not recoverable — data for them is never stored. Round-trip XML reconstruction will omit + Skipped fields are not recoverable: data for them is never stored. Round-trip XML reconstruction will omit any skipped field even when it was present in the source document. !!! example From 7788c4d5ad1a5841f229e21c6f048d8927d16159 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Mon, 22 Jun 2026 14:20:50 +0000 Subject: [PATCH 9/9] docs: cross-reference skip from document_tree_hook descriptions --- docs/configuring.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuring.md b/docs/configuring.md index a10f634..3351287 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -52,10 +52,12 @@ The following options can be passed as top-level keys of the model configuration * `document_tree_hook` (`Callable`): sets a hook function which can modify the data extracted from the XML. It gives direct access to the underlying tree data structure just before it is extracted to be loaded to the database. This can be used, for instance, to prune or modify some parts of the document tree before loading it into the database. The document tree -should of course stay compatible with the data model. +should of course stay compatible with the data model. For simply excluding a field from the schema without custom logic, +the declarative [`"transform": "skip"`](#skipping-fields) option is simpler. * `document_tree_node_hook` (`Callable`): sets a hook function which can modify the data extracted from the XML. It is similar with `document_tree_hook`, but it is called as soon as a node is completed, not waiting for the entire parsing to -finish. It is especially useful if you intend to filter out some nodes and reduce memory footprint while parsing. +finish. It is especially useful if you intend to filter out some nodes and reduce memory footprint while parsing. For +straightforward field exclusion, see [`"transform": "skip"`](#skipping-fields). * `row_numbers` (`bool`): adds `xml2db_row_number` columns either to `n-n` relationships tables, or directly to data tables when deduplication of rows is opted out. This allows recording the original order of elements in the source XML, which is not always respected otherwise. It was implemented primarily for round-trip tests, but could serve other purposes. The