diff --git a/docs/configuring.md b/docs/configuring.md index 86b767b..3351287 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -52,10 +52,12 @@ The following options can be passed as top-level keys of the model configuration * `document_tree_hook` (`Callable`): sets a hook function which can modify the data extracted from the XML. It gives direct access to the underlying tree data structure just before it is extracted to be loaded to the database. This can be used, for instance, to prune or modify some parts of the document tree before loading it into the database. The document tree -should of course stay compatible with the data model. +should of course stay compatible with the data model. For simply excluding a field from the schema without custom logic, +the declarative [`"transform": "skip"`](#skipping-fields) option is simpler. * `document_tree_node_hook` (`Callable`): sets a hook function which can modify the data extracted from the XML. It is similar with `document_tree_hook`, but it is called as soon as a node is completed, not waiting for the entire parsing to -finish. It is especially useful if you intend to filter out some nodes and reduce memory footprint while parsing. +finish. It is especially useful if you intend to filter out some nodes and reduce memory footprint while parsing. For +straightforward field exclusion, see [`"transform": "skip"`](#skipping-fields). * `row_numbers` (`bool`): adds `xml2db_row_number` columns either to `n-n` relationships tables, or directly to data tables when deduplication of rows is opted out. This allows recording the original order of elements in the source XML, which is not always respected otherwise. It was implemented primarily for round-trip tests, but could serve other purposes. The @@ -123,6 +125,44 @@ defined as `sqlalchemy` types and will be passed to the `sqlalchemy.Column` cons You can infer `my_table` and `my_column` when visualizing the data model. +### Renaming columns + +The physical database column name for any field can be overridden while keeping the original XML element name as the +internal logical key. This is useful when XSD element names are awkward, conflict with reserved SQL words, or need to +follow a naming convention that differs from the source schema. + +The rename applies to both the target table and the staging table. All internal references (data dict keys, foreign key +lookups, merge statements) continue to use the original logical name, so only the visible DB column name changes. + +Configuration: `"rename":` `"new_column_name"` (no default; omit to keep the original name) + +!!! example + Rename the `orderid` attribute to `order_id` in the `shiporder` table: + ```python + model_config = { + "tables": { + "shiporder": { + "fields": { + "orderid": {"rename": "order_id"} + } + } + } + } + ``` + + Elevated fields (those pulled up from a child table) are renamed using their prefixed name in the parent table: + ```python + model_config = { + "tables": { + "shiporder": { + "fields": { + "orderperson_name": {"rename": "contact_name"} + } + } + } + } + ``` + ### Joining values for simple types By default, XML simple type elements with types in `["string", "date", "dateTime", "NMTOKEN", "time", "base64Binary", "decimal"]` and max @@ -148,6 +188,36 @@ automatically applied `join`, as it would require a complex process of adding a } ``` +### Skipping fields + +Any field (column or relation) can be excluded from the data model entirely by setting its transform to `"skip"`. +The field will be absent from the target table schema and all data for it will be silently dropped during XML +parsing. This is useful for PII columns, large binary blobs, or fields that are irrelevant for analysis. + +For a skipped relation, the child table is also pruned from the model unless it is referenced by another relation +elsewhere in the schema. + +Configuration: `"transform": "skip"` + +!!! warning + Skipped fields are not recoverable: data for them is never stored. Round-trip XML reconstruction will omit + any skipped field even when it was present in the source document. + +!!! example + Skip an optional scalar column and an optional relation: + ```python + model_config = { + "tables": { + "item": { + "fields": { + "note": {"transform": "skip"}, + "delivery": {"transform": "skip"}, + } + } + } + } + ``` + ### Elevate children to upper level A mandatory child (min occurrences = 1, i.e. `[1, 1]`) is always elevated to its parent by default, as long as it diff --git a/pyproject.toml b/pyproject.toml index b6253bd..92f5c64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xml2db" -version = "0.13.3" +version = "0.14.0" authors = [ { name="Commission de régulation de l'énergie", email="opensource@cre.fr" }, ] diff --git a/src/xml2db/dialect/base.py b/src/xml2db/dialect/base.py index 27f5f94..afcc9a7 100644 --- a/src/xml2db/dialect/base.py +++ b/src/xml2db/dialect/base.py @@ -47,7 +47,7 @@ def __init__(self, **kwargs): # Identifier handling # ------------------------------------------------------------------ - def db_identifier(self, logical_name: str, temp_prefix: bool = False) -> str: + def db_identifier(self, logical_name: str) -> str: """Return the physical database identifier for a logical name. Names longer than :attr:`MAX_IDENTIFIER_LENGTH` are truncated using a @@ -57,15 +57,12 @@ def db_identifier(self, logical_name: str, temp_prefix: bool = False) -> str: Args: logical_name: The full logical name used inside the Python model (e.g. ``"very_long_table_name_derived_from_xsd"``). - temp_prefix: Should we save 14 more characters for the temp prefix? Returns: A string that is safe to use as a database identifier for this backend. Guaranteed to be stable across calls with the same input. """ max_len = self.MAX_IDENTIFIER_LENGTH - if temp_prefix: - max_len += 14 if len(logical_name) <= max_len: return logical_name diff --git a/src/xml2db/model.py b/src/xml2db/model.py index 05e70cf..79818f1 100644 --- a/src/xml2db/model.py +++ b/src/xml2db/model.py @@ -224,7 +224,7 @@ def _build_model(self): # compute a text representation of the original data model and store it self.source_tree = str(root_table) # check user-provided configuration for tables - for tb_config in self.model_config.get("tables", {}): + for tb_config in self.tables_config: if tb_config not in self.names_types_map: raise DataModelConfigError( f"Table '{tb_config}' provided in config does not exist" diff --git a/src/xml2db/table/column.py b/src/xml2db/table/column.py index 9579360..877874c 100644 --- a/src/xml2db/table/column.py +++ b/src/xml2db/table/column.py @@ -25,7 +25,7 @@ class DataModelColumn: is_content: is the column used to store the content value of a mixed complex type? allow_empty: is nullable ? ngroup: a string key used to handle nested sequences (``str(hash(parent_node))``) or ``None`` - model_config: the table-level config dict; may contain a ``fields`` entry with a custom column type + config: the table-level config dict; may contain a ``fields`` entry with a custom column type data_model: the DataModel object it belongs to Attributes: @@ -47,7 +47,7 @@ def __init__( is_content: bool, allow_empty: bool, ngroup: Union[str, None], - model_config: dict[str, Any], + config: dict[str, Any], data_model: "DataModel", ): """Constructor method""" @@ -62,7 +62,7 @@ def __init__( self.is_content = is_content self.allow_empty = allow_empty self.ngroup = ngroup - self.model_config = model_config + self.config = config self.data_model = data_model self.other_table = None # just to avoid a linting warning @@ -100,9 +100,7 @@ def get_sqlalchemy_column(self, temp: bool = False) -> Iterable[Column]: Args: temp: temp table or target table ? """ - # use type specified in config if exists - column_type = self.model_config.get("fields", {}).get(self.name, {}).get( - "type" - ) or self.data_model.dialect.column_type(self, temp) - db_col = self.data_model.dialect.db_identifier(self.name) + field_config = self.config.get("fields", {}).get(self.name, {}) + column_type = field_config.get("type") or self.data_model.dialect.column_type(self, temp) + db_col = self.data_model.dialect.db_identifier(field_config.get("rename", self.name)) yield Column(db_col, column_type, key=self.name) diff --git a/src/xml2db/table/transformed_table.py b/src/xml2db/table/transformed_table.py index 031d858..65382dd 100644 --- a/src/xml2db/table/transformed_table.py +++ b/src/xml2db/table/transformed_table.py @@ -140,6 +140,8 @@ def _get_field_transform( if "transform" in field_config: if field_config["transform"] is False: return None + if field_config["transform"] == "skip": + return "skip" if self._can_transform_field( field_type, field_name, field_config["transform"] ): @@ -292,12 +294,16 @@ def simplify_table(self) -> Tuple[dict, dict]: fields_transforms = {} for field_type, field_name, field in self.fields: if field_type == "col": - if self._get_field_transform("col", field_name) == "join": - fields_transforms[(self.type_name, field_name)] = ( - None, - "join", - ) - out_fields.append(("col", field_name, field)) + transform = self._get_field_transform("col", field_name) + if transform == "skip": + del self.columns[field_name] + else: + if transform == "join": + fields_transforms[(self.type_name, field_name)] = ( + None, + "join", + ) + out_fields.append(("col", field_name, field)) else: # simplify child table @@ -310,7 +316,13 @@ def simplify_table(self) -> Tuple[dict, dict]: # check if children can be "elevated" to the upper level transform = self._get_field_transform(field_type, field_name) - if transform is not None: + if transform == "skip": + if field_type == "rel1": + del self.relations_1[field_name] + elif field_type == "reln": + del self.relations_n[field_name] + # don't add to out_fields, don't set keep_table, don't add to fields_transforms + elif transform is not None: if field_type == "rel1": elevated_fields = self._elevate_relation_1( field_name, transform diff --git a/src/xml2db/xml_converter.py b/src/xml2db/xml_converter.py index a16f401..67a137f 100644 --- a/src/xml2db/xml_converter.py +++ b/src/xml2db/xml_converter.py @@ -46,7 +46,7 @@ def parse_xml( self, xml_file: Union[str, BytesIO], file_path: str = None, - skip_validation: bool = False, + skip_validation: bool = True, recover: bool = False, iterparse: bool = True, ) -> tuple: @@ -57,7 +57,8 @@ def parse_xml( Args: xml_file: An XML file path or file content to be converted file_path: The file path to be printed in logs - skip_validation: Whether we should validate XML against the schema before parsing + skip_validation: Whether to skip XML validation against the schema before parsing + (default ``True``; set to ``False`` to validate) recover: Try to process malformed XML (lxml option) iterparse: Parse XML using iterative parsing, which is a bit slower but uses less memory diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index 50cd865..af1e4c6 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -23,8 +23,14 @@ def wrapped(): { "config": { "tables": { - "shiporder": {"fields": {"orderperson": {"transform": False}}}, - "item": None, + "shiporder": { + "fields": { + "orderperson": {"transform": False}, + "orderid": {"rename": "order_id"}, + } + }, + "item": {"fields": {"comment": {"transform": "skip"}}}, + "orderperson": None, }, "record_hash_column_name": "record_hash", "metadata_columns": [ @@ -39,7 +45,13 @@ def wrapped(): "config": { "row_numbers": True, "tables": { - "item": {"reuse": False}, + "item": { + "reuse": False, + "fields": { + "product_name": {"rename": "prd_name"}, + "comment": {"transform": "skip"}, + }, + }, "shiporder": {"fields": {"orderperson": {"transform": False}}}, "companyId": {"choice_transform": False}, }, @@ -63,7 +75,12 @@ def wrapped(): "config": { "tables": { "shiporder": {"reuse": False}, - "item": {"fields": {"product": {"transform": False}}}, + "item": { + "fields": { + "product": {"transform": False}, + "comment": {"transform": "skip"}, + } + }, }, "metadata_columns": [ { diff --git a/tests/sample_models/orders/equivalent_xml/order1a.xml b/tests/sample_models/orders/equivalent_xml/order1a.xml index 7a6f7c1..82234d8 100644 --- a/tests/sample_models/orders/equivalent_xml/order1a.xml +++ b/tests/sample_models/orders/equivalent_xml/order1a.xml @@ -48,6 +48,7 @@ regular string + this comment should be skipped 8 56.2 EUR diff --git a/tests/sample_models/orders/orders.xsd b/tests/sample_models/orders/orders.xsd index 151fc76..beb0724 100644 --- a/tests/sample_models/orders/orders.xsd +++ b/tests/sample_models/orders/orders.xsd @@ -87,6 +87,7 @@ + diff --git a/tests/sample_models/orders/orders_ddl_mssql_version0.sql b/tests/sample_models/orders/orders_ddl_mssql_version0.sql index a4c4121..e7c4a05 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version0.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version0.sql @@ -76,7 +76,7 @@ CREATE TABLE item_product_features_stringfeature ( CREATE TABLE shiporder ( pk_shiporder INTEGER NOT NULL IDENTITY, - orderid VARCHAR(1000) NULL, + order_id VARCHAR(1000) NULL, processed_at DATETIMEOFFSET NULL, fk_orderperson INTEGER NULL, shipto_fk_orderperson INTEGER NULL, diff --git a/tests/sample_models/orders/orders_ddl_mssql_version1.sql b/tests/sample_models/orders/orders_ddl_mssql_version1.sql index 7ec3ed2..51f3837 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version1.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version1.sql @@ -81,7 +81,7 @@ CREATE TABLE item ( temp_pk_item INTEGER NULL, fk_parent_shiporder INTEGER NULL, xml2db_row_number INTEGER NOT NULL, - product_name VARCHAR(1000) NULL, + prd_name VARCHAR(1000) NULL, product_version VARCHAR(1000) NULL, note VARCHAR(1000) NULL, quantity INTEGER NULL, diff --git a/tests/sample_models/orders/orders_ddl_mysql_version0.sql b/tests/sample_models/orders/orders_ddl_mysql_version0.sql index 7b7b79f..6da9028 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version0.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version0.sql @@ -76,7 +76,7 @@ CREATE TABLE item_product_features_stringfeature ( CREATE TABLE shiporder ( pk_shiporder INTEGER NOT NULL AUTO_INCREMENT, - orderid VARCHAR(255), + order_id VARCHAR(255), processed_at DATETIME, fk_orderperson INTEGER, shipto_fk_orderperson INTEGER, diff --git a/tests/sample_models/orders/orders_ddl_mysql_version1.sql b/tests/sample_models/orders/orders_ddl_mysql_version1.sql index 07865a6..8a381d3 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version1.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version1.sql @@ -81,7 +81,7 @@ CREATE TABLE item ( temp_pk_item INTEGER, fk_parent_shiporder INTEGER, xml2db_row_number INTEGER NOT NULL, - product_name VARCHAR(255), + prd_name VARCHAR(255), product_version VARCHAR(255), note VARCHAR(255), quantity INTEGER, diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version0.sql b/tests/sample_models/orders/orders_ddl_postgresql_version0.sql index ee1c43d..fdbd56c 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version0.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version0.sql @@ -76,7 +76,7 @@ CREATE TABLE item_product_features_stringfeature ( CREATE TABLE shiporder ( pk_shiporder SERIAL NOT NULL, - orderid VARCHAR(1000), + order_id VARCHAR(1000), processed_at TIMESTAMP WITH TIME ZONE, fk_orderperson INTEGER, shipto_fk_orderperson INTEGER, diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version1.sql b/tests/sample_models/orders/orders_ddl_postgresql_version1.sql index f7f8fc2..03dce19 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version1.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version1.sql @@ -81,7 +81,7 @@ CREATE TABLE item ( temp_pk_item INTEGER, fk_parent_shiporder INTEGER, xml2db_row_number INTEGER NOT NULL, - product_name VARCHAR(1000), + prd_name VARCHAR(1000), product_version VARCHAR(1000), note VARCHAR(1000), quantity INTEGER, diff --git a/tests/sample_models/orders/orders_source_tree_version0.txt b/tests/sample_models/orders/orders_source_tree_version0.txt index aaead3a..5563bcf 100644 --- a/tests/sample_models/orders/orders_source_tree_version0.txt +++ b/tests/sample_models/orders/orders_source_tree_version0.txt @@ -52,6 +52,7 @@ orders: id[1, 1]: string value[1, 1]: string note[0, 1]: string + comment[0, 1]: string quantity[1, 1]: integer price[1, 1]: decimal currency[1, 1]: string diff --git a/tests/sample_models/orders/orders_source_tree_version1.txt b/tests/sample_models/orders/orders_source_tree_version1.txt index aaead3a..5563bcf 100644 --- a/tests/sample_models/orders/orders_source_tree_version1.txt +++ b/tests/sample_models/orders/orders_source_tree_version1.txt @@ -52,6 +52,7 @@ orders: id[1, 1]: string value[1, 1]: string note[0, 1]: string + comment[0, 1]: string quantity[1, 1]: integer price[1, 1]: decimal currency[1, 1]: string diff --git a/tests/sample_models/orders/orders_source_tree_version2.txt b/tests/sample_models/orders/orders_source_tree_version2.txt index aaead3a..5563bcf 100644 --- a/tests/sample_models/orders/orders_source_tree_version2.txt +++ b/tests/sample_models/orders/orders_source_tree_version2.txt @@ -52,6 +52,7 @@ orders: id[1, 1]: string value[1, 1]: string note[0, 1]: string + comment[0, 1]: string quantity[1, 1]: integer price[1, 1]: decimal currency[1, 1]: string diff --git a/tests/test_conversions.py b/tests/test_conversions.py index b279cfa..b880b34 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -117,6 +117,67 @@ def test_document_tree_to_xml(test_config): assert xml == ref_xml +def test_field_rename(): + """Test that 'rename' in field config sets the physical DB column name without affecting internal logic""" + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "tables": { + "shiporder": { + "fields": { + "orderid": {"rename": "order_id"}, + } + } + } + }, + ) + shiporder_table = model.tables["shipordertype"] + + # physical column name in the DB uses the renamed value + assert shiporder_table.table.c["orderid"].name == "order_id" + # temp table should also use the renamed physical name + assert shiporder_table.temp_table.c["orderid"].name == "order_id" + + # parsing still works and data dict uses the original logical name + doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order1.xml"))) + records = doc.data["shipordertype"]["records"] + assert len(records) > 0 + assert "orderid" in records[0] + + +def test_field_skip_relation(): + """Test that transform='skip' on a relation removes its generated columns and prunes the child table""" + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "tables": { + "item": { + "fields": { + "delivery": {"transform": "skip"}, + } + } + } + }, + ) + item_table = model.tables["itemtype"] + + # delivery is a rel1 that would normally be elevated → its FK columns must be absent + assert "delivery_from_fk_orderperson" not in item_table.table.c + assert "delivery_to_fk_orderperson" not in item_table.table.c + # 'delivery' is also absent from relations_1 (no FK column, no merge statements) + assert "delivery" not in item_table.relations_1 + + # deliveryType has no other parents, so it should be pruned from the model + assert "deliveryType" not in model.tables + + # parsing works even for XML files that contain ; delivery data is silently dropped + doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order3.xml"))) + records = doc.data["itemtype"]["records"] + assert len(records) > 0 + assert all("delivery" not in key for key in records[0]) + + + @pytest.mark.parametrize( "test_config", [