From a4b23a3b61feef2621477f21bf21757dd1e72c71 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 18:16:46 +0000 Subject: [PATCH 01/30] feat: YAML/typed config support and interactive CLI schema explorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `config.py` with `ModelConfig`, `TableConfig`, `FieldConfig` TypedDicts and `load_config(path)` / `parse_yaml_config(text)` YAML helpers; callable-only keys (hooks, custom hash fn) raise `DataModelConfigError` when present in YAML - Add `cli.py` with two subcommands: - `xml2db render [--config] [--format erd|target-tree|source-tree]` - `xml2db serve [--config] [--port]` — launches a browser-based ERD explorer with in-page YAML editing, Apply/Save buttons, and SSE-driven live reload when the XSD changes on disk All plumbing uses stdlib only (http.server, threading, SSE); Mermaid.js loaded from CDN - Export new public symbols from `xml2db.__init__` - Add PyYAML ≥6.0 as a runtime dependency; register `xml2db` console script Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B94Mo7W3K5YiT4MuTDgg5c --- pyproject.toml | 4 + src/xml2db/__init__.py | 6 + src/xml2db/cli.py | 450 +++++++++++++++++++++++++++++++++++++++++ src/xml2db/config.py | 91 +++++++++ 4 files changed, 551 insertions(+) create mode 100644 src/xml2db/cli.py create mode 100644 src/xml2db/config.py diff --git a/pyproject.toml b/pyproject.toml index 92f5c64..f73a444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,12 +20,16 @@ dependencies = [ "sqlalchemy>1.4", "xmlschema>=3.3.2", "lxml>=5.1.0", + "pyyaml>=6.0", ] [project.optional-dependencies] docs = ["mkdocs-material>=9.5.34", "mkdocstrings-python>=1.11.1"] tests = ["pytest>=7.0"] +[project.scripts] +xml2db = "xml2db.cli:main" + [project.urls] "Documentation" = "https://cre-dev.github.io/xml2db" "Repository" = "https://github.com/cre-dev/xml2db" diff --git a/src/xml2db/__init__.py b/src/xml2db/__init__.py index 07bd4ad..aafaf59 100644 --- a/src/xml2db/__init__.py +++ b/src/xml2db/__init__.py @@ -8,6 +8,7 @@ DataModelRelationN, DataModelRelation1, ) +from .config import ModelConfig, TableConfig, FieldConfig, load_config, parse_yaml_config __all__ = [ "DataModel", @@ -20,4 +21,9 @@ "DataModelColumn", "DataModelRelation1", "DataModelRelationN", + "ModelConfig", + "TableConfig", + "FieldConfig", + "load_config", + "parse_yaml_config", ] diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py new file mode 100644 index 0000000..60a22fc --- /dev/null +++ b/src/xml2db/cli.py @@ -0,0 +1,450 @@ +"""Command-line interface for xml2db.""" +from __future__ import annotations + +import argparse +import html as _html +import json +import os +import queue +import threading +import time +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Optional + +from .config import load_config, parse_yaml_config +from .model import DataModel + + +# --------------------------------------------------------------------------- +# render command +# --------------------------------------------------------------------------- + +def cmd_render(args: argparse.Namespace) -> None: + config = load_config(args.config) if args.config else None + model = DataModel( + xsd_file=args.xsd_file, + short_name=args.short_name, + model_config=config, + ) + fmt = args.format + if fmt == "erd": + output = model.get_entity_rel_diagram(text_context=False) + elif fmt == "target-tree": + output = model.target_tree + else: + output = model.source_tree + + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(output) + print(f"Written to {args.output}") + else: + print(output) + + +# --------------------------------------------------------------------------- +# serve command — shared state +# --------------------------------------------------------------------------- + +class _State: + def __init__(self, xsd_file: str, config_file: Optional[str], short_name: str): + self.xsd_file = xsd_file + self.config_file = config_file + self.short_name = short_name + self.lock = threading.Lock() + self.sse_clients: list[queue.Queue] = [] + + self.config_yaml = "" + if config_file and os.path.exists(config_file): + with open(config_file, encoding="utf-8") as f: + self.config_yaml = f.read() + + self.erd = "" + self.build_error = "" + self.xsd_mtime = self._xsd_mtime() + self._rebuild() + + def _xsd_mtime(self) -> Optional[float]: + try: + return os.path.getmtime(self.xsd_file) + except OSError: + return None + + def _rebuild(self) -> None: + try: + cfg = parse_yaml_config(self.config_yaml) if self.config_yaml.strip() else None + model = DataModel( + xsd_file=self.xsd_file, + short_name=self.short_name, + model_config=cfg, + ) + self.erd = model.get_entity_rel_diagram(text_context=False) + self.build_error = "" + except Exception as exc: + self.build_error = str(exc) + + def apply_yaml(self, text: str) -> tuple[str, str]: + """Parse + validate YAML text and rebuild ERD. Returns (erd, error).""" + try: + cfg = parse_yaml_config(text) if text.strip() else None + model = DataModel( + xsd_file=self.xsd_file, + short_name=self.short_name, + model_config=cfg, + ) + erd = model.get_entity_rel_diagram(text_context=False) + with self.lock: + self.config_yaml = text + self.erd = erd + self.build_error = "" + return erd, "" + except Exception as exc: + return "", str(exc) + + def save_config(self, text: str) -> str: + """Save YAML text to config file. Returns error string or empty string.""" + if not self.config_file: + return "No --config file was specified; cannot save." + try: + with open(self.config_file, "w", encoding="utf-8") as f: + f.write(text) + with self.lock: + self.config_yaml = text + return "" + except OSError as exc: + return str(exc) + + def watch_xsd(self) -> None: + while True: + time.sleep(1) + mtime = self._xsd_mtime() + if mtime is not None and mtime != self.xsd_mtime: + self.xsd_mtime = mtime + self._rebuild() + self._push("reload") + + def _push(self, event: str) -> None: + with self.lock: + clients = list(self.sse_clients) + for q in clients: + try: + q.put_nowait(event) + except queue.Full: + pass + + def add_sse_client(self, q: queue.Queue) -> None: + with self.lock: + self.sse_clients.append(q) + + def remove_sse_client(self, q: queue.Queue) -> None: + with self.lock: + try: + self.sse_clients.remove(q) + except ValueError: + pass + + +# --------------------------------------------------------------------------- +# HTTP handler +# --------------------------------------------------------------------------- + +def _make_handler(state: _State): + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): # silence default stderr logging + pass + + def do_GET(self): + if self.path == "/": + self._serve_html() + elif self.path == "/erd": + self._json({"erd": state.erd, "error": state.build_error}) + elif self.path == "/events": + self._sse() + else: + self.send_error(404) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length).decode("utf-8", errors="replace") + if self.path == "/config": + erd, error = state.apply_yaml(body) + self._json({"erd": erd, "error": error}) + elif self.path == "/save": + self._json({"error": state.save_config(body)}) + else: + self.send_error(404) + + def _json(self, data: dict): + body = json.dumps(data).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _serve_html(self): + body = _build_html(state).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _sse(self): + q: queue.Queue = queue.Queue(maxsize=16) + state.add_sse_client(q) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "keep-alive") + self.end_headers() + try: + while True: + try: + event = q.get(timeout=25) + self.wfile.write(f"event: {event}\ndata:\n\n".encode()) + self.wfile.flush() + except queue.Empty: + self.wfile.write(b": keepalive\n\n") + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + pass + finally: + state.remove_sse_client(q) + + return Handler + + +# --------------------------------------------------------------------------- +# HTML template +# --------------------------------------------------------------------------- + +_HTML = """\ + + + + +xml2db — TMPL_TITLE + + + + +
+ xml2db + TMPL_HEADER + ready +
+
+
+
model_config (YAML)
+ +
+
+ + +
+
+ +
+ + + +""" + + +def _build_html(state: _State) -> str: + header = os.path.basename(state.xsd_file) + if state.config_file: + header += f" · {os.path.basename(state.config_file)}" + save_label = "Save to file" if state.config_file else "Save (no file)" + return ( + _HTML + .replace("TMPL_TITLE", _html.escape(header)) + .replace("TMPL_HEADER", _html.escape(header)) + .replace("TMPL_CONFIG_YAML", _html.escape(state.config_yaml)) + .replace("TMPL_SAVE_LABEL", _html.escape(save_label)) + .replace("TMPL_HAS_SAVE_TARGET", "true" if state.config_file else "false") + .replace("TMPL_INITIAL_ERD", json.dumps(state.erd)) + .replace("TMPL_INITIAL_ERROR", json.dumps(state.build_error)) + ) + + +# --------------------------------------------------------------------------- +# serve command +# --------------------------------------------------------------------------- + +def cmd_serve(args: argparse.Namespace) -> None: + print(f"Loading schema: {args.xsd_file}") + state = _State( + xsd_file=args.xsd_file, + config_file=args.config, + short_name=args.short_name, + ) + if state.build_error: + print(f"Warning: initial build error: {state.build_error}") + + threading.Thread(target=state.watch_xsd, daemon=True).start() + + server = ThreadingHTTPServer(("127.0.0.1", args.port), _make_handler(state)) + url = f"http://127.0.0.1:{args.port}" + print(f"Serving at {url} (Ctrl+C to stop)") + + if not args.no_browser: + threading.Timer(0.4, webbrowser.open, args=[url]).start() + + try: + server.serve_forever() + except KeyboardInterrupt: + server.shutdown() + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + prog="xml2db", + description="xml2db — explore and configure XSD-to-database mappings", + ) + sub = parser.add_subparsers(dest="command", required=True) + + r = sub.add_parser("render", help="Print ERD or tree representation to stdout or a file") + r.add_argument("xsd_file", help="Path to the XSD schema file") + r.add_argument("--config", "-c", metavar="FILE", help="YAML model config file") + r.add_argument( + "--format", "-f", + choices=["erd", "target-tree", "source-tree"], + default="erd", + help="Output format (default: erd)", + ) + r.add_argument("--output", "-o", metavar="FILE", help="Write to file instead of stdout") + r.add_argument("--short-name", default="DocumentRoot", metavar="NAME", + help="Data model short name (default: DocumentRoot)") + + s = sub.add_parser("serve", help="Launch an interactive schema explorer in the browser") + s.add_argument("xsd_file", help="Path to the XSD schema file") + s.add_argument("--config", "-c", metavar="FILE", + help="YAML model config file (editable and saveable from the browser)") + s.add_argument("--port", "-p", type=int, default=8765, metavar="PORT", + help="HTTP port (default: 8765)") + s.add_argument("--no-browser", action="store_true", + help="Do not open the browser automatically") + s.add_argument("--short-name", default="DocumentRoot", metavar="NAME", + help="Data model short name (default: DocumentRoot)") + + args = parser.parse_args() + if args.command == "render": + cmd_render(args) + else: + cmd_serve(args) + + +if __name__ == "__main__": + main() diff --git a/src/xml2db/config.py b/src/xml2db/config.py new file mode 100644 index 0000000..3608e44 --- /dev/null +++ b/src/xml2db/config.py @@ -0,0 +1,91 @@ +"""Typed config definitions and YAML loading utilities for xml2db.""" +from __future__ import annotations + +from typing import Any, TypedDict + +from .exceptions import DataModelConfigError + +# Keys that require Python callables — cannot be expressed in YAML +_CALLABLE_ONLY_KEYS: frozenset[str] = frozenset( + {"document_tree_hook", "document_tree_node_hook", "record_hash_constructor"} +) + + +class FieldConfig(TypedDict, total=False): + type: Any # SQLAlchemy column type — Python only, not settable via YAML + rename: str + transform: Any # str | False + + +class TableConfig(TypedDict, total=False): + reuse: bool + as_columnstore: bool + choice_transform: bool + extra_args: Any # list | tuple | callable — callable is Python only + fields: dict[str, FieldConfig] + + +class ModelConfig(TypedDict, total=False): + as_columnstore: bool + row_numbers: bool + document_tree_hook: Any # callable — Python only + document_tree_node_hook: Any # callable — Python only + record_hash_column_name: str + record_hash_constructor: Any # callable — Python only + record_hash_size: int + metadata_columns: list + tables: dict[str, TableConfig] + + +def parse_yaml_config(text: str) -> ModelConfig: + """Parse a YAML string into a model config dict. + + Args: + text: YAML text representing a model config mapping. + + Returns: + A ModelConfig dict. + + Raises: + DataModelConfigError: If the YAML contains keys that require Python objects + (callables), or if the top level is not a mapping. + ImportError: If PyYAML is not installed. + """ + try: + import yaml + except ImportError: + raise ImportError( + "PyYAML is required to parse YAML config. Install with: pip install pyyaml" + ) + + data = yaml.safe_load(text) + if data is None: + return {} + if not isinstance(data, dict): + raise DataModelConfigError( + "Config must be a YAML mapping at the top level." + ) + for key in _CALLABLE_ONLY_KEYS: + if key in data: + raise DataModelConfigError( + f"'{key}' requires a Python callable and cannot be set in a YAML config " + "file. Pass a Python dict with the callable directly instead." + ) + return data + + +def load_config(path: str) -> ModelConfig: + """Load a model config from a YAML file. + + Args: + path: Path to a YAML file containing the model config. + + Returns: + A ModelConfig dict. + + Raises: + DataModelConfigError: If the file contains keys that require Python objects. + ImportError: If PyYAML is not installed. + """ + with open(path, encoding="utf-8") as f: + return parse_yaml_config(f.read()) From 43e5211a4411d1e7d8a3cf9f4feda256b23b2703 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 20:14:27 +0000 Subject: [PATCH 02/30] feat: string SQLAlchemy types, IndexConfig TypedDict, and full YAML validation - Add resolve_sa_type() to config.py: resolves "String(100)", "Integer", "DateTime(timezone=True)" etc. to SQLAlchemy type instances; non-string values are passed through unchanged so Python dict configs are unaffected - field.type in YAML config is now resolved via resolve_sa_type() at column build time (column.py); supports all common SQLAlchemy types with args - metadata_columns[*].type is now resolved via resolve_sa_type() at table build time (reused_table.py) - Add IndexConfig TypedDict and accept dict-form extra_args entries {"name": ..., "columns": [...], "unique": ...} in YAML config; converted to sqlalchemy.Index objects in DataModelTable._validate_config() (table.py) - Add MetadataColumnConfig TypedDict with typed name/type and common Column kwargs (nullable, default, server_default, comment, index, unique) - Extend YAML validation in parse_yaml_config() to cover all config levels: metadata_columns[*].type and tables[*].fields[*].type must be strings; model-level callable-only keys already checked Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B94Mo7W3K5YiT4MuTDgg5c --- src/xml2db/config.py | 200 ++++++++++++++++++++++++++++--- src/xml2db/table/column.py | 8 +- src/xml2db/table/reused_table.py | 3 +- src/xml2db/table/table.py | 18 ++- 4 files changed, 207 insertions(+), 22 deletions(-) diff --git a/src/xml2db/config.py b/src/xml2db/config.py index 3608e44..758d408 100644 --- a/src/xml2db/config.py +++ b/src/xml2db/config.py @@ -1,8 +1,11 @@ """Typed config definitions and YAML loading utilities for xml2db.""" from __future__ import annotations +import re from typing import Any, TypedDict +import sqlalchemy as sa + from .exceptions import DataModelConfigError # Keys that require Python callables — cannot be expressed in YAML @@ -10,33 +13,197 @@ {"document_tree_hook", "document_tree_node_hook", "record_hash_constructor"} ) +# --------------------------------------------------------------------------- +# SQLAlchemy type resolver +# --------------------------------------------------------------------------- + +_SA_TYPE_MAP: dict[str, type] = { + "String": sa.String, + "Text": sa.Text, + "Integer": sa.Integer, + "BigInteger": sa.BigInteger, + "SmallInteger": sa.SmallInteger, + "Float": sa.Float, + "Double": sa.Double, + "Numeric": sa.Numeric, + "Boolean": sa.Boolean, + "DateTime": sa.DateTime, + "Date": sa.Date, + "Time": sa.Time, + "LargeBinary": sa.LargeBinary, + "JSON": sa.JSON, + "Uuid": sa.Uuid, + "UUID": sa.Uuid, +} + +_TYPE_RE = re.compile(r"^(\w+)(?:\(([^)]*)\))?$") + + +def _parse_type_arg(s: str) -> Any: + s = s.strip() + if s in ("True", "true"): + return True + if s in ("False", "false"): + return False + try: + return int(s) + except ValueError: + pass + try: + return float(s) + except ValueError: + pass + if len(s) >= 2 and s[0] in ('"', "'") and s[-1] == s[0]: + return s[1:-1] + raise DataModelConfigError(f"Cannot parse type argument value: '{s}'") + + +def resolve_sa_type(type_spec: Any) -> Any: + """Resolve a SQLAlchemy type name string to a type instance, or pass through. + + Args: + type_spec: A string such as ``"String(100)"``, ``"Integer"``, + ``"DateTime(timezone=True)"``, or an existing SQLAlchemy type + instance / class (returned unchanged). + + Returns: + A SQLAlchemy type instance. + + Raises: + DataModelConfigError: If the string cannot be parsed or names an + unknown type. + """ + if not isinstance(type_spec, str): + return type_spec # already a SQLAlchemy type — pass through + + m = _TYPE_RE.match(type_spec.strip()) + if not m: + raise DataModelConfigError(f"Cannot parse SQLAlchemy type: '{type_spec}'") + + type_name, args_str = m.group(1), m.group(2) + if type_name not in _SA_TYPE_MAP: + raise DataModelConfigError( + f"Unknown SQLAlchemy type '{type_name}'. " + f"Supported: {', '.join(sorted(_SA_TYPE_MAP))}" + ) + + type_cls = _SA_TYPE_MAP[type_name] + if not args_str or not args_str.strip(): + return type_cls() + + args: list = [] + kwargs: dict = {} + for part in args_str.split(","): + part = part.strip() + if not part: + continue + if "=" in part: + k, _, v = part.partition("=") + kwargs[k.strip()] = _parse_type_arg(v) + else: + args.append(_parse_type_arg(part)) + + return type_cls(*args, **kwargs) + + +# --------------------------------------------------------------------------- +# TypedDicts +# --------------------------------------------------------------------------- class FieldConfig(TypedDict, total=False): - type: Any # SQLAlchemy column type — Python only, not settable via YAML + type: Any # str (e.g. "String(100)") or SQLAlchemy type — Python only when not a str rename: str transform: Any # str | False +class IndexConfig(TypedDict, total=False): + name: str # Index name (required) + columns: list[str] # Column names (required) + unique: bool + + class TableConfig(TypedDict, total=False): reuse: bool as_columnstore: bool choice_transform: bool - extra_args: Any # list | tuple | callable — callable is Python only + # In YAML: list of IndexConfig dicts. In Python: list/tuple of SQLAlchemy + # schema objects, or a zero-argument callable returning such a list. + extra_args: list[IndexConfig] | list[Any] | Any fields: dict[str, FieldConfig] +class MetadataColumnConfig(TypedDict, total=False): + name: str # Required — column name + type: Any # Required — str (e.g. "String(100)") or SQLAlchemy type + nullable: bool + default: Any + server_default: Any + comment: str + index: bool + unique: bool + + class ModelConfig(TypedDict, total=False): as_columnstore: bool row_numbers: bool - document_tree_hook: Any # callable — Python only - document_tree_node_hook: Any # callable — Python only + document_tree_hook: Any # callable — Python only + document_tree_node_hook: Any # callable — Python only record_hash_column_name: str - record_hash_constructor: Any # callable — Python only + record_hash_constructor: Any # callable — Python only record_hash_size: int - metadata_columns: list + metadata_columns: list[MetadataColumnConfig] tables: dict[str, TableConfig] +# --------------------------------------------------------------------------- +# YAML parsing + validation +# --------------------------------------------------------------------------- + +def _check_config(data: dict, *, from_yaml: bool) -> None: + """Validate a raw config dict, raising DataModelConfigError on Python-only values.""" + if not from_yaml: + return # dict configs are trusted; no checks needed + + # Model-level callable-only keys + for key in _CALLABLE_ONLY_KEYS: + if key in data: + raise DataModelConfigError( + f"'{key}' requires a Python callable and cannot be set in a YAML config " + "file. Pass a Python dict with the callable directly instead." + ) + + # metadata_columns — each entry's 'type' must be a string + for i, col_cfg in enumerate(data.get("metadata_columns", [])): + if not isinstance(col_cfg, dict): + raise DataModelConfigError( + f"metadata_columns[{i}] must be a mapping with at least 'name' and 'type'." + ) + if "type" in col_cfg and not isinstance(col_cfg["type"], str): + raise DataModelConfigError( + f"metadata_columns[{i}].type must be a string (e.g. 'String(100)') " + "in a YAML config file." + ) + + # Table- and field-level checks + for table_name, table_cfg in data.get("tables", {}).items(): + if not isinstance(table_cfg, dict): + continue + # extra_args must be a list (callable not possible from YAML) + if "extra_args" in table_cfg and callable(table_cfg["extra_args"]): + raise DataModelConfigError( + f"tables.{table_name}.extra_args is a callable and cannot be set in a " + "YAML config file. Pass a Python dict with the callable directly instead." + ) + for field_name, field_cfg in table_cfg.get("fields", {}).items(): + if not isinstance(field_cfg, dict): + continue + if "type" in field_cfg and not isinstance(field_cfg["type"], str): + raise DataModelConfigError( + f"tables.{table_name}.fields.{field_name}.type must be a string " + "(e.g. 'String(100)') in a YAML config file." + ) + + def parse_yaml_config(text: str) -> ModelConfig: """Parse a YAML string into a model config dict. @@ -44,11 +211,12 @@ def parse_yaml_config(text: str) -> ModelConfig: text: YAML text representing a model config mapping. Returns: - A ModelConfig dict. + A :class:`ModelConfig` dict. Raises: - DataModelConfigError: If the YAML contains keys that require Python objects - (callables), or if the top level is not a mapping. + DataModelConfigError: If the config contains keys that require Python + objects (callables or SQLAlchemy types) or if the top level is not + a mapping. ImportError: If PyYAML is not installed. """ try: @@ -62,15 +230,9 @@ def parse_yaml_config(text: str) -> ModelConfig: if data is None: return {} if not isinstance(data, dict): - raise DataModelConfigError( - "Config must be a YAML mapping at the top level." - ) - for key in _CALLABLE_ONLY_KEYS: - if key in data: - raise DataModelConfigError( - f"'{key}' requires a Python callable and cannot be set in a YAML config " - "file. Pass a Python dict with the callable directly instead." - ) + raise DataModelConfigError("Config must be a YAML mapping at the top level.") + + _check_config(data, from_yaml=True) return data @@ -81,7 +243,7 @@ def load_config(path: str) -> ModelConfig: path: Path to a YAML file containing the model config. Returns: - A ModelConfig dict. + A :class:`ModelConfig` dict. Raises: DataModelConfigError: If the file contains keys that require Python objects. diff --git a/src/xml2db/table/column.py b/src/xml2db/table/column.py index 877874c..2eedc4d 100644 --- a/src/xml2db/table/column.py +++ b/src/xml2db/table/column.py @@ -3,6 +3,8 @@ from sqlalchemy import Column +from ..config import resolve_sa_type + if TYPE_CHECKING: from ..model import DataModel @@ -101,6 +103,10 @@ def get_sqlalchemy_column(self, temp: bool = False) -> Iterable[Column]: temp: temp table or target table ? """ field_config = self.config.get("fields", {}).get(self.name, {}) - column_type = field_config.get("type") or self.data_model.dialect.column_type(self, temp) + raw_type = field_config.get("type") + column_type = ( + resolve_sa_type(raw_type) if raw_type is not None + else self.data_model.dialect.column_type(self, temp) + ) db_col = self.data_model.dialect.db_identifier(field_config.get("rename", self.name)) yield Column(db_col, column_type, key=self.name) diff --git a/src/xml2db/table/reused_table.py b/src/xml2db/table/reused_table.py index a0a9fe3..a0ea178 100644 --- a/src/xml2db/table/reused_table.py +++ b/src/xml2db/table/reused_table.py @@ -12,6 +12,7 @@ from .column import DataModelColumn from .transformed_table import DataModelTableTransformed +from ..config import resolve_sa_type def shorten_str(x: str, max_len: int = 30) -> str: @@ -61,7 +62,7 @@ def get_col(temp=False): for metadata_col in self.data_model.model_config["metadata_columns"]: yield Column( metadata_col["name"], - metadata_col["type"], + resolve_sa_type(metadata_col["type"]), **{ k: v for k, v in metadata_col.items() diff --git a/src/xml2db/table/table.py b/src/xml2db/table/table.py index db64a99..64417bb 100644 --- a/src/xml2db/table/table.py +++ b/src/xml2db/table/table.py @@ -14,6 +14,22 @@ logger = logging.getLogger(__name__) +def _resolve_extra_args(extra_args): + """Convert dict-form index specs to sqlalchemy.Index objects, pass others through.""" + if callable(extra_args): + return extra_args # callable — Python-only, unchanged + result = [] + for item in extra_args: + if isinstance(item, dict): + name = item.get("name") + columns = item.get("columns", []) + kwargs = {k: v for k, v in item.items() if k not in ("name", "columns")} + result.append(sqlalchemy.Index(name, *columns, **kwargs)) + else: + result.append(item) + return result + + class DataModelTable: """A class representing a database table translated from an XML schema complex type @@ -106,7 +122,7 @@ def _validate_config(self, cfg): or callable(cfg["extra_args"]) ): raise DataModelConfigError("extra_args must be a list, a tuple or callable") - config["extra_args"] = cfg.get("extra_args", []) + config["extra_args"] = _resolve_extra_args(cfg.get("extra_args", [])) if "choice_transform" in cfg: config["choice_transform"] = check_type( cfg, "choice_transform", bool, False From c5ffb4d4253e10326901d6b40ca540a545e192b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 21:40:22 +0000 Subject: [PATCH 03/30] Add --format ddl to xml2db render CLI command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds DDL SQL output (CREATE TABLE + CREATE INDEX) to the render subcommand, matching the dialect-specific output already tested in snapshot .sql files. A new --db-type option (postgresql, mssql, mysql, …) selects the SQLAlchemy dialect used for compilation; defaults to generic if omitted. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B94Mo7W3K5YiT4MuTDgg5c --- src/xml2db/cli.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py index 60a22fc..6cddcf2 100644 --- a/src/xml2db/cli.py +++ b/src/xml2db/cli.py @@ -20,20 +20,42 @@ # render command # --------------------------------------------------------------------------- +def _get_sa_dialect(db_type: str | None): + """Return a SQLAlchemy dialect instance for DDL compilation, or None for generic.""" + if db_type is None: + return None + from sqlalchemy.dialects import postgresql, mssql, mysql + _map = { + "postgresql": postgresql, + "mssql": mssql, + "mysql": mysql, + "mariadb": mysql, + } + module = _map.get(db_type) + return module.dialect() if module is not None else None + + def cmd_render(args: argparse.Namespace) -> None: config = load_config(args.config) if args.config else None + db_type = getattr(args, "db_type", None) model = DataModel( xsd_file=args.xsd_file, short_name=args.short_name, model_config=config, + db_type=db_type, ) fmt = args.format if fmt == "erd": output = model.get_entity_rel_diagram(text_context=False) elif fmt == "target-tree": output = model.target_tree - else: + elif fmt == "source-tree": output = model.source_tree + else: # ddl + sa_dialect = _get_sa_dialect(db_type) + parts = [str(s.compile(dialect=sa_dialect)) for s in model.get_all_create_table_statements()] + parts += [str(s.compile(dialect=sa_dialect)) + "\n\n" for s in model.get_all_create_index_statements()] + output = "".join(parts) if args.output: with open(args.output, "w", encoding="utf-8") as f: @@ -420,13 +442,15 @@ def main() -> None: r.add_argument("--config", "-c", metavar="FILE", help="YAML model config file") r.add_argument( "--format", "-f", - choices=["erd", "target-tree", "source-tree"], + choices=["erd", "target-tree", "source-tree", "ddl"], default="erd", help="Output format (default: erd)", ) r.add_argument("--output", "-o", metavar="FILE", help="Write to file instead of stdout") r.add_argument("--short-name", default="DocumentRoot", metavar="NAME", help="Data model short name (default: DocumentRoot)") + r.add_argument("--db-type", metavar="BACKEND", default=None, + help="Database backend for DDL output (postgresql, mssql, mysql, …)") s = sub.add_parser("serve", help="Launch an interactive schema explorer in the browser") s.add_argument("xsd_file", help="Path to the XSD schema file") From d601ee729bec20f4c3c789013a87f6cc85d40f6c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 21:52:19 +0000 Subject: [PATCH 04/30] Rework serve command: debounced auto-rebuild, output tabs, simplified state - Removes XSD file watching and SSE/EventSource (XSD is immutable) - Removes the Apply button; textarea changes now trigger a debounced rebuild (500 ms) - Adds four output tabs: ERD, Target tree, Source tree, DDL - On build error: shows error in red below editor, keeps last successful output visible - Save button writes to --config path or ./model_config.yml if none was given - Adds --db-type option to serve (for DDL tab dialect) - All outputs (erd, target_tree, source_tree, ddl) computed in one rebuild pass Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B94Mo7W3K5YiT4MuTDgg5c --- src/xml2db/cli.py | 337 ++++++++++++++++++++++------------------------ 1 file changed, 158 insertions(+), 179 deletions(-) diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py index 6cddcf2..a3d26a1 100644 --- a/src/xml2db/cli.py +++ b/src/xml2db/cli.py @@ -5,9 +5,7 @@ import html as _html import json import os -import queue import threading -import time import webbrowser from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Optional @@ -70,102 +68,81 @@ def cmd_render(args: argparse.Namespace) -> None: # --------------------------------------------------------------------------- class _State: - def __init__(self, xsd_file: str, config_file: Optional[str], short_name: str): + def __init__( + self, + xsd_file: str, + config_file: Optional[str], + short_name: str, + db_type: Optional[str], + ): self.xsd_file = xsd_file - self.config_file = config_file + self.config_file = config_file if config_file else "model_config.yml" self.short_name = short_name + self.db_type = db_type self.lock = threading.Lock() - self.sse_clients: list[queue.Queue] = [] self.config_yaml = "" if config_file and os.path.exists(config_file): with open(config_file, encoding="utf-8") as f: self.config_yaml = f.read() - self.erd = "" + self.outputs: dict = {"erd": "", "target_tree": "", "source_tree": "", "ddl": ""} self.build_error = "" - self.xsd_mtime = self._xsd_mtime() - self._rebuild() + self._rebuild(self.config_yaml) - def _xsd_mtime(self) -> Optional[float]: + def _rebuild(self, yaml_text: str) -> tuple[dict | None, str]: + """Build all outputs from yaml_text. On success updates state and returns (outputs, ""). + On failure updates build_error and returns (None, error).""" try: - return os.path.getmtime(self.xsd_file) - except OSError: - return None - - def _rebuild(self) -> None: - try: - cfg = parse_yaml_config(self.config_yaml) if self.config_yaml.strip() else None - model = DataModel( - xsd_file=self.xsd_file, - short_name=self.short_name, - model_config=cfg, - ) - self.erd = model.get_entity_rel_diagram(text_context=False) - self.build_error = "" - except Exception as exc: - self.build_error = str(exc) - - def apply_yaml(self, text: str) -> tuple[str, str]: - """Parse + validate YAML text and rebuild ERD. Returns (erd, error).""" - try: - cfg = parse_yaml_config(text) if text.strip() else None + cfg = parse_yaml_config(yaml_text) if yaml_text.strip() else None model = DataModel( xsd_file=self.xsd_file, short_name=self.short_name, model_config=cfg, + db_type=self.db_type, ) - erd = model.get_entity_rel_diagram(text_context=False) + sa_dialect = _get_sa_dialect(self.db_type) + ddl_parts = [ + str(s.compile(dialect=sa_dialect)) + for s in model.get_all_create_table_statements() + ] + ddl_parts += [ + str(s.compile(dialect=sa_dialect)) + "\n\n" + for s in model.get_all_create_index_statements() + ] + outputs = { + "erd": model.get_entity_rel_diagram(text_context=False), + "target_tree": model.target_tree, + "source_tree": model.source_tree, + "ddl": "".join(ddl_parts), + } with self.lock: - self.config_yaml = text - self.erd = erd + self.outputs = outputs self.build_error = "" - return erd, "" + self.config_yaml = yaml_text + return outputs, "" except Exception as exc: - return "", str(exc) + error = str(exc) + with self.lock: + self.build_error = error + return None, error + + def apply_yaml(self, text: str) -> tuple[dict, str]: + """Rebuild from new YAML text. Always returns (current_outputs, error) so the + browser keeps the last successful output visible when a build fails.""" + self._rebuild(text) + with self.lock: + return dict(self.outputs), self.build_error def save_config(self, text: str) -> str: - """Save YAML text to config file. Returns error string or empty string.""" - if not self.config_file: - return "No --config file was specified; cannot save." + """Write text to config_file. Returns error string or "".""" try: with open(self.config_file, "w", encoding="utf-8") as f: f.write(text) - with self.lock: - self.config_yaml = text return "" except OSError as exc: return str(exc) - def watch_xsd(self) -> None: - while True: - time.sleep(1) - mtime = self._xsd_mtime() - if mtime is not None and mtime != self.xsd_mtime: - self.xsd_mtime = mtime - self._rebuild() - self._push("reload") - - def _push(self, event: str) -> None: - with self.lock: - clients = list(self.sse_clients) - for q in clients: - try: - q.put_nowait(event) - except queue.Full: - pass - - def add_sse_client(self, q: queue.Queue) -> None: - with self.lock: - self.sse_clients.append(q) - - def remove_sse_client(self, q: queue.Queue) -> None: - with self.lock: - try: - self.sse_clients.remove(q) - except ValueError: - pass - # --------------------------------------------------------------------------- # HTTP handler @@ -179,10 +156,6 @@ def log_message(self, fmt, *args): # silence default stderr logging def do_GET(self): if self.path == "/": self._serve_html() - elif self.path == "/erd": - self._json({"erd": state.erd, "error": state.build_error}) - elif self.path == "/events": - self._sse() else: self.send_error(404) @@ -190,10 +163,11 @@ def do_POST(self): length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length).decode("utf-8", errors="replace") if self.path == "/config": - erd, error = state.apply_yaml(body) - self._json({"erd": erd, "error": error}) + outputs, error = state.apply_yaml(body) + self._json({"outputs": outputs, "error": error}) elif self.path == "/save": - self._json({"error": state.save_config(body)}) + error = state.save_config(body) + self._json({"error": error, "saved_to": state.config_file}) else: self.send_error(404) @@ -213,28 +187,6 @@ def _serve_html(self): self.end_headers() self.wfile.write(body) - def _sse(self): - q: queue.Queue = queue.Queue(maxsize=16) - state.add_sse_client(q) - self.send_response(200) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Cache-Control", "no-cache") - self.send_header("Connection", "keep-alive") - self.end_headers() - try: - while True: - try: - event = q.get(timeout=25) - self.wfile.write(f"event: {event}\ndata:\n\n".encode()) - self.wfile.flush() - except queue.Empty: - self.wfile.write(b": keepalive\n\n") - self.wfile.flush() - except (BrokenPipeError, ConnectionResetError): - pass - finally: - state.remove_sse_client(q) - return Handler @@ -256,11 +208,10 @@ def _sse(self): header { padding: 6px 14px; background: #1a1a2e; color: #cdd; font-size: 13px; display: flex; align-items: center; gap: 10px; flex-shrink: 0; } header strong { color: #fff; } - #hdr-file { opacity: .75; } - #status { margin-left: auto; font-size: 11px; opacity: .65; } main { display: flex; flex: 1; overflow: hidden; } #left { width: 360px; min-width: 140px; display: flex; flex-direction: column; - padding: 8px; gap: 6px; border-right: 1px solid #ddd; background: #fff; } + padding: 8px; gap: 6px; border-right: 1px solid #ddd; background: #fff; + flex-shrink: 0; } #editor-label { font-size: 11px; font-weight: 600; color: #666; } #editor { flex: 1; font-family: 'Menlo', 'Consolas', monospace; font-size: 12px; border: 1px solid #ccc; border-radius: 3px; padding: 6px; @@ -269,111 +220,142 @@ def _sse(self): #msg { font-size: 11px; min-height: 14px; white-space: pre-wrap; word-break: break-word; } #msg.ok { color: #2a7a2a; } #msg.err { color: #c00; } - #buttons { display: flex; gap: 6px; } - button { padding: 5px 14px; border: 1px solid transparent; border-radius: 3px; - cursor: pointer; font-size: 12px; } - #apply-btn { background: #4a90e2; color: #fff; border-color: #357abd; } - #apply-btn:hover { background: #357abd; } - #save-btn { background: #f0f0f0; border-color: #bbb; } + #save-btn { padding: 5px 14px; border: 1px solid #bbb; border-radius: 3px; + cursor: pointer; font-size: 12px; background: #f0f0f0; } #save-btn:hover { background: #e0e0e0; } - #right { flex: 1; overflow: auto; padding: 14px; } - #right svg { max-width: 100%; } + #right { flex: 1; display: flex; flex-direction: column; overflow: hidden; } + #tabs { display: flex; gap: 2px; padding: 6px 8px 0; background: #f0f0f4; + border-bottom: 1px solid #ddd; flex-shrink: 0; } + .tab { padding: 5px 14px; border: 1px solid #ccc; border-bottom: none; + border-radius: 3px 3px 0 0; cursor: pointer; font-size: 12px; + background: #e8e8ee; color: #444; } + .tab.active { background: #fff; border-color: #ddd; border-bottom-color: #fff; + margin-bottom: -1px; color: #000; font-weight: 600; } + #content { flex: 1; overflow: auto; padding: 14px; } + #content svg { max-width: 100%; } + #content pre { font-family: 'Menlo', 'Consolas', monospace; font-size: 12px; + white-space: pre; line-height: 1.5; }
xml2db - TMPL_HEADER - ready + TMPL_HEADER
model_config (YAML)
-
- - + +
+ -
@@ -382,17 +364,13 @@ def _sse(self): def _build_html(state: _State) -> str: header = os.path.basename(state.xsd_file) - if state.config_file: - header += f" · {os.path.basename(state.config_file)}" - save_label = "Save to file" if state.config_file else "Save (no file)" return ( _HTML .replace("TMPL_TITLE", _html.escape(header)) .replace("TMPL_HEADER", _html.escape(header)) .replace("TMPL_CONFIG_YAML", _html.escape(state.config_yaml)) - .replace("TMPL_SAVE_LABEL", _html.escape(save_label)) - .replace("TMPL_HAS_SAVE_TARGET", "true" if state.config_file else "false") - .replace("TMPL_INITIAL_ERD", json.dumps(state.erd)) + .replace("TMPL_SAVE_PATH", _html.escape(state.config_file)) + .replace("TMPL_INITIAL_OUTPUTS", json.dumps(state.outputs)) .replace("TMPL_INITIAL_ERROR", json.dumps(state.build_error)) ) @@ -407,12 +385,11 @@ def cmd_serve(args: argparse.Namespace) -> None: xsd_file=args.xsd_file, config_file=args.config, short_name=args.short_name, + db_type=getattr(args, "db_type", None), ) if state.build_error: print(f"Warning: initial build error: {state.build_error}") - threading.Thread(target=state.watch_xsd, daemon=True).start() - server = ThreadingHTTPServer(("127.0.0.1", args.port), _make_handler(state)) url = f"http://127.0.0.1:{args.port}" print(f"Serving at {url} (Ctrl+C to stop)") @@ -437,7 +414,7 @@ def main() -> None: ) sub = parser.add_subparsers(dest="command", required=True) - r = sub.add_parser("render", help="Print ERD or tree representation to stdout or a file") + r = sub.add_parser("render", help="Print ERD, tree or DDL to stdout or a file") r.add_argument("xsd_file", help="Path to the XSD schema file") r.add_argument("--config", "-c", metavar="FILE", help="YAML model config file") r.add_argument( @@ -455,13 +432,15 @@ def main() -> None: s = sub.add_parser("serve", help="Launch an interactive schema explorer in the browser") s.add_argument("xsd_file", help="Path to the XSD schema file") s.add_argument("--config", "-c", metavar="FILE", - help="YAML model config file (editable and saveable from the browser)") + help="YAML model config file (loaded on startup; Save button writes it back)") s.add_argument("--port", "-p", type=int, default=8765, metavar="PORT", help="HTTP port (default: 8765)") s.add_argument("--no-browser", action="store_true", help="Do not open the browser automatically") s.add_argument("--short-name", default="DocumentRoot", metavar="NAME", help="Data model short name (default: DocumentRoot)") + s.add_argument("--db-type", metavar="BACKEND", default=None, + help="Database backend for DDL tab (postgresql, mssql, mysql, …)") args = parser.parse_args() if args.command == "render": From e6c7eefda6d08c49a40426cf747dd152dbe81f2a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 04:02:36 +0000 Subject: [PATCH 05/30] Add xml2db import CLI command Parses a single XML file and loads it into a database in one step: xml2db import --connection-string Options: --config (YAML model config), --short-name, --db-schema, --metadata KEY=VALUE (for metadata_columns), --validate, --no-iterparse, --recover. Prints inserted/existing row counts and per-phase timings on completion. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B94Mo7W3K5YiT4MuTDgg5c --- src/xml2db/cli.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py index a3d26a1..3ea25f7 100644 --- a/src/xml2db/cli.py +++ b/src/xml2db/cli.py @@ -33,6 +33,34 @@ def _get_sa_dialect(db_type: str | None): return module.dialect() if module is not None else None +def cmd_import(args: argparse.Namespace) -> None: + config = load_config(args.config) if args.config else None + metadata = dict(kv.split("=", 1) for kv in args.metadata) if args.metadata else None + + model = DataModel( + xsd_file=args.xsd_file, + short_name=args.short_name, + model_config=config, + connection_string=args.connection_string, + db_schema=args.db_schema, + ) + doc = model.parse_xml( + xml_file=args.xml_file, + metadata=metadata, + skip_validation=not args.validate, + iterparse=not args.no_iterparse, + recover=args.recover, + ) + stats = doc.insert_into_target_tables() + print( + f"Imported {args.xml_file}: " + f"{stats.inserted} rows inserted, {stats.existing} rows already existed " + f"({stats.duration_temp_insert:.2f}s staging, " + f"{stats.duration_merge:.2f}s merge, " + f"{stats.duration_cleanup:.2f}s cleanup)" + ) + + def cmd_render(args: argparse.Namespace) -> None: config = load_config(args.config) if args.config else None db_type = getattr(args, "db_type", None) @@ -414,6 +442,25 @@ def main() -> None: ) sub = parser.add_subparsers(dest="command", required=True) + i = sub.add_parser("import", help="Parse an XML file and load it into a database") + i.add_argument("xml_file", help="Path to the XML file to import") + i.add_argument("xsd_file", help="Path to the XSD schema file") + i.add_argument("--connection-string", "-d", required=True, metavar="DSN", + help="SQLAlchemy connection string (e.g. postgresql+psycopg2://user:pw@host/db)") + i.add_argument("--config", "-c", metavar="FILE", help="YAML model config file") + i.add_argument("--short-name", default="DocumentRoot", metavar="NAME", + help="Data model short name (default: DocumentRoot)") + i.add_argument("--db-schema", metavar="SCHEMA", default=None, + help="Database schema to use") + i.add_argument("--metadata", "-m", nargs="*", metavar="KEY=VALUE", + help="Metadata values for root table metadata_columns (e.g. -m source=file.xml)") + i.add_argument("--validate", action="store_true", + help="Validate the XML against the schema before importing") + i.add_argument("--no-iterparse", action="store_true", + help="Use recursive parser instead of iterparse (higher memory usage)") + i.add_argument("--recover", action="store_true", + help="Attempt to parse malformed XML") + r = sub.add_parser("render", help="Print ERD, tree or DDL to stdout or a file") r.add_argument("xsd_file", help="Path to the XSD schema file") r.add_argument("--config", "-c", metavar="FILE", help="YAML model config file") @@ -443,7 +490,9 @@ def main() -> None: help="Database backend for DDL tab (postgresql, mssql, mysql, …)") args = parser.parse_args() - if args.command == "render": + if args.command == "import": + cmd_import(args) + elif args.command == "render": cmd_render(args) else: cmd_serve(args) From d577850c3e3cad92f8b53d46763759ed4bc78b3a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 04:39:58 +0000 Subject: [PATCH 06/30] Add CodeMirror 6 YAML editor with schema-aware autocomplete to serve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the plain textarea with a CodeMirror 6 editor (loaded from esm.sh CDN — no new Python deps) featuring: - YAML syntax highlighting via @codemirror/lang-yaml - Context-aware autocompletion using indent-level heuristics: - Top-level model_config keys - Table names from the parsed XSD (injected as SCHEMA_INFO) - Table config keys (reuse, as_columnstore, choice_transform, …) - Field names per table from the parsed XSD - Field config keys (type, rename, transform) - metadata_columns and extra_args keys - Value completions: true/false for booleans, SQLAlchemy type names for 'type', transform options for 'transform' SCHEMA_INFO is extracted from model.tables on each rebuild and injected into the page as JSON, so completions always reflect the actual XSD schema. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B94Mo7W3K5YiT4MuTDgg5c --- src/xml2db/cli.py | 190 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 140 insertions(+), 50 deletions(-) diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py index 3ea25f7..a39b967 100644 --- a/src/xml2db/cli.py +++ b/src/xml2db/cli.py @@ -115,6 +115,7 @@ def __init__( self.config_yaml = f.read() self.outputs: dict = {"erd": "", "target_tree": "", "source_tree": "", "ddl": ""} + self.schema_info: dict = {} self.build_error = "" self._rebuild(self.config_yaml) @@ -144,8 +145,17 @@ def _rebuild(self, yaml_text: str) -> tuple[dict | None, str]: "source_tree": model.source_tree, "ddl": "".join(ddl_parts), } + schema_info = { + table.name: sorted( + list(table.columns.keys()) + + list(table.relations_1.keys()) + + list(table.relations_n.keys()) + ) + for table in model.tables.values() + } with self.lock: self.outputs = outputs + self.schema_info = schema_info self.build_error = "" self.config_yaml = yaml_text return outputs, "" @@ -237,14 +247,15 @@ def _serve_html(self): display: flex; align-items: center; gap: 10px; flex-shrink: 0; } header strong { color: #fff; } main { display: flex; flex: 1; overflow: hidden; } - #left { width: 360px; min-width: 140px; display: flex; flex-direction: column; + #left { width: 380px; min-width: 160px; display: flex; flex-direction: column; padding: 8px; gap: 6px; border-right: 1px solid #ddd; background: #fff; flex-shrink: 0; } #editor-label { font-size: 11px; font-weight: 600; color: #666; } - #editor { flex: 1; font-family: 'Menlo', 'Consolas', monospace; font-size: 12px; - border: 1px solid #ccc; border-radius: 3px; padding: 6px; - resize: none; outline: none; } - #editor:focus { border-color: #4a90e2; box-shadow: 0 0 0 2px #4a90e230; } + #editor { flex: 1; border: 1px solid #ccc; border-radius: 3px; overflow: hidden; + min-height: 0; } + #editor:focus-within { border-color: #4a90e2; box-shadow: 0 0 0 2px #4a90e230; } + .cm-editor { height: 100%; font-size: 12px; } + .cm-scroller { overflow: auto !important; } #msg { font-size: 11px; min-height: 14px; white-space: pre-wrap; word-break: break-word; } #msg.ok { color: #2a7a2a; } #msg.err { color: #c00; } @@ -273,7 +284,7 @@ def _serve_html(self):
model_config (YAML)
- +
@@ -287,34 +298,129 @@ def _serve_html(self):
- @@ -346,6 +353,11 @@ def _serve_html(self): +
+ Names: + + +
@@ -434,13 +446,19 @@ def _serve_html(self): // ---- editor setup ---- mermaid.initialize({ startOnLoad: false, theme: 'default' }); -const msg = document.getElementById('msg'); -const contentEl = document.getElementById('content'); -let outputs = TMPL_INITIAL_OUTPUTS; -let currentTab = 'erd'; +const msg = document.getElementById('msg'); +const contentEl = document.getElementById('content'); +const erdNamesEl = document.getElementById('erd-names'); +let outputs = TMPL_INITIAL_OUTPUTS; +let currentTab = 'erd'; let debounceTimer = null; let mermaidCounter = 0; +function erdKey() { + return document.querySelector('input[name="erd_names"]:checked').value === 'db' + ? 'erd_db' : 'erd'; +} + const view = new EditorView({ doc: TMPL_CONFIG_YAML_JSON, extensions: [ @@ -469,7 +487,8 @@ def _serve_html(self): async function renderTab() { if (currentTab === 'erd') { - const erd = (outputs && outputs.erd) || ''; + erdNamesEl.classList.add('visible'); + const erd = (outputs && outputs[erdKey()]) || ''; if (!erd) { contentEl.innerHTML = '

No ERD available.

'; return; } try { const id = 'g' + (++mermaidCounter); @@ -479,6 +498,7 @@ def _serve_html(self): contentEl.innerHTML = '
' + escapeHtml(String(e)) + '
'; } } else { + erdNamesEl.classList.remove('visible'); contentEl.innerHTML = '
' + escapeHtml((outputs && outputs[currentTab]) || '') + '
'; } } @@ -492,6 +512,10 @@ def _serve_html(self): }); }); +document.querySelectorAll('input[name="erd_names"]').forEach(radio => { + radio.addEventListener('change', () => { if (currentTab === 'erd') renderTab(); }); +}); + async function doRebuild() { try { const r = await fetch('/config', { @@ -616,6 +640,8 @@ def main() -> None: help="Data model short name (default: DocumentRoot)") r.add_argument("--db-type", metavar="BACKEND", default=None, help="Database backend for DDL output (postgresql, mssql, mysql, …)") + r.add_argument("--db-names", action="store_true", + help="Use physical database identifiers in the ERD instead of logical names") s = sub.add_parser("serve", help="Launch an interactive schema explorer in the browser") s.add_argument("xsd_file", help="Path to the XSD schema file") diff --git a/src/xml2db/model.py b/src/xml2db/model.py index 79818f1..829629f 100644 --- a/src/xml2db/model.py +++ b/src/xml2db/model.py @@ -568,21 +568,27 @@ def get_occurs(particle): return parent_table - def get_entity_rel_diagram(self, text_context: bool = True) -> str: + def get_entity_rel_diagram( + self, text_context: bool = True, use_db_names: bool = False, sa_dialect=None + ) -> str: """Build an entity relationship diagram for the data model The ERD syntax is used by mermaid.js to create a visual representation of the diagram, which is supported by Pycharm IDE or GitHub in markdown files, among others Args: - text_context: Should we add a title, a text explanation, etc. or just the ERD? + text_context: should we add a title, a text explanation, etc. or just the ERD? + use_db_names: if True, use the physical database identifier for table and + column names, and the compiled SQL type for column types. + sa_dialect: SQLAlchemy dialect instance for SQL type compilation when + use_db_names is True. Falls back to generic type names when None. Returns: A string representation of the ERD """ out = ["erDiagram"] for tb in self.fk_ordered_tables_reversed: - out += tb.get_entity_rel_diagram() + out += tb.get_entity_rel_diagram(use_db_names=use_db_names, sa_dialect=sa_dialect) if text_context: out = ( diff --git a/src/xml2db/table/table.py b/src/xml2db/table/table.py index 783a773..535ec19 100644 --- a/src/xml2db/table/table.py +++ b/src/xml2db/table/table.py @@ -393,30 +393,64 @@ def drop_temp_tables(self, engine: sqlalchemy.engine.base.Engine) -> None: rel.temp_rel_table.drop(engine, checkfirst=True) self.temp_table.drop(engine, checkfirst=True) - def get_entity_rel_diagram(self) -> List: + def get_entity_rel_diagram(self, use_db_names: bool = False, sa_dialect=None) -> List: """Build ERD representation for a single table and its relationships The string representation is used by mermaid.js to create a visual diagram. + Args: + use_db_names: if True, use the physical database identifier for table and + column names, and the compiled SQL type for column types. + sa_dialect: SQLAlchemy dialect instance used to compile SQL types when + use_db_names is True. Falls back to generic SQL type names when None. + Returns: a list of strings (lines) """ + d = self.data_model.dialect + if use_db_names: + tname = d.db_identifier(self.name) + col_by_key = {col.key: col for col in self.table.c} + else: + tname = self.name + + def other_name(tb): + return d.db_identifier(tb.name) if use_db_names else tb.name + + def col_name(logical): + if use_db_names: + sa_col = col_by_key.get(logical) + return sa_col.name if sa_col is not None else logical.replace(".", "_") + return logical.replace(".", "_") + + def col_type(logical): + if use_db_names: + sa_col = col_by_key.get(logical) + if sa_col is not None: + raw = ( + sa_col.type.compile(dialect=sa_dialect) + if sa_dialect is not None + else str(sa_col.type) + ) + return raw.split("(")[0] + return self.columns[logical].data_type + out = ( [ - f"{self.name} ||--{'o' if rel.occurs[0] == 0 else '|'}| {rel.other_table.name} : " + f"{tname} ||--{'o' if rel.occurs[0] == 0 else '|'}| {other_name(rel.other_table)} : " f'"{rel.name}"' for rel in self.relations_1.values() ] + [ - f"{self.name} ||--{'o' if rel.occurs[0] == 0 else '|'}{{ {rel.other_table.name} : " + f"{tname} ||--{'o' if rel.occurs[0] == 0 else '|'}{{ {other_name(rel.other_table)} : " f"\"{rel.name}{'*' if rel.other_table.is_reused else ''}\"" for rel in self.relations_n.values() ] - + [f"{self.name} {{"] + + [f"{tname} {{"] + [ ( - f" {self.columns[field[1]].data_type}{'-N' if self.columns[field[1]].occurs[1] is None else ''} " - f"{field[1].replace('.', '_')}" + f" {col_type(field[1])}{'-N' if self.columns[field[1]].occurs[1] is None else ''} " + f"{col_name(field[1])}" ) for field in self.fields if field[0] == "col" From 3f150c9a8c4dda2886bdda48be6ad427e4e6a1f9 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Tue, 23 Jun 2026 08:58:18 +0000 Subject: [PATCH 14/30] Refine DB-names ERD: full type size, no -N suffix, updated label - Show full SQL type including size (VARCHAR(255) not just VARCHAR), since the size is meaningful in DB context - Drop the -N suffix in DB mode: it represents multi-value CSV storage, a logical concept with no DB equivalent - Rename radio label to "Names & types" to reflect that both change Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01Kc7NkAzxyFPd4sWfCXcvZp --- src/xml2db/cli.py | 2 +- src/xml2db/table/table.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py index 00e1b58..e9e934a 100644 --- a/src/xml2db/cli.py +++ b/src/xml2db/cli.py @@ -354,7 +354,7 @@ def _serve_html(self):
- Names: + Names & types:
diff --git a/src/xml2db/table/table.py b/src/xml2db/table/table.py index 535ec19..88adfe1 100644 --- a/src/xml2db/table/table.py +++ b/src/xml2db/table/table.py @@ -427,14 +427,18 @@ def col_type(logical): if use_db_names: sa_col = col_by_key.get(logical) if sa_col is not None: - raw = ( + return ( sa_col.type.compile(dialect=sa_dialect) if sa_dialect is not None else str(sa_col.type) ) - return raw.split("(")[0] return self.columns[logical].data_type + def col_type_suffix(logical): + if use_db_names: + return "" + return "-N" if self.columns[logical].occurs[1] is None else "" + out = ( [ f"{tname} ||--{'o' if rel.occurs[0] == 0 else '|'}| {other_name(rel.other_table)} : " @@ -449,7 +453,7 @@ def col_type(logical): + [f"{tname} {{"] + [ ( - f" {col_type(field[1])}{'-N' if self.columns[field[1]].occurs[1] is None else ''} " + f" {col_type(field[1])}{col_type_suffix(field[1])} " f"{col_name(field[1])}" ) for field in self.fields From a97eb3149b3a8e5a46bba5b4eb84dc550be66b84 Mon Sep 17 00:00:00 2001 From: Martin Vergier Date: Tue, 23 Jun 2026 09:14:41 +0000 Subject: [PATCH 15/30] Clarify source vs target field names in docs and autocomplete Two different name spaces apply to fields. in model_config: - transform uses the source (XSD/pre-simplification) name - type and rename use the target (post-elevation/logical) name This distinction is invisible when a field is not elevated (same name in both), but matters whenever elevation collapses a child relation into prefixed columns in the parent (e.g. orderperson -> orderperson_*). Docs (configuring.md): - Add "Source names vs target names" section with a table and example - Add "Uses source/target name" note to each sub-section serve autocomplete: - schema_info now carries both {source: [...], target: [...]} per table - source fields come from model.fields_transforms keyed by type_name - Field completions at tables..fields. show source-only fields labelled "source" and target-only fields labelled "target"; fields present in both (non-elevated) have no label Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01Kc7NkAzxyFPd4sWfCXcvZp --- docs/configuring.md | 30 ++++++++++++++++++++++++++++++ src/xml2db/cli.py | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/docs/configuring.md b/docs/configuring.md index 240ccf0..b907699 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -122,8 +122,26 @@ functions (defaults to `hashlib.sha1`). These configuration options are defined for a specific field of a specific table. A "field" refers to a column in the table, or a child table. +### Source names vs target names + +Field names in `model_config` come from two different points in the processing pipeline, and **which one to use depends on the config key**: + +| Config key | Name to use | Where to look | +|---|---|---| +| `transform` | **Source name**: the original XSD element or relation name, before any simplification | **Source tree** tab | +| `type`, `rename` | **Target name**: the logical column name after elevation and prefixing | **Target tree** tab | + +**Why this matters:** elevation (the default for small mandatory children) collapses a child relation into prefixed columns in the parent. For example, if `timeInterval` is elevated, the target model has `timeInterval_start` and `timeInterval_end` — but `timeInterval` itself no longer appears in the target tree. To opt out of elevation you configure `fields.timeInterval.transform: false` using the **source name**. To rename an elevated result you configure `fields.timeInterval_start.rename: "start"` using the **target name**. + +If a field is not elevated (a direct column or a kept relation), its source and target names are identical and there is no ambiguity. + +The browser explorer autocomplete (`xml2db serve`) offers both source and target field names and labels each accordingly. + ### Data types +!!! note "Uses target name" + Use the field name as it appears in the **Target tree** tab. + By default, the data type defined in the database table for each column is based on a mapping between the data type indicated in the XSD and a corresponding `sqlalchemy` type implemented in the following three methods: @@ -168,6 +186,9 @@ defined as `sqlalchemy` types and will be passed to the `sqlalchemy.Column` cons ### Renaming columns +!!! note "Uses target name" + Use the field name as it appears in the **Target tree** tab. For elevated fields, this is the prefixed name (e.g. `orderperson_name`), not the original child relation name. + The physical database column name for any field can be overridden while keeping the original XML element name as the internal logical key. This is useful when XSD element names are awkward, conflict with reserved SQL words, or need to follow a naming convention that differs from the source schema. @@ -206,6 +227,9 @@ Configuration: `"rename":` `"new_column_name"` (no default; omit to keep the ori ### Joining values for simple types +!!! note "Uses source name" + Use the field name as it appears in the **Source tree** tab. + By default, XML simple type elements with types in `["string", "date", "dateTime", "NMTOKEN", "time", "base64Binary", "decimal"]` and max occurrences >= 1 are joined in one column as comma separated values and optionally wrapped in double quotes if they contain commas (an Excel-like csv format, which can be queried with `LIKE` statements in SQL). @@ -231,6 +255,9 @@ automatically applied `join`, as it would require a complex process of adding a ### Skipping fields +!!! note "Uses source name" + Use the field name as it appears in the **Source tree** tab. + Any field (column or relation) can be excluded from the data model entirely by setting its transform to `"skip"`. The field will be absent from the target table schema and all data for it will be silently dropped during XML parsing. This is useful for PII columns, large binary blobs, or fields that are irrelevant for analysis. @@ -261,6 +288,9 @@ Configuration: `"transform": "skip"` ### Elevate children to upper level +!!! note "Uses source name" + Use the child relation name as it appears in the **Source tree** tab (the parent's field pointing to the child, before any elevation). This name may not appear in the Target tree at all if default elevation has already collapsed it. + A mandatory child (min occurrences = 1, i.e. `[1, 1]`) is always elevated to its parent by default, as long as it is not involved in a 1-n relationship elsewhere in the schema. diff --git a/src/xml2db/cli.py b/src/xml2db/cli.py index e9e934a..7a21c1d 100644 --- a/src/xml2db/cli.py +++ b/src/xml2db/cli.py @@ -192,14 +192,21 @@ def _rebuild(self, yaml_text: str) -> tuple[dict | None, str]: "source_tree": model.source_tree, "ddl": "".join(ddl_parts), } - schema_info = { - table.name: sorted( + schema_info = {} + for table in model.tables.values(): + target_fields = sorted( list(table.columns.keys()) + list(table.relations_1.keys()) + list(table.relations_n.keys()) ) - for table in model.tables.values() - } + source_fields = sorted(set( + fn for (tn, fn) in model.fields_transforms + if tn == table.type_name + )) + schema_info[table.name] = { + "target": target_fields, + "source": source_fields, + } with self.lock: self.outputs = outputs self.schema_info = schema_info @@ -364,7 +371,9 @@ def _serve_html(self): +