From b7bed24da0b1b6c16f951b47d48cef53bdecff6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Zim=C3=A1nyi?= Date: Thu, 10 Sep 2026 19:46:06 +0200 Subject: [PATCH] Describe the temporal coverings as TemporalParquet 2.0 struct columns The temporal-covering descriptor states the covering columns of TemporalParquet 2.0.0. Each covering is a struct column at the root of the schema, named after its temporal column: `bbox` is a GeoParquet bounding box column with the fields xmin, ymin, [zmin,] xmax, ymax[, zmax] in that order, and `tspan` and `vspan` hold the time and value bounds. The spatial class carries bbox, tspan and the plain srid column, the number class vspan and tspan, the time-only class tspan. The parser rejects a bbox whose fields are not a GeoParquet bounding box column's, in its order, and a class that declares a covering twice. The projection generator gives, per temporal type, each covering with its column name and its fields in order, every field with the MEOS expression that derives it from the value; a binding substitutes `VALUE` with the column reference and `{col}` with its name. --- docs/temporal-covering.md | 78 +++++++++++++++++------------- generator/covering.py | 55 +++++++++++++-------- meta/temporal-covering.json | 74 +++++++++++++++++----------- meta/temporal-covering.schema.json | 52 ++++++++++++++------ parser/covering.py | 55 +++++++++++++++++---- tests/test_covering.py | 78 +++++++++++++++++++++++++----- tests/test_covering_projection.py | 75 ++++++++++++++++++++-------- 7 files changed, 324 insertions(+), 143 deletions(-) diff --git a/docs/temporal-covering.md b/docs/temporal-covering.md index d5bac18..a01c365 100644 --- a/docs/temporal-covering.md +++ b/docs/temporal-covering.md @@ -2,34 +2,39 @@ `meta/temporal-covering.json` is the **single codegen source of truth** (RFC #870 TemporalParquet / #913 Temporal Data Lake) for projecting a MEOS -temporal column into Parquet/Iceberg **covering columns**. The pipeline -folds it into `meos-idl.json` as `temporalCovering`. Every binding/engine -(PyMEOS, JMEOS, MobilityDuck, MobilitySpark, …) generates the **identical** -covering schema from this one mapping, so a temporal table prunes the same -way on every platform — no per-engine covering code to maintain. +temporal column into the **covering columns** of TemporalParquet 2.0.0. The +pipeline folds it into `meos-idl.json` as `temporalCovering`. Every +binding/engine (PyMEOS, JMEOS, MobilityDuck, MobilitySpark, …) generates the +**identical** covering schema from this one mapping, so a temporal table +prunes the same way on every platform — no per-engine covering code to +maintain. ## What it is A temporal value is stored on disk as a canonical MEOS-WKB `BLOB`. Iceberg and Parquet cannot prune on a `BLOB`. The covering descriptor names, per -temporal-type **class**, the primitive columns to *materialise alongside* -the value — the bounding box and SRID — which Iceberg collects as manifest -statistics and Parquet as row-group min/max. A bbox/time predicate then -prunes whole files and row groups with **no spatial-aware engine** -(GeoParquet 1.1 `covering.bbox`; MVB v3 measured this as ~10× faster than -the `ST_Intersects` path). +temporal-type **class**, the columns to *materialise alongside* the value — +struct columns at the root of the schema, named after the temporal column, +whose fields Iceberg collects as manifest statistics and Parquet as +row-group min/max. A bbox/time predicate on those fields then prunes whole +files and row groups with **no spatial-aware engine** (MVB v3 measured this +as ~10× faster than the `ST_Intersects` path). The mapping is keyed by **class**, not by type — adding a type is one entry in its class: | Class | Box | Types | Covering columns | |---|---|---|---| -| `spatial` | `STBOX` via `tspatial_to_stbox` | tgeompoint, tgeogpoint, tgeometry, tgeography, tcbuffer, tnpoint, tpose, trgeometry | `xmin xmax ymin ymax [zmin zmax] tmin tmax srid` | -| `number` | `TBOX` via `tnumber_to_tbox` | tint, tfloat, tbigint | `vmin vmax tmin tmax` | - +| `spatial` | `STBOX` via `tspatial_to_stbox` | tgeompoint, tgeogpoint, tgeometry, tgeography, tcbuffer, tnpoint, tpose, trgeometry | `{col}_bbox` {`xmin`, `ymin`, [`zmin`,] `xmax`, `ymax`[, `zmax`]} · `{col}_tspan` {`tmin`, `tmax`} · `srid` | +| `number` | `TBOX` via `tnumber_to_tbox` | tint, tfloat, tbigint | `{col}_vspan` {`vmin`, `vmax`} · `{col}_tspan` {`tmin`, `tmax`} | +| `timeOnly` | — | tbool, ttext | `{col}_tspan` {`tmin`, `tmax`} | + +`{col}_bbox` is a GeoParquet bounding box column: its fields are `DOUBLE`, in +the order shown, it has the repetition of its temporal column, and it holds a +value exactly when the temporal column does. `zmin`/`zmax` are emitted only +for 3D values (`when: hasZ`). `srid` is a plain column beside the coverings. The canonical value column is unchanged and lossless; covering columns are -denormalised derivations of the value's box. `zmin`/`zmax` are emitted only -for 3D values (`when: hasZ`). +denormalised derivations of the value's box. ## In the catalog @@ -41,41 +46,48 @@ codegen: "valueCodec": { "asHexWkb": "temporal_as_hexwkb", "fromHexWkb": "temporal_from_hexwkb" }, "byType": { "tgeompoint": { "class": "spatial", "box": {...}, - "srid": "tspatial_srid", "columns": [...] }, ... }, + "srid": "tspatial_srid", + "coverings": [...], "columns": [...] }, ... }, "symbols": ["stbox_xmin", "tbox_xmin", "tspatial_to_stbox", ...], - "count": 11 + "count": 13 } ``` -- `byType` — `"tgeompoint"` → its class, box converter, SRID accessor, and - covering columns (each with its MEOS bbox accessor and SQL type). A - generator reads this directly; it never re-derives the mapping. +- `byType` — `"tgeompoint"` → its class, box converter, SRID accessor, its + coverings (each with its key, its column name and its fields, every field + with its MEOS accessor and SQL type) and its plain columns. A generator + reads this directly; it never re-derives the mapping. - `symbols` — every MEOS C symbol the descriptor depends on. The covering parity audit (`tools/covering_parity.py`) checks each is exported by the catalog and each covered type is a real `MeosType` — a miss is reported as a worklist (add/export the accessor in MEOS), never a fabricated pass. +The parser rejects a `bbox` covering whose fields are not a GeoParquet +bounding box column's, in its order, and a class that declares a covering +twice. + ## How a generator uses it -For a column `traj TGEOMPOINT`, emit alongside the WKB value column: +`generate_covering.py` projects the catalog onto the language-agnostic +contract: per type, each covering with its column name and its fields in +order, every field with the MEOS expression that derives it from `VALUE`. +For a column `traj TGEOMPOINT`, a generator substitutes `VALUE` with the +column reference and `{col}` with its name, and emits alongside the WKB +value column: ```sql -xmin = stbox_xmin(tspatial_to_stbox(traj)), xmax = stbox_xmax(...), -ymin = stbox_ymin(...), ymax = stbox_ymax(...), -tmin = stbox_tmin(...), tmax = stbox_tmax(...), -srid = tspatial_srid(traj) +traj_bbox = {xmin: stbox_xmin(tspatial_to_stbox(traj)), + ymin: stbox_ymin(...), xmax: stbox_xmax(...), ymax: stbox_ymax(...)}, +traj_tspan = {tmin: stbox_tmin(tspatial_to_stbox(traj)), tmax: stbox_tmax(...)}, +srid = tspatial_srid(traj) ``` -(each engine in its own idiom — DuckDB generated columns, a Spark UDF -projection, a PyMEOS writer), plus the `temporal` and GeoParquet `geo` / -`covering.bbox` file metadata keys from `metadataKeys`. +(each engine in its own idiom — a DuckDB struct column, a Spark UDF +projection, a PyMEOS writer), and declares the coverings in the `temporal` +file metadata key under `covering`, with `bbox` in GeoParquet's form. ## Not yet covered -- **Time-only** (`tbool`, `ttext`): a `tmin`/`tmax` covering needs a span - lower/upper bound accessor; `temporal_to_tstzspan` is exported but a span - bound accessor is not. Surfaced as a MEOS export gap (close in MEOS C), - not filled binding-side. - **Point-cloud / cell-index** (`tpcpoint`, `tpcpatch`, `th3index`, `tquadbin`): fold into the `spatial` class once the catalog confirms a uniform temporal→`STBOX` converter for these families. diff --git a/generator/covering.py b/generator/covering.py index 5d9e755..ababa3d 100644 --- a/generator/covering.py +++ b/generator/covering.py @@ -2,32 +2,46 @@ Projects the ``temporalCovering`` block of the MEOS catalog (``meos-idl.json``, produced by ``parser/covering.py``) onto the canonical, -language-agnostic covering-column contract: per temporal type, the ordered -covering columns with the fully-composed MEOS expression that derives each -from the value. +language-agnostic covering-column contract of TemporalParquet 2.0.0: per +temporal type, each covering struct column with its name, its fields in +order, and the fully-composed MEOS expression that derives each field from +the value, plus the plain columns beside them. Every binding generator (PyMEOS, JMEOS, MobilityDuck, MobilitySpark, …) -renders this same contract in its own idiom — a DuckDB ``GENERATED`` column, -a Spark UDF projection, a PyMEOS writer — so a temporal table prunes the +renders this same contract in its own idiom — a DuckDB struct column, a +Spark UDF projection, a PyMEOS writer — so a temporal table prunes the same way on every platform (Iceberg manifest + Parquet row-group min/max). -The ``VALUE`` placeholder is the temporal column reference the binding -substitutes. +Two placeholders are the binding's to substitute: ``VALUE`` is the temporal +column reference, and ``{col}`` in a covering's column name is the temporal +column's name. -Pure ``dict`` → ``dict``; no libclang and no MEOS runtime. +Pure ``dict`` → ``dict``: it reads the catalog only and needs no MEOS runtime. """ from __future__ import annotations def _column_expr(column: dict, box_from: str) -> str: - """Compose the MEOS expression that derives one covering column from the - temporal value (``VALUE``). A ``box`` column is read off the value's box; - a ``value`` column is read off the value directly.""" + """Compose the MEOS expression that derives one field from the temporal + value (``VALUE``). A ``box`` field is read off the value's box; a + ``value`` field is read off the value directly.""" if column["source"] == "value": return f"{column['accessor']}(VALUE)" return f"{column['accessor']}({box_from}(VALUE))" +def _field(field: dict, box_from: str) -> dict: + """Project one covering field or plain column.""" + entry = { + "name": field["name"], + "sqlType": field["sqlType"], + "expr": _column_expr(field, box_from), + } + if field.get("when"): + entry["when"] = field["when"] + return entry + + def build_covering_projection(catalog: dict) -> dict: """Project ``temporalCovering`` onto the canonical covering-column contract.""" cov = catalog.get("temporalCovering") @@ -38,20 +52,19 @@ def build_covering_projection(catalog: dict) -> dict: for tname, spec in cov["byType"].items(): box = spec.get("box") box_from = box["from"] if box else None - columns = [] - for col in spec["columns"]: - entry = { - "name": col["name"], - "sqlType": col["sqlType"], - "expr": _column_expr(col, box_from), + coverings = [ + { + "key": covering["key"], + "column": covering["column"], + "fields": [_field(f, box_from) for f in covering["fields"]], } - if col.get("when"): - entry["when"] = col["when"] - columns.append(entry) + for covering in spec["coverings"] + ] types[tname] = { "class": spec["class"], "boxType": box["type"] if box else None, - "columns": columns, + "coverings": coverings, + "columns": [_field(c, box_from) for c in spec.get("columns", [])], } return { diff --git a/meta/temporal-covering.json b/meta/temporal-covering.json index 111282f..1029417 100644 --- a/meta/temporal-covering.json +++ b/meta/temporal-covering.json @@ -1,12 +1,13 @@ { - "_comment": "Temporal-covering descriptor — the single codegen source of truth for projecting a MEOS temporal column into Parquet/Iceberg covering columns (GeoParquet 1.1 `covering.bbox`). Every binding/engine generates the IDENTICAL covering schema from this mapping, so a temporal table prunes the same way on every platform (Iceberg manifest pruning + Parquet row-group min/max) with no spatial-aware engine. Curated canonical data keyed by temporal-type FAMILY (a `class`), not per type — adding a type is one entry in its class. The canonical MEOS-WKB value column is unchanged and lossless; the covering columns are denormalised derivations of the value's bounding box. RFC #870 (TemporalParquet) / #913 (Temporal Data Lake).", + "_comment": "Temporal-covering descriptor — the single codegen source of truth for projecting a MEOS temporal column into the covering columns of TemporalParquet 2.0.0. Every binding/engine generates the IDENTICAL covering schema from this mapping, so a temporal table prunes the same way on every platform (Iceberg manifest pruning + Parquet row-group min/max) with no spatial-aware engine. Each covering is a struct column at the root of the schema, named after its temporal column: `bbox` is a GeoParquet 2.0 bounding box column, and `tspan` and `vspan` are built the same way for the time and value bounds. Curated canonical data keyed by temporal-type FAMILY (a `class`), not per type — adding a type is one entry in its class. The canonical MEOS-WKB value column is unchanged and lossless; the covering columns are denormalised derivations of the value's bounding box.", "provenance": { "rfc": "MobilityDB RFC #870 (TemporalParquet) + #913 (Temporal Data Lake)", "discussion": "MobilityDB#861 (edge-to-cloud SQL portability: one query, three platforms)", - "geoParquet": "GeoParquet 1.1 covering.bbox (geoparquet.org/releases/v1.1.0)", + "temporalParquet": "TemporalParquet 2.0.0 covering columns (github.com/MobilityDB/MobilityLakehouse/blob/main/spec/covering-columns.md)", + "geoParquet": "GeoParquet 2.0 bounding box column, declared under covering.bbox (github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md)", "benchmark": "MVB v3 — the scalar AND-chain on materialised covering columns prunes row groups identically to the spatial-aware path and ~10x faster, with no DuckDB spatial extension" }, - "version": "1.0.0", + "version": "2.0.0", "valueCodec": { "asHexWkb": "temporal_as_hexwkb", "fromHexWkb": "temporal_from_hexwkb", @@ -14,48 +15,61 @@ }, "metadataKeys": { "temporal": "temporal", - "geo": "geo", - "covering": "bbox" + "geo": "geo" }, "classes": { "spatial": { - "doc": "Spatial temporal types — STBOX covering (x/y[/z] extent + time extent + SRID).", + "doc": "Spatial temporal types — STBOX covering: a GeoParquet bounding box column, the time extent, and the SRID.", "box": {"type": "STBOX", "from": "tspatial_to_stbox"}, "srid": "tspatial_srid", "types": ["tgeompoint", "tgeogpoint", "tgeometry", "tgeography", "tcbuffer", "tnpoint", "tpose", "trgeometry"], + "coverings": [ + {"key": "bbox", "column": "{col}_bbox", "fields": [ + {"name": "xmin", "sqlType": "double", "accessor": "stbox_xmin", "source": "box"}, + {"name": "ymin", "sqlType": "double", "accessor": "stbox_ymin", "source": "box"}, + {"name": "zmin", "sqlType": "double", "accessor": "stbox_zmin", "source": "box", "when": "hasZ"}, + {"name": "xmax", "sqlType": "double", "accessor": "stbox_xmax", "source": "box"}, + {"name": "ymax", "sqlType": "double", "accessor": "stbox_ymax", "source": "box"}, + {"name": "zmax", "sqlType": "double", "accessor": "stbox_zmax", "source": "box", "when": "hasZ"} + ]}, + {"key": "tspan", "column": "{col}_tspan", "fields": [ + {"name": "tmin", "sqlType": "timestamptz", "accessor": "stbox_tmin", "source": "box"}, + {"name": "tmax", "sqlType": "timestamptz", "accessor": "stbox_tmax", "source": "box"} + ]} + ], "columns": [ - {"name": "xmin", "sqlType": "double", "accessor": "stbox_xmin", "source": "box"}, - {"name": "xmax", "sqlType": "double", "accessor": "stbox_xmax", "source": "box"}, - {"name": "ymin", "sqlType": "double", "accessor": "stbox_ymin", "source": "box"}, - {"name": "ymax", "sqlType": "double", "accessor": "stbox_ymax", "source": "box"}, - {"name": "zmin", "sqlType": "double", "accessor": "stbox_zmin", "source": "box", "when": "hasZ"}, - {"name": "zmax", "sqlType": "double", "accessor": "stbox_zmax", "source": "box", "when": "hasZ"}, - {"name": "tmin", "sqlType": "timestamptz", "accessor": "stbox_tmin", "source": "box"}, - {"name": "tmax", "sqlType": "timestamptz", "accessor": "stbox_tmax", "source": "box"}, - {"name": "srid", "sqlType": "int", "accessor": "tspatial_srid", "source": "value"} + {"name": "srid", "sqlType": "int", "accessor": "tspatial_srid", "source": "value"} ] }, "number": { - "doc": "Numeric temporal types — TBOX covering (value range + time extent).", + "doc": "Numeric temporal types — TBOX covering: the value range and the time extent.", "box": {"type": "TBOX", "from": "tnumber_to_tbox"}, "srid": null, "types": ["tint", "tfloat", "tbigint"], - "columns": [ - {"name": "vmin", "sqlType": "double", "accessor": "tbox_xmin", "source": "box"}, - {"name": "vmax", "sqlType": "double", "accessor": "tbox_xmax", "source": "box"}, - {"name": "tmin", "sqlType": "timestamptz", "accessor": "tbox_tmin", "source": "box"}, - {"name": "tmax", "sqlType": "timestamptz", "accessor": "tbox_tmax", "source": "box"} - ] + "coverings": [ + {"key": "vspan", "column": "{col}_vspan", "fields": [ + {"name": "vmin", "sqlType": "double", "accessor": "tbox_xmin", "source": "box"}, + {"name": "vmax", "sqlType": "double", "accessor": "tbox_xmax", "source": "box"} + ]}, + {"key": "tspan", "column": "{col}_tspan", "fields": [ + {"name": "tmin", "sqlType": "timestamptz", "accessor": "tbox_tmin", "source": "box"}, + {"name": "tmax", "sqlType": "timestamptz", "accessor": "tbox_tmax", "source": "box"} + ]} + ], + "columns": [] }, "timeOnly": { - "doc": "Time-only temporal types — no spatial box; time extent only.", + "doc": "Time-only temporal types — no box; the time extent only.", "box": null, "srid": null, "types": ["tbool", "ttext"], - "columns": [ - {"name": "tmin", "sqlType": "timestamptz", "accessor": "temporal_start_timestamptz", "source": "value"}, - {"name": "tmax", "sqlType": "timestamptz", "accessor": "temporal_end_timestamptz", "source": "value"} - ] + "coverings": [ + {"key": "tspan", "column": "{col}_tspan", "fields": [ + {"name": "tmin", "sqlType": "timestamptz", "accessor": "temporal_start_timestamptz", "source": "value"}, + {"name": "tmax", "sqlType": "timestamptz", "accessor": "temporal_end_timestamptz", "source": "value"} + ]} + ], + "columns": [] } }, "deferred": { @@ -66,8 +80,10 @@ }, "notes": [ "The covering columns are a denormalisation of the value's bounding box; the canonical MEOS-WKB BLOB remains the lossless source of truth.", - "Materialising the covering columns as primitive Parquet columns gives Iceberg manifest-level file pruning and Parquet row-group min/max pruning, with no spatial-aware engine.", - "zmin/zmax are emitted only for 3D values (`when: hasZ`); 2D values omit them or store null.", + "Each covering is a struct column at the root of the schema; `{col}` in its `column` is the name of the temporal column it covers. The statistics of its fields give Iceberg manifest-level file pruning and Parquet row-group min/max pruning, with no spatial-aware engine.", + "The `bbox` covering is a GeoParquet bounding box column: its fields are DOUBLE, in the order xmin, ymin, [zmin,] xmax, ymax[, zmax], it has the repetition of its temporal column, and it holds a value exactly when the temporal column does.", + "zmin/zmax are emitted only for 3D values (`when: hasZ`); the bbox of a 2D value has four fields.", + "`columns` are plain columns at the root beside the coverings; the spatial class carries `srid` there.", "`source: box` accessors take the box returned by `class.box.from(value)`; `source: value` accessors take the temporal value directly.", "This descriptor is type-agnostic per class exactly as `portable-aliases.json` is type-agnostic per operator family — codegen consumes it identically across every binding." ] diff --git a/meta/temporal-covering.schema.json b/meta/temporal-covering.schema.json index ffb516b..243efd9 100644 --- a/meta/temporal-covering.schema.json +++ b/meta/temporal-covering.schema.json @@ -2,10 +2,24 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/MobilityDB/MEOS-API/blob/main/meta/temporal-covering.schema.json", "title": "Temporal-covering descriptor — canonical SoT", - "description": "Schema for `meta/temporal-covering.json` (RFC #870/#913). Catches shape regressions earlier than the unit tests; validated as a test step in `tests/test_covering.py`.", + "description": "Schema for `meta/temporal-covering.json` (RFC #870/#913, TemporalParquet 2.0.0 covering columns). Catches shape regressions earlier than the unit tests; validated as a test step in `tests/test_covering.py`.", "type": "object", "additionalProperties": true, "required": ["provenance", "version", "valueCodec", "metadataKeys", "classes", "notes"], + "$defs": { + "field": { + "type": "object", + "additionalProperties": false, + "required": ["name", "sqlType", "accessor", "source"], + "properties": { + "name": {"type": "string", "pattern": "^[a-z][a-z0-9]*$"}, + "sqlType": {"enum": ["double", "int", "timestamptz"]}, + "accessor": {"type": "string"}, + "source": {"enum": ["box", "value"]}, + "when": {"enum": ["hasZ"]} + } + } + }, "properties": { "_comment": {"type": "string"}, "provenance": { @@ -13,10 +27,11 @@ "additionalProperties": true, "required": ["rfc"], "properties": { - "rfc": {"type": "string"}, - "discussion": {"type": "string"}, - "geoParquet": {"type": "string"}, - "benchmark": {"type": "string"} + "rfc": {"type": "string"}, + "discussion": {"type": "string"}, + "temporalParquet": {"type": "string"}, + "geoParquet": {"type": "string"}, + "benchmark": {"type": "string"} } }, "version": {"type": "string"}, @@ -33,11 +48,10 @@ "metadataKeys": { "type": "object", "additionalProperties": true, - "required": ["temporal", "covering"], + "required": ["temporal"], "properties": { "temporal": {"type": "string"}, - "geo": {"type": "string"}, - "covering": {"type": "string"} + "geo": {"type": "string"} } }, "classes": { @@ -46,7 +60,7 @@ "additionalProperties": { "type": "object", "additionalProperties": true, - "required": ["types", "columns"], + "required": ["types", "coverings"], "properties": { "doc": {"type": "string"}, "srid": {"type": ["string", "null"]}, @@ -63,21 +77,27 @@ "minItems": 1, "items": {"type": "string", "pattern": "^t[a-z0-9]+$"} }, - "columns": { + "coverings": { "type": "array", "minItems": 1, "items": { "type": "object", "additionalProperties": false, - "required": ["name", "sqlType", "accessor", "source"], + "required": ["key", "column", "fields"], "properties": { - "name": {"type": "string", "pattern": "^[a-z][a-z0-9]*$"}, - "sqlType": {"enum": ["double", "int", "timestamptz"]}, - "accessor": {"type": "string"}, - "source": {"enum": ["box", "value"]}, - "when": {"enum": ["hasZ"]} + "key": {"enum": ["bbox", "tspan", "vspan"]}, + "column": {"type": "string", "pattern": "^\\{col\\}_[a-z]+$"}, + "fields": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/field"} + } } } + }, + "columns": { + "type": "array", + "items": {"$ref": "#/$defs/field"} } } } diff --git a/parser/covering.py b/parser/covering.py index 789d453..9d0ae46 100644 --- a/parser/covering.py +++ b/parser/covering.py @@ -1,14 +1,15 @@ """Temporal-covering descriptor — the single codegen source of truth for -projecting a MEOS temporal column into Parquet/Iceberg covering columns. +projecting a MEOS temporal column into the covering columns of +TemporalParquet 2.0.0. `meta/temporal-covering.json` is the curated, authoritative mapping (RFC #870 TemporalParquet / #913 Temporal Data Lake): per temporal-type *class* -(spatial → STBOX, number → TBOX) it names the box converter, the SRID -accessor, and the covering columns with their MEOS bbox accessors. Folding -it into the catalog means every binding/engine generates the *identical* -covering schema, so a temporal table prunes the same way on every platform -(Iceberg manifest pruning + Parquet row-group min/max) with no spatial-aware -engine. +(spatial → STBOX, number → TBOX, timeOnly → no box) it names the box +converter, the SRID accessor, the covering struct columns with their fields +and MEOS accessors, and the plain columns beside them. Folding it into the +catalog means every binding/engine generates the *identical* covering +schema, so a temporal table prunes the same way on every platform (Iceberg +manifest pruning + Parquet row-group min/max) with no spatial-aware engine. This is curated canonical data, not a heuristic — it is preserved verbatim and only *derived* lookups are added (a flat `byType` index and the set of @@ -19,6 +20,35 @@ import json from pathlib import Path +# The field names of a GeoParquet bounding box column, in their required order. +BBOX_2D = ("xmin", "ymin", "xmax", "ymax") +BBOX_3D = ("xmin", "ymin", "zmin", "xmax", "ymax", "zmax") + + +def _check_coverings(class_name: str, coverings: list) -> None: + """Reject a class whose coverings a generator could not render as the + TemporalParquet 2.0.0 covering columns.""" + keys = [c["key"] for c in coverings] + if len(keys) != len(set(keys)): + raise ValueError( + f"temporal-covering: class {class_name!r} declares a covering " + f"twice ({keys})") + for covering in coverings: + if covering["key"] != "bbox": + continue + fields = covering["fields"] + names = tuple(f["name"] for f in fields) + planar = tuple(f["name"] for f in fields if f.get("when") != "hasZ") + if names not in (BBOX_2D, BBOX_3D) or planar != BBOX_2D: + raise ValueError( + f"temporal-covering: class {class_name!r} declares the bbox " + f"fields {names}, where a GeoParquet bounding box column has " + f"{BBOX_2D} or {BBOX_3D}, the z fields only for 3D values") + if any(f["sqlType"] != "double" for f in fields): + raise ValueError( + f"temporal-covering: class {class_name!r} declares a bbox " + f"field that is not double") + def attach_temporal_covering(idl: dict, path: Path) -> dict: """Attach ``idl["temporalCovering"]`` from the canonical mapping file.""" @@ -32,6 +62,7 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: # two classes claiming the same type would make codegen ambiguous. by_type = {} for class_name, spec in classes.items(): + _check_coverings(class_name, spec["coverings"]) for t in spec["types"]: if t in by_type: raise ValueError( @@ -41,7 +72,8 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: "class": class_name, "box": spec.get("box"), "srid": spec.get("srid"), - "columns": spec["columns"], + "coverings": spec["coverings"], + "columns": spec.get("columns", []), } # The complete set of MEOS C symbols this descriptor depends on — the @@ -52,7 +84,10 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: symbols.add(spec["box"]["from"]) if spec.get("srid"): symbols.add(spec["srid"]) - for col in spec["columns"]: + for covering in spec["coverings"]: + for field in covering["fields"]: + symbols.add(field["accessor"]) + for col in spec.get("columns", []): symbols.add(col["accessor"]) idl["temporalCovering"] = { @@ -63,7 +98,7 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: "classes": classes, "deferred": data.get("deferred", {}), "notes": data["notes"], - "byType": by_type, # "tgeompoint" -> class + columns + "byType": by_type, # "tgeompoint" -> class + coverings "types": sorted(by_type), "symbols": sorted(symbols), # referenced C symbols (audit set) "count": len(by_type), diff --git a/tests/test_covering.py b/tests/test_covering.py index 768a10c..a389081 100644 --- a/tests/test_covering.py +++ b/tests/test_covering.py @@ -2,6 +2,7 @@ python3 tests/test_covering.py """ +import copy import json import sys import unittest @@ -10,12 +11,25 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from parser.covering import attach_temporal_covering +from parser.covering import attach_temporal_covering, BBOX_3D MAP = ROOT / "meta" / "temporal-covering.json" SCHEMA = ROOT / "meta" / "temporal-covering.schema.json" +def _attach_variant(testcase, mutate): + """Attach a mutated copy of the descriptor written to a scratch file.""" + bad = json.loads(MAP.read_text()) + mutate(bad) + p = ROOT / "output" / "_variant_covering.json" + p.parent.mkdir(exist_ok=True) + p.write_text(json.dumps(bad)) + try: + return attach_temporal_covering({}, p) + finally: + p.unlink() + + class AttachTests(unittest.TestCase): def test_attaches_and_indexes(self): idl = attach_temporal_covering({"functions": []}, MAP) @@ -23,22 +37,43 @@ def test_attaches_and_indexes(self): # tgeompoint resolves to the spatial class with an STBOX box self.assertEqual(cov["byType"]["tgeompoint"]["class"], "spatial") self.assertEqual(cov["byType"]["tgeompoint"]["box"]["type"], "STBOX") + self.assertEqual( + [c["key"] for c in cov["byType"]["tgeompoint"]["coverings"]], + ["bbox", "tspan"]) # tfloat resolves to the number class with a TBOX box self.assertEqual(cov["byType"]["tfloat"]["class"], "number") self.assertEqual(cov["byType"]["tfloat"]["box"]["type"], "TBOX") + self.assertEqual( + [c["key"] for c in cov["byType"]["tfloat"]["coverings"]], + ["vspan", "tspan"]) # tbool resolves to the time-only class with no box self.assertEqual(cov["byType"]["tbool"]["class"], "timeOnly") self.assertIsNone(cov["byType"]["tbool"]["box"]) + self.assertEqual( + [c["key"] for c in cov["byType"]["tbool"]["coverings"]], ["tspan"]) # count == number of covered types; types sorted self.assertEqual(cov["count"], len(cov["byType"])) self.assertEqual(cov["types"], sorted(cov["byType"])) + def test_bbox_is_a_geoparquet_bounding_box_column(self): + cov = attach_temporal_covering({}, MAP)["temporalCovering"] + bbox = next(c for c in cov["byType"]["tgeompoint"]["coverings"] + if c["key"] == "bbox") + self.assertEqual(bbox["column"], "{col}_bbox") + self.assertEqual(tuple(f["name"] for f in bbox["fields"]), BBOX_3D) + self.assertEqual({f["sqlType"] for f in bbox["fields"]}, {"double"}) + self.assertEqual( + [f["name"] for f in bbox["fields"] if f.get("when") == "hasZ"], + ["zmin", "zmax"]) + def test_symbols_collected(self): cov = attach_temporal_covering({}, MAP)["temporalCovering"] - # the value codec and both box converters are in the audit set + # the value codec, both box converters, and the field accessors are + # in the audit set for sym in ("temporal_as_hexwkb", "temporal_from_hexwkb", "tspatial_to_stbox", "tnumber_to_tbox", "stbox_xmin", - "tbox_xmin", "tspatial_srid"): + "stbox_tmin", "tbox_xmin", "tspatial_srid", + "temporal_start_timestamptz"): self.assertIn(sym, cov["symbols"]) def test_missing_file_is_noop(self): @@ -46,17 +81,34 @@ def test_missing_file_is_noop(self): self.assertEqual(idl, {"x": 1}) def test_duplicate_type_rejected(self): - bad = json.loads(MAP.read_text()) # claim tfloat in a second class too -> ambiguous codegen - bad["classes"]["spatial"]["types"].append("tfloat") - p = ROOT / "output" / "_dup_covering.json" - p.parent.mkdir(exist_ok=True) - p.write_text(json.dumps(bad)) - try: - with self.assertRaises(ValueError): - attach_temporal_covering({}, p) - finally: - p.unlink() + with self.assertRaises(ValueError): + _attach_variant(self, lambda d: d["classes"]["spatial"]["types"] + .append("tfloat")) + + def test_misordered_bbox_rejected(self): + # xmax before ymin is not the GeoParquet field order + def swap(d): + fields = d["classes"]["spatial"]["coverings"][0]["fields"] + fields[1], fields[3] = fields[3], fields[1] + with self.assertRaises(ValueError): + _attach_variant(self, swap) + + def test_time_bounds_in_bbox_rejected(self): + # a bbox carrying tmin is not a GeoParquet bounding box column + def add_time(d): + spatial = d["classes"]["spatial"] + spatial["coverings"][0]["fields"].append( + copy.deepcopy(spatial["coverings"][1]["fields"][0])) + with self.assertRaises(ValueError): + _attach_variant(self, add_time) + + def test_duplicate_covering_rejected(self): + def dup(d): + number = d["classes"]["number"] + number["coverings"].append(copy.deepcopy(number["coverings"][1])) + with self.assertRaises(ValueError): + _attach_variant(self, dup) class SchemaTests(unittest.TestCase): diff --git a/tests/test_covering_projection.py b/tests/test_covering_projection.py index d27cc69..b0b431e 100644 --- a/tests/test_covering_projection.py +++ b/tests/test_covering_projection.py @@ -2,7 +2,7 @@ python3 tests/test_covering_projection.py Also the CI gate: when the enriched catalog with `temporalCovering` is -present, every covered type projects to a well-formed covering expression +present, every covered type projects to well-formed covering columns composed against the value. """ @@ -14,7 +14,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from parser.covering import attach_temporal_covering +from parser.covering import attach_temporal_covering, BBOX_2D, BBOX_3D from generator.covering import build_covering_projection MAP = ROOT / "meta" / "temporal-covering.json" @@ -25,25 +25,52 @@ def _projected(): return build_covering_projection(attach_temporal_covering({}, MAP)) +def _covering(projection, tname, key): + return next(c for c in projection["types"][tname]["coverings"] + if c["key"] == key) + + class ProjectionTests(unittest.TestCase): def test_spatial_box_composition(self): - cols = {c["name"]: c - for c in _projected()["types"]["tgeompoint"]["columns"]} - # box columns compose accessor(box_from(VALUE)) - self.assertEqual(cols["xmin"]["expr"], + p = _projected() + bbox = _covering(p, "tgeompoint", "bbox") + self.assertEqual(bbox["column"], "{col}_bbox") + # the fields are in GeoParquet bounding box order + self.assertEqual(tuple(f["name"] for f in bbox["fields"]), BBOX_3D) + fields = {f["name"]: f for f in bbox["fields"]} + # box fields compose accessor(box_from(VALUE)) + self.assertEqual(fields["xmin"]["expr"], "stbox_xmin(tspatial_to_stbox(VALUE))") - self.assertEqual(cols["xmin"]["sqlType"], "double") - # srid is read off the value, not the box - self.assertEqual(cols["srid"]["expr"], "tspatial_srid(VALUE)") + self.assertEqual(fields["xmin"]["sqlType"], "double") # zmin is conditional on 3D - self.assertEqual(cols["zmin"]["when"], "hasZ") + self.assertEqual(fields["zmin"]["when"], "hasZ") + tspan = _covering(p, "tgeompoint", "tspan") + self.assertEqual(tspan["column"], "{col}_tspan") + self.assertEqual([f["expr"] for f in tspan["fields"]], + ["stbox_tmin(tspatial_to_stbox(VALUE))", + "stbox_tmax(tspatial_to_stbox(VALUE))"]) + # srid is a plain column read off the value, not the box + cols = {c["name"]: c for c in p["types"]["tgeompoint"]["columns"]} + self.assertEqual(cols["srid"]["expr"], "tspatial_srid(VALUE)") def test_number_box_composition(self): - t = _projected()["types"]["tfloat"] - cols = {c["name"]: c for c in t["columns"]} + p = _projected() + t = p["types"]["tfloat"] self.assertEqual(t["boxType"], "TBOX") - self.assertEqual(cols["vmin"]["expr"], "tbox_xmin(tnumber_to_tbox(VALUE))") - self.assertEqual(cols["tmax"]["expr"], "tbox_tmax(tnumber_to_tbox(VALUE))") + self.assertEqual([c["key"] for c in t["coverings"]], ["vspan", "tspan"]) + vspan = {f["name"]: f for f in _covering(p, "tfloat", "vspan")["fields"]} + self.assertEqual(vspan["vmin"]["expr"], "tbox_xmin(tnumber_to_tbox(VALUE))") + tspan = {f["name"]: f for f in _covering(p, "tfloat", "tspan")["fields"]} + self.assertEqual(tspan["tmax"]["expr"], "tbox_tmax(tnumber_to_tbox(VALUE))") + self.assertEqual(t["columns"], []) + + def test_time_only_composition(self): + p = _projected() + t = p["types"]["tbool"] + self.assertIsNone(t["boxType"]) + self.assertEqual([c["key"] for c in t["coverings"]], ["tspan"]) + tspan = {f["name"]: f for f in _covering(p, "tbool", "tspan")["fields"]} + self.assertEqual(tspan["tmin"]["expr"], "temporal_start_timestamptz(VALUE)") def test_count_and_codec(self): p = _projected() @@ -61,16 +88,22 @@ def test_every_type_projects_wellformed(self): cat = attach_temporal_covering(json.loads(_CATALOG.read_text()), MAP) p = build_covering_projection(cat) self.assertEqual(p["count"], 13) - # time-only types (tbool/ttext) project to tmin/tmax via the value, no box + # time-only types (tbool/ttext) project to a tspan read off the value self.assertEqual(p["types"]["tbool"]["boxType"], None) self.assertEqual( - {c["name"] for c in p["types"]["tbool"]["columns"]}, {"tmin", "tmax"}) + [f["name"] for f in _covering(p, "tbool", "tspan")["fields"]], + ["tmin", "tmax"]) for spec in p["types"].values(): - self.assertTrue(spec["columns"]) - for c in spec["columns"]: - # composed against the value, balanced parentheses - self.assertIn("(VALUE)", c["expr"]) - self.assertEqual(c["expr"].count("("), c["expr"].count(")")) + self.assertTrue(spec["coverings"]) + for covering in spec["coverings"]: + self.assertTrue(covering["column"].startswith("{col}_")) + if covering["key"] == "bbox": + names = tuple(f["name"] for f in covering["fields"]) + self.assertIn(names, (BBOX_2D, BBOX_3D)) + for f in covering["fields"] + spec["columns"]: + # composed against the value, balanced parentheses + self.assertIn("(VALUE)", f["expr"]) + self.assertEqual(f["expr"].count("("), f["expr"].count(")")) if __name__ == "__main__":