From 900b3c2aeeea6d6607e6dc9d2f7aecedb14b4ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Zim=C3=A1nyi?= Date: Thu, 10 Sep 2026 19:55:35 +0200 Subject: [PATCH] Read the value bounds of a temporal number in its base type The `vspan` covering holds the value bounds of a temporal number in its base type, read off the value: `tint_min_value`/`tint_max_value` as int, `tbigint_min_value`/`tbigint_max_value` as bigint, and `tfloat_min_value`/`tfloat_max_value` as double. A bigint bound stays exact above 2^53 and no bound is rounded inward, so the covering stays conservative and pruning never skips a matching row. The three number types differ in base type, so a covering gives its fields per type (`byType`) where they differ and once for the class (`fields`) where they do not. The parser resolves them into the fields of each type and requires a per-type covering to name exactly the types of its class. The descriptor schema admits `bigint` as a field type. --- docs/temporal-covering.md | 10 ++++ meta/temporal-covering.json | 19 ++++-- meta/temporal-covering.schema.json | 23 +++++-- parser/covering.py | 96 ++++++++++++++++++++---------- tests/test_covering.py | 35 ++++++++++- tests/test_covering_projection.py | 14 ++++- 6 files changed, 151 insertions(+), 46 deletions(-) diff --git a/docs/temporal-covering.md b/docs/temporal-covering.md index a01c365..1cfe49f 100644 --- a/docs/temporal-covering.md +++ b/docs/temporal-covering.md @@ -33,6 +33,16 @@ in its class: the order shown, it has the repetition of its temporal column, and it holds a value exactly when the temporal column does. `zmin`/`zmax` are emitted only for 3D values (`when: hasZ`). `srid` is a plain column beside the coverings. + +`{col}_vspan` holds the value bounds in the base type of the temporal type, +read off the value with `tint_min_value`/`tint_max_value` (`int`), +`tbigint_min_value`/`tbigint_max_value` (`bigint`) and +`tfloat_min_value`/`tfloat_max_value` (`double`). A bound that went through +`double` would lose exactness above 2^53 for `tbigint`, and a minimum +rounded upwards would let pruning skip a matching row. Because the three +number types differ in base type, the `vspan` covering gives its fields per +type (`byType`); a covering whose fields hold for the whole class gives them +once (`fields`). The canonical value column is unchanged and lossless; covering columns are denormalised derivations of the value's box. diff --git a/meta/temporal-covering.json b/meta/temporal-covering.json index 1029417..4ff85f9 100644 --- a/meta/temporal-covering.json +++ b/meta/temporal-covering.json @@ -47,10 +47,20 @@ "srid": null, "types": ["tint", "tfloat", "tbigint"], "coverings": [ - {"key": "vspan", "column": "{col}_vspan", "fields": [ - {"name": "vmin", "sqlType": "double", "accessor": "tbox_xmin", "source": "box"}, - {"name": "vmax", "sqlType": "double", "accessor": "tbox_xmax", "source": "box"} - ]}, + {"key": "vspan", "column": "{col}_vspan", "byType": { + "tint": [ + {"name": "vmin", "sqlType": "int", "accessor": "tint_min_value", "source": "value"}, + {"name": "vmax", "sqlType": "int", "accessor": "tint_max_value", "source": "value"} + ], + "tfloat": [ + {"name": "vmin", "sqlType": "double", "accessor": "tfloat_min_value", "source": "value"}, + {"name": "vmax", "sqlType": "double", "accessor": "tfloat_max_value", "source": "value"} + ], + "tbigint": [ + {"name": "vmin", "sqlType": "bigint", "accessor": "tbigint_min_value", "source": "value"}, + {"name": "vmax", "sqlType": "bigint", "accessor": "tbigint_max_value", "source": "value"} + ] + }}, {"key": "tspan", "column": "{col}_tspan", "fields": [ {"name": "tmin", "sqlType": "timestamptz", "accessor": "tbox_tmin", "source": "box"}, {"name": "tmax", "sqlType": "timestamptz", "accessor": "tbox_tmax", "source": "box"} @@ -83,6 +93,7 @@ "Each covering is a struct column at the root of the schema; `{col}` in its `column` is the name of the temporal column it covers. The statistics of its fields give Iceberg manifest-level file pruning and Parquet row-group min/max pruning, with no spatial-aware engine.", "The `bbox` covering is a GeoParquet bounding box column: its fields are DOUBLE, in the order xmin, ymin, [zmin,] xmax, ymax[, zmax], it has the repetition of its temporal column, and it holds a value exactly when the temporal column does.", "zmin/zmax are emitted only for 3D values (`when: hasZ`); the bbox of a 2D value has four fields.", + "The `vspan` bounds are of the base type of each number type (int, bigint, double), read off the value, so a bigint bound stays exact and no bound is rounded inward; a covering whose fields differ by type gives them per type (`byType`), one that holds for the whole class gives them once (`fields`).", "`columns` are plain columns at the root beside the coverings; the spatial class carries `srid` there.", "`source: box` accessors take the box returned by `class.box.from(value)`; `source: value` accessors take the temporal value directly.", "This descriptor is type-agnostic per class exactly as `portable-aliases.json` is type-agnostic per operator family — codegen consumes it identically across every binding." diff --git a/meta/temporal-covering.schema.json b/meta/temporal-covering.schema.json index 243efd9..d85936e 100644 --- a/meta/temporal-covering.schema.json +++ b/meta/temporal-covering.schema.json @@ -13,11 +13,16 @@ "required": ["name", "sqlType", "accessor", "source"], "properties": { "name": {"type": "string", "pattern": "^[a-z][a-z0-9]*$"}, - "sqlType": {"enum": ["double", "int", "timestamptz"]}, + "sqlType": {"enum": ["double", "int", "bigint", "timestamptz"]}, "accessor": {"type": "string"}, "source": {"enum": ["box", "value"]}, "when": {"enum": ["hasZ"]} } + }, + "fields": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/field"} } }, "properties": { @@ -83,14 +88,20 @@ "items": { "type": "object", "additionalProperties": false, - "required": ["key", "column", "fields"], + "required": ["key", "column"], + "oneOf": [ + {"required": ["fields"]}, + {"required": ["byType"]} + ], "properties": { "key": {"enum": ["bbox", "tspan", "vspan"]}, "column": {"type": "string", "pattern": "^\\{col\\}_[a-z]+$"}, - "fields": { - "type": "array", - "minItems": 1, - "items": {"$ref": "#/$defs/field"} + "fields": {"$ref": "#/$defs/fields"}, + "byType": { + "type": "object", + "minProperties": 1, + "propertyNames": {"pattern": "^t[a-z0-9]+$"}, + "additionalProperties": {"$ref": "#/$defs/fields"} } } } diff --git a/parser/covering.py b/parser/covering.py index 9d0ae46..84afbcf 100644 --- a/parser/covering.py +++ b/parser/covering.py @@ -6,15 +6,18 @@ #870 TemporalParquet / #913 Temporal Data Lake): per temporal-type *class* (spatial → STBOX, number → TBOX, timeOnly → no box) it names the box converter, the SRID accessor, the covering struct columns with their fields -and MEOS accessors, and the plain columns beside them. Folding it into the -catalog means every binding/engine generates the *identical* covering -schema, so a temporal table prunes the same way on every platform (Iceberg -manifest pruning + Parquet row-group min/max) with no spatial-aware engine. +and MEOS accessors, and the plain columns beside them. A covering gives its +fields once for the whole class, or per type (`byType`) where the types of a +class differ in base type. Folding it into the catalog means every +binding/engine generates the *identical* covering schema, so a temporal +table prunes the same way on every platform (Iceberg manifest pruning + +Parquet row-group min/max) with no spatial-aware engine. This is curated canonical data, not a heuristic — it is preserved verbatim -and only *derived* lookups are added (a flat `byType` index and the set of -referenced C symbols), so a generator never has to re-derive the mapping. -Pure dict → dict; no libclang. +and only *derived* lookups are added (a flat `byType` index, where every +covering carries the fields of that type, and the set of referenced C +symbols), so a generator never has to re-derive the mapping. Pure dict → +dict; no header parsing. """ import json @@ -25,29 +28,59 @@ BBOX_3D = ("xmin", "ymin", "zmin", "xmax", "ymax", "zmax") -def _check_coverings(class_name: str, coverings: list) -> None: - """Reject a class whose coverings a generator could not render as the - TemporalParquet 2.0.0 covering columns.""" - keys = [c["key"] for c in coverings] +def _resolve(class_name: str, types: list, covering: dict) -> dict: + """Return, per type of the class, the fields a covering holds.""" + has_fields, has_by_type = "fields" in covering, "byType" in covering + if has_fields == has_by_type: + raise ValueError( + f"temporal-covering: class {class_name!r} covering " + f"{covering['key']!r} gives neither or both of fields and byType") + if has_fields: + return {t: covering["fields"] for t in types} + by_type = covering["byType"] + if set(by_type) != set(types): + raise ValueError( + f"temporal-covering: class {class_name!r} covering " + f"{covering['key']!r} gives fields for {sorted(by_type)}, where " + f"the class holds {sorted(types)}") + return by_type + + +def _check_bbox(class_name: str, fields: list) -> None: + """Reject bbox fields that are not a GeoParquet bounding box column's.""" + names = tuple(f["name"] for f in fields) + planar = tuple(f["name"] for f in fields if f.get("when") != "hasZ") + if names not in (BBOX_2D, BBOX_3D) or planar != BBOX_2D: + raise ValueError( + f"temporal-covering: class {class_name!r} declares the bbox " + f"fields {names}, where a GeoParquet bounding box column has " + f"{BBOX_2D} or {BBOX_3D}, the z fields only for 3D values") + if any(f["sqlType"] != "double" for f in fields): + raise ValueError( + f"temporal-covering: class {class_name!r} declares a bbox " + f"field that is not double") + + +def _coverings_by_type(class_name: str, spec: dict) -> dict: + """Return, per type of the class, its coverings with their fields.""" + keys = [c["key"] for c in spec["coverings"]] if len(keys) != len(set(keys)): raise ValueError( f"temporal-covering: class {class_name!r} declares a covering " f"twice ({keys})") - for covering in coverings: - if covering["key"] != "bbox": - continue - fields = covering["fields"] - names = tuple(f["name"] for f in fields) - planar = tuple(f["name"] for f in fields if f.get("when") != "hasZ") - if names not in (BBOX_2D, BBOX_3D) or planar != BBOX_2D: - raise ValueError( - f"temporal-covering: class {class_name!r} declares the bbox " - f"fields {names}, where a GeoParquet bounding box column has " - f"{BBOX_2D} or {BBOX_3D}, the z fields only for 3D values") - if any(f["sqlType"] != "double" for f in fields): - raise ValueError( - f"temporal-covering: class {class_name!r} declares a bbox " - f"field that is not double") + result = {t: [] for t in spec["types"]} + for covering in spec["coverings"]: + fields_by_type = _resolve(class_name, spec["types"], covering) + for t in spec["types"]: + fields = fields_by_type[t] + if covering["key"] == "bbox": + _check_bbox(class_name, fields) + result[t].append({ + "key": covering["key"], + "column": covering["column"], + "fields": fields, + }) + return result def attach_temporal_covering(idl: dict, path: Path) -> dict: @@ -62,7 +95,7 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: # two classes claiming the same type would make codegen ambiguous. by_type = {} for class_name, spec in classes.items(): - _check_coverings(class_name, spec["coverings"]) + coverings = _coverings_by_type(class_name, spec) for t in spec["types"]: if t in by_type: raise ValueError( @@ -72,7 +105,7 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: "class": class_name, "box": spec.get("box"), "srid": spec.get("srid"), - "coverings": spec["coverings"], + "coverings": coverings[t], "columns": spec.get("columns", []), } @@ -84,11 +117,12 @@ def attach_temporal_covering(idl: dict, path: Path) -> dict: symbols.add(spec["box"]["from"]) if spec.get("srid"): symbols.add(spec["srid"]) - for covering in spec["coverings"]: - for field in covering["fields"]: - symbols.add(field["accessor"]) for col in spec.get("columns", []): symbols.add(col["accessor"]) + for entry in by_type.values(): + for covering in entry["coverings"]: + for field in covering["fields"]: + symbols.add(field["accessor"]) idl["temporalCovering"] = { "provenance": data["provenance"], diff --git a/tests/test_covering.py b/tests/test_covering.py index a389081..32a6bba 100644 --- a/tests/test_covering.py +++ b/tests/test_covering.py @@ -30,6 +30,10 @@ def _attach_variant(testcase, mutate): p.unlink() +def _covering(cov, tname, key): + return next(c for c in cov["byType"][tname]["coverings"] if c["key"] == key) + + class AttachTests(unittest.TestCase): def test_attaches_and_indexes(self): idl = attach_temporal_covering({"functions": []}, MAP) @@ -57,8 +61,7 @@ def test_attaches_and_indexes(self): def test_bbox_is_a_geoparquet_bounding_box_column(self): cov = attach_temporal_covering({}, MAP)["temporalCovering"] - bbox = next(c for c in cov["byType"]["tgeompoint"]["coverings"] - if c["key"] == "bbox") + bbox = _covering(cov, "tgeompoint", "bbox") self.assertEqual(bbox["column"], "{col}_bbox") self.assertEqual(tuple(f["name"] for f in bbox["fields"]), BBOX_3D) self.assertEqual({f["sqlType"] for f in bbox["fields"]}, {"double"}) @@ -66,13 +69,32 @@ def test_bbox_is_a_geoparquet_bounding_box_column(self): [f["name"] for f in bbox["fields"] if f.get("when") == "hasZ"], ["zmin", "zmax"]) + def test_vspan_bounds_are_of_the_base_type(self): + # the value bounds are read off the value in its base type, so a + # bigint bound stays exact and no bound is rounded inward + cov = attach_temporal_covering({}, MAP)["temporalCovering"] + expected = { + "tint": ("int", "tint_min_value", "tint_max_value"), + "tbigint": ("bigint", "tbigint_min_value", "tbigint_max_value"), + "tfloat": ("double", "tfloat_min_value", "tfloat_max_value"), + } + for tname, (sql_type, vmin, vmax) in expected.items(): + vspan = _covering(cov, tname, "vspan") + self.assertEqual(vspan["column"], "{col}_vspan") + self.assertEqual( + [(f["name"], f["sqlType"], f["accessor"], f["source"]) + for f in vspan["fields"]], + [("vmin", sql_type, vmin, "value"), + ("vmax", sql_type, vmax, "value")]) + def test_symbols_collected(self): cov = attach_temporal_covering({}, MAP)["temporalCovering"] # the value codec, both box converters, and the field accessors are # in the audit set for sym in ("temporal_as_hexwkb", "temporal_from_hexwkb", "tspatial_to_stbox", "tnumber_to_tbox", "stbox_xmin", - "stbox_tmin", "tbox_xmin", "tspatial_srid", + "stbox_tmin", "tbox_tmin", "tspatial_srid", + "tint_min_value", "tbigint_max_value", "tfloat_min_value", "temporal_start_timestamptz"): self.assertIn(sym, cov["symbols"]) @@ -110,6 +132,13 @@ def dup(d): with self.assertRaises(ValueError): _attach_variant(self, dup) + def test_by_type_missing_a_type_rejected(self): + # a per-type covering must give fields for every type of its class + def drop(d): + del d["classes"]["number"]["coverings"][0]["byType"]["tbigint"] + with self.assertRaises(ValueError): + _attach_variant(self, drop) + class SchemaTests(unittest.TestCase): def test_descriptor_validates(self): diff --git a/tests/test_covering_projection.py b/tests/test_covering_projection.py index b0b431e..32a7aa1 100644 --- a/tests/test_covering_projection.py +++ b/tests/test_covering_projection.py @@ -58,12 +58,22 @@ def test_number_box_composition(self): t = p["types"]["tfloat"] self.assertEqual(t["boxType"], "TBOX") self.assertEqual([c["key"] for c in t["coverings"]], ["vspan", "tspan"]) - vspan = {f["name"]: f for f in _covering(p, "tfloat", "vspan")["fields"]} - self.assertEqual(vspan["vmin"]["expr"], "tbox_xmin(tnumber_to_tbox(VALUE))") tspan = {f["name"]: f for f in _covering(p, "tfloat", "tspan")["fields"]} self.assertEqual(tspan["tmax"]["expr"], "tbox_tmax(tnumber_to_tbox(VALUE))") self.assertEqual(t["columns"], []) + def test_number_value_bounds_per_base_type(self): + # each numeric type reads its value bounds off the value, typed as + # its base type + p = _projected() + for tname, sql_type in (("tint", "int"), ("tbigint", "bigint"), + ("tfloat", "double")): + vspan = _covering(p, tname, "vspan") + self.assertEqual( + [(f["name"], f["sqlType"], f["expr"]) for f in vspan["fields"]], + [("vmin", sql_type, f"{tname}_min_value(VALUE)"), + ("vmax", sql_type, f"{tname}_max_value(VALUE)")]) + def test_time_only_composition(self): p = _projected() t = p["types"]["tbool"]