feat(zarr-metadata): Zarr metadata model layer

.github/workflows/zarr-metadata.yml

@@ -82,7 +82,10 @@ jobs:
       - name: Sync test dependency group
         run: uv sync --group test --python 3.11
       - name: Run pyright
-        run: uv run --group test --with pyright pyright src
+        # Pinned to the last version that types PEP 661 sentinels in class
+        # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115).
+        # Unpin when the fix lands.
+        run: uv run --group test --with 'pyright==1.1.404' pyright src
 
   zarr-metadata-complete:
     name: zarr-metadata complete

packages/zarr-metadata/changes/210.feature.md

@@ -0,0 +1,69 @@
+Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`,
+`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`,
+`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`)
+that are canonical, lossless representations of Zarr metadata documents, plus
+structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension
+point (data type, chunk grid, chunk key encoding, codecs, storage transformers)
+is held as a name + configuration pair; nothing is interpreted. Model fields
+are annotated with the role alias `MetadataFieldModelV3` (today exactly
+`NamedConfigModelV3`), so the annotations convey the logical meaning of the
+field and stay put if the spec ever adds a new field form.
+
+Validation is strict about what the types declare: v2 `dtype` / `order` /
+`compressor` / `filters` / `dimension_separator` shapes and the fixed
+`zarr_format` / `node_type` literals are all enforced. Every
+`ValidationProblem` carries a machine-readable `kind`
+(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so
+consumers can dispatch on the failure mode without matching message strings,
+and every ingestion failure — including missing store keys and undecodable
+bytes in `from_key_value` — surfaces as `MetadataValidationError`. An
+adversarial review added further structural checks: JSON booleans are not
+accepted as dimension lengths, dimensions are non-negative,
+`dimension_names` must have one entry per dimension of `shape`, `attributes`
+and `configuration` values are JSON-checked recursively (like `fill_value`),
+and the inline consolidated-metadata envelope and entries are deep-validated
+so the group validator's verdict always agrees with the model constructor.
+
+The v3 models expose `must_understand_fields`: the subset of `extra_fields`
+not explicitly waived with `must_understand: false` (fields are implicitly
+must-understand per the spec). Readers discharge the spec's fail-to-open
+duty by subtracting the extension names they recognize; the model only
+partitions by obligation, since recognition is reader-specific.
+
+Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it
+requires pydantic v2; the core package does not depend on it): one `Annotated`
+field type per model, validating raw documents through `from_json`, passing
+core-model instances through unchanged, and serializing via `to_json`. The
+instances are the core model classes, so values interoperate freely with
+non-pydantic code.
+
+`create_default` keeps its output self-consistent: overriding `shape` without
+a chunk grid derives one regular chunk covering the array (v3
+`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the
+scalar default's 0-d grid.
+
+A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2
+convention's default `"."` (the model previously normalized absence to `"/"`,
+which would misaddress the chunks of real-world default-separator arrays).
+The value is never null: absent, `"."`, or `"/"` are the only spellings.
+
+Optional document keys use `UNSET` — a PEP 661 sentinel
+(`typing_extensions.Sentinel`), usable directly in type expressions — never
+`None`: in a model, `None` always corresponds to a JSON `null` in the
+document (a v2 `compressor`, an unnamed dimension inside `dimension_names`),
+and `UNSET` always means the key is absent. Checker note: ty types the
+sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115
+is fixed (this package's CI pins it); mypy users need a `cast` or
+`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings
+distinct — an absent `dimension_names` ("there are no dimension names") and
+an explicit `[null, null]` ("every dimension has a name, which is null") are
+different documents and round-trip as such. The `consolidated_metadata: null`
+written by a historical zarr-python bug is the one deliberate exception to
+faithful round-tripping: those stores remain readable, but the bug spelling
+is repaired to absence on read and never written back.
+
+The v2 models treat the `.zattrs` file's presence as part of the store:
+`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value`
+emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a
+file. Previously `to_key_value` always emitted `.zattrs`, silently adding a
+file to stores that never had one.

packages/zarr-metadata/pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 ]
 keywords = ["zarr"]
 dependencies = [
-    "typing_extensions>=4.13",
+    "typing_extensions>=4.14",
 ]
 
 [project.urls]
@@ -82,6 +82,9 @@ checks = [
     "PR06",
 ]
 
+# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in
+# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel
+# relies on. Use the same pin locally; unpin when the fix lands.
 [tool.pyright]
 include = ["src"]
 enableExperimentalFeatures = true

packages/zarr-metadata/src/zarr_metadata/__init__.py

@@ -1,6 +1,24 @@
 from importlib.metadata import version
 
 from zarr_metadata._common import JSONValue, NamedConfigV3
+from zarr_metadata.model import (
+    UNSET,
+    ArrayMetadataModelV2,
+    ArrayMetadataModelV2Partial,
+    ArrayMetadataModelV3,
+    ArrayMetadataModelV3Partial,
+    ConsolidatedMetadataModelV2,
+    ConsolidatedMetadataModelV3,
+    GroupMetadataModelV2,
+    GroupMetadataModelV2Partial,
+    GroupMetadataModelV3,
+    GroupMetadataModelV3Partial,
+    MetadataFieldModelV3,
+    MetadataValidationError,
+    NamedConfigModelV3,
+    ProblemKind,
+    ValidationProblem,
+)
 from zarr_metadata.v2.array import (
     ARRAY_DIMENSION_SEPARATOR_V2,
     ARRAY_ORDER_V2,
@@ -231,10 +249,15 @@
     "UINT16_DATA_TYPE_NAME",
     "UINT32_DATA_TYPE_NAME",
     "UINT64_DATA_TYPE_NAME",
+    "UNSET",
     "V2_CHUNK_KEY_ENCODING_NAME",
     "V2_CHUNK_KEY_ENCODING_SEPARATOR",
     "ZSTD_CODEC_NAME",
     "ArrayDimensionSeparatorV2",
+    "ArrayMetadataModelV2",
+    "ArrayMetadataModelV2Partial",
+    "ArrayMetadataModelV3",
+    "ArrayMetadataModelV3Partial",
     "ArrayMetadataV2",
     "ArrayMetadataV2Partial",
     "ArrayMetadataV3",
@@ -259,6 +282,8 @@
     "Complex64FillValue",
     "Complex128DataTypeName",
     "Complex128FillValue",
+    "ConsolidatedMetadataModelV2",
+    "ConsolidatedMetadataModelV3",
     "ConsolidatedMetadataV2",
     "ConsolidatedMetadataV3",
     "Crc32cCodecMetadata",
@@ -275,6 +300,10 @@
     "Float32FillValue",
     "Float64DataTypeName",
     "Float64FillValue",
+    "GroupMetadataModelV2",
+    "GroupMetadataModelV2Partial",
+    "GroupMetadataModelV3",
+    "GroupMetadataModelV3Partial",
     "GroupMetadataV2",
     "GroupMetadataV2Partial",
     "GroupMetadataV3",
@@ -290,13 +319,17 @@
     "Int64DataTypeName",
     "Int64FillValue",
     "JSONValue",
+    "MetadataFieldModelV3",
     "MetadataV3",
+    "MetadataValidationError",
+    "NamedConfigModelV3",
     "NamedConfigV3",
     "NumpyDatetime64DataTypeName",
     "NumpyDatetime64FillValue",
     "NumpyTimeUnit",
     "NumpyTimedelta64DataTypeName",
     "NumpyTimedelta64FillValue",
+    "ProblemKind",
     "RawBytesDataTypeName",
     "RawBytesFillValue",
     "RectilinearChunkGridMetadata",
@@ -325,6 +358,7 @@
     "V2ChunkKeyEncodingMetadata",
     "V2ChunkKeyEncodingName",
     "V2ChunkKeyEncodingSeparator",
+    "ValidationProblem",
     "ZArrayMetadata",
     "ZAttrsMetadata",
     "ZGroupMetadata",

packages/zarr-metadata/src/zarr_metadata/_common.py

@@ -13,7 +13,7 @@
 
 JSONValue = TypeAliasType(
     "JSONValue",
-    "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]",  # type: ignore[reportInvalidTypeForm]
+    "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]",
 )
 """A recursive type alias for JSON-encodable values.
 

packages/zarr-metadata/src/zarr_metadata/model/__init__.py

@@ -0,0 +1,120 @@
+"""In-memory models for Zarr metadata documents.
+
+Models are frozen dataclasses that hold a canonical, lossless representation
+of the JSON documents; they never interpret extension points (codecs, chunk
+grids, data types). Validators check JSON structure, not domain validity.
+Each document concept gets a `validate_*` function returning every problem
+found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an
+`is_*` type guard, and a `parse_*` function that narrows or raises
+`MetadataValidationError`. Model `from_json` / `from_key_value` constructors
+raise `MetadataValidationError` for every ingestion failure, including
+missing store keys and undecodable bytes.
+"""
+
+from zarr_metadata.model._array import (
+    ARRAY_METADATA_STORE_KEY_V2,
+    ARRAY_METADATA_STORE_KEY_V3,
+    ATTRIBUTES_STORE_KEY_V2,
+    ArrayMetadataModelV2,
+    ArrayMetadataModelV2Partial,
+    ArrayMetadataModelV3,
+    ArrayMetadataModelV3Partial,
+    MetadataFieldModelV3,
+    NamedConfigModelV3,
+)
+from zarr_metadata.model._group import (
+    CONSOLIDATED_METADATA_KEY_V3,
+    CONSOLIDATED_METADATA_STORE_KEY_V2,
+    GROUP_METADATA_STORE_KEY_V2,
+    GROUP_METADATA_STORE_KEY_V3,
+    ConsolidatedMetadataModelV2,
+    ConsolidatedMetadataModelV3,
+    GroupMetadataModelV2,
+    GroupMetadataModelV2Partial,
+    GroupMetadataModelV3,
+    GroupMetadataModelV3Partial,
+)
+from zarr_metadata.model._sentinel import UNSET
+from zarr_metadata.model._validation import (
+    ARRAY_METADATA_OPTIONAL_KEYS_V3,
+    ARRAY_METADATA_REQUIRED_KEYS_V2,
+    ARRAY_METADATA_REQUIRED_KEYS_V3,
+    ARRAY_METADATA_STANDARD_KEYS_V3,
+    GROUP_METADATA_OPTIONAL_KEYS_V3,
+    GROUP_METADATA_REQUIRED_KEYS_V2,
+    GROUP_METADATA_REQUIRED_KEYS_V3,
+    GROUP_METADATA_STANDARD_KEYS_V3,
+    MetadataValidationError,
+    ProblemKind,
+    ValidationProblem,
+    is_array_metadata_v2,
+    is_array_metadata_v3,
+    is_group_metadata_v2,
+    is_group_metadata_v3,
+    is_json,
+    is_metadata_field_v3,
+    parse_array_metadata_v2,
+    parse_array_metadata_v3,
+    parse_group_metadata_v2,
+    parse_group_metadata_v3,
+    parse_json,
+    parse_metadata_field_v3,
+    validate_array_metadata_v2,
+    validate_array_metadata_v3,
+    validate_group_metadata_v2,
+    validate_group_metadata_v3,
+    validate_json,
+    validate_metadata_field_v3,
+)
+
+__all__ = [
+    "ARRAY_METADATA_OPTIONAL_KEYS_V3",
+    "ARRAY_METADATA_REQUIRED_KEYS_V2",
+    "ARRAY_METADATA_REQUIRED_KEYS_V3",
+    "ARRAY_METADATA_STANDARD_KEYS_V3",
+    "ARRAY_METADATA_STORE_KEY_V2",
+    "ARRAY_METADATA_STORE_KEY_V3",
+    "ATTRIBUTES_STORE_KEY_V2",
+    "CONSOLIDATED_METADATA_KEY_V3",
+    "CONSOLIDATED_METADATA_STORE_KEY_V2",
+    "GROUP_METADATA_OPTIONAL_KEYS_V3",
+    "GROUP_METADATA_REQUIRED_KEYS_V2",
+    "GROUP_METADATA_REQUIRED_KEYS_V3",
+    "GROUP_METADATA_STANDARD_KEYS_V3",
+    "GROUP_METADATA_STORE_KEY_V2",
+    "GROUP_METADATA_STORE_KEY_V3",
+    "UNSET",
+    "ArrayMetadataModelV2",
+    "ArrayMetadataModelV2Partial",
+    "ArrayMetadataModelV3",
+    "ArrayMetadataModelV3Partial",
+    "ConsolidatedMetadataModelV2",
+    "ConsolidatedMetadataModelV3",
+    "GroupMetadataModelV2",
+    "GroupMetadataModelV2Partial",
+    "GroupMetadataModelV3",
+    "GroupMetadataModelV3Partial",
+    "MetadataFieldModelV3",
+    "MetadataValidationError",
+    "NamedConfigModelV3",
+    "ProblemKind",
+    "ValidationProblem",
+    "is_array_metadata_v2",
+    "is_array_metadata_v3",
+    "is_group_metadata_v2",
+    "is_group_metadata_v3",
+    "is_json",
+    "is_metadata_field_v3",
+    "parse_array_metadata_v2",
+    "parse_array_metadata_v3",
+    "parse_group_metadata_v2",
+    "parse_group_metadata_v3",
+    "parse_json",
+    "parse_metadata_field_v3",
+    "validate_array_metadata_v2",
+    "validate_array_metadata_v3",
+    "validate_group_metadata_v2",
+    "validate_group_metadata_v3",
+    "validate_json",
+    "validate_metadata_field_v3",
+]