From 8da13336dfa60c2b4412097bc5e58488aebbb66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 31 May 2026 19:28:07 +0200 Subject: [PATCH 01/33] chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/downstream.yml | 2 +- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/zarr-metadata-release.yml | 12 ++++++------ .github/workflows/zizmor.yml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 74026233c4..3eb6898895 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -34,7 +34,7 @@ jobs: persist-credentials: false - name: Set up pixi - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0 # v0.9.5 + uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 with: manifest-path: xarray/pixi.toml diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 403441b306..333769cb9e 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -76,7 +76,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: gpu diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 4f9467be7d..a456b2aa0a 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -93,7 +93,7 @@ jobs: key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 14fba5b9ec..510849ef3e 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github/issue-metrics@c9e9838147fd355dace335ba787f01b6641a400a # v4.2.2 + uses: github/issue-metrics@1e38d5e62363e14db8019ed7d106b9855bdba6cc # v4.2.7 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 768e660ec2..fec211b4dd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,4 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true - - uses: j178/prek-action@6ad80277337ad479fe43bd70701c3f7f8aa74db3 # v2.0.3 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03143d3e5b..62e571856b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests @@ -125,7 +125,7 @@ jobs: run: | hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 809d502f16..9639fcfdd3 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: hatch build - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: zarr-metadata-dist path: packages/zarr-metadata/dist @@ -45,7 +45,7 @@ jobs: needs: [build] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -76,7 +76,7 @@ jobs: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -87,7 +87,7 @@ jobs: subject-path: dist/* - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 upload_testpypi: name: Upload to TestPyPI @@ -101,7 +101,7 @@ jobs: id-token: write attestations: write steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -112,6 +112,6 @@ jobs: subject-path: dist/* - name: Publish package to TestPyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index da19f22421..7ac4fe5d0e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 From ce2cfd7ba44bc6f9b44e63ec372e026a4860c409 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 4 Jul 2026 22:41:46 +0200 Subject: [PATCH 02/33] =?UTF-8?q?feat(zarr-metadata):=20add=20model.=5Fval?= =?UTF-8?q?idation=20=E2=80=94=20structural=20validators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/__init__.py | 1 + .../src/zarr_metadata/model/_validation.py | 271 ++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_validation.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py new file mode 100644 index 0000000000..e1382a6f0a --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -0,0 +1 @@ +"""In-memory models for Zarr metadata documents.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py new file mode 100644 index 0000000000..d0e30e1cda --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -0,0 +1,271 @@ +"""Structural validation for Zarr metadata documents. + +Validators check JSON structure (shapes, key presence, primitive kinds), +not domain validity. Each concept gets a `validate_*` function returning +every problem found, an `is_*` type guard, and a `parse_*` function that +narrows or raises `MetadataValidationError`. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Final, cast + +from typing_extensions import TypeIs + +from zarr_metadata import ArrayMetadataV2, ArrayMetadataV3, MetadataV3 +from zarr_metadata._common import JSONValue + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single structural problem found while validating a metadata document. + + `loc` is the path from the document root to the offending value, e.g. + `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + """ + + loc: tuple[str | int, ...] + message: str + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails structural metadata validation. + + Carries every problem found (not just the first) in `.problems`. + """ + + def __init__(self, problems: list[ValidationProblem]) -> None: + self.problems = problems + super().__init__("\n".join(str(problem) for problem in problems)) + + +def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return [ValidationProblem((loc_head, *p.loc), p.message) for p in problems] + + +def validate_json(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not JSON-serializable (recursively).""" + if isinstance(value, (str, int, float, bool)) or value is None: + return [] + problems: list[ValidationProblem] = [] + if isinstance(value, Mapping): + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append(ValidationProblem((), f"non-string key {key!r} in JSON object")) + continue + problems.extend(_prefix(key, validate_json(item))) + return problems + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for index, item in enumerate(cast("Sequence[object]", value)): + problems.extend(_prefix(index, validate_json(item))) + return problems + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}")] + + +def is_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` is a JSON-serializable structure (recursively).""" + return not validate_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return `value` narrowed to `JSONValue`, or raise `MetadataValidationError`.""" + problems = validate_json(value) + if problems: + raise MetadataValidationError(problems) + return cast(JSONValue, value) + + +# The standard top-level keys of a v3 array metadata document. Anything outside +# this set is an extension field. Built from the TypedDict's required/optional +# key sets (which resolve inherited keys, unlike `__annotations__`). +ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 +) + +ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ArrayMetadataV2.__required_keys__ +) + + +def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a `{name, configuration}` mapping. + """ + if isinstance(value, str): + return [] + if not isinstance(value, Mapping): + return [ + ValidationProblem((), "expected a metadata field (string or {name, configuration})") + ] + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append(ValidationProblem(("configuration",), "expected a mapping")) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append(ValidationProblem(("configuration",), "expected string keys")) + return problems + + +def is_metadata_field_v3(value: object) -> TypeIs[MetadataV3]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + return not validate_metadata_field_v3(value) + + +def parse_metadata_field_v3(value: object) -> MetadataV3: + """Return `value` narrowed to `MetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_metadata_field_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(MetadataV3, value) + + +def _is_int_sequence(value: object) -> bool: + """Whether `value` is a non-string sequence of integers.""" + return ( + not isinstance(value, str) + and isinstance(value, Sequence) + and all(isinstance(item, int) for item in cast("Sequence[object]", value)) + ) + + +def _validate_attributes(value: object) -> list[ValidationProblem]: + """Validate an `attributes` value: a mapping with string keys. + + Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the + v2 and v3 validators. Unlike the other `validate_*` functions (which + return value-relative locs for the caller to `_prefix`), this emits the + already-parent-relative `("attributes",)` loc, since it is only ever called + with a document's `attributes` value. + """ + if not isinstance(value, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", value) + ): + return [ValidationProblem(("attributes",), "expected a mapping with string keys")] + return [] + + +def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 array doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V3 - doc.keys()) + ] + if "shape" in doc and not _is_int_sequence(doc["shape"]): + problems.append(ValidationProblem(("shape",), "expected a sequence of int")) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if key in doc: + problems.extend(_prefix(key, validate_metadata_field_v3(doc[key]))) + for key in ("codecs", "storage_transformers"): + if key in doc: + entries = doc[key] + if isinstance(entries, str) or not isinstance(entries, Sequence): + problems.append(ValidationProblem((key,), "expected a sequence")) + else: + for index, entry in enumerate(cast("Sequence[object]", entries)): + problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "dimension_names" in doc: + # Simple typed sequences (dimension_names, shape, chunks) report a single + # field-level loc, not per-bad-item locs; per-index locs are reserved for + # the metadata-field lists (codecs, storage_transformers). + names = doc["dimension_names"] + if isinstance(names, str) or not isinstance(names, Sequence): + problems.append(ValidationProblem(("dimension_names",), "expected a sequence")) + elif not all( + item is None or isinstance(item, str) for item in cast("Sequence[object]", names) + ): + problems.append( + ValidationProblem(("dimension_names",), "expected items of str or None") + ) + return problems + + +def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: + """Whether `value` is a structurally-valid v3 array metadata document.""" + return not validate_array_metadata_v3(value) + + +def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: + """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV3, value) + + +def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 array doc. + + Checks structure, not domain validity. `compressor`/`filters` are required + keys but may be `None`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V2 - doc.keys()) + ] + problems.extend( + ValidationProblem((key,), "expected a sequence of int") + for key in ("shape", "chunks") + if key in doc and not _is_int_sequence(doc[key]) + ) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: + """Whether `value` is a structurally-valid v2 array metadata document.""" + return not validate_array_metadata_v2(value) + + +def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: + """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV2, value) + + +def _arrays_to_tuples(obj: object) -> object: + """Recursively convert every list in a JSON-decoded structure to a tuple.""" + if isinstance(obj, list): + return tuple(_arrays_to_tuples(item) for item in cast("list[object]", obj)) + if isinstance(obj, dict): + return { + key: _arrays_to_tuples(value) + for key, value in cast("dict[object, object]", obj).items() + } + return obj From eee768bdb419fe9c4f4f703da61956f54e176a77 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 10:19:35 +0200 Subject: [PATCH 03/33] =?UTF-8?q?feat(zarr-metadata):=20add=20model.=5Farr?= =?UTF-8?q?ay=20=E2=80=94=20array=20metadata=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_array.py | 348 ++++++++++++++++++ .../src/zarr_metadata/model/_validation.py | 7 +- 2 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_array.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py new file mode 100644 index 0000000000..ea2745284d --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -0,0 +1,348 @@ +"""In-memory models for Zarr array metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + arrays_to_tuples, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_metadata_field_v3, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import ( + ArrayDimensionSeparatorV2, + ArrayMetadataV2, + ArrayMetadataV3, + ArrayOrderV2, + ExtensionFieldV3, + MetadataV3, + ) + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2, DataTypeMetadataV2 + +ArrayMetadataStoreKeyV3 = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" + +ArrayMetadataStoreKeyV2 = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ArrayMetadataStoreKeyV2] = ".zarray" + +AttributesStoreKeyV2 = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[AttributesStoreKeyV2] = ".zattrs" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrMetadataV3: + """A v3 metadata field in normalized form: a name plus a configuration. + + This is the in-memory model of `MetadataV3` (a bare name string or a + `{name, configuration}` mapping): the bare-name and missing-configuration + forms normalize to an empty configuration. + """ + + name: str + configuration: dict[str, JSONValue] + + def to_json(self) -> MetadataV3: + return {"name": self.name, "configuration": self.configuration} + + @classmethod + def from_json(cls, data: object) -> ZarrMetadataV3: + field = parse_metadata_field_v3(data) + if isinstance(field, str): + return cls(name=field, configuration={}) + configuration = arrays_to_tuples(dict(field.get("configuration", {}))) + return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] + + +class ArrayMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. + + Every key is optional and typed with the model's own (not serialized) + value types, so it describes valid keyword arguments to + `ArrayMetadataModelV3.update`. The `init=False` fields `zarr_format` and + `node_type` are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrMetadataV3 + chunk_grid: ZarrMetadataV3 + codecs: tuple[ZarrMetadataV3, ...] + chunk_key_encoding: ZarrMetadataV3 + dimension_names: tuple[str | None, ...] | None + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrMetadataV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV3: + """In-memory model of a v3 array metadata document. + + A canonical, lossless representation of the `zarr.json` content for an + array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, + `codecs`, `storage_transformers`) are held as `ZarrMetadataV3` name + + configuration pairs and are never interpreted; `fill_value` is held + verbatim in its JSON form. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrMetadataV3 + chunk_grid: ZarrMetadataV3 + codecs: tuple[ZarrMetadataV3, ...] + chunk_key_encoding: ZarrMetadataV3 + dimension_names: tuple[str | None, ...] | None + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrMetadataV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV3Partial] + ) -> ArrayMetadataModelV3: + """ + Create a default (empty) v3 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` array — the array + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls( + shape=(), + fill_value=0, + data_type=ZarrMetadataV3(name="uint8", configuration={}), + chunk_grid=ZarrMetadataV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrMetadataV3(name="bytes", configuration={}),), + chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}), + dimension_names=None, + attributes={}, + storage_transformers=(), + extra_fields={}, + ) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadataModelV3: + """ + Return a new `ArrayMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV3Partial` can be updated; any attempt to update + other fields (including the fixed `zarr_format` / `node_type`) is + rejected at the type level. Each given field fully replaces its + previous value, including `extra_fields`. + + This is useful for test fixtures that want to override a few fields of a + base template without having to re-specify the entire document. + """ + return dataclasses.replace(self, **kwargs) + + def __post_init__(self) -> None: + if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3): + raise ValueError("Extra fields cannot overlap with standard ArrayMetadataV3 fields") + + def to_json(self) -> ArrayMetadataV3: + out: ArrayMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + "shape": self.shape, + "fill_value": self.fill_value, + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + } + if self.dimension_names is not None: + out["dimension_names"] = self.dimension_names + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if len(self.storage_transformers) > 0: + out["storage_transformers"] = tuple( + transformer.to_json() for transformer in self.storage_transformers + ) + # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them + # by key rather than `out.update(**...)`: type checkers understand the + # indexed-write path against `extra_items`, but not the `update(**...)` + # overload. + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV3: + parsed = parse_array_metadata_v3(arrays_to_tuples(data)) + extra_fields: dict[str, ExtensionFieldV3] = { + k: v # type: ignore[misc] + for k, v in parsed.items() + if k not in ARRAY_METADATA_STANDARD_KEYS_V3 + } + return cls( + shape=parsed["shape"], + fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + data_type=ZarrMetadataV3.from_json(parsed["data_type"]), + chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]), + dimension_names=parsed.get("dimension_names"), + attributes=dict(parsed.get("attributes", {})), + storage_transformers=tuple( + ZarrMetadataV3.from_json(t) for t in parsed.get("storage_transformers", ()) + ), + extra_fields=extra_fields, + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: + return cls.from_json(json.loads(mapping[ARRAY_METADATA_STORE_KEY_V3])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + ARRAY_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +class ArrayMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ArrayMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_v2_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 + attributes: dict[str, JSONValue] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV2: + """In-memory model of a v2 array metadata document. + + A canonical, lossless representation of the `.zarray` content plus the + sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are + held in their raw JSON forms and are never interpreted; `fill_value` is + held verbatim in its JSON form. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 = field(default="/") + attributes: dict[str, JSONValue] + + def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: + """ + Return a new `ArrayMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV2Partial` can be updated; the fixed `zarr_format` is + rejected at the type level. Each given field fully replaces its previous + value. + """ + return dataclasses.replace(self, **kwargs) + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV2Partial] + ) -> ArrayMetadataModelV2: + """ + Create a default (empty) v2 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the + array analog of `list()` returning `[]`. Any field can be overridden by + keyword (the same fields accepted by `update`). + """ + default = cls( + shape=(), + dtype="|u1", + chunks=(), + fill_value=0, + order="C", + compressor=None, + filters=None, + attributes={}, + ) + return default.update(**overrides) + + def to_json(self) -> ArrayMetadataV2: + out: ArrayMetadataV2 = { + "zarr_format": self.zarr_format, + "shape": self.shape, + "dtype": self.dtype, + "order": self.order, + "chunks": self.chunks, + "fill_value": self.fill_value, + "dimension_separator": self.dimension_separator, + "attributes": self.attributes, + "compressor": self.compressor, + "filters": self.filters, + } + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV2: + parsed = parse_array_metadata_v2(arrays_to_tuples(data)) + return cls( + shape=parsed["shape"], + dtype=parsed["dtype"], + chunks=parsed["chunks"], + fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + order=parsed["order"], + compressor=parsed["compressor"], + filters=parsed["filters"], + dimension_separator=parsed.get("dimension_separator", "/"), + attributes=dict(parsed.get("attributes", {})), + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: + zarray = json.loads(mapping[ARRAY_METADATA_STORE_KEY_V2]) + zattrs: dict[str, JSONValue] = ( + json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + if ATTRIBUTES_STORE_KEY_V2 in mapping + else {} + ) + return cls.from_json({**zarray, "attributes": zattrs}) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zarray` + # document must exclude them. + zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + return { + ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8"), + ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index d0e30e1cda..25f3422c50 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -259,13 +259,12 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) -def _arrays_to_tuples(obj: object) -> object: +def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): - return tuple(_arrays_to_tuples(item) for item in cast("list[object]", obj)) + return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) if isinstance(obj, dict): return { - key: _arrays_to_tuples(value) - for key, value in cast("dict[object, object]", obj).items() + key: arrays_to_tuples(value) for key, value in cast("dict[object, object]", obj).items() } return obj From 6b391474ea5be5c9ef41ee791bcfa502b8c75640 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 10:23:53 +0200 Subject: [PATCH 04/33] test(zarr-metadata): port array model test suite Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/__init__.py | 60 + .../zarr-metadata/tests/model/__init__.py | 0 packages/zarr-metadata/tests/model/_cases.py | 45 + .../zarr-metadata/tests/model/test_array.py | 1019 +++++++++++++++++ 4 files changed, 1124 insertions(+) create mode 100644 packages/zarr-metadata/tests/model/__init__.py create mode 100644 packages/zarr-metadata/tests/model/_cases.py create mode 100644 packages/zarr-metadata/tests/model/test_array.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index e1382a6f0a..8776b4fab3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -1 +1,61 @@ """In-memory models for Zarr metadata documents.""" + +from zarr_metadata.model._array import ( + ARRAY_METADATA_STORE_KEY_V2, + ARRAY_METADATA_STORE_KEY_V3, + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ZarrMetadataV3, +) +from zarr_metadata.model._validation import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V2, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) + +__all__ = [ + "ARRAY_METADATA_OPTIONAL_KEYS_V3", + "ARRAY_METADATA_REQUIRED_KEYS_V2", + "ARRAY_METADATA_REQUIRED_KEYS_V3", + "ARRAY_METADATA_STANDARD_KEYS_V3", + "ARRAY_METADATA_STORE_KEY_V2", + "ARRAY_METADATA_STORE_KEY_V3", + "ATTRIBUTES_STORE_KEY_V2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "MetadataValidationError", + "ValidationProblem", + "ZarrMetadataV3", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_json", + "is_metadata_field_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_json", + "parse_metadata_field_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_json", + "validate_metadata_field_v3", +] diff --git a/packages/zarr-metadata/tests/model/__init__.py b/packages/zarr-metadata/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py new file mode 100644 index 0000000000..f925c69dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import pytest + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + +TIn = TypeVar("TIn") +TOut = TypeVar("TOut") + + +@dataclass(frozen=True) +class Expect(Generic[TIn, TOut]): + """A test case with explicit input, expected output, and a human-readable id.""" + + input: TIn + output: TOut + id: str + + +@dataclass(frozen=True) +class ExpectFail(Generic[TIn]): + """A test case that should raise an exception. + + `msg` is a regex matched against the exception text (pytest's native + `match=` semantics). Leave it `None` to assert only the exception type. Set + `escape=True` when `msg` is a literal that contains regex metacharacters + such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. + """ + + input: TIn + exception: type[Exception] + id: str + msg: str | None = None + escape: bool = False + + def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: + if self.msg is None: + return pytest.raises(self.exception) + pattern = re.escape(self.msg) if self.escape else self.msg + return pytest.raises(self.exception, match=pattern) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py new file mode 100644 index 0000000000..80dac709de --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -0,0 +1,1019 @@ +"""Tests for the metadata models in ``zarr_metadata.model``.""" + +import dataclasses +import json +from collections.abc import Callable +from typing import TYPE_CHECKING + +import pytest + +from tests.model._cases import Expect, ExpectFail +from zarr_metadata.model import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + MetadataValidationError, + ValidationProblem, + ZarrMetadataV3, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) +from zarr_metadata.model._validation import _prefix, arrays_to_tuples + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2 + +# --- public exports -------------------------------------------------------- + + +def test_guards_exported_from_package() -> None: + """The wire-type guard/parser functions are exported from the package.""" + import zarr_metadata.model + + for name in ( + "is_json", + "parse_json", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "is_array_metadata_v3", + "parse_array_metadata_v3", + "is_array_metadata_v2", + "parse_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_validation_diagnostics_exported_from_package() -> None: + """The validation-diagnostic types and validators are exported from the package.""" + import zarr_metadata.model + + for name in ( + "ValidationProblem", + "MetadataValidationError", + "validate_json", + "validate_metadata_field_v3", + "validate_array_metadata_v3", + "validate_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_expect_expectfail_smoke() -> None: + """The Expect/ExpectFail test-case dataclasses behave as expected.""" + e = Expect(input=1, output=2, id="x") + assert (e.input, e.output, e.id) == (1, 2, "x") + f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") + with f.raises(): + raise ValueError("boom") + + +def test_v3_from_json_error_lists_all_problems() -> None: + """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["shape"] + doc["data_type"] = 5 + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_json(doc) + locs = {p.loc for p in exc_info.value.problems} + assert ("shape",) in locs + assert ("data_type",) in locs + + +# --- JSON type / fill_value contract --------------------------------------- + + +def test_json_value_type_accepts_json_shapes() -> None: + # JSONValue is the package's public JSON type alias; assigning JSON-shaped + # values to it is valid. + """The JSONValue type alias accepts JSON-shaped values.""" + value: JSONValue = {"a": [1, 2.0, "x", True, None]} + assert value == {"a": [1, 2.0, "x", True, None]} + + +def test_string_nan_fill_value_roundtrips() -> None: + # Non-finite floats are represented as the spec strings ("NaN", "Infinity", + # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # The string form round-trips cleanly under default dataclass equality, + # unlike a raw float('nan') (which is an invalid fill_value the caller must + # not pass). + """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" + m = ArrayMetadataModelV3.create_default(fill_value="NaN") + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" + + +# --- ZarrMetadataV3.to_json ------------------------------------------------ + +ZARR_TO_JSON_CASES = [ + Expect( + ZarrMetadataV3(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + ZarrMetadataV3(name="bytes", configuration={}), + {"name": "bytes", "configuration": {}}, + id="without-configuration", + ), +] + + +@pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_to_json(case: Expect[ZarrMetadataV3, dict[str, object]]) -> None: + """ZarrMetadataV3.to_json emits the canonical object form.""" + assert case.input.to_json() == case.output + + +# --- ZarrMetadataV3.from_json ----------------------------------------------- + +ZARR_FROM_JSON_CASES = [ + Expect("bytes", ZarrMetadataV3(name="bytes", configuration={}), id="bare-string"), + Expect( + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + ZarrMetadataV3(name="regular", configuration={"chunk_shape": (1,)}), + id="object-with-config", + ), + Expect( + {"name": "bytes"}, + ZarrMetadataV3(name="bytes", configuration={}), + id="object-without-config", + ), +] + + +@pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> None: + """ZarrMetadataV3.from_json parses both the bare-string and object forms.""" + assert ZarrMetadataV3.from_json(case.input) == case.output + + +# --- V3 baseline ----------------------------------------------------------- + + +def test_v3_to_json_includes_required_fields() -> None: + """V3 to_json emits all required fields with the expected values.""" + out = ArrayMetadataModelV3.create_default( + shape=(10,), data_type=ZarrMetadataV3(name="int32", configuration={}) + ).to_json() + assert out["zarr_format"] == 3 + assert out["node_type"] == "array" + assert out["shape"] == (10,) + assert out["fill_value"] == 0 + assert out["data_type"] == {"name": "int32", "configuration": {}} + assert out["codecs"] == ({"name": "bytes", "configuration": {}},) + + +def test_v3_dimension_names_included_when_present() -> None: + """V3 to_json includes dimension_names when they are set.""" + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(dimension_names=("x",)).to_json() + ) + assert out["dimension_names"] == ("x",) + + +def test_v3_dimension_names_omitted_when_none() -> None: + """V3 to_json omits dimension_names when they are None.""" + out = ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + assert "dimension_names" not in out + + +# --- BUG 1: attributes gated on dimension_names ---------------------------- + + +def test_v3_attributes_included_when_dimension_names_is_none() -> None: + """Attributes must be emitted regardless of dimension_names. + + Regression: attributes were gated on ``dimension_names is not None``, + so non-empty attributes were silently dropped when there were no + dimension names. + """ + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default( + dimension_names=None, attributes={"foo": "bar"} + ).to_json() + ) + assert out["attributes"] == {"foo": "bar"} + + +# --- BUG 2: single storage transformer dropped ----------------------------- + + +def test_v3_single_storage_transformer_included() -> None: + """A single storage transformer must be emitted. + + Regression: the guard used ``> 1`` instead of ``> 0``, dropping a + lone storage transformer. + """ + st = ZarrMetadataV3(name="some_transformer", configuration={}) + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() + ) + assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) + + +def test_v3_no_storage_transformers_omitted() -> None: + """V3 to_json omits storage_transformers when empty.""" + out = ArrayMetadataModelV3.create_default(storage_transformers=()).to_json() + assert "storage_transformers" not in out + + +# --- V3 extra fields ------------------------------------------------------- + + +def test_v3_extra_fields_merged() -> None: + """V3 to_json merges extra_fields into the top-level document.""" + out = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + assert out["my_ext"] == {"must_understand": False} + + +def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: + """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" + with pytest.raises(ValueError): + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + + +# --- V3 key/value ---------------------------------------------------------- + + +def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: + """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" + kv = ArrayMetadataModelV3.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {"zarr.json"} + parsed = json.loads(kv["zarr.json"].decode("utf-8")) + assert parsed["zarr_format"] == 3 + assert parsed["attributes"] == {"a": 1} + + +# --- V3 standard-key sets -------------------------------------------------- + + +def test_standard_keys_is_union_of_required_and_optional() -> None: + """The standard-key set is the union of the required and optional key sets.""" + assert ( + ARRAY_METADATA_STANDARD_KEYS_V3 + == ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 + ) + + +def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: + """The standard-key set contains known fields and excludes extension keys.""" + assert { + "zarr_format", + "node_type", + "shape", + "codecs", + } <= ARRAY_METADATA_STANDARD_KEYS_V3 + assert "my_ext" not in ARRAY_METADATA_STANDARD_KEYS_V3 + + +# --- create_default -------------------------------------------------------- + + +def test_v3_create_default_is_valid_empty_array() -> None: + """V3 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV3.create_default() + assert m.shape == () + assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.fill_value == 0 + assert m.attributes == {} + assert m.extra_fields == {} + # the default document is structurally valid and round-trips + assert validate_array_metadata_v3(m.to_json()) == [] + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + + +def test_v3_create_default_applies_overrides() -> None: + """V3 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV3.create_default(shape=(4, 4), attributes={"a": 1}) + assert m.shape == (4, 4) + assert m.attributes == {"a": 1} + # un-overridden fields keep their defaults + assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + + +def test_v2_create_default_is_valid_empty_array() -> None: + """V2 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV2.create_default() + assert m.shape == () + assert m.chunks == () + assert m.fill_value == 0 + assert m.compressor is None + assert m.filters is None + assert m.attributes == {} + assert validate_array_metadata_v2(m.to_json()) == [] + assert ArrayMetadataModelV2.from_json(m.to_json()) == m + + +def test_v2_create_default_applies_overrides() -> None: + """V2 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV2.create_default(shape=(8,), attributes={"k": "v"}) + assert m.shape == (8,) + assert m.attributes == {"k": "v"} + assert m.dtype == "|u1" # default dtype unchanged + + +# --- V3 update ------------------------------------------------------------- + +# Cluster 3: update same-shape pairs across versions — parametrized + +UPDATE_NEW_INSTANCE_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) +def test_update_returns_new_instance( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update returns a new instance with the field replaced, leaving the original unchanged.""" + base = model_cls.create_default(shape=(10,)) + updated = base.update(shape=(20,)) + assert updated.shape == (20,) + assert base.shape == (10,) # original unchanged + assert isinstance(updated, model_cls) + + +UPDATE_NO_ARGS_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) +def test_update_no_args_returns_equal_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update with no arguments returns a model equal to the original.""" + base = model_cls.create_default() + assert base.update() == base + + +# V3-only update tests — kept direct (extra_fields is v3-specific) + + +def test_update_can_replace_extra_fields() -> None: + """update can replace the extra_fields mapping.""" + base = ArrayMetadataModelV3.create_default(extra_fields={}) + updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) + assert updated.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_update_replaces_extra_fields_rather_than_merging() -> None: + """update replaces extra_fields wholesale rather than merging.""" + base = ArrayMetadataModelV3.create_default(extra_fields={"a": {"must_understand": False}}) + updated = base.update(extra_fields={"b": {"must_understand": True}}) + assert updated.extra_fields == {"b": {"must_understand": True}} + + +def test_partial_keys_match_settable_model_fields() -> None: + """The partial TypedDict must list exactly the constructor-settable fields. + + Guards against drift: adding/removing a settable field on the model + without updating ``ArrayMetadataModelV3Partial`` fails here. + """ + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV3) if f.init} + assert set(ArrayMetadataModelV3Partial.__annotations__) == settable + + +# --- V2 model -------------------------------------------------------------- + + +def test_v2_partial_keys_match_settable_model_fields() -> None: + """The v2 partial TypedDict must list exactly the settable fields.""" + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV2) if f.init} + assert set(ArrayMetadataModelV2Partial.__annotations__) == settable + + +def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: + """V2 to_key_value splits the document into .zarray and .zattrs.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {".zarray", ".zattrs"} + zarray = json.loads(kv[".zarray"].decode("utf-8")) + zattrs = json.loads(kv[".zattrs"].decode("utf-8")) + assert zarray["zarr_format"] == 2 + assert zattrs == {"a": 1} + + +def test_v2_zarray_excludes_attributes() -> None: + """The on-disk ``.zarray`` document must not contain user attributes. + + In v2, attributes live only in the sibling ``.zattrs`` file. The bundled + ``ArrayMetadataV2`` / ``to_json()`` carry attributes for convenience, but + ``to_key_value()`` must split them out. + """ + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + zarray = json.loads(kv[".zarray"].decode("utf-8")) + assert "attributes" not in zarray + + +def test_v2_to_json_still_includes_attributes() -> None: + """``to_json()`` is the bundled in-memory form and keeps attributes.""" + out: dict[str, object] = dict( + ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + ) + assert out["attributes"] == {"a": 1} + + +# --- arrays_to_tuples helper ---------------------------------------------- + +ARRAYS_TO_TUPLES_CASES = [ + Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), + Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), + Expect(5, 5, id="scalar-int"), + Expect("s", "s", id="scalar-str"), + Expect(None, None, id="scalar-none"), + Expect( + {"name": "bytes", "configuration": {"nums": [1, 2]}}, + {"name": "bytes", "configuration": {"nums": (1, 2)}}, + id="dict-keys-preserved", + ), +] + + +@pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) +def test_arrays_to_tuples(case: Expect[object, object]) -> None: + """arrays_to_tuples recursively converts JSON arrays to tuples.""" + assert arrays_to_tuples(case.input) == case.output + + +# --- ArrayMetadataModelV3.from_json ---------------------------------------- + + +def test_v3_from_json_reconstructs_required_fields() -> None: + """V3 from_json reconstructs the required fields from a document.""" + doc = ArrayMetadataModelV3.create_default( + shape=(7,), + attributes={"a": 1}, + data_type=ZarrMetadataV3(name="int32", configuration={}), + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.shape == (7,) + assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.attributes == {"a": 1} + + +def test_v3_from_json_defaults_for_omitted_optionals() -> None: + """V3 from_json supplies defaults for omitted optional fields.""" + doc = ArrayMetadataModelV3.create_default( + attributes={}, storage_transformers=(), dimension_names=None + ).to_json() + # to_json omits these entirely; from_json must restore defaults + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {} + assert model.storage_transformers == () + assert model.dimension_names is None + + +def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: + """V3 from_json routes unknown top-level keys into extra_fields.""" + doc = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: + """V3 from_json keeps standard keys out of extra_fields.""" + doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {} + + +def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V3 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV3.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {"scale": ((1, 2), (3, 4))} + + +# --- ArrayMetadataModelV3.from_key_value ---------------------------------- + + +def test_v3_from_key_value_parses_zarr_json() -> None: + """V3 from_key_value parses the zarr.json entry into a model.""" + kv = ArrayMetadataModelV3.create_default(shape=(3,)).to_key_value() + model = ArrayMetadataModelV3.from_key_value(kv) + assert model.shape == (3,) + + +# --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- + +FROM_KEY_VALUE_MISSING_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail({}, KeyError, id="v3-missing-zarr-json"), + id="v3-missing-zarr-json", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail({}, KeyError, id="v2-missing-zarray"), + id="v2-missing-zarray", + ), +] + + +@pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) +def test_from_key_value_missing_key_raises( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + case: ExpectFail[dict[str, bytes]], +) -> None: + """from_key_value raises KeyError when the required store key is absent.""" + with case.raises(): + model_cls.from_key_value(case.input) + + +# --- Cluster 1: round-trips (model → json → model, parametrized) ----------- + +ROUNDTRIP_MODEL_JSON_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={"a": 1}, + dimension_names=("x",), + storage_transformers=(ZarrMetadataV3(name="t", configuration={}),), + extra_fields={"ext": {"must_understand": False}}, + ), + id="v3-full", + ), + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={}, + dimension_names=None, + storage_transformers=(), + extra_fields={}, + ), + id="v3-empty-optionals", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}, filters=None, compressor=None), + id="v2-basic", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) +def test_roundtrip_model_json_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_json/from_json back to an equal model.""" + assert model_cls.from_json(model.to_json()) == model + + +# --- Round-trips (model → key_value → model, parametrized) ----------------- + +ROUNDTRIP_KEY_VALUE_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default(attributes={"a": 1}), + id="v3", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}), + id="v2", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) +def test_roundtrip_via_key_value( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_key_value/from_key_value back to an equal model.""" + assert model_cls.from_key_value(model.to_key_value()) == model + + +# --- Round-trips (json → model → json, direction distinct — kept direct) --- + + +def test_v3_roundtrip_json_model_json() -> None: + """A v3 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + assert ArrayMetadataModelV3.from_json(doc).to_json() == doc + + +def test_v2_roundtrip_json_model_json() -> None: + """A v2 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + assert ArrayMetadataModelV2.from_json(doc).to_json() == doc + + +def test_v3_parser_accepts_bare_string_data_type() -> None: + """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" + doc = ArrayMetadataModelV3.create_default().to_json() + doc["data_type"] = "int32" # bare-string form, not canonical object form + model = ArrayMetadataModelV3.from_json(doc) + # parses correctly, re-serializes to canonical object form + assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} + + +def test_v2_roundtrip_with_compressor_and_filters() -> None: + # Non-None compressor/filters must round-trip; extra assertion on .compressor. + """A v2 model with non-None compressor and filters round-trips.""" + compressor: CodecMetadataV2 = {"id": "blosc", "clevel": 5} + filters: tuple[CodecMetadataV2, ...] = ({"id": "delta"},) + m = ArrayMetadataModelV2.create_default(compressor=compressor, filters=filters) + restored = ArrayMetadataModelV2.from_json(m.to_json()) + assert restored == m + assert restored.compressor == {"id": "blosc", "clevel": 5} + + +# --- ArrayMetadataModelV2.from_json ---------------------------------------- + + +def test_v2_from_json_reconstructs_fields() -> None: + """V2 from_json reconstructs the fields from a document.""" + doc = ArrayMetadataModelV2.create_default( + shape=(4,), attributes={"a": 1}, dtype=" None: + """V2 from_json defaults attributes to empty when the key is absent.""" + doc = ArrayMetadataModelV2.create_default().to_json() + del doc["attributes"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {} + + +# --- ArrayMetadataModelV2.from_key_value -------------------------------- + + +def test_v2_from_key_value_remerges_zattrs() -> None: + """V2 from_key_value re-merges .zattrs back into attributes.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {"a": 1} + assert model.shape == (10,) + + +def test_v2_from_key_value_absent_zattrs_gives_empty_attributes() -> None: + """V2 from_key_value yields empty attributes when .zattrs is absent.""" + kv: dict[str, bytes] = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + del kv[".zattrs"] + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {} + + +def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V2 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV2.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {"axes": ((0, 1), (2, 3))} + + +# --- scalar wire-type guards (is_/validate_/parse_) ------------------------ +# +# Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] +# where `output` is the set of expected problem locs validate_* must report — +# frozenset() means VALID. Valid iff output == frozenset(). + +JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("s", frozenset(), id="str"), + Expect(1, frozenset(), id="int"), + Expect(1.5, frozenset(), id="float"), + Expect(True, frozenset(), id="bool"), + Expect(None, frozenset(), id="none"), + Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), + Expect((1, 2, 3), frozenset(), id="tuple-array"), + Expect(float("nan"), frozenset(), id="nan"), + Expect(float("inf"), frozenset(), id="inf"), + Expect(object(), frozenset({()}), id="object"), + Expect(b"abc", frozenset({()}), id="bytes"), + Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), + Expect({1: "x"}, frozenset({()}), id="non-str-key"), + Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), + Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), +] + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_json reports whether a value is JSON-serializable.""" + assert is_json(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """validate_json reports the problems (and their locs) for a value.""" + problems = validate_json(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """parse_json returns valid JSON values and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_json(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_json(case.input) + + +def test_validate_json_reports_json_in_message() -> None: + """validate_json's message for a non-JSON value mentions JSON.""" + problems = validate_json(object()) + assert problems[0].loc == () + assert "JSON" in problems[0].message + + +METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("bytes", frozenset(), id="bare-string"), + Expect({"name": "x", "configuration": {"a": 1}}, frozenset(), id="named-config"), + Expect({"name": "bytes"}, frozenset(), id="name-only"), + Expect(5, frozenset({()}), id="not-str-or-mapping"), + Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), + Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), + Expect( + {"name": "x", "configuration": [1]}, + frozenset({("configuration",)}), + id="config-not-mapping", + ), + Expect( + {"name": "x", "configuration": {1: "y"}}, + frozenset({("configuration",)}), + id="config-non-str-key", + ), +] + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_metadata_field_v3(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_metadata_field_v3 reports whether a value is a v3 metadata field.""" + assert is_metadata_field_v3(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """validate_metadata_field_v3 reports the problems for a metadata-field value.""" + problems = validate_metadata_field_v3(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """parse_metadata_field_v3 returns valid fields and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_metadata_field_v3(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_metadata_field_v3(case.input) + + +# --- array-document wire-type guards (is_/validate_/parse_) ---------------- +# +# Each case starts from a valid document (built by `make`) and applies a +# mutation. `expected_locs` are loc paths `validate_*` must report for the +# invalid cases (a subset check, so accumulation of OTHER problems is allowed). + + +def _build_v3(**overrides: object) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) # type: ignore[arg-type] + + +def _build_v2(**overrides: object) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) # type: ignore[arg-type] + + +def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: + def _factory() -> dict: + doc = build() + mutate(doc) + return doc + + return _factory + + +def _del(key: str) -> Callable[[dict], object]: + return lambda doc: doc.pop(key) + + +def _set(key: str, value: object) -> Callable[[dict], object]: + return lambda doc: doc.__setitem__(key, value) + + +V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v3, frozenset(), id="valid"), + Expect( + lambda: _build_v3(attributes={"a": 1}, dimension_names=("x",)), + frozenset(), + id="valid-with-attributes-and-dim-names", + ), + Expect( + lambda: _build_v3(extra_fields={"my_ext": {"must_understand": False}}), + frozenset(), + id="valid-with-extra-fields", + ), + Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), + Expect( + _mutate(_build_v3, _set("data_type", 5)), + frozenset({("data_type",)}), + id="bad-data-type", + ), + Expect( + _mutate(_build_v3, _set("shape", "not-a-shape")), + frozenset({("shape",)}), + id="shape-not-sequence", + ), + Expect( + _mutate(_build_v3, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="shape-non-int-item", + ), + Expect( + _mutate(_build_v3, _set("codecs", (5,))), + frozenset({("codecs", 0)}), + id="bad-codec-entry", + ), + Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), + Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), + Expect( + _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), + frozenset({("shape",), ("data_type",)}), + id="missing-shape-and-bad-data-type", + ), +] + +V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v2, frozenset(), id="valid"), + Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), + Expect( + lambda: _build_v2(compressor=None, filters=None), + frozenset(), + id="valid-none-compressor-filters", + ), + Expect( + _mutate(_build_v2, _del("chunks")), + frozenset({("chunks",)}), + id="missing-chunks", + ), + Expect( + _mutate(_build_v2, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="bad-shape", + ), + Expect( + _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), + frozenset({("chunks",), ("shape",)}), + id="missing-chunks-and-bad-shape", + ), +] + +ALL_DOC_CASES = [ + *( + pytest.param( + is_array_metadata_v3, + validate_array_metadata_v3, + parse_array_metadata_v3, + c, + id=f"v3-{c.id}", + ) + for c in V3_DOC_CASES + ), + *( + pytest.param( + is_array_metadata_v2, + validate_array_metadata_v2, + parse_array_metadata_v2, + c, + id=f"v2-{c.id}", + ) + for c in V2_DOC_CASES + ), +] + + +@pytest.mark.parametrize(("is_fn", "validate_fn", "parse_fn", "case"), ALL_DOC_CASES) +def test_array_metadata_guards( + is_fn: Callable[[object], bool], + validate_fn: Callable[[object], list[ValidationProblem]], + parse_fn: Callable[[object], object], + case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], +) -> None: + """is_/validate_/parse_ array-metadata guards agree on validity and locs for each case.""" + doc = case.input() + valid = case.output == frozenset() + assert is_fn(doc) is valid + problems = validate_fn(doc) + assert (problems == []) is valid + assert {p.loc for p in problems} >= case.output + if valid: + assert parse_fn(doc) is doc + else: + with pytest.raises(MetadataValidationError): + parse_fn(doc) + + +# --- strict from_json validation ------------------------------------------- + + +FROM_JSON_REJECT_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), + id="v3-missing-required", + ), + pytest.param( + ArrayMetadataModelV3, + ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), + id="v3-bad-field-type", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), + id="v2-missing-required", + ), + pytest.param( + ZarrMetadataV3, + ExpectFail(lambda: 5, MetadataValidationError, id="x"), + id="zarr-metadata-bad-input", + ), +] + + +@pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) +def test_from_json_rejects_malformed( + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | ZarrMetadataV3], + case: ExpectFail[Callable[[], object]], +) -> None: + """from_json raises MetadataValidationError on a malformed document.""" + with case.raises(): + model.from_json(case.input()) + + +# --- ValidationProblem / MetadataValidationError / _prefix ----------------- +# Small structural tests — not "parametrize over inputs" shaped, kept direct. + + +def test_validation_problem_str_with_loc() -> None: + """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str") + assert str(p) == "codecs.0.name: expected str" + + +def test_validation_problem_str_empty_loc() -> None: + """ValidationProblem.__str__ renders an empty loc as .""" + p = ValidationProblem(loc=(), message="not a mapping") + assert str(p) == ": not a mapping" + + +def test_validation_problem_is_frozen() -> None: + """ValidationProblem is immutable (frozen dataclass).""" + p = ValidationProblem(loc=("shape",), message="x") + with pytest.raises(dataclasses.FrozenInstanceError): + p.message = "y" # type: ignore[misc] + + +def test_metadata_validation_error_holds_problems() -> None: + """MetadataValidationError carries its problem list and renders them in its message.""" + problems = [ + ValidationProblem(loc=("shape",), message="missing required key"), + ValidationProblem(loc=("data_type",), message="expected a metadata field"), + ] + err = MetadataValidationError(problems) + assert err.problems == problems + assert "shape: missing required key" in str(err) + assert "data_type: expected a metadata field" in str(err) + + +def test_prefix_prepends_loc_head() -> None: + """_prefix prepends a loc head to each problem's loc.""" + problems = [ValidationProblem(loc=("name",), message="expected str")] + prefixed = _prefix(0, problems) + assert prefixed == [ValidationProblem(loc=(0, "name"), message="expected str")] From 9f32f14f75f9e97204c01e8ad28b34f6515e2a44 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:24:31 +0200 Subject: [PATCH 05/33] feat(zarr-metadata): add group metadata models Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_group.py | 370 ++++++++++++++++++ .../src/zarr_metadata/model/_validation.py | 89 ++++- .../zarr-metadata/tests/model/test_group.py | 137 +++++++ 3 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_group.py create mode 100644 packages/zarr-metadata/tests/model/test_group.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py new file mode 100644 index 0000000000..8eeaed1a89 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -0,0 +1,370 @@ +"""In-memory models for Zarr group and consolidated metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._array import ( + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV3, +) +from zarr_metadata.model._validation import ( + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + parse_group_metadata_v2, + parse_group_metadata_v3, +) + +if TYPE_CHECKING: + from zarr_metadata import ( + ConsolidatedMetadataV3, + ExtensionFieldV3, + GroupMetadataV2, + GroupMetadataV3, + ) + from zarr_metadata._common import JSONValue + +GroupMetadataStoreKeyV3 = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" + +GroupMetadataStoreKeyV2 = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[GroupMetadataStoreKeyV2] = ".zgroup" + +ConsolidatedMetadataStoreKeyV2 = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ConsolidatedMetadataStoreKeyV2] = ".zmetadata" + +# The key under which consolidated metadata is embedded in a v3 group document. +# This is a reference-implementation convention (not a spec artifact), stored +# as an extension field on the group's `zarr.json`. +CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" + + +class GroupMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV3`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV3.update` and + `create_default`. The `init=False` fields `zarr_format` and `node_type` + are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | None + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV3: + """In-memory model of a v3 group metadata document. + + A canonical, lossless representation of the `zarr.json` content for a + group. The `consolidated_metadata` reference-implementation convention is + modeled as a typed field holding thin child models; every other unknown + top-level key lands in `extra_fields` verbatim. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["group"] = field(default="group", init=False) + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | None + extra_fields: dict[str, ExtensionFieldV3] + + def __post_init__(self) -> None: + reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} + if set(self.extra_fields.keys()).intersection(reserved): + raise ValueError("Extra fields cannot overlap with standard GroupMetadataV3 fields") + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV3Partial] + ) -> GroupMetadataModelV3: + """ + Create a default (empty) v3 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}, consolidated_metadata=None, extra_fields={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: + """ + Return a new `GroupMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV3Partial` can be updated; the fixed `zarr_format` / + `node_type` are rejected at the type level. Each given field fully + replaces its previous value, including `extra_fields`. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV3: + out: GroupMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + } + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if self.consolidated_metadata is not None: + # The consolidated-metadata shape ({kind, must_understand, metadata}, + # no `name`) predates the strict v3.1 extension-field rules, so it is + # not assignable to `ExtensionFieldV3`; see the discussion on + # `zarr_metadata.v3.consolidated`. + out[CONSOLIDATED_METADATA_KEY_V3] = cast( + "ExtensionFieldV3", self.consolidated_metadata.to_json() + ) + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV3: + parsed = parse_group_metadata_v3(arrays_to_tuples(data)) + consolidated_raw = parsed.get(CONSOLIDATED_METADATA_KEY_V3) + consolidated = ( + None + if consolidated_raw is None + else ConsolidatedMetadataModelV3.from_json(consolidated_raw) + ) + extra_fields: dict[str, ExtensionFieldV3] = { + k: v # type: ignore[misc] + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + } + return cls( + attributes=dict(parsed.get("attributes", {})), + consolidated_metadata=consolidated, + extra_fields=extra_fields, + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: + return cls.from_json(json.loads(mapping[GROUP_METADATA_STORE_KEY_V3])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + GROUP_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV3: + """In-memory model of v3 inline consolidated metadata. + + Models the reference-implementation convention where consolidated metadata + is embedded as an extension field on a group's `zarr.json`. Each entry in + `metadata` is a complete child document, held as a thin array or group + model. `must_understand` is typed permissively as `bool` to mirror the + document shape, but only `False` is valid; this is enforced at runtime. + """ + + kind: Literal["inline"] = field(default="inline", init=False) + must_understand: bool = False + metadata: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] + + def __post_init__(self) -> None: + if self.must_understand is not False: + raise ValueError( + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}." + ) + + def to_json(self) -> ConsolidatedMetadataV3: + # `must_understand` is emitted as the literal False: the field is typed + # permissively as `bool`, but `__post_init__` guarantees the value. + return { + "kind": self.kind, + "must_understand": False, + "metadata": {key: node.to_json() for key, node in self.metadata.items()}, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: + if not isinstance(data, Mapping): + raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + doc = cast("Mapping[str, object]", data) + entries_raw = doc.get("metadata") + if not isinstance(entries_raw, Mapping): + raise MetadataValidationError([ValidationProblem(("metadata",), "expected a mapping")]) + entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} + problems: list[ValidationProblem] = [] + for key, entry in cast("Mapping[object, object]", entries_raw).items(): + if not isinstance(key, str): + problems.append(ValidationProblem(("metadata",), f"non-string key {key!r}")) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + entries[key] = ArrayMetadataModelV3.from_json(entry_obj) + elif node_type == "group": + entries[key] = GroupMetadataModelV3.from_json(entry_obj) + else: + problems.append( + ValidationProblem(("metadata", key, "node_type"), "expected 'array' or 'group'") + ) + if problems: + raise MetadataValidationError(problems) + must_understand = doc.get("must_understand", False) + if must_understand is not False: + raise MetadataValidationError( + [ValidationProblem(("must_understand",), "expected False")] + ) + return cls(must_understand=must_understand, metadata=entries) + + +class GroupMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV2: + """In-memory model of a v2 group metadata document. + + A canonical, lossless representation of the `.zgroup` content plus the + sibling `.zattrs` attributes, folded into a single in-memory value + (mirroring the merged `GroupMetadataV2` document form). + """ + + zarr_format: Literal[2] = field(default=2, init=False) + attributes: dict[str, JSONValue] + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV2Partial] + ) -> GroupMetadataModelV2: + """ + Create a default (empty) v2 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: + """ + Return a new `GroupMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV2Partial` can be updated; the fixed `zarr_format` + is rejected at the type level. Each given field fully replaces its + previous value. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV2: + out: GroupMetadataV2 = {"zarr_format": self.zarr_format} + if len(self.attributes) > 0: + out["attributes"] = self.attributes + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV2: + parsed = parse_group_metadata_v2(arrays_to_tuples(data)) + return cls(attributes=dict(parsed.get("attributes", {}))) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: + zgroup = json.loads(mapping[GROUP_METADATA_STORE_KEY_V2]) + zattrs: dict[str, JSONValue] = ( + json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + if ATTRIBUTES_STORE_KEY_V2 in mapping + else {} + ) + return cls.from_json({**zgroup, "attributes": zattrs}) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zgroup` + # document must exclude them. + zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + return { + GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8"), + ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV2: + """In-memory model of a v2 `.zmetadata` document. + + The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, + `"path/.zattrs"`, ...) verbatim, preserving byte-faithful round-tripping. + Entries are deliberately NOT merged into per-node models: which nodes had + a `.zattrs` file at all is information the canonical representation must + keep. Interpreting entries into node models is consumer work. + """ + + zarr_consolidated_format: Literal[1] = field(default=1, init=False) + metadata: dict[str, JSONValue] + + def to_json(self) -> dict[str, JSONValue]: + return { + "zarr_consolidated_format": self.zarr_consolidated_format, + "metadata": self.metadata, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: + if not isinstance(data, Mapping): + raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + doc = cast("Mapping[str, object]", data) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in ("zarr_consolidated_format", "metadata") + if key not in doc + ] + if "metadata" in doc: + entries = doc["metadata"] + if not isinstance(entries, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", entries) + ): + problems.append( + ValidationProblem(("metadata",), "expected a mapping with string keys") + ) + if problems: + raise MetadataValidationError(problems) + entries_tupled = cast( + "dict[str, JSONValue]", + arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), + ) + return cls(metadata=entries_tupled) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: + return cls.from_json(json.loads(mapping[CONSOLIDATED_METADATA_STORE_KEY_V2])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + CONSOLIDATED_METADATA_STORE_KEY_V2: json.dumps(self.to_json(), indent=indent).encode( + "utf-8" + ) + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 25f3422c50..c69a80c144 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,7 +14,13 @@ from typing_extensions import TypeIs -from zarr_metadata import ArrayMetadataV2, ArrayMetadataV3, MetadataV3 +from zarr_metadata import ( + ArrayMetadataV2, + ArrayMetadataV3, + GroupMetadataV2, + GroupMetadataV3, + MetadataV3, +) from zarr_metadata._common import JSONValue @@ -99,6 +105,22 @@ def parse_json(value: object) -> JSONValue: ArrayMetadataV2.__required_keys__ ) +# The standard top-level keys of a v3 group metadata document. Anything outside +# this set is an extension field. +GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 +) + +GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + GroupMetadataV2.__required_keys__ +) + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -259,6 +281,71 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) +def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 group doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`); a `consolidated_metadata` key, if present, + must be a mapping (its entries are validated by the consolidated model). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V3 - doc.keys()) + ] + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): + problems.append(ValidationProblem(("consolidated_metadata",), "expected a mapping")) + return problems + + +def is_group_metadata_v3(value: object) -> TypeIs[GroupMetadataV3]: + """Whether `value` is a structurally-valid v3 group metadata document.""" + return not validate_group_metadata_v3(value) + + +def parse_group_metadata_v3(value: object) -> GroupMetadataV3: + """Return `value` narrowed to `GroupMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV3, value) + + +def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 group doc. + + Validates the in-memory merged form: the `.zgroup` fields plus an + optional `attributes` mapping folded in from `.zattrs`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V2 - doc.keys()) + ] + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_group_metadata_v2(value: object) -> TypeIs[GroupMetadataV2]: + """Whether `value` is a structurally-valid v2 group metadata document.""" + return not validate_group_metadata_v2(value) + + +def parse_group_metadata_v2(value: object) -> GroupMetadataV2: + """Return `value` narrowed to `GroupMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV2, value) + + def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py new file mode 100644 index 0000000000..1e98cf3e47 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -0,0 +1,137 @@ +"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" + +import json + +import pytest + +from zarr_metadata.model._group import ( + GroupMetadataModelV2, + GroupMetadataModelV3, +) +from zarr_metadata.model._validation import ( + MetadataValidationError, + parse_group_metadata_v2, + parse_group_metadata_v3, +) + +# --- GroupMetadataModelV3 --------------------------------------------------- + + +def test_group_v3_roundtrip() -> None: + """A v3 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} + model = GroupMetadataModelV3.from_json(doc) + assert model.to_json() == doc + + +def test_group_v3_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + model = GroupMetadataModelV3.create_default() + assert "attributes" not in model.to_json() + + +def test_group_v3_lists_become_tuples() -> None: + """from_json converts JSON arrays in attributes to tuples.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} + model = GroupMetadataModelV3.from_json(doc) + assert model.attributes == {"a": (1, 2)} + + +def test_group_v3_extra_fields_roundtrip() -> None: + """Unknown top-level keys land in extra_fields and reappear in to_json.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "my_extension": {"name": "thing", "must_understand": False}, + } + model = GroupMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} + assert model.to_json() == doc + + +def test_group_v3_extra_fields_overlap_rejected() -> None: + """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=None, + extra_fields={"node_type": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_consolidated_extra_field_rejected() -> None: + """extra_fields may not shadow the consolidated_metadata convention key.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=None, + extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_missing_required_key() -> None: + """parse_group_metadata_v3 reports each missing required key.""" + with pytest.raises(MetadataValidationError, match="node_type"): + parse_group_metadata_v3({"zarr_format": 3}) + + +def test_group_v3_bad_attributes() -> None: + """parse_group_metadata_v3 rejects a non-mapping attributes value.""" + with pytest.raises(MetadataValidationError, match="attributes"): + parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) + + +def test_group_v3_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for v3 groups.""" + model = GroupMetadataModelV3.create_default(attributes={"a": 1}) + assert GroupMetadataModelV3.from_key_value(model.to_key_value()) == model + + +def test_group_v3_update() -> None: + """update replaces the given fields and returns a new instance.""" + base = GroupMetadataModelV3.create_default() + updated = base.update(attributes={"a": 1}) + assert updated.attributes == {"a": 1} + assert base.attributes == {} + + +# --- GroupMetadataModelV2 --------------------------------------------------- + + +def test_group_v2_key_value_split() -> None: + """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" + model = GroupMetadataModelV2.create_default(attributes={"a": 1}) + kv = model.to_key_value() + assert set(kv) == {".zgroup", ".zattrs"} + assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} + assert GroupMetadataModelV2.from_key_value(kv) == model + + +def test_group_v2_from_key_value_without_zattrs() -> None: + """A v2 group with no .zattrs file parses with empty attributes.""" + model = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert model.attributes == {} + + +def test_group_v2_json_roundtrip() -> None: + """A merged-form v2 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 2, "attributes": {"a": 1}} + model = GroupMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_group_v2_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + assert "attributes" not in GroupMetadataModelV2.create_default().to_json() + + +def test_group_v2_not_a_mapping() -> None: + """parse_group_metadata_v2 rejects a non-mapping document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + parse_group_metadata_v2([1, 2, 3]) + + +def test_group_v2_missing_required_key() -> None: + """parse_group_metadata_v2 reports a missing zarr_format key.""" + with pytest.raises(MetadataValidationError, match="zarr_format"): + parse_group_metadata_v2({}) From a85319f69d381d648fe28e9411e093228789a022 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:27:15 +0200 Subject: [PATCH 06/33] test(zarr-metadata): add consolidated metadata model tests Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/tests/model/test_group.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 1e98cf3e47..909346586b 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -1,12 +1,18 @@ """Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" +import dataclasses import json import pytest +from zarr_metadata.model._array import ArrayMetadataModelV3 from zarr_metadata.model._group import ( + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, GroupMetadataModelV2, + GroupMetadataModelV2Partial, GroupMetadataModelV3, + GroupMetadataModelV3Partial, ) from zarr_metadata.model._validation import ( MetadataValidationError, @@ -135,3 +141,127 @@ def test_group_v2_missing_required_key() -> None: """parse_group_metadata_v2 reports a missing zarr_format key.""" with pytest.raises(MetadataValidationError, match="zarr_format"): parse_group_metadata_v2({}) + + +# --- Partial TypedDict drift guards ----------------------------------------- + + +def test_group_partial_keys_match_settable_model_fields() -> None: + """Each group partial TypedDict must list exactly the settable model fields. + + Guards against drift: adding/removing a settable field on a group model + without updating its `*Partial` TypedDict fails here. + """ + for model_cls, partial_cls in ( + (GroupMetadataModelV3, GroupMetadataModelV3Partial), + (GroupMetadataModelV2, GroupMetadataModelV2Partial), + ): + settable = {f.name for f in dataclasses.fields(model_cls) if f.init} + assert set(partial_cls.__annotations__) == settable + + +# --- ConsolidatedMetadataModelV3 -------------------------------------------- + + +def test_consolidated_v3_roundtrip() -> None: + """A v3 group with inline consolidated metadata round-trips, with child + entries parsed into array/group models.""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + model = GroupMetadataModelV3.from_json(doc) + assert isinstance(model.consolidated_metadata, ConsolidatedMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["a"], ArrayMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["g"], GroupMetadataModelV3) + assert model.to_json() == doc + + +def test_consolidated_v3_must_understand_true_rejected() -> None: + """ConsolidatedMetadataModelV3 enforces must_understand=False at runtime.""" + with pytest.raises(ValueError, match="must_understand"): + ConsolidatedMetadataModelV3(must_understand=True, metadata={}) + + +def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: + """from_json rejects a consolidated document carrying must_understand=true.""" + with pytest.raises(MetadataValidationError, match="must_understand"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": True, "metadata": {}} + ) + + +def test_consolidated_v3_entry_without_node_type_rejected() -> None: + """from_json rejects a consolidated entry lacking a recognizable node_type.""" + with pytest.raises(MetadataValidationError, match="node_type"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} + ) + + +def test_consolidated_v3_not_a_mapping() -> None: + """from_json rejects a non-mapping consolidated document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV3.from_json(5) + + +# --- ConsolidatedMetadataModelV2 -------------------------------------------- + + +def test_consolidated_v2_verbatim_roundtrip() -> None: + """The v2 .zmetadata model holds the flat file-keyed map verbatim, + including nodes that have no .zattrs entry.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": {"zarr_format": 2}, + "a/.zarray": { + "zarr_format": 2, + "shape": (2,), + "chunks": (2,), + "dtype": "|u1", + "fill_value": 0, + "order": "C", + "compressor": None, + "filters": None, + }, + }, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_consolidated_v2_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" + model = ConsolidatedMetadataModelV2.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} + ) + assert ConsolidatedMetadataModelV2.from_key_value(model.to_key_value()) == model + + +def test_consolidated_v2_lists_become_tuples() -> None: + """from_json converts JSON arrays inside entries to tuples.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zarray": {"shape": [2, 3]}}, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} + + +def test_consolidated_v2_envelope_validation() -> None: + """from_json rejects a .zmetadata document missing the metadata key.""" + with pytest.raises(MetadataValidationError, match="metadata"): + ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 1}) + + +def test_consolidated_v2_not_a_mapping() -> None: + """from_json rejects a non-mapping .zmetadata document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV2.from_json([1]) From fbfb425ad913cdc2424f51db029c57ec7377207d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:30:18 +0200 Subject: [PATCH 07/33] feat(zarr-metadata): export model layer from package front door Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/__init__.py | 28 +++++++++++++ .../src/zarr_metadata/model/__init__.py | 42 +++++++++++++++++++ .../src/zarr_metadata/model/_array.py | 12 +++--- .../src/zarr_metadata/model/_group.py | 10 ++--- .../src/zarr_metadata/model/_validation.py | 12 +++--- .../zarr-metadata/tests/test_public_api.py | 14 +++++++ 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 46949570a2..d522c89cd2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,6 +1,21 @@ from importlib.metadata import version from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, + MetadataValidationError, + ValidationProblem, + ZarrMetadataV3, +) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, @@ -235,6 +250,10 @@ "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", "ArrayDimensionSeparatorV2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", "ArrayMetadataV2", "ArrayMetadataV2Partial", "ArrayMetadataV3", @@ -259,6 +278,8 @@ "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", "ConsolidatedMetadataV2", "ConsolidatedMetadataV3", "Crc32cCodecMetadata", @@ -275,6 +296,10 @@ "Float32FillValue", "Float64DataTypeName", "Float64FillValue", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "GroupMetadataV2", "GroupMetadataV2Partial", "GroupMetadataV3", @@ -291,6 +316,7 @@ "Int64FillValue", "JSONValue", "MetadataV3", + "MetadataValidationError", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", @@ -325,9 +351,11 @@ "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", + "ValidationProblem", "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrMetadataV3", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 8776b4fab3..68e36be0a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -10,23 +10,45 @@ ArrayMetadataModelV3Partial, ZarrMetadataV3, ) +from zarr_metadata.model._group import ( + CONSOLIDATED_METADATA_KEY_V3, + CONSOLIDATED_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, +) from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_OPTIONAL_KEYS_V3, + GROUP_METADATA_REQUIRED_KEYS_V2, + GROUP_METADATA_REQUIRED_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, is_json, is_metadata_field_v3, parse_array_metadata_v2, parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, parse_json, parse_metadata_field_v3, validate_array_metadata_v2, validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, validate_json, validate_metadata_field_v3, ) @@ -39,23 +61,43 @@ "ARRAY_METADATA_STORE_KEY_V2", "ARRAY_METADATA_STORE_KEY_V3", "ATTRIBUTES_STORE_KEY_V2", + "CONSOLIDATED_METADATA_KEY_V3", + "CONSOLIDATED_METADATA_STORE_KEY_V2", + "GROUP_METADATA_OPTIONAL_KEYS_V3", + "GROUP_METADATA_REQUIRED_KEYS_V2", + "GROUP_METADATA_REQUIRED_KEYS_V3", + "GROUP_METADATA_STANDARD_KEYS_V3", + "GROUP_METADATA_STORE_KEY_V2", + "GROUP_METADATA_STORE_KEY_V3", "ArrayMetadataModelV2", "ArrayMetadataModelV2Partial", "ArrayMetadataModelV3", "ArrayMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "MetadataValidationError", "ValidationProblem", "ZarrMetadataV3", "is_array_metadata_v2", "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", "is_json", "is_metadata_field_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", "parse_json", "parse_metadata_field_v3", "validate_array_metadata_v2", "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", "validate_json", "validate_metadata_field_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index ea2745284d..e5b23918d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -20,16 +20,16 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata import ( + from zarr_metadata._common import JSONValue + from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, ArrayMetadataV2, - ArrayMetadataV3, ArrayOrderV2, - ExtensionFieldV3, - MetadataV3, + DataTypeMetadataV2, ) - from zarr_metadata._common import JSONValue - from zarr_metadata.v2 import CodecMetadataV2, DataTypeMetadataV2 + from zarr_metadata.v2.codec import CodecMetadataV2 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 ArrayMetadataStoreKeyV3 = Literal["zarr.json"] ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 8eeaed1a89..a061ed5d67 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -24,13 +24,11 @@ ) if TYPE_CHECKING: - from zarr_metadata import ( - ConsolidatedMetadataV3, - ExtensionFieldV3, - GroupMetadataV2, - GroupMetadataV3, - ) from zarr_metadata._common import JSONValue + from zarr_metadata.v2.group import GroupMetadataV2 + from zarr_metadata.v3.array import ExtensionFieldV3 + from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 + from zarr_metadata.v3.group import GroupMetadataV3 GroupMetadataStoreKeyV3 = Literal["zarr.json"] GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index c69a80c144..a38258316b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,14 +14,12 @@ from typing_extensions import TypeIs -from zarr_metadata import ( - ArrayMetadataV2, - ArrayMetadataV3, - GroupMetadataV2, - GroupMetadataV3, - MetadataV3, -) from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ArrayMetadataV2 +from zarr_metadata.v2.group import GroupMetadataV2 +from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.group import GroupMetadataV3 @dataclass(frozen=True, slots=True) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index d3270579c3..7a6db7ab30 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -39,6 +39,20 @@ def _group_rank(s: str) -> int: "NamedConfigV3", "MetadataV3", "JSONValue", + # Category A' — metadata models (in-memory dataclasses over the documents) + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "ZarrMetadataV3", + "ValidationProblem", + "MetadataValidationError", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 99608ce38789fcd9c677d34cdcfd9fa6cce17fb0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:31:40 +0200 Subject: [PATCH 08/33] docs(zarr-metadata): changelog entry for the model layer Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/zarr-metadata/changes/210.feature.md diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md new file mode 100644 index 0000000000..35835c9ebe --- /dev/null +++ b/packages/zarr-metadata/changes/210.feature.md @@ -0,0 +1,7 @@ +Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, +`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `ZarrMetadataV3`) +that are canonical, lossless representations of Zarr metadata documents, plus +structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension +point (data type, chunk grid, chunk key encoding, codecs, storage transformers) +is held as a name + configuration pair; nothing is interpreted. From c98a1220493813935a2e451725117952dbe01eed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 14:58:41 +0200 Subject: [PATCH 09/33] feat(zarr-metadata): harden model validation and error reporting Findings from an API-ergonomics exercise (a fresh agent consuming defective metadata documents): - ValidationProblem gains a machine-readable kind (missing_key / invalid_type / invalid_value / invalid_json), ending message string-matching in consumers. - The v2 array validator now enforces what its types declare (dtype, order, compressor, filters, dimension_separator), and all four document validators check the fixed zarr_format / node_type literals. - All ingestion failures surface as MetadataValidationError: missing store keys and undecodable bytes in from_key_value (previously KeyError / JSONDecodeError) and constructor invariants (previously bare ValueError). - ZarrMetadataV3 is renamed NamedConfigModelV3: it models a name + configuration pair, and the old name read as a whole-document type. - Discoverability: the validate_*/is_*/parse_* contract is documented on zarr_metadata.model itself; update() documents that it does not re-validate; the v2 to_json/to_key_value attributes split is documented on both. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 11 +- .../src/zarr_metadata/__init__.py | 6 +- .../src/zarr_metadata/model/__init__.py | 19 +- .../src/zarr_metadata/model/_array.py | 76 ++++-- .../src/zarr_metadata/model/_group.py | 67 +++-- .../src/zarr_metadata/model/_validation.py | 237 +++++++++++++++--- .../zarr-metadata/tests/model/test_array.py | 187 +++++++++++--- .../zarr-metadata/tests/model/test_group.py | 19 ++ .../zarr-metadata/tests/test_public_api.py | 3 +- 9 files changed, 501 insertions(+), 124 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 35835c9ebe..13517ce38a 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -1,7 +1,16 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, `ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, -`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `ZarrMetadataV3`) +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`) that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) is held as a name + configuration pair; nothing is interpreted. + +Validation is strict about what the types declare: v2 `dtype` / `order` / +`compressor` / `filters` / `dimension_separator` shapes and the fixed +`zarr_format` / `node_type` literals are all enforced. Every +`ValidationProblem` carries a machine-readable `kind` +(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so +consumers can dispatch on the failure mode without matching message strings, +and every ingestion failure — including missing store keys and undecodable +bytes in `from_key_value` — surfaces as `MetadataValidationError`. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index d522c89cd2..ecd7e342cf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -13,8 +13,9 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, MetadataValidationError, + NamedConfigModelV3, + ProblemKind, ValidationProblem, - ZarrMetadataV3, ) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, @@ -317,12 +318,14 @@ "JSONValue", "MetadataV3", "MetadataValidationError", + "NamedConfigModelV3", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", @@ -355,7 +358,6 @@ "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", - "ZarrMetadataV3", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 68e36be0a7..aa8555d623 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -1,4 +1,15 @@ -"""In-memory models for Zarr metadata documents.""" +"""In-memory models for Zarr metadata documents. + +Models are frozen dataclasses that hold a canonical, lossless representation +of the JSON documents; they never interpret extension points (codecs, chunk +grids, data types). Validators check JSON structure, not domain validity. +Each document concept gets a `validate_*` function returning every problem +found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an +`is_*` type guard, and a `parse_*` function that narrows or raises +`MetadataValidationError`. Model `from_json` / `from_key_value` constructors +raise `MetadataValidationError` for every ingestion failure, including +missing store keys and undecodable bytes. +""" from zarr_metadata.model._array import ( ARRAY_METADATA_STORE_KEY_V2, @@ -8,7 +19,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, - ZarrMetadataV3, + NamedConfigModelV3, ) from zarr_metadata.model._group import ( CONSOLIDATED_METADATA_KEY_V3, @@ -32,6 +43,7 @@ GROUP_METADATA_REQUIRED_KEYS_V3, GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, + ProblemKind, ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, @@ -80,8 +92,9 @@ "GroupMetadataModelV3", "GroupMetadataModelV3Partial", "MetadataValidationError", + "NamedConfigModelV3", + "ProblemKind", "ValidationProblem", - "ZarrMetadataV3", "is_array_metadata_v2", "is_array_metadata_v3", "is_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index e5b23918d4..4ec02137ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -11,7 +11,10 @@ from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, arrays_to_tuples, + load_store_json, parse_array_metadata_v2, parse_array_metadata_v3, parse_metadata_field_v3, @@ -42,7 +45,7 @@ @dataclass(frozen=True, slots=True, kw_only=True) -class ZarrMetadataV3: +class NamedConfigModelV3: """A v3 metadata field in normalized form: a name plus a configuration. This is the in-memory model of `MetadataV3` (a bare name string or a @@ -57,7 +60,7 @@ def to_json(self) -> MetadataV3: return {"name": self.name, "configuration": self.configuration} @classmethod - def from_json(cls, data: object) -> ZarrMetadataV3: + def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) @@ -81,13 +84,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: ZarrMetadataV3 - chunk_grid: ZarrMetadataV3 - codecs: tuple[ZarrMetadataV3, ...] - chunk_key_encoding: ZarrMetadataV3 + data_type: NamedConfigModelV3 + chunk_grid: NamedConfigModelV3 + codecs: tuple[NamedConfigModelV3, ...] + chunk_key_encoding: NamedConfigModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[ZarrMetadataV3, ...] + storage_transformers: tuple[NamedConfigModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -97,7 +100,7 @@ class ArrayMetadataModelV3: A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `ZarrMetadataV3` name + + `codecs`, `storage_transformers`) are held as `NamedConfigModelV3` name + configuration pairs and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -106,13 +109,13 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: ZarrMetadataV3 - chunk_grid: ZarrMetadataV3 - codecs: tuple[ZarrMetadataV3, ...] - chunk_key_encoding: ZarrMetadataV3 + data_type: NamedConfigModelV3 + chunk_grid: NamedConfigModelV3 + codecs: tuple[NamedConfigModelV3, ...] + chunk_key_encoding: NamedConfigModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[ZarrMetadataV3, ...] + storage_transformers: tuple[NamedConfigModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @classmethod @@ -129,10 +132,10 @@ def create_default( default = cls( shape=(), fill_value=0, - data_type=ZarrMetadataV3(name="uint8", configuration={}), - chunk_grid=ZarrMetadataV3(name="regular", configuration={"chunk_shape": ()}), - codecs=(ZarrMetadataV3(name="bytes", configuration={}),), - chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}), + data_type=NamedConfigModelV3(name="uint8", configuration={}), + chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(NamedConfigModelV3(name="bytes", configuration={}),), + chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), dimension_names=None, attributes={}, storage_transformers=(), @@ -152,12 +155,25 @@ def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadata This is useful for test fixtures that want to override a few fields of a base template without having to re-specify the entire document. + + No re-validation is performed (`update` is `dataclasses.replace`), so + a repair or edit can produce an invalid document; validity is checked + on `from_json`, not on field replacement. """ return dataclasses.replace(self, **kwargs) def __post_init__(self) -> None: - if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3): - raise ValueError("Extra fields cannot overlap with standard ArrayMetadataV3 fields") + overlap = set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3) + if overlap: + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard ArrayMetadataV3 fields", + "invalid_value", + ) + ] + ) def to_json(self) -> ArrayMetadataV3: out: ArrayMetadataV3 = { @@ -197,21 +213,21 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: return cls( shape=parsed["shape"], fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict - data_type=ZarrMetadataV3.from_json(parsed["data_type"]), - chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]), - codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]), - chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]), + data_type=NamedConfigModelV3.from_json(parsed["data_type"]), + chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), + codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), dimension_names=parsed.get("dimension_names"), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( - ZarrMetadataV3.from_json(t) for t in parsed.get("storage_transformers", ()) + NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) ), extra_fields=extra_fields, ) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: - return cls.from_json(json.loads(mapping[ARRAY_METADATA_STORE_KEY_V3])) + return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { @@ -299,6 +315,12 @@ def create_default( return default.update(**overrides) def to_json(self) -> ArrayMetadataV2: + """Return the merged in-memory document form, INCLUDING `attributes`. + + This is not the on-disk `.zarray` content: a conforming `.zarray` must + exclude `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ out: ArrayMetadataV2 = { "zarr_format": self.zarr_format, "shape": self.shape, @@ -330,9 +352,9 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: - zarray = json.loads(mapping[ARRAY_METADATA_STORE_KEY_V2]) + zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) zattrs: dict[str, JSONValue] = ( - json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping else {} ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index a061ed5d67..6128b0306b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -19,6 +19,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, ) @@ -83,7 +84,15 @@ class GroupMetadataModelV3: def __post_init__(self) -> None: reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} if set(self.extra_fields.keys()).intersection(reserved): - raise ValueError("Extra fields cannot overlap with standard GroupMetadataV3 fields") + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard GroupMetadataV3 fields", + "invalid_value", + ) + ] + ) @classmethod def create_default( @@ -151,7 +160,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: - return cls.from_json(json.loads(mapping[GROUP_METADATA_STORE_KEY_V3])) + return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { @@ -176,9 +185,15 @@ class ConsolidatedMetadataModelV3: def __post_init__(self) -> None: if self.must_understand is not False: - raise ValueError( - f"Invalid value for 'must_understand'. Expected False. " - f"Got {self.must_understand!r}." + raise MetadataValidationError( + [ + ValidationProblem( + ("must_understand",), + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}.", + "invalid_value", + ) + ] ) def to_json(self) -> ConsolidatedMetadataV3: @@ -193,16 +208,22 @@ def to_json(self) -> ConsolidatedMetadataV3: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: if not isinstance(data, Mapping): - raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) doc = cast("Mapping[str, object]", data) entries_raw = doc.get("metadata") if not isinstance(entries_raw, Mapping): - raise MetadataValidationError([ValidationProblem(("metadata",), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem(("metadata",), "expected a mapping", "invalid_type")] + ) entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} problems: list[ValidationProblem] = [] for key, entry in cast("Mapping[object, object]", entries_raw).items(): if not isinstance(key, str): - problems.append(ValidationProblem(("metadata",), f"non-string key {key!r}")) + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) continue entry_obj: object = entry node_type: object = None @@ -214,14 +235,18 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: entries[key] = GroupMetadataModelV3.from_json(entry_obj) else: problems.append( - ValidationProblem(("metadata", key, "node_type"), "expected 'array' or 'group'") + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) ) if problems: raise MetadataValidationError(problems) must_understand = doc.get("must_understand", False) if must_understand is not False: raise MetadataValidationError( - [ValidationProblem(("must_understand",), "expected False")] + [ValidationProblem(("must_understand",), "expected False", "invalid_value")] ) return cls(must_understand=must_understand, metadata=entries) @@ -280,6 +305,12 @@ def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadata return dataclasses.replace(self, **kwargs) def to_json(self) -> GroupMetadataV2: + """Return the merged in-memory document form, INCLUDING `attributes`. + + This is not the on-disk `.zgroup` content: a conforming `.zgroup` must + exclude `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ out: GroupMetadataV2 = {"zarr_format": self.zarr_format} if len(self.attributes) > 0: out["attributes"] = self.attributes @@ -292,9 +323,9 @@ def from_json(cls, data: object) -> GroupMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: - zgroup = json.loads(mapping[GROUP_METADATA_STORE_KEY_V2]) + zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) zattrs: dict[str, JSONValue] = ( - json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping else {} ) @@ -333,10 +364,12 @@ def to_json(self) -> dict[str, JSONValue]: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: if not isinstance(data, Mapping): - raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) doc = cast("Mapping[str, object]", data) problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") + ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") if key not in doc ] @@ -346,7 +379,9 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: isinstance(k, str) for k in cast("Mapping[object, object]", entries) ): problems.append( - ValidationProblem(("metadata",), "expected a mapping with string keys") + ValidationProblem( + ("metadata",), "expected a mapping with string keys", "invalid_type" + ) ) if problems: raise MetadataValidationError(problems) @@ -358,7 +393,7 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: - return cls.from_json(json.loads(mapping[CONSOLIDATED_METADATA_STORE_KEY_V2])) + return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a38258316b..304a80d4ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -1,16 +1,22 @@ """Structural validation for Zarr metadata documents. -Validators check JSON structure (shapes, key presence, primitive kinds), -not domain validity. Each concept gets a `validate_*` function returning -every problem found, an `is_*` type guard, and a `parse_*` function that -narrows or raises `MetadataValidationError`. +Validators check JSON structure (key presence, value shapes, and fixed +literals like `zarr_format`), not domain validity. Each concept gets a +`validate_*` function returning every problem found, an `is_*` type guard, +and a `parse_*` function that narrows or raises `MetadataValidationError`. + +Every `ValidationProblem` carries a machine-readable `kind` alongside its +human-readable `message`, so consumers can dispatch on the failure mode +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without +string-matching messages. """ from __future__ import annotations +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Final, cast +from typing import Any, Final, Literal, cast from typing_extensions import TypeIs @@ -21,6 +27,17 @@ from zarr_metadata.v3.array import ArrayMetadataV3 from zarr_metadata.v3.group import GroupMetadataV3 +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +""" + @dataclass(frozen=True, slots=True) class ValidationProblem: @@ -28,10 +45,13 @@ class ValidationProblem: `loc` is the path from the document root to the offending value, e.g. `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. """ loc: tuple[str | int, ...] message: str + kind: ProblemKind def __str__(self) -> str: location = ".".join(str(part) for part in self.loc) if self.loc else "" @@ -51,7 +71,7 @@ def __init__(self, problems: list[ValidationProblem]) -> None: def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" - return [ValidationProblem((loc_head, *p.loc), p.message) for p in problems] + return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] def validate_json(value: object) -> list[ValidationProblem]: @@ -62,7 +82,9 @@ def validate_json(value: object) -> list[ValidationProblem]: if isinstance(value, Mapping): for key, item in cast("Mapping[object, object]", value).items(): if not isinstance(key, str): - problems.append(ValidationProblem((), f"non-string key {key!r} in JSON object")) + problems.append( + ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + ) continue problems.extend(_prefix(key, validate_json(item))) return problems @@ -70,7 +92,7 @@ def validate_json(value: object) -> list[ValidationProblem]: for index, item in enumerate(cast("Sequence[object]", value)): problems.extend(_prefix(index, validate_json(item))) return problems - return [ValidationProblem((), f"not a JSON-serializable value: {value!r}")] + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] def is_json(value: object) -> TypeIs[JSONValue]: @@ -120,6 +142,25 @@ def parse_json(value: object) -> JSONValue: ) +def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: + """One `missing_key` problem per required key absent from `doc`.""" + return [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in sorted(required - doc.keys()) + ] + + +def _check_literal( + doc: Mapping[str, object], key: str, expected: object +) -> list[ValidationProblem]: + """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" + if key in doc and doc[key] != expected: + return [ + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") + ] + return [] + + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -129,18 +170,26 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: return [] if not isinstance(value, Mapping): return [ - ValidationProblem((), "expected a metadata field (string or {name, configuration})") + ValidationProblem( + (), + "expected a metadata field (string or {name, configuration})", + "invalid_type", + ) ] field = cast("Mapping[object, object]", value) problems: list[ValidationProblem] = [] if not isinstance(field.get("name"), str): - problems.append(ValidationProblem(("name",), "expected a string name")) + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) if "configuration" in field: configuration = field["configuration"] if not isinstance(configuration, Mapping): - problems.append(ValidationProblem(("configuration",), "expected a mapping")) + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): - problems.append(ValidationProblem(("configuration",), "expected string keys")) + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) return problems @@ -166,6 +215,40 @@ def _is_int_sequence(value: object) -> bool: ) +def _is_dtype_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 dtype: a string or field records. + + A field record is a `(name, dtype)` or `(name, dtype, shape)` sequence, + where `dtype` is itself a string or nested field records and `shape` is a + sequence of int. The string content is NOT interpreted — whether the + string names a real dtype is domain validity, not structure. + """ + if isinstance(value, str): + return True + if not isinstance(value, Sequence): + return False + for record in cast("Sequence[object]", value): + if isinstance(record, str) or not isinstance(record, Sequence): + return False + fields = cast("Sequence[object]", record) + if len(fields) not in (2, 3): + return False + if not isinstance(fields[0], str): + return False + if not _is_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not _is_int_sequence(fields[2]): + return False + return True + + +def _is_codec_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" + return isinstance(value, Mapping) and isinstance( + cast("Mapping[object, object]", value).get("id"), str + ) + + def _validate_attributes(value: object) -> list[ValidationProblem]: """Validate an `attributes` value: a mapping with string keys. @@ -178,7 +261,11 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping) or not all( isinstance(k, str) for k in cast("Mapping[object, object]", value) ): - return [ValidationProblem(("attributes",), "expected a mapping with string keys")] + return [ + ValidationProblem( + ("attributes",), "expected a mapping with string keys", "invalid_type" + ) + ] return [] @@ -189,14 +276,13 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: (they map to `extra_fields`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V3 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "array")) if "shape" in doc and not _is_int_sequence(doc["shape"]): - problems.append(ValidationProblem(("shape",), "expected a sequence of int")) + problems.append(ValidationProblem(("shape",), "expected a sequence of int", "invalid_type")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): @@ -206,7 +292,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: if key in doc: entries = doc[key] if isinstance(entries, str) or not isinstance(entries, Sequence): - problems.append(ValidationProblem((key,), "expected a sequence")) + problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: for index, entry in enumerate(cast("Sequence[object]", entries)): problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) @@ -218,12 +304,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: # the metadata-field lists (codecs, storage_transformers). names = doc["dimension_names"] if isinstance(names, str) or not isinstance(names, Sequence): - problems.append(ValidationProblem(("dimension_names",), "expected a sequence")) + problems.append( + ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type") + ) elif not all( item is None or isinstance(item, str) for item in cast("Sequence[object]", names) ): problems.append( - ValidationProblem(("dimension_names",), "expected items of str or None") + ValidationProblem( + ("dimension_names",), "expected items of str or None", "invalid_type" + ) ) return problems @@ -244,21 +334,67 @@ def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a structurally-valid v2 array doc. - Checks structure, not domain validity. `compressor`/`filters` are required - keys but may be `None`. + Checks structure, not domain validity: `dtype` must be a string or field + records, but the string content is not interpreted; `compressor` and + `filters` are required keys that may be `None`, and otherwise must be + codec configurations (mappings with a string `id`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V2 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) problems.extend( - ValidationProblem((key,), "expected a sequence of int") + ValidationProblem((key,), "expected a sequence of int", "invalid_type") for key in ("shape", "chunks") if key in doc and not _is_int_sequence(doc[key]) ) + if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): + problems.append( + ValidationProblem( + ("dtype",), + "expected a v2 dtype string or a sequence of field records", + "invalid_type", + ) + ) + if "order" in doc and doc["order"] not in ("C", "F"): + problems.append( + ValidationProblem( + ("order",), f"expected 'C' or 'F', got {doc['order']!r}", "invalid_value" + ) + ) + if "compressor" in doc: + compressor = doc["compressor"] + if compressor is not None and not _is_codec_v2(compressor): + problems.append( + ValidationProblem( + ("compressor",), + "expected null or a codec configuration with a string 'id'", + "invalid_type", + ) + ) + if "filters" in doc: + filters = doc["filters"] + if filters is not None and ( + isinstance(filters, str) + or not isinstance(filters, Sequence) + or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters)) + ): + problems.append( + ValidationProblem( + ("filters",), + "expected null or a sequence of codec configurations with string 'id's", + "invalid_type", + ) + ) + if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): + problems.append( + ValidationProblem( + ("dimension_separator",), + f"expected '.' or '/', got {doc['dimension_separator']!r}", + "invalid_value", + ) + ) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) if "attributes" in doc: @@ -287,16 +423,17 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: must be a mapping (its entries are validated by the consolidated model). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V3 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): - problems.append(ValidationProblem(("consolidated_metadata",), "expected a mapping")) + problems.append( + ValidationProblem(("consolidated_metadata",), "expected a mapping", "invalid_type") + ) return problems @@ -320,12 +457,10 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: optional `attributes` mapping folded in from `.zattrs`. """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V2 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) return problems @@ -344,6 +479,26 @@ def parse_group_metadata_v2(value: object) -> GroupMetadataV2: return cast(GroupMetadataV2, value) +def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: + """Decode the JSON document stored at `key` in `mapping`. + + Every ingestion failure surfaces as `MetadataValidationError`: a missing + store key is a `missing_key` problem and undecodable bytes are an + `invalid_json` problem, rather than leaking `KeyError` / + `json.JSONDecodeError` to callers. + """ + if key not in mapping: + raise MetadataValidationError( + [ValidationProblem((key,), "missing store key", "missing_key")] + ) + try: + return json.loads(mapping[key]) + except json.JSONDecodeError as exc: + raise MetadataValidationError( + [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] + ) from exc + + def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 80dac709de..904e0b12fa 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -17,8 +17,8 @@ ArrayMetadataModelV3, ArrayMetadataModelV3Partial, MetadataValidationError, + NamedConfigModelV3, ValidationProblem, - ZarrMetadataV3, is_array_metadata_v2, is_array_metadata_v3, is_json, @@ -119,16 +119,16 @@ def test_string_nan_fill_value_roundtrips() -> None: assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" -# --- ZarrMetadataV3.to_json ------------------------------------------------ +# --- NamedConfigModelV3.to_json ------------------------------------------------ ZARR_TO_JSON_CASES = [ Expect( - ZarrMetadataV3(name="regular", configuration={"chunk_shape": [1]}), + NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), {"name": "regular", "configuration": {"chunk_shape": [1]}}, id="with-configuration", ), Expect( - ZarrMetadataV3(name="bytes", configuration={}), + NamedConfigModelV3(name="bytes", configuration={}), {"name": "bytes", "configuration": {}}, id="without-configuration", ), @@ -136,32 +136,32 @@ def test_string_nan_fill_value_roundtrips() -> None: @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[ZarrMetadataV3, dict[str, object]]) -> None: - """ZarrMetadataV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: + """NamedConfigModelV3.to_json emits the canonical object form.""" assert case.input.to_json() == case.output -# --- ZarrMetadataV3.from_json ----------------------------------------------- +# --- NamedConfigModelV3.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ - Expect("bytes", ZarrMetadataV3(name="bytes", configuration={}), id="bare-string"), + Expect("bytes", NamedConfigModelV3(name="bytes", configuration={}), id="bare-string"), Expect( {"name": "regular", "configuration": {"chunk_shape": [1]}}, - ZarrMetadataV3(name="regular", configuration={"chunk_shape": (1,)}), + NamedConfigModelV3(name="regular", configuration={"chunk_shape": (1,)}), id="object-with-config", ), Expect( {"name": "bytes"}, - ZarrMetadataV3(name="bytes", configuration={}), + NamedConfigModelV3(name="bytes", configuration={}), id="object-without-config", ), ] @pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> None: - """ZarrMetadataV3.from_json parses both the bare-string and object forms.""" - assert ZarrMetadataV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> None: + """NamedConfigModelV3.from_json parses both the bare-string and object forms.""" + assert NamedConfigModelV3.from_json(case.input) == case.output # --- V3 baseline ----------------------------------------------------------- @@ -170,7 +170,7 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> Non def test_v3_to_json_includes_required_fields() -> None: """V3 to_json emits all required fields with the expected values.""" out = ArrayMetadataModelV3.create_default( - shape=(10,), data_type=ZarrMetadataV3(name="int32", configuration={}) + shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() assert out["zarr_format"] == 3 assert out["node_type"] == "array" @@ -221,7 +221,7 @@ def test_v3_single_storage_transformer_included() -> None: Regression: the guard used ``> 1`` instead of ``> 0``, dropping a lone storage transformer. """ - st = ZarrMetadataV3(name="some_transformer", configuration={}) + st = NamedConfigModelV3(name="some_transformer", configuration={}) out: dict[str, object] = dict( ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() ) @@ -292,7 +292,7 @@ def test_v3_create_default_is_valid_empty_array() -> None: """V3 create_default builds a structurally valid empty array that round-trips.""" m = ArrayMetadataModelV3.create_default() assert m.shape == () - assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) assert m.fill_value == 0 assert m.attributes == {} assert m.extra_fields == {} @@ -307,7 +307,7 @@ def test_v3_create_default_applies_overrides() -> None: assert m.shape == (4, 4) assert m.attributes == {"a": 1} # un-overridden fields keep their defaults - assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) def test_v2_create_default_is_valid_empty_array() -> None: @@ -464,11 +464,11 @@ def test_v3_from_json_reconstructs_required_fields() -> None: doc = ArrayMetadataModelV3.create_default( shape=(7,), attributes={"a": 1}, - data_type=ZarrMetadataV3(name="int32", configuration={}), + data_type=NamedConfigModelV3(name="int32", configuration={}), ).to_json() model = ArrayMetadataModelV3.from_json(doc) assert model.shape == (7,) - assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) assert model.attributes == {"a": 1} @@ -522,12 +522,12 @@ def test_v3_from_key_value_parses_zarr_json() -> None: FROM_KEY_VALUE_MISSING_PARAMS = [ pytest.param( ArrayMetadataModelV3, - ExpectFail({}, KeyError, id="v3-missing-zarr-json"), + ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), id="v3-missing-zarr-json", ), pytest.param( ArrayMetadataModelV2, - ExpectFail({}, KeyError, id="v2-missing-zarray"), + ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), id="v2-missing-zarray", ), ] @@ -538,7 +538,7 @@ def test_from_key_value_missing_key_raises( model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], case: ExpectFail[dict[str, bytes]], ) -> None: - """from_key_value raises KeyError when the required store key is absent.""" + """from_key_value raises MetadataValidationError when the required store key is absent.""" with case.raises(): model_cls.from_key_value(case.input) @@ -551,7 +551,7 @@ def test_from_key_value_missing_key_raises( ArrayMetadataModelV3.create_default( attributes={"a": 1}, dimension_names=("x",), - storage_transformers=(ZarrMetadataV3(name="t", configuration={}),), + storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), extra_fields={"ext": {"must_understand": False}}, ), id="v3-full", @@ -629,7 +629,7 @@ def test_v3_parser_accepts_bare_string_data_type() -> None: doc["data_type"] = "int32" # bare-string form, not canonical object form model = ArrayMetadataModelV3.from_json(doc) # parses correctly, re-serializes to canonical object form - assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} @@ -960,7 +960,7 @@ def test_array_metadata_guards( id="v2-missing-required", ), pytest.param( - ZarrMetadataV3, + NamedConfigModelV3, ExpectFail(lambda: 5, MetadataValidationError, id="x"), id="zarr-metadata-bad-input", ), @@ -969,7 +969,7 @@ def test_array_metadata_guards( @pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) def test_from_json_rejects_malformed( - model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | ZarrMetadataV3], + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | NamedConfigModelV3], case: ExpectFail[Callable[[], object]], ) -> None: """from_json raises MetadataValidationError on a malformed document.""" @@ -983,19 +983,19 @@ def test_from_json_rejects_malformed( def test_validation_problem_str_with_loc() -> None: """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" - p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str") + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") assert str(p) == "codecs.0.name: expected str" def test_validation_problem_str_empty_loc() -> None: """ValidationProblem.__str__ renders an empty loc as .""" - p = ValidationProblem(loc=(), message="not a mapping") + p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") assert str(p) == ": not a mapping" def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" - p = ValidationProblem(loc=("shape",), message="x") + p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): p.message = "y" # type: ignore[misc] @@ -1003,8 +1003,10 @@ def test_validation_problem_is_frozen() -> None: def test_metadata_validation_error_holds_problems() -> None: """MetadataValidationError carries its problem list and renders them in its message.""" problems = [ - ValidationProblem(loc=("shape",), message="missing required key"), - ValidationProblem(loc=("data_type",), message="expected a metadata field"), + ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), + ValidationProblem( + loc=("data_type",), message="expected a metadata field", kind="invalid_type" + ), ] err = MetadataValidationError(problems) assert err.problems == problems @@ -1014,6 +1016,125 @@ def test_metadata_validation_error_holds_problems() -> None: def test_prefix_prepends_loc_head() -> None: """_prefix prepends a loc head to each problem's loc.""" - problems = [ValidationProblem(loc=("name",), message="expected str")] + problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] prefixed = _prefix(0, problems) - assert prefixed == [ValidationProblem(loc=(0, "name"), message="expected str")] + assert prefixed == [ + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") + ] + + +# --- Stricter v2/v3 field validation and error kinds ------------------------- + + +def test_v2_dtype_must_be_string_or_records() -> None: + """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": 42} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] + + +def test_v2_structured_dtype_records_accepted() -> None: + """A structured v2 dtype (field records, optionally nested/shaped) validates.""" + dtype = (("a", " None: + """A field record with the wrong arity is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": (("a",),)} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("dtype",)] + + +def test_v2_order_literal_enforced() -> None: + """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"order": "Q"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] + + +def test_v2_compressor_must_be_codec_or_none() -> None: + """A compressor that is not null or a codec config mapping is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": "zlib"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] + + +def test_v2_compressor_requires_string_id() -> None: + """A compressor mapping without a string id is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": {"level": 3}} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("compressor",)] + + +def test_v2_filters_must_be_codec_sequence_or_none() -> None: + """Filters that are not null or a sequence of codec configs are rejected.""" + for bad in (7, (5,), "gzip"): + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"filters": bad} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad + + +def test_v2_dimension_separator_literal_enforced() -> None: + """A dimension_separator other than '.' or '/' is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": "-"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_v2_zarr_format_literal_enforced() -> None: + """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"zarr_format": 3} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_zarr_format_literal_enforced() -> None: + """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"zarr_format": 2} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_node_type_literal_enforced() -> None: + """A v3 array document claiming node_type 'group' is rejected.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"node_type": "group"} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] + + +def test_missing_key_kind_is_machine_readable() -> None: + """A missing required key is distinguishable by kind, without message matching.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["chunk_key_encoding"] + problems = validate_array_metadata_v3(doc) + assert problems == [ + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") + ] + + +# --- Unified error channels --------------------------------------------------- + + +def test_from_key_value_invalid_json_raises_metadata_error() -> None: + """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_key_value({"zarr.json": b"{not json"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_from_key_value_missing_key_kind() -> None: + """A missing store key surfaces as a missing_key problem at the store-key loc.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV2.from_key_value({}) + assert exc_info.value.problems == [ + ValidationProblem((".zarray",), "missing store key", "missing_key") + ] + + +def test_extra_fields_overlap_raises_metadata_error() -> None: + """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" + with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 909346586b..829d09b11a 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -18,6 +18,8 @@ MetadataValidationError, parse_group_metadata_v2, parse_group_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, ) # --- GroupMetadataModelV3 --------------------------------------------------- @@ -265,3 +267,20 @@ def test_consolidated_v2_not_a_mapping() -> None: """from_json rejects a non-mapping .zmetadata document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): ConsolidatedMetadataModelV2.from_json([1]) + + +# --- Literal-value enforcement ----------------------------------------------- + + +def test_group_v3_literals_enforced() -> None: + """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" + base = GroupMetadataModelV3.create_default().to_json() + for key, bad in (("zarr_format", 2), ("node_type", "array")): + problems = validate_group_metadata_v3(dict(base) | {key: bad}) + assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key + + +def test_group_v2_zarr_format_literal_enforced() -> None: + """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" + problems = validate_group_metadata_v2({"zarr_format": 3}) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 7a6db7ab30..cd0196d47b 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -50,9 +50,10 @@ def _group_rank(s: str) -> int: "GroupMetadataModelV3Partial", "ConsolidatedMetadataModelV2", "ConsolidatedMetadataModelV3", - "ZarrMetadataV3", + "NamedConfigModelV3", "ValidationProblem", "MetadataValidationError", + "ProblemKind", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From be736dabd317fda8da16075c697450393aa4c906 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:10:26 +0200 Subject: [PATCH 10/33] feat(zarr-metadata): annotate metadata fields with role alias MetadataFieldModelV3 Model fields and consumer signatures should convey the logical meaning of the type (a metadata-document field), not the form it takes when JSON-serialized (a named configuration). MetadataFieldModelV3 is today exactly NamedConfigModelV3; if a future spec revision adds a field form that cannot normalize to name + configuration, the alias widens to a union and annotation sites do not move. Mirrors the raw-layer split between NamedConfigV3 (shape) and MetadataV3 (field union). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 ++- .../src/zarr_metadata/__init__.py | 2 + .../src/zarr_metadata/model/__init__.py | 2 + .../src/zarr_metadata/model/_array.py | 41 +++++++++++++------ .../zarr-metadata/tests/model/test_array.py | 13 ++++++ .../zarr-metadata/tests/test_public_api.py | 1 + 6 files changed, 50 insertions(+), 14 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 13517ce38a..326aee1b42 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -4,7 +4,10 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) -is held as a name + configuration pair; nothing is interpreted. +is held as a name + configuration pair; nothing is interpreted. Model fields +are annotated with the role alias `MetadataFieldModelV3` (today exactly +`NamedConfigModelV3`), so the annotations convey the logical meaning of the +field and stay put if the spec ever adds a new field form. Validation is strict about what the types declare: v2 `dtype` / `order` / `compressor` / `filters` / `dimension_separator` shapes and the fixed diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index ecd7e342cf..354f2a97f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -12,6 +12,7 @@ GroupMetadataModelV2Partial, GroupMetadataModelV3, GroupMetadataModelV3Partial, + MetadataFieldModelV3, MetadataValidationError, NamedConfigModelV3, ProblemKind, @@ -316,6 +317,7 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", + "MetadataFieldModelV3", "MetadataV3", "MetadataValidationError", "NamedConfigModelV3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index aa8555d623..42fbc233d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -19,6 +19,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, + MetadataFieldModelV3, NamedConfigModelV3, ) from zarr_metadata.model._group import ( @@ -91,6 +92,7 @@ "GroupMetadataModelV2Partial", "GroupMetadataModelV3", "GroupMetadataModelV3Partial", + "MetadataFieldModelV3", "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 4ec02137ec..33682c2008 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -5,7 +5,7 @@ import dataclasses import json from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from typing_extensions import TypedDict, Unpack @@ -68,6 +68,20 @@ def from_json(cls, data: object) -> NamedConfigModelV3: return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] +MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 +"""The in-memory model of one field of a v3 metadata document. + +This is the role-named alias for annotation positions: model fields and +consumer signatures should say `MetadataFieldModelV3` (the logical meaning) +rather than `NamedConfigModelV3` (the serialized form the field currently +takes). Today every metadata field normalizes to a named configuration, so +the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a +field form that cannot be normalized to name + configuration, this alias +widens to a union and annotation sites do not change. Mirrors the raw-layer +split between `NamedConfigV3` (shape) and `MetadataV3` (field union). +""" + + class ArrayMetadataModelV3Partial(TypedDict, total=False): """ Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. @@ -84,13 +98,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: NamedConfigModelV3 - chunk_grid: NamedConfigModelV3 - codecs: tuple[NamedConfigModelV3, ...] - chunk_key_encoding: NamedConfigModelV3 + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[NamedConfigModelV3, ...] + storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -100,8 +114,9 @@ class ArrayMetadataModelV3: A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `NamedConfigModelV3` name + - configuration pairs and are never interpreted; `fill_value` is held + `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` + values (currently always `NamedConfigModelV3` name + configuration pairs) + and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -109,13 +124,13 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: NamedConfigModelV3 - chunk_grid: NamedConfigModelV3 - codecs: tuple[NamedConfigModelV3, ...] - chunk_key_encoding: NamedConfigModelV3 + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[NamedConfigModelV3, ...] + storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @classmethod diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 904e0b12fa..e825f0211f 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -16,6 +16,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, + MetadataFieldModelV3, MetadataValidationError, NamedConfigModelV3, ValidationProblem, @@ -1138,3 +1139,15 @@ def test_extra_fields_overlap_raises_metadata_error() -> None: with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] + + +def test_extension_point_fields_annotated_with_role_alias() -> None: + """Extension-point fields are annotated with MetadataFieldModelV3 (the + logical role), not NamedConfigModelV3 (the current serialized form), so a + future widening of the field union does not move annotation sites.""" + assert MetadataFieldModelV3 is NamedConfigModelV3 + annotations = ArrayMetadataModelV3.__annotations__ + for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): + assert annotations[field_name] == "MetadataFieldModelV3" + for field_name in ("codecs", "storage_transformers"): + assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index cd0196d47b..fb2f574a4f 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -51,6 +51,7 @@ def _group_rank(s: str) -> int: "ConsolidatedMetadataModelV2", "ConsolidatedMetadataModelV3", "NamedConfigModelV3", + "MetadataFieldModelV3", "ValidationProblem", "MetadataValidationError", "ProblemKind", From 7fc0211e4ca329ddbab3b4b3ee718fc173719072 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:15:18 +0200 Subject: [PATCH 11/33] test(zarr-metadata): assert required-key coverage via the typed constant test_v3_to_json_includes_required_fields hand-enumerated keys with chained asserts, restating what ARRAY_METADATA_REQUIRED_KEYS_V3 already defines. Now: one coverage assert driven by the constant (tracks the TypedDict automatically) and one whole-document equality for the values. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/tests/model/test_array.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e825f0211f..8806eb8490 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -169,16 +169,22 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> def test_v3_to_json_includes_required_fields() -> None: - """V3 to_json emits all required fields with the expected values.""" + """V3 to_json emits every spec-required key (per the typed constant), + and the whole document matches the model's values.""" out = ArrayMetadataModelV3.create_default( shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() - assert out["zarr_format"] == 3 - assert out["node_type"] == "array" - assert out["shape"] == (10,) - assert out["fill_value"] == 0 - assert out["data_type"] == {"name": "int32", "configuration": {}} - assert out["codecs"] == ({"name": "bytes", "configuration": {}},) + assert out.keys() >= ARRAY_METADATA_REQUIRED_KEYS_V3 + assert out == { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "fill_value": 0, + "data_type": {"name": "int32", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": ()}}, + "codecs": ({"name": "bytes", "configuration": {}},), + "chunk_key_encoding": {"name": "default", "configuration": {}}, + } def test_v3_dimension_names_included_when_present() -> None: From 15e02189def66f60923209aac44368c60535349b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:16:12 +0200 Subject: [PATCH 12/33] test(zarr-metadata): single whole-document comparison for v3 to_json The subset assert against ARRAY_METADATA_REQUIRED_KEYS_V3 was redundant: equality with a literal that spells out the full document already covers every required key. One dict, one assert. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 8806eb8490..0df935144b 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -168,13 +168,12 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> # --- V3 baseline ----------------------------------------------------------- -def test_v3_to_json_includes_required_fields() -> None: - """V3 to_json emits every spec-required key (per the typed constant), - and the whole document matches the model's values.""" +def test_v3_to_json_emits_canonical_document() -> None: + """V3 to_json emits exactly the expected document (which covers every + spec-required key by construction).""" out = ArrayMetadataModelV3.create_default( shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() - assert out.keys() >= ARRAY_METADATA_REQUIRED_KEYS_V3 assert out == { "zarr_format": 3, "node_type": "array", From c0a40cf6bd1c3e9d2b355a86279ddb25c0db557b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 16:52:10 +0200 Subject: [PATCH 13/33] fix(zarr-metadata): close validation holes found by adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invalid documents that previously passed validation: - shape/chunks containing JSON booleans (bool is an int subclass in Python but not an integer in a metadata document) or negative values - dimension_names whose length does not match shape - attributes and configuration values that are not JSON-serializable — now checked recursively like fill_value, so an int-keyed dict cannot be silently rewritten by json.dumps on round-trip and a set() cannot escape as a TypeError from to_key_value - consolidated_metadata envelopes: the group validator now deep-validates the envelope and its entries via the shared validate_consolidated_metadata_v3, which ConsolidatedMetadataModelV3 .from_json also uses, so is_group_metadata_v3 never vouches for a document the model constructor would reject Three pre-existing test fixtures paired dimension_names=('x',) with the default scalar shape () and were themselves spec-invalid; they now use a matching 1-d shape. Deliberately unchanged, pending a design decision: unknown extension fields with must_understand: true still pass (which layer owns the spec's refusal duty), and empty v2 dtype records / empty codec names still pass (domain territory). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +- .../src/zarr_metadata/model/_group.py | 49 ++------ .../src/zarr_metadata/model/_validation.py | 119 +++++++++++++++--- .../zarr-metadata/tests/model/test_array.py | 65 +++++++++- .../zarr-metadata/tests/model/test_group.py | 51 ++++++++ 5 files changed, 235 insertions(+), 57 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 326aee1b42..fbecb7b4a4 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -16,4 +16,10 @@ Validation is strict about what the types declare: v2 `dtype` / `order` / (`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so consumers can dispatch on the failure mode without matching message strings, and every ingestion failure — including missing store keys and undecodable -bytes in `from_key_value` — surfaces as `MetadataValidationError`. +bytes in `from_key_value` — surfaces as `MetadataValidationError`. An +adversarial review added further structural checks: JSON booleans are not +accepted as dimension lengths, dimensions are non-negative, +`dimension_names` must have one entry per dimension of `shape`, `attributes` +and `configuration` values are JSON-checked recursively (like `fill_value`), +and the inline consolidated-metadata envelope and entries are deep-validated +so the group validator's verdict always agrees with the model constructor. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 6128b0306b..b138a7d0f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -22,6 +22,7 @@ load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, + validate_consolidated_metadata_v3, ) if TYPE_CHECKING: @@ -207,48 +208,18 @@ def to_json(self) -> ConsolidatedMetadataV3: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: - if not isinstance(data, Mapping): - raise MetadataValidationError( - [ValidationProblem((), "expected a mapping", "invalid_type")] - ) - doc = cast("Mapping[str, object]", data) - entries_raw = doc.get("metadata") - if not isinstance(entries_raw, Mapping): - raise MetadataValidationError( - [ValidationProblem(("metadata",), "expected a mapping", "invalid_type")] - ) + problems = validate_consolidated_metadata_v3(data) + if problems: + raise MetadataValidationError(problems) + env = cast("Mapping[str, object]", data) entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} - problems: list[ValidationProblem] = [] - for key, entry in cast("Mapping[object, object]", entries_raw).items(): - if not isinstance(key, str): - problems.append( - ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") - ) - continue - entry_obj: object = entry - node_type: object = None - if isinstance(entry, Mapping): - node_type = cast("Mapping[str, object]", entry).get("node_type") + for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): + node_type = cast("Mapping[str, object]", entry).get("node_type") if node_type == "array": - entries[key] = ArrayMetadataModelV3.from_json(entry_obj) - elif node_type == "group": - entries[key] = GroupMetadataModelV3.from_json(entry_obj) + entries[key] = ArrayMetadataModelV3.from_json(entry) else: - problems.append( - ValidationProblem( - ("metadata", key, "node_type"), - "expected 'array' or 'group'", - "invalid_value", - ) - ) - if problems: - raise MetadataValidationError(problems) - must_understand = doc.get("must_understand", False) - if must_understand is not False: - raise MetadataValidationError( - [ValidationProblem(("must_understand",), "expected False", "invalid_value")] - ) - return cls(must_understand=must_understand, metadata=entries) + entries[key] = GroupMetadataModelV3.from_json(entry) + return cls(metadata=entries) class GroupMetadataModelV2Partial(TypedDict, total=False): diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 304a80d4ab..fc781c7899 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -190,6 +190,9 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: problems.append( ValidationProblem(("configuration",), "expected string keys", "invalid_type") ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) return problems @@ -207,14 +210,36 @@ def parse_metadata_field_v3(value: object) -> MetadataV3: def _is_int_sequence(value: object) -> bool: - """Whether `value` is a non-string sequence of integers.""" + """Whether `value` is a non-string sequence of integers. + + JSON booleans decode to `bool`, which is an `int` subclass in Python but + is not an integer in a metadata document, so booleans are excluded. + """ return ( not isinstance(value, str) and isinstance(value, Sequence) - and all(isinstance(item, int) for item in cast("Sequence[object]", value)) + and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast("Sequence[object]", value) + ) ) +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: + """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. + + Dimension lengths are non-negative integers. + """ + if key not in doc: + return [] + value = doc[key] + if not _is_int_sequence(value): + return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + if any(item < 0 for item in cast("Sequence[int]", value)): + return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] + return [] + + def _is_dtype_v2(value: object) -> bool: """Whether `value` is shaped like a v2 dtype: a string or field records. @@ -266,7 +291,10 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: ("attributes",), "expected a mapping with string keys", "invalid_type" ) ] - return [] + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[str, object]", value).items(): + problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + return problems def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: @@ -281,8 +309,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "array")) - if "shape" in doc and not _is_int_sequence(doc["shape"]): - problems.append(ValidationProblem(("shape",), "expected a sequence of int", "invalid_type")) + problems.extend(_validate_dim_sequence(doc, "shape")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): @@ -315,6 +342,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: ("dimension_names",), "expected items of str or None", "invalid_type" ) ) + elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( + cast("Sequence[int]", doc["shape"]) + ): + problems.append( + ValidationProblem( + ("dimension_names",), + "expected one name per dimension of shape", + "invalid_value", + ) + ) return problems @@ -344,11 +381,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) problems.extend(_check_literal(doc, "zarr_format", 2)) - problems.extend( - ValidationProblem((key,), "expected a sequence of int", "invalid_type") - for key in ("shape", "chunks") - if key in doc and not _is_int_sequence(doc[key]) - ) + problems.extend(_validate_dim_sequence(doc, "shape")) + problems.extend(_validate_dim_sequence(doc, "chunks")) if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): problems.append( ValidationProblem( @@ -415,12 +449,66 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) +def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a valid inline consolidated envelope. + + Locs are value-relative (the caller prefixes with `consolidated_metadata` + where appropriate). Entries recurse into the array and group document + validators, so a validator verdict always agrees with what + `ConsolidatedMetadataModelV3.from_json` accepts. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + env = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("kind", "must_understand", "metadata") + if key not in env + ] + problems.extend(_check_literal(env, "kind", "inline")) + if "must_understand" in env and env["must_understand"] is not False: + problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) + if "metadata" in env: + entries = env["metadata"] + if not isinstance(entries, Mapping): + problems.append(ValidationProblem(("metadata",), "expected a mapping", "invalid_type")) + else: + for key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + problems.extend( + _prefix("metadata", _prefix(key, validate_array_metadata_v3(entry_obj))) + ) + elif node_type == "group": + problems.extend( + _prefix("metadata", _prefix(key, validate_group_metadata_v3(entry_obj))) + ) + else: + problems.append( + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) + ) + return problems + + def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a structurally-valid v3 group doc. Checks structure, not domain validity. Unknown top-level keys are allowed (they map to `extra_fields`); a `consolidated_metadata` key, if present, - must be a mapping (its entries are validated by the consolidated model). + is deep-validated (envelope and entries) via + `validate_consolidated_metadata_v3`. """ if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -430,9 +518,12 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): - problems.append( - ValidationProblem(("consolidated_metadata",), "expected a mapping", "invalid_type") + if "consolidated_metadata" in doc: + problems.extend( + _prefix( + "consolidated_metadata", + validate_consolidated_metadata_v3(doc["consolidated_metadata"]), + ) ) return problems diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 0df935144b..9032f18638 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -501,7 +501,9 @@ def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: """V3 from_json keeps standard keys out of extra_fields.""" - doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() model = ArrayMetadataModelV3.from_json(doc) assert model.extra_fields == {} @@ -555,6 +557,7 @@ def test_from_key_value_missing_key_raises( pytest.param( ArrayMetadataModelV3, ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",), storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), @@ -619,7 +622,9 @@ def test_roundtrip_via_key_value( def test_v3_roundtrip_json_model_json() -> None: """A v3 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() assert ArrayMetadataModelV3.from_json(doc).to_json() == doc @@ -837,7 +842,7 @@ def _set(key: str, value: object) -> Callable[[dict], object]: V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ Expect(_build_v3, frozenset(), id="valid"), Expect( - lambda: _build_v3(attributes={"a": 1}, dimension_names=("x",)), + lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), frozenset(), id="valid-with-attributes-and-dim-names", ), @@ -1156,3 +1161,57 @@ def test_extension_point_fields_annotated_with_role_alias() -> None: assert annotations[field_name] == "MetadataFieldModelV3" for field_name in ("codecs", "storage_transformers"): assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" + + +# --- Adversarial-probe fixes: documents that used to pass validation --------- + + +def test_shape_rejects_json_booleans() -> None: + """JSON booleans are not integers: shape/chunks containing true/false are + rejected (bool is an int subclass in Python, so isinstance alone passes).""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (True, True)} + assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (True,)} + assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] + + +def test_shape_rejects_negative_dimensions() -> None: + """Dimension lengths must be non-negative; a negative entry is invalid_value.""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (-1,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_value") + ] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (-5,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_value") + ] + + +def test_dimension_names_length_must_match_shape() -> None: + """dimension_names must have one entry per dimension of shape.""" + doc = dict(ArrayMetadataModelV3.create_default(shape=(10,)).to_json()) | { + "dimension_names": ("x", "y", "z") + } + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + (("dimension_names",), "invalid_value") + ] + + +def test_attributes_values_must_be_json() -> None: + """Attribute values are JSON-checked recursively (like fill_value), so a + non-serializable value is a validation problem, not a later TypeError.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] + + +def test_configuration_values_must_be_json() -> None: + """Configuration values are JSON-checked recursively, so an int-keyed dict + cannot pass validation and be silently rewritten by json.dumps.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | { + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") + ] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 829d09b11a..329471cd94 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -284,3 +284,54 @@ def test_group_v2_zarr_format_literal_enforced() -> None: """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" problems = validate_group_metadata_v2({"zarr_format": 3}) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +# --- Consolidated envelope validated by the group validator ------------------ + + +def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: + """The group validator validates the consolidated envelope and entries, so + is_group_metadata_v3 never vouches for a document from_json would reject.""" + bad_docs = ( + # empty envelope: missing kind/must_understand/metadata + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, + # entry without a recognizable node_type + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {"zarr_format": 3}}, + }, + }, + # must_understand: true + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": True, + "metadata": {}, + }, + }, + ) + for doc in bad_docs: + assert validate_group_metadata_v3(doc) != [], doc + with pytest.raises(MetadataValidationError): + GroupMetadataModelV3.from_json(doc) + + +def test_group_v3_valid_consolidated_passes_validator() -> None: + """A well-formed consolidated group validates cleanly (control case).""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + assert validate_group_metadata_v3(doc) == [] From b6e53ce02b8db5e95d48683ee4e3974081abffce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:09:31 +0200 Subject: [PATCH 14/33] feat(zarr-metadata): expose must_understand_fields on the v3 models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v3 core spec: 'An implementation MUST fail to open Zarr groups or arrays if any metadata fields are present which (a) the implementation does not recognize and (b) are not explicitly set to "must_understand": false' — and fields are implicitly must-understand unless waived. The model layer cannot discharge this itself: recognition is reader-specific (consolidated_metadata is itself an extension field one reader understands and another does not), and a document carrying a must-understand extension is still a valid document. So the models partition by obligation: must_understand_fields is the subset of extra_fields not explicitly waived, and a compliant reader fails to open when must_understand_fields.keys() - recognized is non-empty. The design spec pins that duty on the part-2 resolve layer, matching what zarr-python's parse_extra_fields enforces today. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 44 +++++++++++++++++-- .../src/zarr_metadata/model/_group.py | 13 ++++++ .../zarr-metadata/tests/model/test_array.py | 29 ++++++++++++ .../zarr-metadata/tests/model/test_group.py | 15 +++++++ 5 files changed, 104 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index fbecb7b4a4..f35625558b 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -23,3 +23,9 @@ accepted as dimension lengths, dimensions are non-negative, and `configuration` values are JSON-checked recursively (like `fill_value`), and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. + +The v3 models expose `must_understand_fields`: the subset of `extra_fields` +not explicitly waived with `must_understand: false` (fields are implicitly +must-understand per the spec). Readers discharge the spec's fail-to-open +duty by subtracting the extension names they recognize; the model only +partitions by obligation, since recognition is reader-specific. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 33682c2008..7a4487b7f4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -4,8 +4,9 @@ import dataclasses import json +from collections.abc import Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast from typing_extensions import TypedDict, Unpack @@ -21,8 +22,6 @@ ) if TYPE_CHECKING: - from collections.abc import Mapping - from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, @@ -82,6 +81,33 @@ def from_json(cls, data: object) -> NamedConfigModelV3: """ +def must_understand_subset( + extra_fields: Mapping[str, ExtensionFieldV3], +) -> dict[str, ExtensionFieldV3]: + """The subset of `extra_fields` the reader is obligated to understand. + + Per the v3 spec, an extension field is implicitly `must_understand: True` + unless it explicitly says otherwise, and an implementation MUST fail to + open a group or array carrying fields it does not recognize that are not + explicitly `must_understand: false`. A non-mapping field value cannot + carry the explicit waiver, so it always requires understanding (the + runtime isinstance check defends against values looser than the declared + `ExtensionFieldV3`). + """ + fields = cast("Mapping[str, object]", extra_fields) + return cast( + "dict[str, ExtensionFieldV3]", + { + name: value + for name, value in fields.items() + if not ( + isinstance(value, Mapping) + and cast("Mapping[str, object]", value).get("must_understand") is False + ) + }, + ) + + class ArrayMetadataModelV3Partial(TypedDict, total=False): """ Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. @@ -240,6 +266,18 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: extra_fields=extra_fields, ) + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the array if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index b138a7d0f5..b8cca3849d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -13,6 +13,7 @@ from zarr_metadata.model._array import ( ATTRIBUTES_STORE_KEY_V2, ArrayMetadataModelV3, + must_understand_subset, ) from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, @@ -159,6 +160,18 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: extra_fields=extra_fields, ) + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the group if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 9032f18638..e73739929a 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1215,3 +1215,32 @@ def test_configuration_values_must_be_json() -> None: assert [(p.loc, p.kind) for p in problems] == [ (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") ] + + +# --- must_understand partition (spec: MUST fail to open unrecognized fields) -- + + +def test_must_understand_fields_partition() -> None: + """must_understand_fields contains every extra field not explicitly waived + with must_understand: false, including implicitly-true and non-mapping + fields, so a reader can discharge the spec's fail-to-open duty by + subtracting the extensions it recognizes.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={ + "ext_a": {"name": "a", "must_understand": False}, + "ext_b": {"name": "b"}, + "ext_c": {"name": "c", "must_understand": True}, + "ext_d": 123, + } + ) + assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} + recognized = {"ext_b"} + assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} + + +def test_must_understand_fields_empty_when_all_waived() -> None: + """must_understand_fields is empty when every extra field is explicitly waived.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={"ext_a": {"name": "a", "must_understand": False}} + ) + assert model.must_understand_fields == {} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 329471cd94..d2399220be 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -335,3 +335,18 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: }, } assert validate_group_metadata_v3(doc) == [] + + +# --- must_understand partition ------------------------------------------------ + + +def test_group_must_understand_fields_partition() -> None: + """The group model partitions extra fields by the spec's implicit-true rule, + like the array model.""" + model = GroupMetadataModelV3.create_default( + extra_fields={ + "waived": {"name": "w", "must_understand": False}, + "implicit": {"name": "i"}, + } + ) + assert set(model.must_understand_fields) == {"implicit"} From 22188c9fddb703450049ae335c039de600fc5c11 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:18:27 +0200 Subject: [PATCH 15/33] test(zarr-metadata): executable example of pydantic integration Delegate wholesale rather than letting pydantic introspect the dataclass: InstanceOf (is-instance core schema) + BeforeValidator(from_json) + PlainSerializer(to_json, return_type=dict). Field-by-field validation is impossible anyway (the models' annotation-only imports live behind TYPE_CHECKING, so pydantic raises class-not-fully-defined) and would diverge from the library's structural validation via coercion if it weren't. MetadataValidationError subclasses ValueError, so failed parses surface as pydantic ValidationError with the loc-annotated messages. pydantic is already in the package's test dependency group. Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 packages/zarr-metadata/tests/model/test_pydantic.py diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py new file mode 100644 index 0000000000..23f69aaf52 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -0,0 +1,130 @@ +"""Executable example: integrating the metadata models with pydantic (v2). + +Do NOT hand the dataclass to pydantic for field-by-field validation: the +models keep their annotation-only imports behind `TYPE_CHECKING` (so the +string annotations are unresolvable at runtime, and pydantic raises +`class-not-fully-defined`), and even if they resolved, pydantic's coercion +rules would diverge from the library's structural validation. Delegate +wholesale instead — treat the model as an opaque value: + +- `InstanceOf` makes pydantic's core schema an is-instance check (no field + introspection), +- validation goes through `from_json` (the single source of truth for what + a well-formed document is, including normalization: bare-string metadata + fields, arrays-to-tuples), +- serialization goes through `to_json` (the canonical document form). + +`MetadataValidationError` subclasses `ValueError`, so pydantic converts a +failed parse into its own `ValidationError` with the loc-annotated problem +messages intact. +""" + +from typing import Annotated + +import pytest +from pydantic import ( + BaseModel, + BeforeValidator, + InstanceOf, + PlainSerializer, + TypeAdapter, + ValidationError, +) + +from zarr_metadata.model import ArrayMetadataModelV3 + +# --- the integration (this is the example) ----------------------------------- + + +def _as_array_metadata_v3(value: object) -> ArrayMetadataModelV3: + """Accept an existing model instance or a raw metadata document.""" + if isinstance(value, ArrayMetadataModelV3): + return value + return ArrayMetadataModelV3.from_json(value) + + +# return_type is explicit because to_json's own annotation (`ArrayMetadataV3`) +# is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. +ArrayMetadataV3Field = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_as_array_metadata_v3), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), +] +"""A pydantic-ready field type for v3 array metadata. + +Validates raw documents via `from_json`, passes model instances through, +and serializes to the canonical document form via `to_json`. +""" + + +class ArrayManifest(BaseModel): + """Example consumer model: a named array with its metadata document.""" + + path: str + metadata: ArrayMetadataV3Field + + +# --- tests pinning the example ------------------------------------------------ + +VALID_DOC = { + "zarr_format": 3, + "node_type": "array", + "shape": [10], + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": [{"name": "bytes"}], +} + + +def test_raw_document_is_validated_into_a_model() -> None: + """A raw metadata document on a pydantic field is parsed by from_json, + with the library's normalization applied (tuples, canonical field form).""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + assert isinstance(manifest.metadata, ArrayMetadataModelV3) + assert manifest.metadata.shape == (10,) + assert manifest.metadata.data_type.name == "uint8" + + +def test_model_instance_passes_through() -> None: + """An already-constructed model instance is accepted unchanged.""" + model = ArrayMetadataModelV3.from_json(VALID_DOC) + manifest = ArrayManifest(path="a/b", metadata=model) + assert manifest.metadata is model + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A structurally-invalid document fails pydantic validation, carrying the + loc-annotated problem messages from MetadataValidationError.""" + doc = dict(VALID_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError) as exc_info: + ArrayManifest(path="a/b", metadata=doc) # type: ignore[arg-type] + assert "chunk_key_encoding: missing required key" in str(exc_info.value) + + +def test_dump_emits_canonical_document() -> None: + """model_dump serializes the field via to_json — the canonical document, + not pydantic's field-by-field view of the dataclass.""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + dumped = manifest.model_dump() + assert dumped["metadata"] == manifest.metadata.to_json() + # the bare-string data_type was normalized to the canonical object form + assert dumped["metadata"]["data_type"] == {"name": "uint8", "configuration": {}} + + +def test_json_roundtrip_through_pydantic() -> None: + """model_dump_json output re-validates to an equal manifest (JSON emits + tuples as arrays; from_json converts them back).""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) + assert revived == manifest + + +def test_type_adapter_standalone() -> None: + """The annotated alias also works without a BaseModel, via TypeAdapter.""" + adapter = TypeAdapter(ArrayMetadataV3Field) + model = adapter.validate_python(VALID_DOC) + assert isinstance(model, ArrayMetadataModelV3) + assert adapter.dump_python(model) == model.to_json() From 6ea15335d2ce732ba49efd614308c568ad1b212e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:24:03 +0200 Subject: [PATCH 16/33] test(zarr-metadata): document pydantic's native dataclass path and why not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correcting the previous commit's too-strong claim: pydantic CAN introspect the model dataclass — TypeAdapter(...).rebuild() with the TYPE_CHECKING-only names supplied as _types_namespace resolves the schema, and __post_init__ invariants still run. A new test exercises that path and pins why it is not the recommended integration: it validates the model shape, not the document (bare-string data_type rejected — no from_json normalization), and pydantic's lax coercion silently re-opens holes the library validators close (shape=[True, -5] coerces to (1, -5); a wrong dimension_names count passes). Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 74 +++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 23f69aaf52..cfd077651a 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -1,11 +1,16 @@ """Executable example: integrating the metadata models with pydantic (v2). -Do NOT hand the dataclass to pydantic for field-by-field validation: the -models keep their annotation-only imports behind `TYPE_CHECKING` (so the -string annotations are unresolvable at runtime, and pydantic raises -`class-not-fully-defined`), and even if they resolved, pydantic's coercion -rules would diverge from the library's structural validation. Delegate -wholesale instead — treat the model as an opaque value: +Pydantic's native dataclass introspection CAN be made to work (see +`test_native_dataclass_introspection_is_possible_but_diverges`): the models +keep their annotation-only imports behind `TYPE_CHECKING`, so a bare +`TypeAdapter(ArrayMetadataModelV3)` raises `class-not-fully-defined`, but +`rebuild(_types_namespace=...)` with the names supplied resolves the schema. +It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — +no `from_json` normalization (a bare-string `data_type` is rejected), and +pydantic's lax coercion silently re-opens holes the library's validators +close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` +count passes). The recommended integration delegates wholesale — treat the +model as an opaque value: - `InstanceOf` makes pydantic's core schema an is-instance check (no field introspection), @@ -128,3 +133,60 @@ def test_type_adapter_standalone() -> None: model = adapter.validate_python(VALID_DOC) assert isinstance(model, ArrayMetadataModelV3) assert adapter.dump_python(model) == model.to_json() + + +# --- the road not taken: native dataclass introspection ---------------------- + + +def test_native_dataclass_introspection_is_possible_but_diverges() -> None: + """Pydantic CAN introspect the model dataclass after a namespace rebuild, + but that path validates the model shape, not the document: it rejects the + document form, coerces booleans into dimensions, and skips the library's + cross-field checks. This test documents why the delegation pattern above + is the recommended integration.""" + from zarr_metadata._common import JSONValue + from zarr_metadata.model import NamedConfigModelV3 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + }, + ) + model_shaped = { + "shape": [10], + "fill_value": 0, + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "codecs": [{"name": "bytes", "configuration": {}}], + "chunk_key_encoding": {"name": "default", "configuration": {}}, + "dimension_names": None, + "attributes": {}, + "storage_transformers": [], + "extra_fields": {}, + } + # model-shaped data validates, nested named configs and all + model = adapter.validate_python(model_shaped) + assert isinstance(model.data_type, NamedConfigModelV3) + + # divergence 1: the DOCUMENT form is rejected — no from_json normalization + with pytest.raises(ValidationError): + adapter.validate_python(model_shaped | {"data_type": "uint8"}) + + # divergence 2: lax coercion re-opens holes the library validators close + coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]}) + assert coerced.shape == (1, -5) # from_json would reject both entries + + # __post_init__ invariants DO still run under pydantic construction + with pytest.raises(ValidationError, match="Extra fields"): + adapter.validate_python( + model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} + ) From 03ee0b15fa8b821f2f326e003add95c009b67483 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:31:15 +0200 Subject: [PATCH 17/33] test(zarr-metadata): engine-backed pydantic BaseModel example (pydantic-zarr pattern) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consumers that want a first-class BaseModel — JSON schema generation and generics for typed attributes, as in pydantic-zarr's ArraySpec — the example adds a third pattern: pydantic-native fields as the user-facing surface, with the library as the engine. A mode='before' validator canonicalizes every input via from_json(...).to_json(), so structural validation and normalization run before pydantic parses fields (the [True, -5] coercion divergence cannot occur), and to_metadata_model / to_document bridge both ways through the document form. One translation noted at the bridge: the document spells 'no dimension names' as key absence, the pydantic side as None. Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index cfd077651a..fa4fb5f590 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -24,19 +24,23 @@ messages intact. """ -from typing import Annotated +from collections.abc import Mapping +from typing import Annotated, Generic, TypeVar import pytest from pydantic import ( BaseModel, BeforeValidator, + ConfigDict, InstanceOf, PlainSerializer, TypeAdapter, ValidationError, + model_validator, ) -from zarr_metadata.model import ArrayMetadataModelV3 +from zarr_metadata import JSONValue +from zarr_metadata.model import ArrayMetadataModelV3, NamedConfigModelV3 # --- the integration (this is the example) ----------------------------------- @@ -145,7 +149,6 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import NamedConfigModelV3 from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -190,3 +193,117 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: adapter.validate_python( model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} ) + + +# --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- +# +# When a consumer wants a real BaseModel — JSON schema generation, and +# generics for typed attributes, as in pydantic-zarr's ArraySpec — the model +# fields are pydantic-native, but validation and serialization still route +# through the library: a mode="before" validator canonicalizes every input +# document with from_json(...).to_json(), so the structural validators and +# normalization run BEFORE pydantic parses fields (no coercion divergence), +# and the document form is the bridge in both directions. + +AttrsT = TypeVar("AttrsT") + + +class NamedConfig(BaseModel): + """Pydantic mirror of a normalized metadata field (name + configuration).""" + + name: str + configuration: dict[str, JSONValue] = {} + + +class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): + """A pydantic-native, attribute-typed view of a v3 array metadata document. + + The library is the engine: every input is canonicalized and structurally + validated by `ArrayMetadataModelV3.from_json` before pydantic sees the + fields, and `to_document` / `to_metadata_model` emit through the library. + """ + + model_config = ConfigDict(frozen=True) + + zarr_format: int = 3 + node_type: str = "array" + shape: tuple[int, ...] + data_type: NamedConfig + chunk_grid: NamedConfig + chunk_key_encoding: NamedConfig + fill_value: JSONValue + codecs: tuple[NamedConfig, ...] + attributes: AttrsT + dimension_names: tuple[str | None, ...] | None = None + storage_transformers: tuple[NamedConfig, ...] = () + + @model_validator(mode="before") + @classmethod + def _canonicalize(cls, data: object) -> object: + """Route every input document through the library's validation and + normalization; pydantic then parses only canonical documents.""" + if isinstance(data, Mapping): + doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + doc.setdefault("attributes", {}) + return doc + return data + + def to_metadata_model(self) -> ArrayMetadataModelV3: + """Bridge back to the canonical model, via the document form. + + In the document, "no dimension names" is key-absence, not null; the + pydantic-side None translates to dropping the key. + """ + doc = self.model_dump() + if doc["dimension_names"] is None: + del doc["dimension_names"] + return ArrayMetadataModelV3.from_json(doc) + + def to_document(self) -> dict[str, object]: + """The canonical document (omit-empty conventions applied).""" + return dict(self.to_metadata_model().to_json()) + + +class MicroscopyAttrs(BaseModel): + """Example of consumer-typed attributes, pydantic-zarr style.""" + + resolution_um: float + + +def test_spec_typed_attributes() -> None: + """The generic parameter types the attributes, so consumers get validated, + attribute-level access — the pydantic-zarr ArraySpec pattern.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + assert spec.attributes.resolution_um == 0.5 + assert spec.data_type == NamedConfig(name="uint8") + + +def test_spec_engine_validates_before_pydantic() -> None: + """The library's structural validation runs before pydantic's parsing, so + coercion cannot re-open validator holes (contrast with the native + introspection test above, where [True, -5] coerced to (1, -5)).""" + with pytest.raises(ValidationError, match="shape"): + ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( + dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} + ) + + +def test_spec_bridges_to_canonical_model_and_document() -> None: + """to_metadata_model / to_document round-trip through the document form, + and the emitted document matches what the library itself would emit.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + model = spec.to_metadata_model() + assert isinstance(model, ArrayMetadataModelV3) + assert spec.to_document() == dict(model.to_json()) + # and back: the document revalidates to an equal spec + assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec + + +def test_spec_json_schema_generation() -> None: + """A real BaseModel means model_json_schema works — the capability the + opaque InstanceOf pattern cannot provide.""" + schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() + assert schema["properties"]["shape"]["type"] == "array" + assert "MicroscopyAttrs" in schema["$defs"] From 9193bf47bbf5b596550db6e3a75e754855aef378 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:14:40 +0200 Subject: [PATCH 18/33] test(zarr-metadata): pin that a null dimension_names field is invalid Spec: 'If specified, must be an array of strings or null objects... If dimension_names is not specified, all dimensions are unnamed.' The null object is a permitted element (an unnamed dimension), never the field value; key absence is the only spelling of 'not specified'. Pins the validator's existing rejection so it is not later 'fixed' to accept null-as-absence, and documents that in-memory None maps to key absence on serialization. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e73739929a..3a13aa6bea 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1244,3 +1244,17 @@ def test_must_understand_fields_empty_when_all_waived() -> None: extra_fields={"ext_a": {"name": "a", "must_understand": False}} ) assert model.must_understand_fields == {} + + +def test_dimension_names_null_field_rejected() -> None: + """A dimension_names field whose VALUE is null is invalid: the spec permits + null as an element (an unnamed dimension), never as the field value — "not + specified" is spelled by omitting the key. Consumers bridging from an + in-memory None sentinel must drop the key, not write null.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"dimension_names": None} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] + # and the model's own None spelling correctly maps to key absence + assert ( + "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + ) From eaf2587280f0e2aed664d68bed96e5dfcdfeb601 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:34:41 +0200 Subject: [PATCH 19/33] feat(zarr-metadata): optional pydantic integration as zarr_metadata.pydantic Gamed out three shapes with prototypes before choosing: - dunders on the core classes (works, verified pydantic 2.0-2.13, but puts a framework protocol in the dependency-free layer); - pydantic-aware SUBCLASSES in a namespace (rejected on empirical failures: identity split breaks equality, core instances are rejected by subclass-typed fields, and nested construction produces core-class children unless every cross-reference is overridden); - Annotated field types over the CORE classes in an opt-in module (chosen): instances are the core classes so interop is free, pydantic imports eagerly at the module (loud failure when absent), core stays framework-free, and pydantic-protocol risk is quarantined to one clearly-labeled module. The module exports one field type per model. Validation delegates to from_json (structural validation and normalization cannot be bypassed by pydantic coercion), instances pass through, serialization emits the canonical document, and WithJsonSchema describes the accepted document form so model_json_schema works. Tests cover all seven field types, core-instance interop, error quality, JSON schema, roundtrip, and that importing zarr_metadata does not import pydantic. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 7 + .../src/zarr_metadata/pydantic.py | 131 ++++++++++++++++++ .../tests/model/test_pydantic_module.py | 125 +++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 packages/zarr-metadata/src/zarr_metadata/pydantic.py create mode 100644 packages/zarr-metadata/tests/model/test_pydantic_module.py diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index f35625558b..44b7c1d892 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -29,3 +29,10 @@ not explicitly waived with `must_understand: false` (fields are implicitly must-understand per the spec). Readers discharge the spec's fail-to-open duty by subtracting the extension names they recognize; the model only partitions by obligation, since recognition is reader-specific. + +Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it +requires pydantic v2; the core package does not depend on it): one `Annotated` +field type per model, validating raw documents through `from_json`, passing +core-model instances through unchanged, and serializing via `to_json`. The +instances are the core model classes, so values interoperate freely with +non-pydantic code. diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py new file mode 100644 index 0000000000..5ec837ee23 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -0,0 +1,131 @@ +"""Optional pydantic (v2) integration: field types over the core models. + +Importing this module requires pydantic; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +Each exported name is an `Annotated` field type over the corresponding core +model class — the instances ARE the core classes, so values interoperate +freely with non-pydantic code (equality, isinstance, nesting). Validation +delegates to the library: a raw document routes through `from_json` (the +single source of truth for structural validation and normalization, so +pydantic's field-level coercion can never bypass it), an existing model +instance passes through unchanged, and serialization emits the canonical +document via `to_json`. `MetadataValidationError` subclasses `ValueError`, +so a failed parse surfaces as a pydantic `ValidationError` carrying the +loc-annotated problem messages. + +Usage: + + import zarr_metadata.pydantic as zmp + + class ArrayManifest(BaseModel): + path: str + metadata: zmp.ArrayMetadataV3 + +Static type checkers see each field type as its core model class, so +`manifest.metadata` is an `ArrayMetadataModelV3`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar + +from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema + +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +_M = TypeVar("_M") + + +def _coerce_to(cls: type[_M]) -> Callable[[object], _M]: + """A validator that passes instances through and parses anything else.""" + + def coerce(value: object) -> _M: + if isinstance(value, cls): + return value + return cls.from_json(value) # type: ignore[attr-defined, no-any-return] + + return coerce + + +# JSON schemas describe the DOCUMENT form each field accepts (the validation +# input), not the in-memory model shape. +_DOCUMENT_SCHEMA = {"type": "object"} +_FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} + +ArrayMetadataV3 = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_coerce_to(ArrayMetadataModelV3)), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), +] +"""Field type for a v3 array metadata document (`zarr.json` content).""" + +ArrayMetadataV2 = Annotated[ + InstanceOf[ArrayMetadataModelV2], + BeforeValidator(_coerce_to(ArrayMetadataModelV2)), + PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), +] +"""Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + +GroupMetadataV3 = Annotated[ + InstanceOf[GroupMetadataModelV3], + BeforeValidator(_coerce_to(GroupMetadataModelV3)), + PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), +] +"""Field type for a v3 group metadata document (`zarr.json` content).""" + +GroupMetadataV2 = Annotated[ + InstanceOf[GroupMetadataModelV2], + BeforeValidator(_coerce_to(GroupMetadataModelV2)), + PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), +] +"""Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + +ConsolidatedMetadataV3 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV3], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3)), + PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), +] +"""Field type for v3 inline consolidated metadata.""" + +ConsolidatedMetadataV2 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV2], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2)), + PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), +] +"""Field type for a v2 `.zmetadata` document.""" + +MetadataFieldV3 = Annotated[ + InstanceOf[NamedConfigModelV3], + BeforeValidator(_coerce_to(NamedConfigModelV3)), + PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), + WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), +] +"""Field type for one v3 metadata field (bare name string or name + configuration).""" + +__all__ = [ + "ArrayMetadataV2", + "ArrayMetadataV3", + "ConsolidatedMetadataV2", + "ConsolidatedMetadataV3", + "GroupMetadataV2", + "GroupMetadataV3", + "MetadataFieldV3", +] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py new file mode 100644 index 0000000000..6202020dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -0,0 +1,125 @@ +"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. + +The hand-rolled recipes in `test_pydantic.py` document how the integration +works; this module ships it. Instances are the CORE model classes (no +parallel hierarchy), so values interoperate freely with non-pydantic code. +""" + +import pytest +from pydantic import BaseModel, TypeAdapter, ValidationError + +import zarr_metadata.pydantic as zmp +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +V3_ARRAY_DOC = dict(ArrayMetadataModelV3.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ArrayMetadataModelV2.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmp.ArrayMetadataV3, ArrayMetadataModelV3, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ArrayMetadataV2, ArrayMetadataModelV2, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.GroupMetadataV3, GroupMetadataModelV3, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.GroupMetadataV2, GroupMetadataModelV2, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmp.ConsolidatedMetadataV3, + ConsolidatedMetadataModelV3, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmp.ConsolidatedMetadataV2, + ConsolidatedMetadataModelV2, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmp.MetadataFieldV3, NamedConfigModelV3, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_validates_and_dumps_canonically( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class, + passes existing instances through unchanged, and dumps the canonical + document via to_json.""" + adapter = TypeAdapter(field_type) + model = adapter.validate_python(doc) + assert type(model) is model_cls + assert adapter.validate_python(model) is model + assert adapter.dump_python(model) == dict(model.to_json()) + + +def test_core_instances_interoperate() -> None: + """A core model instance (e.g. handed out by zarr-python) drops straight + into a pydantic field — the reason the module ships Annotated aliases over + the core classes rather than pydantic-aware subclasses.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + core = ArrayMetadataModelV3.from_json(V3_ARRAY_DOC) + manifest = Manifest(metadata=core) + assert manifest.metadata is core + + +def test_validation_error_carries_problems() -> None: + """A defective document fails with the library's loc-annotated messages.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): + Manifest(metadata=doc) # type: ignore[arg-type] + + +def test_json_schema_generation() -> None: + """model_json_schema works, describing the document form each field accepts.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + codec: zmp.MetadataFieldV3 + + schema = Manifest.model_json_schema() + assert schema["properties"]["metadata"] == {"type": "object", "title": "ArrayMetadataV3"} + assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] + + +def test_json_roundtrip() -> None: + """model_dump_json output re-validates to an equal pydantic model.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + manifest = Manifest(metadata=V3_ARRAY_DOC) # type: ignore[arg-type] + assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest + + +def test_core_package_does_not_import_pydantic() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + pydantic: the integration is opt-in via zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) From bd80d7dfaf754628c95ac33206e9955adcc47df1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:46:23 +0200 Subject: [PATCH 20/33] fix(zarr-metadata): create_default derives the chunk grid from shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_default(shape=(100, 100)) silently kept the scalar default's 0-d chunk grid (chunk_shape: ()), producing a structurally-valid but semantically inconsistent document — a footgun for every test fixture built on it. When shape is overridden and the grid is not, the default is now one regular chunk covering the array (v3 chunk_shape == shape, v2 chunks == shape); an explicit chunk_grid/chunks override still wins. update() stays a dumb dataclasses.replace, per its documented contract. One existing whole-document test literal carried exactly this inconsistency (shape (10,) with chunk_shape ()) and was updated. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 +++ .../src/zarr_metadata/model/_array.py | 14 ++++++-- .../zarr-metadata/tests/model/test_array.py | 34 ++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 44b7c1d892..088d6e11a6 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -36,3 +36,8 @@ field type per model, validating raw documents through `from_json`, passing core-model instances through unchanged, and serializing via `to_json`. The instances are the core model classes, so values interoperate freely with non-pydantic code. + +`create_default` keeps its output self-consistent: overriding `shape` without +a chunk grid derives one regular chunk covering the array (v3 +`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the +scalar default's 0-d grid. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 7a4487b7f4..af680e8d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -168,8 +168,14 @@ def create_default( The default is a structurally-valid scalar `uint8` array — the array analog of `list()` returning `[]`. Any field can be overridden by keyword - (the same fields accepted by `update`). + (the same fields accepted by `update`). Overriding `shape` without + `chunk_grid` derives a consistent default grid: one regular chunk + covering the array (`chunk_shape` equal to `shape`). """ + if "shape" in overrides and "chunk_grid" not in overrides: + overrides["chunk_grid"] = NamedConfigModelV3( + name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} + ) default = cls( shape=(), fill_value=0, @@ -353,8 +359,12 @@ def create_default( The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the array analog of `list()` returning `[]`. Any field can be overridden by - keyword (the same fields accepted by `update`). + keyword (the same fields accepted by `update`). Overriding `shape` + without `chunks` derives `chunks` equal to `shape` (one chunk covering + the array). """ + if "shape" in overrides and "chunks" not in overrides: + overrides["chunks"] = tuple(overrides["shape"]) default = cls( shape=(), dtype="|u1", diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3a13aa6bea..535541d180 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -180,7 +180,7 @@ def test_v3_to_json_emits_canonical_document() -> None: "shape": (10,), "fill_value": 0, "data_type": {"name": "int32", "configuration": {}}, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": ()}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, "codecs": ({"name": "bytes", "configuration": {}},), "chunk_key_encoding": {"name": "default", "configuration": {}}, } @@ -1258,3 +1258,35 @@ def test_dimension_names_null_field_rejected() -> None: assert ( "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() ) + + +# --- create_default derives the chunk grid from shape ------------------------ + + +def test_v3_create_default_chunk_grid_follows_shape() -> None: + """Overriding shape without chunk_grid derives a consistent default grid: + one chunk covering the array (chunk_shape == shape), instead of silently + keeping the scalar default's 0-d grid.""" + model = ArrayMetadataModelV3.create_default(shape=(100, 100)) + assert model.chunk_grid == NamedConfigModelV3( + name="regular", configuration={"chunk_shape": (100, 100)} + ) + + +def test_v3_create_default_explicit_chunk_grid_respected() -> None: + """An explicit chunk_grid override wins over the shape-derived default.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ArrayMetadataModelV3.create_default(shape=(100, 100), chunk_grid=grid) + assert model.chunk_grid == grid + + +def test_v2_create_default_chunks_follow_shape() -> None: + """Overriding shape without chunks derives chunks == shape.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100)) + assert model.chunks == (100, 100) + + +def test_v2_create_default_explicit_chunks_respected() -> None: + """An explicit chunks override wins over the shape-derived default.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) + assert model.chunks == (10, 10) From bdec803ac47707e69eb2121b26ca07238c8a1d3f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:49:31 +0200 Subject: [PATCH 21/33] test(zarr-metadata): pin zero-length-dimension case of the derived chunk grid The spec's constraint is conditional ('non-zero when the corresponding dimensions of the arrays have non-zero length'), so chunk_shape == shape is sound for every shape, including empty dimensions. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 535541d180..a5ffbb7080 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1290,3 +1290,12 @@ def test_v2_create_default_explicit_chunks_respected() -> None: """An explicit chunks override wins over the shape-derived default.""" model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) assert model.chunks == (10, 10) + + +def test_v3_create_default_zero_length_dimensions() -> None: + """chunk_shape == shape is spec-sound even with zero-length dimensions: + 'The chunk shape elements are non-zero when the corresponding dimensions + of the arrays have non-zero length' — the constraint is conditional, so a + zero chunk length is permitted exactly where the dimension is empty.""" + model = ArrayMetadataModelV3.create_default(shape=(0, 3)) + assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) From 965ea029cd0d5e5421c5b34a6e01b0d65f1db854 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:53:20 +0200 Subject: [PATCH 22/33] docs(zarr-metadata): document that create_default's derivation is one-way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overriding shape without a grid derives the grid; the reverse does not hold. A user-supplied chunk_grid is an extension point taken verbatim — deriving shape from it would require interpreting grid configurations, which the model layer never does and cannot do for unrecognized grid names. Pinned by test so the asymmetry reads as a decision, not an oversight; the v2 model documents the same one-way rule for chunks for cross-version consistency. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_array.py | 13 +++++++++++++ packages/zarr-metadata/tests/model/test_array.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index af680e8d3f..cfd43571ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -171,6 +171,14 @@ def create_default( (the same fields accepted by `update`). Overriding `shape` without `chunk_grid` derives a consistent default grid: one regular chunk covering the array (`chunk_shape` equal to `shape`). + + The derivation is deliberately one-way. A user-supplied `chunk_grid` + is an extension point and is taken verbatim — deriving `shape` from + it would require interpreting the grid's configuration, which this + layer never does (and cannot do for unrecognized grid names). So + overriding `chunk_grid` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. """ if "shape" in overrides and "chunk_grid" not in overrides: overrides["chunk_grid"] = NamedConfigModelV3( @@ -362,6 +370,11 @@ def create_default( keyword (the same fields accepted by `update`). Overriding `shape` without `chunks` derives `chunks` equal to `shape` (one chunk covering the array). + + The derivation is deliberately one-way, matching the v3 model: + overriding `chunks` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. """ if "shape" in overrides and "chunks" not in overrides: overrides["chunks"] = tuple(overrides["shape"]) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index a5ffbb7080..3f7f35c50c 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1299,3 +1299,17 @@ def test_v3_create_default_zero_length_dimensions() -> None: zero chunk length is permitted exactly where the dimension is empty.""" model = ArrayMetadataModelV3.create_default(shape=(0, 3)) assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) + + +def test_create_default_derivation_is_one_way() -> None: + """Overriding the chunk grid (v3) or chunks (v2) without shape leaves the + scalar default shape=() untouched: a user-supplied chunk_grid is an + extension point taken verbatim, and deriving shape from it would require + interpreting grid configurations, which the model layer never does.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ArrayMetadataModelV3.create_default(chunk_grid=grid) + assert v3.shape == () + assert v3.chunk_grid == grid + v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) + assert v2.shape == () + assert v2.chunks == (10, 10) From d4a0f8eecf7a88a25eb6267cca38f8b938c837dd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 19:02:29 +0200 Subject: [PATCH 23/33] refactor(zarr-metadata): eliminate every type-ignore comment Audited all 24 (15 src, 9 tests); each was either obsolete, replaceable by a sound cast, or avoidable by better-typed code: - Two fill_value arg-type ignores were factually obsolete: their justifying comment said 'fill_value: object in upstream TypedDict', but 0.3.0 narrowed it to JSONValue. - Eight pre-existing call-arg/reportInvalidTypeForm ignores on the PEP 728 TypedDicts and the recursive JSONValue alias were mypy-dialect suppressions that the checker of record (pyright strict with enableExperimentalFeatures) never needed; mypy has never checked this package. - The two extra_fields comprehensions are a genuine checker limitation (a key filter cannot narrow a PEP 728 TypedDict's item-value union), now expressed as casts whose comments state the soundness claim instead of suppressing the diagnostic. - pydantic.py's generic coercer factory takes the parse callable explicitly instead of calling from_json through type[_M]. - NamedConfigModelV3.from_json casts the validated configuration (sound since configuration values are now deep-validated as JSON). - Tests: _build_v2/_build_v3 gained real Unpack[...Partial] signatures; raw-document pydantic inputs go through model_validate (the idiomatic entry point for untyped data) instead of ignoring constructor signatures; the frozen-dataclass test uses setattr for its intentional runtime error. src and tests/model now carry zero type-ignore comments. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/_common.py | 2 +- .../src/zarr_metadata/model/_array.py | 25 ++++++++++++------- .../src/zarr_metadata/model/_group.py | 16 ++++++++---- .../src/zarr_metadata/pydantic.py | 20 +++++++-------- .../src/zarr_metadata/v2/codec.py | 2 +- .../src/zarr_metadata/v3/array.py | 6 ++--- .../src/zarr_metadata/v3/codec/crc32c.py | 2 +- .../src/zarr_metadata/v3/group.py | 4 +-- .../zarr-metadata/tests/model/test_array.py | 13 ++++++---- .../tests/model/test_pydantic.py | 8 +++--- .../tests/model/test_pydantic_module.py | 4 +-- 11 files changed, 59 insertions(+), 43 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 598a12e80c..b335cd0bd6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,7 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] + "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", ) """A recursive type alias for JSON-encodable values. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index cfd43571ec..991dd77d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -63,8 +63,13 @@ def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) - configuration = arrays_to_tuples(dict(field.get("configuration", {}))) - return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] + # Sound cast: parse_metadata_field_v3 checked the configuration is a + # string-keyed mapping of JSON values; arrays_to_tuples only converts + # lists to tuples within that shape. + configuration = cast( + "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) + ) + return cls(name=field["name"], configuration=configuration) MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 @@ -260,14 +265,16 @@ def to_json(self) -> ArrayMetadataV3: @classmethod def from_json(cls, data: object) -> ArrayMetadataModelV3: parsed = parse_array_metadata_v3(arrays_to_tuples(data)) - extra_fields: dict[str, ExtensionFieldV3] = { - k: v # type: ignore[misc] - for k, v in parsed.items() - if k not in ARRAY_METADATA_STANDARD_KEYS_V3 - } + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, + ) return cls( shape=parsed["shape"], - fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + fill_value=parsed["fill_value"], data_type=NamedConfigModelV3.from_json(parsed["data_type"]), chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), @@ -418,7 +425,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: shape=parsed["shape"], dtype=parsed["dtype"], chunks=parsed["chunks"], - fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + fill_value=parsed["fill_value"], order=parsed["order"], compressor=parsed["compressor"], filters=parsed["filters"], diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index b8cca3849d..0980096a8b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -149,11 +149,17 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: if consolidated_raw is None else ConsolidatedMetadataModelV3.from_json(consolidated_raw) ) - extra_fields: dict[str, ExtensionFieldV3] = { - k: v # type: ignore[misc] - for k, v in parsed.items() - if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 - } + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + { + k: v + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + }, + ) return cls( attributes=dict(parsed.get("attributes", {})), consolidated_metadata=consolidated, diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 5ec837ee23..3eada76316 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -48,13 +48,13 @@ class ArrayManifest(BaseModel): _M = TypeVar("_M") -def _coerce_to(cls: type[_M]) -> Callable[[object], _M]: - """A validator that passes instances through and parses anything else.""" +def _coerce_to(cls: type[_M], parse: Callable[[object], _M]) -> Callable[[object], _M]: + """A validator that passes instances of `cls` through and parses anything else.""" def coerce(value: object) -> _M: if isinstance(value, cls): return value - return cls.from_json(value) # type: ignore[attr-defined, no-any-return] + return parse(value) return coerce @@ -66,7 +66,7 @@ def coerce(value: object) -> _M: ArrayMetadataV3 = Annotated[ InstanceOf[ArrayMetadataModelV3], - BeforeValidator(_coerce_to(ArrayMetadataModelV3)), + BeforeValidator(_coerce_to(ArrayMetadataModelV3, ArrayMetadataModelV3.from_json)), PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), ] @@ -74,7 +74,7 @@ def coerce(value: object) -> _M: ArrayMetadataV2 = Annotated[ InstanceOf[ArrayMetadataModelV2], - BeforeValidator(_coerce_to(ArrayMetadataModelV2)), + BeforeValidator(_coerce_to(ArrayMetadataModelV2, ArrayMetadataModelV2.from_json)), PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), ] @@ -82,7 +82,7 @@ def coerce(value: object) -> _M: GroupMetadataV3 = Annotated[ InstanceOf[GroupMetadataModelV3], - BeforeValidator(_coerce_to(GroupMetadataModelV3)), + BeforeValidator(_coerce_to(GroupMetadataModelV3, GroupMetadataModelV3.from_json)), PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), ] @@ -90,7 +90,7 @@ def coerce(value: object) -> _M: GroupMetadataV2 = Annotated[ InstanceOf[GroupMetadataModelV2], - BeforeValidator(_coerce_to(GroupMetadataModelV2)), + BeforeValidator(_coerce_to(GroupMetadataModelV2, GroupMetadataModelV2.from_json)), PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), ] @@ -98,7 +98,7 @@ def coerce(value: object) -> _M: ConsolidatedMetadataV3 = Annotated[ InstanceOf[ConsolidatedMetadataModelV3], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3)), + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3, ConsolidatedMetadataModelV3.from_json)), PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), ] @@ -106,7 +106,7 @@ def coerce(value: object) -> _M: ConsolidatedMetadataV2 = Annotated[ InstanceOf[ConsolidatedMetadataModelV2], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2)), + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2, ConsolidatedMetadataModelV2.from_json)), PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), ] @@ -114,7 +114,7 @@ def coerce(value: object) -> _M: MetadataFieldV3 = Annotated[ InstanceOf[NamedConfigModelV3], - BeforeValidator(_coerce_to(NamedConfigModelV3)), + BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 6d194b7e29..8f7f2b3f88 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -10,7 +10,7 @@ from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class CodecMetadataV2(TypedDict, extra_items=JSONValue): """ A numcodecs configuration dict, used as a v2 compressor or filter. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a8b0fa3358..a3fec24a6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -9,7 +9,7 @@ from zarr_metadata.v3._common import MetadataV3 -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class ExtensionFieldV3(TypedDict, extra_items=JSONValue): """ Required shape of any extension field on a v3 metadata document. @@ -41,7 +41,7 @@ class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-a must_understand: bool -class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 array metadata document (the `zarr.json` content for an array). @@ -63,7 +63,7 @@ class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ dimension_names: NotRequired[tuple[str | None, ...]] -class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `ArrayMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index ea35ae5f1d..6b9b46c43d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -18,7 +18,7 @@ """Literal type of the `name` field of the `crc32c` codec.""" -class Empty(TypedDict, closed=True): # type: ignore[call-arg] +class Empty(TypedDict, closed=True): """An empty mapping""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 27186b6059..be990b1ae7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -12,7 +12,7 @@ from zarr_metadata.v3.array import ExtensionFieldV3 -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 group metadata document (the `zarr.json` content for a group). @@ -26,7 +26,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `GroupMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3f7f35c50c..f1ce967b99 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import pytest +from typing_extensions import Unpack from tests.model._cases import Expect, ExpectFail from zarr_metadata.model import ( @@ -814,12 +815,12 @@ def test_parse_metadata_field_v3( # invalid cases (a subset check, so accumulation of OTHER problems is allowed). -def _build_v3(**overrides: object) -> dict[str, object]: - return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) # type: ignore[arg-type] +def _build_v3(**overrides: Unpack[ArrayMetadataModelV3Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) -def _build_v2(**overrides: object) -> dict[str, object]: - return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) # type: ignore[arg-type] +def _build_v2(**overrides: Unpack[ArrayMetadataModelV2Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: @@ -1008,7 +1009,9 @@ def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): - p.message = "y" # type: ignore[misc] + # setattr: assigning to a frozen field is an intentional runtime error, + # spelled dynamically so it is not also a static type error. + setattr(p, "message", "y") # noqa: B010 def test_metadata_validation_error_holds_problems() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index fa4fb5f590..c1cd3010d4 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -90,7 +90,7 @@ class ArrayManifest(BaseModel): def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, with the library's normalization applied (tuples, canonical field form).""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) assert isinstance(manifest.metadata, ArrayMetadataModelV3) assert manifest.metadata.shape == (10,) assert manifest.metadata.data_type.name == "uint8" @@ -109,14 +109,14 @@ def test_invalid_document_surfaces_problems_in_validation_error() -> None: doc = dict(VALID_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError) as exc_info: - ArrayManifest(path="a/b", metadata=doc) # type: ignore[arg-type] + ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) assert "chunk_key_encoding: missing required key" in str(exc_info.value) def test_dump_emits_canonical_document() -> None: """model_dump serializes the field via to_json — the canonical document, not pydantic's field-by-field view of the dataclass.""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) dumped = manifest.model_dump() assert dumped["metadata"] == manifest.metadata.to_json() # the bare-string data_type was normalized to the canonical object form @@ -126,7 +126,7 @@ def test_dump_emits_canonical_document() -> None: def test_json_roundtrip_through_pydantic() -> None: """model_dump_json output re-validates to an equal manifest (JSON emits tuples as arrays; from_json converts them back).""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) assert revived == manifest diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 6202020dd6..81b9b962af 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -90,7 +90,7 @@ class Manifest(BaseModel): doc = dict(V3_ARRAY_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): - Manifest(metadata=doc) # type: ignore[arg-type] + Manifest.model_validate({"metadata": doc}) def test_json_schema_generation() -> None: @@ -111,7 +111,7 @@ def test_json_roundtrip() -> None: class Manifest(BaseModel): metadata: zmp.ArrayMetadataV3 - manifest = Manifest(metadata=V3_ARRAY_DOC) # type: ignore[arg-type] + manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest From 0a7f674ac29f5cd51dee19ae73d37a44f1b7bb4c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 19:14:23 +0200 Subject: [PATCH 24/33] fix(zarr-metadata): absent v2 dimension_separator means '.', not '/' roborev job 426 (branch review) found that ArrayMetadataModelV2 normalized an ABSENT dimension_separator key to '/', inherited verbatim from the zng prototype. The v2 convention's default is '.': a consumer deriving chunk keys from the model against a real-world v2 array written with the default separator would have looked for '0/0' instead of '0.0'. No test caught it because every fixture started from create_default(), which always carries an explicit separator. Absence is normalized to an explicit '.' -- a semantics-preserving spelling normalization consistent with the model's existing canonical forms (bare-string metadata fields, missing configuration). The field is deliberately NOT modeled as Optional: the document grammar has no null spelling for this key, and a None in the model invites writing 'dimension_separator': null into documents. Pinned by three tests, including explicit-null rejection. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 +++ .../src/zarr_metadata/model/_array.py | 12 ++++-- .../zarr-metadata/tests/model/test_array.py | 37 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 088d6e11a6..3274fd2e3b 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -41,3 +41,8 @@ non-pydantic code. a chunk grid derives one regular chunk covering the array (v3 `chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the scalar default's 0-d grid. + +A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 +convention's default `"."` (the model previously normalized absence to `"/"`, +which would misaddress the chunks of real-world default-separator arrays). +The value is never null: absent, `"."`, or `"/"` are the only spellings. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 991dd77d3f..6d1d2c3d6b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -340,7 +340,9 @@ class ArrayMetadataModelV2: A canonical, lossless representation of the `.zarray` content plus the sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are held in their raw JSON forms and are never interpreted; `fill_value` is - held verbatim in its JSON form. + held verbatim in its JSON form. One spelling normalization: a `.zarray` + that omits `dimension_separator` means `"."` by the v2 convention, and + the model holds and re-emits that value explicitly. """ zarr_format: Literal[2] = field(default=2, init=False) @@ -351,7 +353,11 @@ class ArrayMetadataModelV2: order: ArrayOrderV2 compressor: CodecMetadataV2 | None filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: ArrayDimensionSeparatorV2 = field(default="/") + # "." is the v2 convention's default for an ABSENT dimension_separator key; + # from_json normalizes absence to it (a semantics-preserving spelling + # normalization, like the v3 bare-string metadata-field form). The value + # is never None: the document grammar has no null spelling for this field. + dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") attributes: dict[str, JSONValue] def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: @@ -429,7 +435,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: order=parsed["order"], compressor=parsed["compressor"], filters=parsed["filters"], - dimension_separator=parsed.get("dimension_separator", "/"), + dimension_separator=parsed.get("dimension_separator", "."), attributes=dict(parsed.get("attributes", {})), ) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index f1ce967b99..4805fc2674 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1316,3 +1316,40 @@ def test_create_default_derivation_is_one_way() -> None: v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) assert v2.shape == () assert v2.chunks == (10, 10) + + +# --- v2 dimension_separator default (roborev job 426) ------------------------- + + +def test_v2_absent_dimension_separator_means_dot() -> None: + """A .zarray that omits dimension_separator uses the v2 convention default + '.', not '/': chunk keys of real-world default-separator v2 arrays look + like '0.0'. The model normalizes the absent key to an explicit '.' — a + semantics-preserving spelling normalization.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) + del doc["dimension_separator"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.dimension_separator == "." + assert model.to_json()["dimension_separator"] == "." + + +def test_v2_from_key_value_without_separator_means_dot() -> None: + """The .zarray store-file path applies the same '.' default for an absent + dimension_separator key.""" + doc = { + k: v + for k, v in ArrayMetadataModelV2.create_default().to_json().items() + if k not in ("dimension_separator", "attributes") + } + import json as _json + + model = ArrayMetadataModelV2.from_key_value({".zarray": _json.dumps(doc).encode()}) + assert model.dimension_separator == "." + + +def test_v2_null_dimension_separator_rejected() -> None: + """dimension_separator may be absent, '.', or '/' — never null: the + document grammar has no null spelling for this field.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] From 69ae90192af1bafc07a7905f5f8b7b384a86d883 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 20:58:07 +0200 Subject: [PATCH 25/33] feat(zarr-metadata): UNSET sentinel for absent optional document keys Establishes the models' None/absence invariant: None in a model always corresponds to a JSON null in the document (a v2 compressor/filters value, an unnamed dimension inside dimension_names), and UNSET always means the key is absent. The two are never interchangeable. Applied to the two fields that used None as an absence marker: dimension_names (ArrayMetadataModelV3) and consolidated_metadata (GroupMetadataModelV3). For dimension_names this also preserves a semantic distinction d-v-b identified: an absent field ("there are no dimension names") and an explicit all-null array ("every dimension has a name, which is null") are different documents; both spellings now round-trip faithfully and compare unequal. Normalizing absence to the all-null form was considered and rejected: the spellings' interpretations coincide but interpretation-equivalence is the resolve layer's business, and collapsing document-level distinctions on that basis is the layer violation this package exists to avoid. Verified that current zarr-python never writes "consolidated_metadata": null (GroupMetadata.to_dict pops the key), so None there was purely an absence marker, not a document spelling. UnsetType is a single-member enum (identity-checkable, repr "UNSET", deliberately truthy so `if not x` cannot silently treat it as absent); UNSET and UnsetType are exported from zarr_metadata.model and the package front door. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +++++ .../src/zarr_metadata/__init__.py | 4 +++ .../src/zarr_metadata/model/__init__.py | 3 ++ .../src/zarr_metadata/model/_array.py | 11 +++--- .../src/zarr_metadata/model/_group.py | 15 ++++---- .../src/zarr_metadata/model/_sentinel.py | 30 ++++++++++++++++ .../zarr-metadata/tests/model/test_array.py | 36 +++++++++++++++---- .../zarr-metadata/tests/model/test_group.py | 5 +-- .../tests/model/test_pydantic.py | 4 ++- .../zarr-metadata/tests/test_public_api.py | 2 ++ 10 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 3274fd2e3b..29c893b3c1 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -46,3 +46,11 @@ A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. + +Optional document keys use the `UNSET` sentinel, never `None`: in a model, +`None` always corresponds to a JSON `null` in the document (a v2 +`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` +always means the key is absent. This keeps semantically distinct spellings +distinct — an absent `dimension_names` ("there are no dimension names") and +an explicit `[null, null]` ("every dimension has a name, which is null") are +different documents and round-trip as such. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 354f2a97f5..4e0cfd4e36 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -2,6 +2,7 @@ from zarr_metadata._common import JSONValue, NamedConfigV3 from zarr_metadata.model import ( + UNSET, ArrayMetadataModelV2, ArrayMetadataModelV2Partial, ArrayMetadataModelV3, @@ -16,6 +17,7 @@ MetadataValidationError, NamedConfigModelV3, ProblemKind, + UnsetType, ValidationProblem, ) from zarr_metadata.v2.array import ( @@ -248,6 +250,7 @@ "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", + "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", @@ -353,6 +356,7 @@ "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", + "UnsetType", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 42fbc233d8..76367eae32 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -34,6 +34,7 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, ) +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, @@ -82,6 +83,7 @@ "GROUP_METADATA_STANDARD_KEYS_V3", "GROUP_METADATA_STORE_KEY_V2", "GROUP_METADATA_STORE_KEY_V3", + "UNSET", "ArrayMetadataModelV2", "ArrayMetadataModelV2Partial", "ArrayMetadataModelV3", @@ -96,6 +98,7 @@ "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", + "UnsetType", "ValidationProblem", "is_array_metadata_v2", "is_array_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 6d1d2c3d6b..604c9a7d1e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,6 +10,7 @@ from typing_extensions import TypedDict, Unpack +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -133,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | None + dimension_names: tuple[str | None, ...] | UnsetType attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -159,7 +160,7 @@ class ArrayMetadataModelV3: chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | None + dimension_names: tuple[str | None, ...] | UnsetType attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -196,7 +197,7 @@ def create_default( chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), codecs=(NamedConfigModelV3(name="bytes", configuration={}),), chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), - dimension_names=None, + dimension_names=UNSET, attributes={}, storage_transformers=(), extra_fields={}, @@ -246,7 +247,7 @@ def to_json(self) -> ArrayMetadataV3: "codecs": tuple(codec.to_json() for codec in self.codecs), "chunk_key_encoding": self.chunk_key_encoding.to_json(), } - if self.dimension_names is not None: + if self.dimension_names is not UNSET: out["dimension_names"] = self.dimension_names if len(self.attributes) > 0: out["attributes"] = self.attributes @@ -279,7 +280,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), - dimension_names=parsed.get("dimension_names"), + dimension_names=parsed.get("dimension_names", UNSET), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 0980096a8b..29f32bd330 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -15,6 +15,7 @@ ArrayMetadataModelV3, must_understand_subset, ) +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -63,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -80,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -107,7 +108,7 @@ def create_default( analog of `list()` returning `[]`. Any field can be overridden by keyword (the same fields accepted by `update`). """ - default = cls(attributes={}, consolidated_metadata=None, extra_fields={}) + default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) return default.update(**overrides) def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: @@ -128,7 +129,7 @@ def to_json(self) -> GroupMetadataV3: } if len(self.attributes) > 0: out["attributes"] = self.attributes - if self.consolidated_metadata is not None: + if self.consolidated_metadata is not UNSET: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on @@ -143,10 +144,10 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - consolidated_raw = parsed.get(CONSOLIDATED_METADATA_KEY_V3) + consolidated_raw: object = parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET) consolidated = ( - None - if consolidated_raw is None + UNSET + if consolidated_raw is UNSET else ConsolidatedMetadataModelV3.from_json(consolidated_raw) ) # Sound cast: the TypedDict types all non-standard keys as its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py new file mode 100644 index 0000000000..b0cab883ca --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -0,0 +1,30 @@ +"""The absence sentinel for optional metadata-document keys. + +The models observe one invariant: `None` in a model always corresponds to a +JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable: for keys where the spec +gives `null` no meaning (`dimension_names` itself, `consolidated_metadata`), +a model `None` would invite serializing an invalid `key: null` spelling, +while `UNSET` cannot leak into a document at all. + +Check with identity: `if model.dimension_names is UNSET: ...`. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Final, Literal + + +class UnsetType(Enum): + """The type of `UNSET`; use in annotations as `T | UnsetType`.""" + + UNSET = "UNSET" + + def __repr__(self) -> str: + return "UNSET" + + +UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET +"""Marks a metadata-document key as absent. Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 4805fc2674..d9af8aeff6 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -13,6 +13,7 @@ ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, + UNSET, ArrayMetadataModelV2, ArrayMetadataModelV2Partial, ArrayMetadataModelV3, @@ -196,8 +197,8 @@ def test_v3_dimension_names_included_when_present() -> None: def test_v3_dimension_names_omitted_when_none() -> None: - """V3 to_json omits dimension_names when they are None.""" - out = ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + """V3 to_json omits dimension_names when they are UNSET.""" + out = ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() assert "dimension_names" not in out @@ -213,7 +214,7 @@ def test_v3_attributes_included_when_dimension_names_is_none() -> None: """ out: dict[str, object] = dict( ArrayMetadataModelV3.create_default( - dimension_names=None, attributes={"foo": "bar"} + dimension_names=UNSET, attributes={"foo": "bar"} ).to_json() ) assert out["attributes"] == {"foo": "bar"} @@ -482,13 +483,13 @@ def test_v3_from_json_reconstructs_required_fields() -> None: def test_v3_from_json_defaults_for_omitted_optionals() -> None: """V3 from_json supplies defaults for omitted optional fields.""" doc = ArrayMetadataModelV3.create_default( - attributes={}, storage_transformers=(), dimension_names=None + attributes={}, storage_transformers=(), dimension_names=UNSET ).to_json() # to_json omits these entirely; from_json must restore defaults model = ArrayMetadataModelV3.from_json(doc) assert model.attributes == {} assert model.storage_transformers == () - assert model.dimension_names is None + assert model.dimension_names is UNSET def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: @@ -570,7 +571,7 @@ def test_from_key_value_missing_key_raises( ArrayMetadataModelV3, ArrayMetadataModelV3.create_default( attributes={}, - dimension_names=None, + dimension_names=UNSET, storage_transformers=(), extra_fields={}, ), @@ -1259,7 +1260,8 @@ def test_dimension_names_null_field_rejected() -> None: assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] # and the model's own None spelling correctly maps to key absence assert ( - "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + "dimension_names" + not in ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() ) @@ -1353,3 +1355,23 @@ def test_v2_null_dimension_separator_rejected() -> None: doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_dimension_names_absent_and_all_null_are_distinct() -> None: + """An absent dimension_names field and an explicit all-null one are + semantically different documents: the explicit form says every dimension + has a name, which is null; absence says there are no dimension names. + The model preserves the distinction (UNSET vs a tuple of Nones), and both + spellings round-trip faithfully.""" + absent_doc = dict(ArrayMetadataModelV3.create_default(shape=(2, 3)).to_json()) + explicit_doc = absent_doc | {"dimension_names": (None, None)} + + absent = ArrayMetadataModelV3.from_json(absent_doc) + explicit = ArrayMetadataModelV3.from_json(explicit_doc) + + assert absent.dimension_names is UNSET + assert explicit.dimension_names == (None, None) + assert absent != explicit + assert "dimension_names" not in absent.to_json() + assert absent.to_json() == absent_doc + assert explicit.to_json() == explicit_doc diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index d2399220be..fcf57bddde 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -5,6 +5,7 @@ import pytest +from zarr_metadata.model import UNSET from zarr_metadata.model._array import ArrayMetadataModelV3 from zarr_metadata.model._group import ( ConsolidatedMetadataModelV2, @@ -62,7 +63,7 @@ def test_group_v3_extra_fields_overlap_rejected() -> None: with pytest.raises(ValueError, match="Extra fields"): GroupMetadataModelV3( attributes={}, - consolidated_metadata=None, + consolidated_metadata=UNSET, extra_fields={"node_type": {"name": "x", "must_understand": False}}, ) @@ -72,7 +73,7 @@ def test_group_v3_consolidated_extra_field_rejected() -> None: with pytest.raises(ValueError, match="Extra fields"): GroupMetadataModelV3( attributes={}, - consolidated_metadata=None, + consolidated_metadata=UNSET, extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, ) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index c1cd3010d4..65db4ae419 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -149,6 +149,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue + from zarr_metadata.model import UnsetType from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -162,6 +163,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "ArrayMetadataV3": ArrayMetadataV3, "NamedConfigModelV3": NamedConfigModelV3, "MetadataFieldModelV3": NamedConfigModelV3, + "UnsetType": UnsetType, }, ) model_shaped = { @@ -171,7 +173,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": None, + "dimension_names": UnsetType.UNSET, "attributes": {}, "storage_transformers": [], "extra_fields": {}, diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index fb2f574a4f..16a540b3f9 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -55,6 +55,8 @@ def _group_rank(s: str) -> int: "ValidationProblem", "MetadataValidationError", "ProblemKind", + "UNSET", + "UnsetType", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 672d2bf016fdc39adbf8f23a3fadc3e40daa68ce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:05:38 +0200 Subject: [PATCH 26/33] fix(zarr-metadata): accept and preserve the wild consolidated_metadata null Historical zarr-python versions wrote "consolidated_metadata": null into group documents for groups without consolidated metadata, so real stores contain the spelling; the validator was rejecting those documents ("expected a mapping"). Per the None/UNSET invariant, the field is now honestly three-state: UNSET (key absent), None (the document's literal null, preserved on round-trip), or a ConsolidatedMetadataModelV3. Interpreting null as absence is the consumer's call, not a document rewrite by this layer. Also records an implementation constraint on the sentinel itself: typing_extensions.Sentinel (PEP 661) is the intended spelling, but pyright 1.1.411 degrades a Sentinel to Unknown in dataclass FIELD annotations (function signatures work), verified by probe both with and without enableExperimentalFeatures. Using it would reintroduce suppressions at every use site under the strict gate, so UNSET stays a single-member enum, with the Sentinel switch documented in _sentinel.py for when pyright catches up. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 ++- .../src/zarr_metadata/model/_group.py | 32 +++++++++++++------ .../src/zarr_metadata/model/_sentinel.py | 16 +++++++--- .../src/zarr_metadata/model/_validation.py | 4 ++- .../zarr-metadata/tests/model/test_group.py | 18 +++++++++++ .../tests/model/test_pydantic.py | 4 +-- 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 29c893b3c1..4cb35ad9cf 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -53,4 +53,7 @@ Optional document keys use the `UNSET` sentinel, never `None`: in a model, always means the key is absent. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are -different documents and round-trip as such. +different documents and round-trip as such. A group's +`consolidated_metadata` is accordingly three-state: absent (`UNSET`), +present, or the literal `null` written by historical zarr-python versions, +which is accepted and preserved rather than rejected or silently rewritten. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 29f32bd330..826492e26e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -133,9 +133,14 @@ def to_json(self) -> GroupMetadataV3: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. + # `zarr_metadata.v3.consolidated`. A model None is the document's + # literal null spelling (written by historical zarr-python versions) + # and round-trips as such. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", self.consolidated_metadata.to_json() + "ExtensionFieldV3", + None + if self.consolidated_metadata is None + else self.consolidated_metadata.to_json(), ) for key, value in self.extra_fields.items(): out[key] = value @@ -144,12 +149,19 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - consolidated_raw: object = parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET) - consolidated = ( - UNSET - if consolidated_raw is UNSET - else ConsolidatedMetadataModelV3.from_json(consolidated_raw) - ) + # Cast to object: the TypedDict's extra_items type does not admit null, + # but wild documents (historical zarr-python) contain it. + consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) + consolidated: ConsolidatedMetadataModelV3 | None | UnsetType + if consolidated_raw is UNSET: + consolidated = UNSET + elif consolidated_raw is None: + # Historical zarr-python versions wrote consolidated_metadata: null + # for groups without consolidated metadata; the spelling is + # preserved (interpreting it as absence is the consumer's call). + consolidated = None + else: + consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index b0cab883ca..29eaa104ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -2,13 +2,19 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed -dimension inside `dimension_names`), and `UNSET` always means the document -key is absent. The two are never interchangeable: for keys where the spec -gives `null` no meaning (`dimension_names` itself, `consolidated_metadata`), -a model `None` would invite serializing an invalid `key: null` spelling, -while `UNSET` cannot leak into a document at all. +dimension inside `dimension_names`, a group's historical +`consolidated_metadata: null`), and `UNSET` always means the document key is +absent. The two are never interchangeable, so a model value can never leak +into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. + +Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended +spelling, but pyright (1.1.411) degrades a Sentinel to `Unknown` in dataclass +FIELD annotations (function signatures work), which would force suppressions +under this package's strict gate at every use site. The single-member enum +gives the same identity semantics with exact `Literal` narrowing; switch to +`Sentinel` once pyright supports it in dataclass fields. """ from __future__ import annotations diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fc781c7899..4558b57ef4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -518,7 +518,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - if "consolidated_metadata" in doc: + if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: + # consolidated_metadata: null is accepted: historical zarr-python + # versions wrote it for groups without consolidated metadata. problems.extend( _prefix( "consolidated_metadata", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index fcf57bddde..4286976962 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -351,3 +351,21 @@ def test_group_must_understand_fields_partition() -> None: } ) assert set(model.must_understand_fields) == {"implicit"} + + +def test_group_v3_null_consolidated_metadata_preserved() -> None: + """Historical zarr-python versions wrote consolidated_metadata: null for + groups without consolidated metadata, so null is a real wild spelling: + it is accepted and preserved on round-trip (model None = document null), + distinct from key absence (UNSET). Interpreting null as "no consolidated + metadata" is the consumer's call, not a document rewrite.""" + null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} + absent_doc = {"zarr_format": 3, "node_type": "group"} + assert validate_group_metadata_v3(null_doc) == [] + null_model = GroupMetadataModelV3.from_json(null_doc) + absent_model = GroupMetadataModelV3.from_json(absent_doc) + assert null_model.consolidated_metadata is None + assert absent_model.consolidated_metadata is UNSET + assert null_model != absent_model + assert null_model.to_json() == null_doc + assert absent_model.to_json() == absent_doc diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 65db4ae419..c10d1cb471 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -149,7 +149,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import UnsetType + from zarr_metadata.model import UNSET, UnsetType from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -173,7 +173,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": UnsetType.UNSET, + "dimension_names": UNSET, "attributes": {}, "storage_transformers": [], "extra_fields": {}, From fa7b7cf3660afa02e16b633a6505d10c1b4dac91 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:10:31 +0200 Subject: [PATCH 27/33] fix(zarr-metadata): repair consolidated_metadata null to absence, not preserve it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d-v-b: the bugged spelling should not be preserved or honored. The three-state field reverts to two states (model | UNSET): a document carrying "consolidated_metadata": null — written by a historical zarr-python bug — remains readable (the validator accepts it so real stores open), but the spelling gets no model representation: it is read as absence and never written back. This is the one deliberate exception to faithful round-tripping, pinned as such: from_json(null_doc) equals from_json(absent_doc), and to_json omits the key. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +++--- .../src/zarr_metadata/model/_group.py | 25 +++++++------------ .../src/zarr_metadata/model/_sentinel.py | 5 ++-- .../src/zarr_metadata/model/_validation.py | 5 ++-- .../zarr-metadata/tests/model/test_group.py | 23 +++++++---------- 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 4cb35ad9cf..c4efbe6f01 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -53,7 +53,7 @@ Optional document keys use the `UNSET` sentinel, never `None`: in a model, always means the key is absent. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are -different documents and round-trip as such. A group's -`consolidated_metadata` is accordingly three-state: absent (`UNSET`), -present, or the literal `null` written by historical zarr-python versions, -which is accepted and preserved rather than rejected or silently rewritten. +different documents and round-trip as such. The `consolidated_metadata: null` +written by a historical zarr-python bug is the one deliberate exception to +faithful round-tripping: those stores remain readable, but the bug spelling +is repaired to absence on read and never written back. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 826492e26e..5c2c2287fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -133,14 +133,9 @@ def to_json(self) -> GroupMetadataV3: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. A model None is the document's - # literal null spelling (written by historical zarr-python versions) - # and round-trips as such. + # `zarr_metadata.v3.consolidated`. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", - None - if self.consolidated_metadata is None - else self.consolidated_metadata.to_json(), + "ExtensionFieldV3", self.consolidated_metadata.to_json() ) for key, value in self.extra_fields.items(): out[key] = value @@ -152,14 +147,12 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | None | UnsetType - if consolidated_raw is UNSET: + consolidated: ConsolidatedMetadataModelV3 | UnsetType + if consolidated_raw is UNSET or consolidated_raw is None: + # consolidated_metadata: null was written by a historical + # zarr-python bug; it gets no model representation. It is read as + # absence and never written back — repaired, not preserved. consolidated = UNSET - elif consolidated_raw is None: - # Historical zarr-python versions wrote consolidated_metadata: null - # for groups without consolidated metadata; the spelling is - # preserved (interpreting it as absence is the consumer's call). - consolidated = None else: consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 29eaa104ab..74f5feec64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -2,9 +2,8 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed -dimension inside `dimension_names`, a group's historical -`consolidated_metadata: null`), and `UNSET` always means the document key is -absent. The two are never interchangeable, so a model value can never leak +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable, so a model value can never leak into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 4558b57ef4..fae09187a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -519,8 +519,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: - # consolidated_metadata: null is accepted: historical zarr-python - # versions wrote it for groups without consolidated metadata. + # consolidated_metadata: null (a historical zarr-python bug) is + # structurally accepted so those stores remain readable, but the model + # repairs it to absence on read and never writes it back. problems.extend( _prefix( "consolidated_metadata", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4286976962..0062bf7620 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -353,19 +353,14 @@ def test_group_must_understand_fields_partition() -> None: assert set(model.must_understand_fields) == {"implicit"} -def test_group_v3_null_consolidated_metadata_preserved() -> None: - """Historical zarr-python versions wrote consolidated_metadata: null for - groups without consolidated metadata, so null is a real wild spelling: - it is accepted and preserved on round-trip (model None = document null), - distinct from key absence (UNSET). Interpreting null as "no consolidated - metadata" is the consumer's call, not a document rewrite.""" +def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: + """consolidated_metadata: null was written by a historical zarr-python bug. + Those stores must remain readable, but the bug spelling is not honored: + it is read as absence (UNSET) and never written back — the round-trip + deliberately repairs the document rather than preserving the bug.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} - absent_doc = {"zarr_format": 3, "node_type": "group"} assert validate_group_metadata_v3(null_doc) == [] - null_model = GroupMetadataModelV3.from_json(null_doc) - absent_model = GroupMetadataModelV3.from_json(absent_doc) - assert null_model.consolidated_metadata is None - assert absent_model.consolidated_metadata is UNSET - assert null_model != absent_model - assert null_model.to_json() == null_doc - assert absent_model.to_json() == absent_doc + model = GroupMetadataModelV3.from_json(null_doc) + assert model.consolidated_metadata is UNSET + assert "consolidated_metadata" not in model.to_json() + assert model == GroupMetadataModelV3.from_json({"zarr_format": 3, "node_type": "group"}) From da3094deadf0d3dd88189b00d70eaace19a80e2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:14:45 +0200 Subject: [PATCH 28/33] docs(zarr-metadata): pin the Sentinel blocker to pyright regression #11115 Investigated: the Unknown-degradation of typing_extensions.Sentinel is a confirmed upstream pyright regression, not by-design. Introduced in 1.1.405 (verified: 1.1.404 is clean on the same probe, 1.1.411 fails), affects reads of any class-body attribute annotation (dataclass or plain class), does not affect function signatures or module variables, and Final on the sentinel does not help. Tracked as microsoft/pyright#11115 (open, bug+regression); #11467 closed as its duplicate. The enum sentinel stays until the fix lands. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 74f5feec64..e83d49852a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -9,11 +9,13 @@ Check with identity: `if model.dimension_names is UNSET: ...`. Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but pyright (1.1.411) degrades a Sentinel to `Unknown` in dataclass -FIELD annotations (function signatures work), which would force suppressions -under this package's strict gate at every use site. The single-member enum -gives the same identity semantics with exact `Literal` narrowing; switch to -`Sentinel` once pyright supports it in dataclass fields. +spelling, but a confirmed pyright regression (1.1.405 through at least +1.1.411; worked in <= 1.1.404) degrades a Sentinel to `Unknown` when read +from any class-body attribute annotation — dataclass or not; function +signatures and module variables are unaffected, and `Final` on the sentinel +does not help. Tracked as https://github.com/microsoft/pyright/issues/11115. +The single-member enum gives the same identity semantics with exact +`Literal` narrowing; switch to `Sentinel` once that regression is fixed. """ from __future__ import annotations From a9edd1faf9f9f0e801ea1faacdc659335904608b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:17:38 +0200 Subject: [PATCH 29/33] docs(zarr-metadata): sentinel switch is blocked by mypy too, not just pyright MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinning a working pyright (<= 1.1.404) in CI was considered and does not suffice: the pin controls one of four checker surfaces. Contributor IDEs (Pylance bundles current pyright) and downstream consumers' pyright read the py.typed inline annotations with their own versions, and decisively, mypy 2.1.0 has no PEP 661 support at all — a sentinel in type position is a hard [valid-type] error, which would degrade these fields to Any for mypy consumers, including zarr-python itself. The enum is currently the only spelling with exact types on every surface; switch when pyright#11115 is fixed AND mypy implements PEP 661. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index e83d49852a..2b77b44b95 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -9,13 +9,17 @@ Check with identity: `if model.dimension_names is UNSET: ...`. Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but a confirmed pyright regression (1.1.405 through at least -1.1.411; worked in <= 1.1.404) degrades a Sentinel to `Unknown` when read -from any class-body attribute annotation — dataclass or not; function -signatures and module variables are unaffected, and `Final` on the sentinel -does not help. Tracked as https://github.com/microsoft/pyright/issues/11115. -The single-member enum gives the same identity semantics with exact -`Literal` narrowing; switch to `Sentinel` once that regression is fixed. +spelling, but two independent checker gaps block it. Pyright: a confirmed +regression (1.1.405 through at least 1.1.411; worked in <= 1.1.404, +https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to +`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): +no PEP 661 support at all — a sentinel in type position is a hard +`[valid-type]` error, so downstream mypy users (zarr-python itself) would +see these fields as `Any`. Pinning a working pyright in this package's CI +would fix neither contributors' IDEs nor downstream checkers reading the +py.typed annotations. The single-member enum gives the same identity +semantics with exact `Literal` narrowing on every checker; switch to +`Sentinel` once the pyright regression is fixed AND mypy implements PEP 661. """ from __future__ import annotations From 934095e8bf77fc0f7b4c87d4de0a66405ec19432 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:20:01 +0200 Subject: [PATCH 30/33] docs(zarr-metadata): PEP 661 is Final (Python 3.15), not a draft Corrects the sentinel implementation note: PEP 661 was accepted 2026-04-23 and ships as stdlib sentinel in Python 3.15. The two checker gaps blocking the Sentinel spelling (pyright regression #11115, mypy not yet implementing the PEP) are therefore temporary gaps against a Final standard, and the enum is a stopgap with a defined end state. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 2b77b44b95..5fa80004d2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -8,18 +8,20 @@ Check with identity: `if model.dimension_names is UNSET: ...`. -Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but two independent checker gaps block it. Pyright: a confirmed -regression (1.1.405 through at least 1.1.411; worked in <= 1.1.404, +Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of +2026-04-23, stdlib in Python 3.15) is the intended spelling, but two +independent checker gaps block it for now. Pyright: a confirmed regression +(1.1.405 through at least 1.1.411; worked in <= 1.1.404, https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to `Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): -no PEP 661 support at all — a sentinel in type position is a hard +has not yet implemented PEP 661 — a sentinel in type position is a hard `[valid-type]` error, so downstream mypy users (zarr-python itself) would see these fields as `Any`. Pinning a working pyright in this package's CI would fix neither contributors' IDEs nor downstream checkers reading the py.typed annotations. The single-member enum gives the same identity semantics with exact `Literal` narrowing on every checker; switch to -`Sentinel` once the pyright regression is fixed AND mypy implements PEP 661. +`Sentinel` once the pyright regression is fixed and mypy support lands — +both expected, now that the PEP is Final. """ from __future__ import annotations From 800546aed3db4129d2bee2843859f5ee91127cf5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:25:46 +0200 Subject: [PATCH 31/33] docs(zarr-metadata): ty fully supports typed sentinels; pyright/mypy are the laggards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ty 0.0.56 types the Sentinel spelling perfectly in dataclass fields: exact T | UNSET unions, both-direction is/is-not narrowing, and wrong-typed constructor arguments rejected (verified with reveal_type, so it is real inference, not silent Any). The checker matrix for sentinel-in-type-position is therefore ty full / pyright regressed (#11115) / mypy not implemented — recorded so the switch decision has current calibration. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/src/zarr_metadata/model/_sentinel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 5fa80004d2..749e353c8a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -18,10 +18,12 @@ `[valid-type]` error, so downstream mypy users (zarr-python itself) would see these fields as `Any`. Pinning a working pyright in this package's CI would fix neither contributors' IDEs nor downstream checkers reading the -py.typed annotations. The single-member enum gives the same identity -semantics with exact `Literal` narrowing on every checker; switch to -`Sentinel` once the pyright regression is fixed and mypy support lands — -both expected, now that the PEP is Final. +py.typed annotations. For calibration: ty (0.0.56) +already types the Sentinel spelling perfectly — exact unions and +`is`/`is not` narrowing in dataclass fields — so the standard is landing; +pyright and mypy are the laggards. The single-member enum gives the same +identity semantics with exact `Literal` narrowing on every checker; switch +to `Sentinel` once the pyright regression is fixed and mypy support lands. """ from __future__ import annotations From f1fba4123cf50c404af13daaeafa86af801fbf06 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:34:32 +0200 Subject: [PATCH 32/33] feat(zarr-metadata): adopt the PEP 661 sentinel for UNSET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d-v-b's call: PEP 661 is Final, ty already types the sentinel spelling exactly, mypy support is in review (python/mypy#21647) and treated as imminent, and pyright has a known-good version — so use the standard sentinel today rather than carrying the enum stopgap. - UNSET is now typing_extensions.Sentinel("UNSET"), used directly in type expressions (tuple[str | None, ...] | UNSET); the UnsetType companion enum is gone from the API. - typing_extensions floor bumped to 4.14 (where Sentinel arrived). - CI pins pyright==1.1.404, the last version before the class-attribute sentinel regression (microsoft/pyright#11115); pyproject documents the same pin for local runs. 0 errors on the pin; ty checks the sentinel fields clean (its 2 remaining diagnostics are its incomplete PEP 728 extra_items write support, unrelated). - Known short-term cost, accepted deliberately: mypy-checked consumers need cast/type-ignore at narrowing sites until mypy#21647 merges, and contributors' Pylance may show phantom Unknowns until the pyright fix ships. Recorded in _sentinel.py and the changelog. - The pydantic native-introspection test reverts to documenting that introspection is unsupported (pydantic 2.13 cannot schema a Sentinel); the delegation patterns are unaffected. Assisted-by: ClaudeCode:claude-fable-5 --- .github/workflows/zarr-metadata.yml | 5 +- packages/zarr-metadata/changes/210.feature.md | 12 ++- packages/zarr-metadata/pyproject.toml | 5 +- .../src/zarr_metadata/__init__.py | 2 - .../src/zarr_metadata/model/__init__.py | 3 +- .../src/zarr_metadata/model/_array.py | 6 +- .../src/zarr_metadata/model/_group.py | 8 +- .../src/zarr_metadata/model/_sentinel.py | 48 ++++------- .../tests/model/test_pydantic.py | 79 ++++++++----------- .../zarr-metadata/tests/test_public_api.py | 1 - 10 files changed, 71 insertions(+), 98 deletions(-) diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 95e8251227..e660483220 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -82,7 +82,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.11 - name: Run pyright - run: uv run --group test --with pyright pyright src + # Pinned to the last version that types PEP 661 sentinels in class + # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). + # Unpin when the fix lands. + run: uv run --group test --with 'pyright==1.1.404' pyright src zarr-metadata-complete: name: zarr-metadata complete diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index c4efbe6f01..12f57c1525 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -47,10 +47,14 @@ convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. -Optional document keys use the `UNSET` sentinel, never `None`: in a model, -`None` always corresponds to a JSON `null` in the document (a v2 -`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` -always means the key is absent. This keeps semantically distinct spellings +Optional document keys use `UNSET` — a PEP 661 sentinel +(`typing_extensions.Sentinel`), usable directly in type expressions — never +`None`: in a model, `None` always corresponds to a JSON `null` in the +document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), +and `UNSET` always means the key is absent. Checker note: ty types the +sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 +is fixed (this package's CI pins it); mypy users need a `cast` or +`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are different documents and round-trip as such. The `consolidated_metadata: null` diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 05667d59e3..87f35db1db 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.13", + "typing_extensions>=4.14", ] [project.urls] @@ -82,6 +82,9 @@ checks = [ "PR06", ] +# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in +# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel +# relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] include = ["src"] enableExperimentalFeatures = true diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 4e0cfd4e36..4c1328bfe1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -17,7 +17,6 @@ MetadataValidationError, NamedConfigModelV3, ProblemKind, - UnsetType, ValidationProblem, ) from zarr_metadata.v2.array import ( @@ -356,7 +355,6 @@ "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", - "UnsetType", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 76367eae32..1257109465 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -34,7 +34,7 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, ) -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, @@ -98,7 +98,6 @@ "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", - "UnsetType", "ValidationProblem", "is_array_metadata_v2", "is_array_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 604c9a7d1e..bfa57ddb34 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,7 +10,7 @@ from typing_extensions import TypedDict, Unpack -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -134,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | UnsetType + dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -160,7 +160,7 @@ class ArrayMetadataModelV3: chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | UnsetType + dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 5c2c2287fc..e3a006699b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -15,7 +15,7 @@ ArrayMetadataModelV3, must_understand_subset, ) -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -147,7 +147,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | UnsetType + consolidated: ConsolidatedMetadataModelV3 | UNSET if consolidated_raw is UNSET or consolidated_raw is None: # consolidated_metadata: null was written by a historical # zarr-python bug; it gets no model representation. It is read as diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 749e353c8a..eeae0da97a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -3,43 +3,27 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed dimension inside `dimension_names`), and `UNSET` always means the document -key is absent. The two are never interchangeable, so a model value can never leak -into a document as a spelling the writer did not intend. +key is absent. The two are never interchangeable, so a model value can never +leak into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. -Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of -2026-04-23, stdlib in Python 3.15) is the intended spelling, but two -independent checker gaps block it for now. Pyright: a confirmed regression -(1.1.405 through at least 1.1.411; worked in <= 1.1.404, -https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to -`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): -has not yet implemented PEP 661 — a sentinel in type position is a hard -`[valid-type]` error, so downstream mypy users (zarr-python itself) would -see these fields as `Any`. Pinning a working pyright in this package's CI -would fix neither contributors' IDEs nor downstream checkers reading the -py.typed annotations. For calibration: ty (0.0.56) -already types the Sentinel spelling perfectly — exact unions and -`is`/`is not` narrowing in dataclass fields — so the standard is landing; -pyright and mypy are the laggards. The single-member enum gives the same -identity semantics with exact `Literal` narrowing on every checker; switch -to `Sentinel` once the pyright regression is fixed and mypy support lands. +Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python +3.15): ty types this spelling exactly, including `is`/`is not` narrowing. +Pyright supports it but a regression (1.1.405+, tracked as +https://github.com/microsoft/pyright/issues/11115) degrades class-attribute +reads to `Unknown`, so this package pins pyright to the last good version +until the fix lands. Mypy support is in review +(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked +consumers of these fields need a `cast` or `type: ignore` at narrowing +sites. This is a deliberate short-term cost: the sentinel is the standard, +and the checkers are converging on it. """ from __future__ import annotations -from enum import Enum -from typing import Final, Literal +from typing_extensions import Sentinel - -class UnsetType(Enum): - """The type of `UNSET`; use in annotations as `T | UnsetType`.""" - - UNSET = "UNSET" - - def __repr__(self) -> str: - return "UNSET" - - -UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET -"""Marks a metadata-document key as absent. Test with `is UNSET`.""" +UNSET = Sentinel("UNSET") +"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly +in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index c10d1cb471..7e80d52ee8 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -34,6 +34,8 @@ ConfigDict, InstanceOf, PlainSerializer, + PydanticSchemaGenerationError, + PydanticUserError, TypeAdapter, ValidationError, model_validator, @@ -142,59 +144,40 @@ def test_type_adapter_standalone() -> None: # --- the road not taken: native dataclass introspection ---------------------- -def test_native_dataclass_introspection_is_possible_but_diverges() -> None: - """Pydantic CAN introspect the model dataclass after a namespace rebuild, - but that path validates the model shape, not the document: it rejects the - document form, coerces booleans into dimensions, and skips the library's - cross-field checks. This test documents why the delegation pattern above - is the recommended integration.""" +def test_native_dataclass_introspection_is_not_supported() -> None: + """Pydantic cannot field-introspect the model dataclasses: the UNSET + sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field + annotations has no pydantic schema (as of pydantic 2.13), so even the + rebuild-with-namespace recipe fails. Introspection was already the wrong + tool before the sentinel existed — it validated the model shape rather + than the document, and its lax coercion re-opened validator holes (e.g. + shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above + are the only supported integrations. If this test ever fails because + pydantic learned to handle sentinels, revisit whether the introspection + path needs its divergences documented again.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import UNSET, UnsetType + from zarr_metadata.model import UNSET from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 - adapter = TypeAdapter(ArrayMetadataModelV3) - adapter.rebuild( - force=True, - _types_namespace={ - "JSONValue": JSONValue, - "ExtensionFieldV3": ExtensionFieldV3, - "MetadataV3": MetadataV3, - "ArrayMetadataV3": ArrayMetadataV3, - "NamedConfigModelV3": NamedConfigModelV3, - "MetadataFieldModelV3": NamedConfigModelV3, - "UnsetType": UnsetType, - }, - ) - model_shaped = { - "shape": [10], - "fill_value": 0, - "data_type": {"name": "uint8", "configuration": {}}, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, - "codecs": [{"name": "bytes", "configuration": {}}], - "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": UNSET, - "attributes": {}, - "storage_transformers": [], - "extra_fields": {}, - } - # model-shaped data validates, nested named configs and all - model = adapter.validate_python(model_shaped) - assert isinstance(model.data_type, NamedConfigModelV3) - - # divergence 1: the DOCUMENT form is rejected — no from_json normalization - with pytest.raises(ValidationError): - adapter.validate_python(model_shaped | {"data_type": "uint8"}) - - # divergence 2: lax coercion re-opens holes the library validators close - coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]}) - assert coerced.shape == (1, -5) # from_json would reject both entries - - # __post_init__ invariants DO still run under pydantic construction - with pytest.raises(ValidationError, match="Extra fields"): - adapter.validate_python( - model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} + def build_and_use() -> None: + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + "UNSET": UNSET, + }, ) + adapter.validate_python({}) + + with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): + build_and_use() # --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 16a540b3f9..ee26e80c5a 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -56,7 +56,6 @@ def _group_rank(s: str) -> int: "MetadataValidationError", "ProblemKind", "UNSET", - "UnsetType", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 8156ac0df92ea796cba83c64c555f9290af71ee7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:50:51 +0200 Subject: [PATCH 33/33] fix(zarr-metadata): .zattrs presence is part of the store, not an artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the last flagged round-trip question from the initial port: to_key_value on the v2 models always emitted a .zattrs key, so a store that never had one gained a file on round-trip. Per d-v-b's ruling, attributes on ArrayMetadataModelV2/GroupMetadataModelV2 is now `dict[str, JSONValue] | UNSET`: UNSET means no .zattrs file (and no attributes key in the merged document form) and emits nothing, while any dict — including an explicit empty {} — means the file exists and is emitted. The two spellings stay distinct through round-trips, per the None/UNSET invariant; create_default defaults to UNSET (a fresh minimal node has no .zattrs). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 50 +++++++++++-------- .../src/zarr_metadata/model/_group.py | 45 +++++++++-------- .../zarr-metadata/tests/model/test_array.py | 39 ++++++++++----- .../zarr-metadata/tests/model/test_group.py | 17 +++++-- 5 files changed, 98 insertions(+), 59 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 12f57c1525..f711d46af5 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -61,3 +61,9 @@ different documents and round-trip as such. The `consolidated_metadata: null` written by a historical zarr-python bug is the one deliberate exception to faithful round-tripping: those stores remain readable, but the bug spelling is repaired to absence on read and never written back. + +The v2 models treat the `.zattrs` file's presence as part of the store: +`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` +emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a +file. Previously `to_key_value` always emitted `.zattrs`, silently adding a +file to stores that never had one. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index bfa57ddb34..f68c1b0f5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -331,7 +331,7 @@ class ArrayMetadataModelV2Partial(TypedDict, total=False): compressor: CodecMetadataV2 | None filters: tuple[CodecMetadataV2, ...] | None dimension_separator: ArrayDimensionSeparatorV2 - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) @@ -341,9 +341,12 @@ class ArrayMetadataModelV2: A canonical, lossless representation of the `.zarray` content plus the sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are held in their raw JSON forms and are never interpreted; `fill_value` is - held verbatim in its JSON form. One spelling normalization: a `.zarray` - that omits `dimension_separator` means `"."` by the v2 convention, and - the model holds and re-emits that value explicitly. + held verbatim in its JSON form. `attributes` is `UNSET` when no + `.zattrs` file (or merged `attributes` key) exists — distinct from an + explicit empty `.zattrs`, which is `{}` and round-trips as a file. One + spelling normalization: a `.zarray` that omits `dimension_separator` + means `"."` by the v2 convention, and the model holds and re-emits that + value explicitly. """ zarr_format: Literal[2] = field(default=2, init=False) @@ -359,7 +362,7 @@ class ArrayMetadataModelV2: # normalization, like the v3 bare-string metadata-field form). The value # is never None: the document grammar has no null spelling for this field. dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: """ @@ -400,15 +403,16 @@ def create_default( order="C", compressor=None, filters=None, - attributes={}, + attributes=UNSET, ) return default.update(**overrides) def to_json(self) -> ArrayMetadataV2: - """Return the merged in-memory document form, INCLUDING `attributes`. + """Return the merged in-memory document form. - This is not the on-disk `.zarray` content: a conforming `.zarray` must - exclude `attributes` (they live in the sibling `.zattrs` file). Use + `attributes` is included when set (even empty). This is not the + on-disk `.zarray` content: a conforming `.zarray` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ out: ArrayMetadataV2 = { @@ -419,10 +423,11 @@ def to_json(self) -> ArrayMetadataV2: "chunks": self.chunks, "fill_value": self.fill_value, "dimension_separator": self.dimension_separator, - "attributes": self.attributes, "compressor": self.compressor, "filters": self.filters, } + if self.attributes is not UNSET: + out["attributes"] = self.attributes return out @classmethod @@ -437,24 +442,25 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: compressor=parsed["compressor"], filters=parsed["filters"], dimension_separator=parsed.get("dimension_separator", "."), - attributes=dict(parsed.get("attributes", {})), + attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET), ) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) - zattrs: dict[str, JSONValue] = ( - load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) - if ATTRIBUTES_STORE_KEY_V2 in mapping - else {} - ) - return cls.from_json({**zarray, "attributes": zattrs}) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zarray, "attributes": zattrs}) + return cls.from_json(dict(zarray)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` - # document must exclude them. + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - return { - ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8"), - ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), - } + out = {ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index e3a006699b..222b0b3bf9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -260,7 +260,7 @@ class GroupMetadataModelV2Partial(TypedDict, total=False): `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. """ - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) @@ -269,11 +269,14 @@ class GroupMetadataModelV2: A canonical, lossless representation of the `.zgroup` content plus the sibling `.zattrs` attributes, folded into a single in-memory value - (mirroring the merged `GroupMetadataV2` document form). + (mirroring the merged `GroupMetadataV2` document form). `attributes` is + `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — + distinct from an explicit empty `.zattrs`, which is `{}` and round-trips + as a file. """ zarr_format: Literal[2] = field(default=2, init=False) - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @classmethod def create_default( @@ -286,7 +289,7 @@ def create_default( analog of `list()` returning `[]`. Any field can be overridden by keyword (the same fields accepted by `update`). """ - default = cls(attributes={}) + default = cls(attributes=UNSET) return default.update(**overrides) def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: @@ -301,40 +304,42 @@ def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadata return dataclasses.replace(self, **kwargs) def to_json(self) -> GroupMetadataV2: - """Return the merged in-memory document form, INCLUDING `attributes`. + """Return the merged in-memory document form. - This is not the on-disk `.zgroup` content: a conforming `.zgroup` must - exclude `attributes` (they live in the sibling `.zattrs` file). Use + `attributes` is included when set (even empty). This is not the + on-disk `.zgroup` content: a conforming `.zgroup` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ out: GroupMetadataV2 = {"zarr_format": self.zarr_format} - if len(self.attributes) > 0: + if self.attributes is not UNSET: out["attributes"] = self.attributes return out @classmethod def from_json(cls, data: object) -> GroupMetadataModelV2: parsed = parse_group_metadata_v2(arrays_to_tuples(data)) - return cls(attributes=dict(parsed.get("attributes", {}))) + return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) - zattrs: dict[str, JSONValue] = ( - load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) - if ATTRIBUTES_STORE_KEY_V2 in mapping - else {} - ) - return cls.from_json({**zgroup, "attributes": zattrs}) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zgroup, "attributes": zattrs}) + return cls.from_json(dict(zgroup)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` - # document must exclude them. + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - return { - GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8"), - ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), - } + out = {GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index d9af8aeff6..ff27923635 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -326,7 +326,7 @@ def test_v2_create_default_is_valid_empty_array() -> None: assert m.fill_value == 0 assert m.compressor is None assert m.filters is None - assert m.attributes == {} + assert m.attributes is UNSET assert validate_array_metadata_v2(m.to_json()) == [] assert ArrayMetadataModelV2.from_json(m.to_json()) == m @@ -671,12 +671,16 @@ def test_v2_from_json_reconstructs_fields() -> None: assert model.attributes == {"a": 1} -def test_v2_from_json_defaults_when_attributes_absent() -> None: - """V2 from_json defaults attributes to empty when the key is absent.""" - doc = ArrayMetadataModelV2.create_default().to_json() - del doc["attributes"] - model = ArrayMetadataModelV2.from_json(doc) - assert model.attributes == {} +def test_v2_from_json_attributes_absent_is_unset() -> None: + """V2 from_json reads an absent attributes key as UNSET, distinct from an + explicit empty mapping.""" + absent = ArrayMetadataModelV2.from_json(ArrayMetadataModelV2.create_default().to_json()) + explicit = ArrayMetadataModelV2.from_json( + ArrayMetadataModelV2.create_default(attributes={}).to_json() + ) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert absent != explicit # --- ArrayMetadataModelV2.from_key_value -------------------------------- @@ -690,12 +694,21 @@ def test_v2_from_key_value_remerges_zattrs() -> None: assert model.shape == (10,) -def test_v2_from_key_value_absent_zattrs_gives_empty_attributes() -> None: - """V2 from_key_value yields empty attributes when .zattrs is absent.""" - kv: dict[str, bytes] = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) - del kv[".zattrs"] - model = ArrayMetadataModelV2.from_key_value(kv) - assert model.attributes == {} +def test_v2_zattrs_presence_round_trips() -> None: + """The .zattrs file's presence is part of the store: an absent file reads + as UNSET and emits no .zattrs; an explicit empty file reads as {} and + emits .zattrs — the two stores stay distinct through a round-trip.""" + explicit_kv = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + assert ".zattrs" in explicit_kv + absent_kv = dict(explicit_kv) + del absent_kv[".zattrs"] + + absent = ArrayMetadataModelV2.from_key_value(absent_kv) + explicit = ArrayMetadataModelV2.from_key_value(explicit_kv) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert ".zattrs" not in absent.to_key_value() + assert ".zattrs" in explicit.to_key_value() def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 0062bf7620..58b92f5d71 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -116,10 +116,19 @@ def test_group_v2_key_value_split() -> None: assert GroupMetadataModelV2.from_key_value(kv) == model -def test_group_v2_from_key_value_without_zattrs() -> None: - """A v2 group with no .zattrs file parses with empty attributes.""" - model = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) - assert model.attributes == {} +def test_group_v2_zattrs_presence_round_trips() -> None: + """A v2 group with no .zattrs file parses with UNSET attributes and emits + no .zattrs; an explicit empty .zattrs stays a file — the stores remain + distinct through a round-trip.""" + absent = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert absent.attributes is UNSET + assert ".zattrs" not in absent.to_key_value() + explicit = GroupMetadataModelV2.from_key_value( + {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} + ) + assert explicit.attributes == {} + assert ".zattrs" in explicit.to_key_value() + assert absent != explicit def test_group_v2_json_roundtrip() -> None: