From 20f2ac887a147471f0cc8e9780d6ef9b8bcdad26 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 25 Jul 2026 16:10:44 -0700 Subject: [PATCH 01/14] =?UTF-8?q?feat:=20url-pipeline=20core=20=E2=80=94?= =?UTF-8?q?=20parser,=20adapter=20ABC,=20registry,=20store=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements URL pipeline support (https://github.com/jbms/url-pipeline): '|'-chained URLs resolve through pluggable adapters registered under the 'zarr.url_adapters' entry-point group (entry-point name = URL scheme). - zarr.abc.url_pipeline: PipelineSegment, AdapterResolution, PipelineContext, URLPipelineAdapter (single-classmethod contract) - zarr.storage._url_pipeline: parse_pipeline / resolve_pipeline; the root sub-URL delegates to make_store so existing file/memory/fsspec routing is unchanged - registry: register_url_adapter / get_url_adapter / list_url_adapter_schemes (name check only; no adapter imports) - make_store/make_store_path route strings containing '|' (or a registered root scheme) through the resolver; residual store paths combine with the user-supplied path - StorePath gains a zarr_format attribute (populated by format segments in a follow-up) Co-Authored-By: Claude Fable 5 --- changes/4192.feature.md | 25 + docs/api/zarr/abc/index.md | 1 + docs/api/zarr/abc/url_pipeline.md | 5 + docs/user-guide/storage.md | 25 + mkdocs.yml | 1 + src/zarr/abc/url_pipeline.py | 256 ++++++++ src/zarr/api/asynchronous.py | 39 +- src/zarr/errors.py | 7 + src/zarr/registry.py | 77 ++- src/zarr/storage/__init__.py | 3 + src/zarr/storage/_common.py | 49 +- src/zarr/storage/_url_pipeline.py | 311 ++++++++++ .../entry_points.txt | 4 +- tests/package_with_entrypoint/__init__.py | 15 + tests/test_api.py | 5 +- tests/test_url_pipeline/__init__.py | 0 tests/test_url_pipeline/conftest.py | 22 + tests/test_url_pipeline/test_parser.py | 232 +++++++ tests/test_url_pipeline/test_resolver.py | 569 ++++++++++++++++++ tests/test_url_pipeline/test_spec_examples.py | 258 ++++++++ 20 files changed, 1887 insertions(+), 17 deletions(-) create mode 100644 changes/4192.feature.md create mode 100644 docs/api/zarr/abc/url_pipeline.md create mode 100644 src/zarr/abc/url_pipeline.py create mode 100644 src/zarr/storage/_url_pipeline.py create mode 100644 tests/test_url_pipeline/__init__.py create mode 100644 tests/test_url_pipeline/conftest.py create mode 100644 tests/test_url_pipeline/test_parser.py create mode 100644 tests/test_url_pipeline/test_resolver.py create mode 100644 tests/test_url_pipeline/test_spec_examples.py diff --git a/changes/4192.feature.md b/changes/4192.feature.md new file mode 100644 index 0000000000..757aa13895 --- /dev/null +++ b/changes/4192.feature.md @@ -0,0 +1,25 @@ +Added core support for URL pipelines (https://github.com/jbms/url-pipeline): +`|`-chained URLs that address zarr data through nested storage layers, e.g. +`s3://bucket/data.zip|zip:|zarr3:`. This PR adds the parser, the single-method +`zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the +`zarr.url_adapters` entry-point group through which third-party packages +(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme +are loaded lazily and individually. Builtin adapters (`zip:`, +`zarr2:`/`zarr3:`) follow in separate pull requests. + +Behavior notes: + +- The `|` character is now reserved as the pipeline delimiter in every string + store specification, and no percent-escape is decoded; pass a `pathlib.Path` + to address a local file whose name contains `|`. URLs without a `|` (and + without a registered root adapter scheme) are handled exactly as before — + registered adapters cannot intercept zarr's native `file:`/`memory:` + routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. +- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline + spec's semantics (spelling equivalences; `file:` must be absolute, with at + most a `localhost` authority). +- Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store + now serves the "open" half instead of raising upfront, for all stores; + unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. +- For root-adapter URLs (e.g. `gh://org/repo`), `storage_options` are handed + to the adapter and are not validated as used by `make_store`. diff --git a/docs/api/zarr/abc/index.md b/docs/api/zarr/abc/index.md index 7e15cb2a51..d155f60a5a 100644 --- a/docs/api/zarr/abc/index.md +++ b/docs/api/zarr/abc/index.md @@ -11,3 +11,4 @@ Abstract base classes for extending Zarr-Python. - **[zarr.abc.metadata](./metadata.md)** - Creating metadata classes compatible with the Zarr API - **[zarr.abc.numcodec](./numcodec.md)** - Protocols and classes for modeling codec interface used by numcodecs - **[zarr.abc.store](./store.md)** - ABC for implementing Zarr stores and managing getting and setting bytes in a store +- **[zarr.abc.url_pipeline](./url_pipeline.md)** - ABC for implementing [URL pipeline](https://github.com/jbms/url-pipeline) adapters diff --git a/docs/api/zarr/abc/url_pipeline.md b/docs/api/zarr/abc/url_pipeline.md new file mode 100644 index 0000000000..2606d4d5d4 --- /dev/null +++ b/docs/api/zarr/abc/url_pipeline.md @@ -0,0 +1,5 @@ +--- +title: url_pipeline +--- + +::: zarr.abc.url_pipeline diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index a34e2e2874..0b86bef2b0 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -103,6 +103,31 @@ print(group) - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. +## URL Pipelines {#user-guide-url-pipelines} + +Zarr supports [URL pipelines](https://github.com/jbms/url-pipeline): `|`-chained URLs +that address zarr data through nested storage layers, read left to right. The first +sub-URL locates a resource with a conventional URL; each subsequent sub-URL names an +*adapter* that reinterprets everything to its left (e.g. +`s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the +`zarr.url_adapters` entry-point group — see +[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. Builtin +adapters (`zip:`, `zarr2:`/`zarr3:`) are under development and will expand this +section. URLs without a `|` (and without a registered root scheme) are handled +exactly as before. + +`storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec +options for `s3://...`); adapters may consume adapter-specific, namespaced keys. +Non-dict forms of `storage_options` are reserved for future per-segment +configuration. + +The `|` character is reserved as the pipeline delimiter in every string store +specification, and no percent-escape is decoded: to address a local file whose +*name* contains `|` (or `#`), pass a `pathlib.Path` instead of a string. +Registered adapters cannot intercept zarr's native `file:` and `memory:` root +schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to +fsspec. + ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers diff --git a/mkdocs.yml b/mkdocs.yml index 4d06701a87..58b5d3344a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - ' zarr.abc.metadata': api/zarr/abc/metadata.md - ' zarr.abc.numcodec': api/zarr/abc/numcodec.md - ' zarr.abc.store': api/zarr/abc/store.md + - ' zarr.abc.url_pipeline': api/zarr/abc/url_pipeline.md - ' zarr.api': - api/zarr/api/index.md - ' zarr.api.asynchronous': api/zarr/api/asynchronous.md diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py new file mode 100644 index 0000000000..0e0363bcf6 --- /dev/null +++ b/src/zarr/abc/url_pipeline.py @@ -0,0 +1,256 @@ +""" +Abstract base class and data model for URL pipeline adapters. + +A URL pipeline is a `|`-separated chain of sub-URLs, read outer-to-inner, +as specified by https://github.com/jbms/url-pipeline. The first sub-URL (the +*root*) locates a resource using a conventional URL, and each subsequent +sub-URL names an *adapter* that reinterprets everything to its left: + + s3://bucket/data.zip|zip:path/inside|zarr3: + +Third-party packages provide adapters by subclassing +[`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] and +registering the class under the `zarr.url_adapters` entry-point group, +using the URL scheme as the entry-point name. +""" + +from __future__ import annotations + +import enum +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from zarr.errors import URLPipelineError + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import AccessModeLiteral, ZarrFormat + +__all__ = [ + "AdapterResolution", + "PipelineContext", + "PipelineSegment", + "URLPipelineAdapter", +] + + +class _Unset(enum.Enum): + token = 0 + + +_UNSET = _Unset.token + + +@dataclass(frozen=True) +class PipelineSegment: + """ + One `|`-delimited sub-URL of a URL pipeline. + + Attributes + ---------- + scheme : str + The lowercased URL scheme. Empty string only for a schemeless root + (a bare local path), which is treated as opaque text. + body : str + The text after `scheme:` and before any `?`. Interpretation is + scheme-defined; it is **not** URL-normalized, so case-significant + content (e.g. icechunk snapshot IDs) is preserved. + query : str | None + The raw query string after `?`, or None. Interpretation is + scheme-defined. + raw : str + The exact original sub-URL text, preserved for lossless + reconstruction of the pipeline. + """ + + scheme: str + body: str + query: str | None + raw: str + + def __str__(self) -> str: + return self.raw + + +@dataclass(frozen=True) +class AdapterResolution: + """ + The result of resolving a URL pipeline (or a prefix of one). + + Attributes + ---------- + store : Store + The resolved store. + path : str + Residual path *within* the store that the pipeline addresses + (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`). + Empty string when the pipeline addresses the store root. + zarr_format : ZarrFormat | None + Zarr format selected by a format segment (`zarr2:`/`zarr3:`), + or None if unspecified. A wrapper adapter that re-wraps a preceding + resolution must carry every field it does not change forward — + prefer `dataclasses.replace(preceding, store=..., path=...)` over + reconstructing, so fields added later are never silently dropped. + """ + + store: Store + path: str = "" + zarr_format: ZarrFormat | None = None + + +@dataclass(frozen=True) +class PipelineContext: + """ + Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + describing the pipeline to the left of its segment. + + Attributes + ---------- + preceding : tuple[PipelineSegment, ...] + The parsed sub-URLs to the left of the adapter's segment, outer to + inner. Empty when the adapter's segment is the pipeline root. + mode : AccessModeLiteral | None + The access mode requested by the caller (e.g. `zarr.open(mode=...)`), + or None when unspecified. Adapters for read-only resources should + raise for unambiguous write modes (`"w"`, `"w-"`, `"r+"`) and + open read-only otherwise. `"a"` (the `zarr.open` default) means + open-or-create: read-only adapters serve the "open" half, and any + subsequent write fails at the store level. + storage_options : dict[str, Any] | None + Options passed by the caller. By convention these configure the + *root* sub-URL (e.g. fsspec options); adapters may consume + adapter-specific keys, and should namespace them (e.g. + `myscheme_credentials`) to avoid collisions with other segments' + backends. An adapter that consumes keys should strip them before + resolving the rest of the pipeline, by passing the reduced mapping + to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + Non-dict forms of the caller-facing `storage_options` argument are + reserved for future per-segment configuration (one mapping per + pipeline segment); this attribute will remain a single mapping — + the one addressed to this adapter's segment. + """ + + preceding: tuple[PipelineSegment, ...] + mode: AccessModeLiteral | None + storage_options: dict[str, Any] | None + + @property + def read_only(self) -> bool: + """ + True when the caller requires a read-only store (`mode == "r"`). + + Adapters must construct their store read-only when this is set + (the resolver enforces it afterwards); when it is False, they may + construct a writable store if the underlying resource supports + writing. + """ + return self.mode == "r" + + @property + def preceding_url(self) -> str: + """ + The pipeline to the left of this segment, reconstructed exactly. + + An adapter that consumes this string instead of calling + [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding] + takes ownership of the *entire* preceding pipeline: it must + validate every preceding segment itself and raise + [`URLPipelineError`][zarr.errors.URLPipelineError] for segments it + does not understand, so that no segment is ever silently ignored. + """ + return "|".join(segment.raw for segment in self.preceding) + + async def resolve_preceding( + self, + *, + mode: AccessModeLiteral | _Unset | None = _UNSET, + storage_options: dict[str, Any] | _Unset | None = _UNSET, + ) -> AdapterResolution: + """ + Resolve the preceding pipeline into a store. + + This is the entry point for *wrapper* adapters (e.g. `zip:`) that + operate on the resource produced by the segments to their left. It + composes with any preceding adapters, because each segment is + resolved by its own adapter. Adapters backed by their own I/O + machinery (e.g. `icechunk:`) may instead consume + [`preceding_url`][zarr.abc.url_pipeline.PipelineContext.preceding_url] + and never materialize the intermediate store — subject to the + ownership contract documented there. + + Parameters + ---------- + mode : AccessModeLiteral | None, optional + Override the mode used to resolve the preceding pipeline. + Wrapper adapters that only read the preceding resource should + pass `mode="r"` so the root is opened read-only and without + create-on-open side effects, regardless of the caller's mode. + When omitted, the caller's mode is used. + storage_options : dict | None, optional + Override the options forwarded to the preceding pipeline. An + adapter that consumed adapter-specific keys should pass the + remaining mapping here (or None when nothing remains), so the + root store never sees keys that were not addressed to it. + When omitted, the caller's options are forwarded unchanged. + """ + from zarr.storage._url_pipeline import _resolve + + if not self.preceding: + raise URLPipelineError( + "this adapter segment is at the pipeline root; " + "there is no preceding sub-URL to resolve" + ) + return await _resolve( + self.preceding, + mode=self.mode if isinstance(mode, _Unset) else mode, + storage_options=( + self.storage_options if isinstance(storage_options, _Unset) else storage_options + ), + ) + + +class URLPipelineAdapter(ABC): + """ + Handler for one URL pipeline scheme. + + Subclasses implement a single classmethod, + [`open_pipeline_segment`][zarr.abc.url_pipeline.URLPipelineAdapter.open_pipeline_segment], + and are registered under the `zarr.url_adapters` entry-point group with + the URL scheme as the entry-point name: + + [project.entry-points."zarr.url_adapters"] + mypackage.myscheme = "mypackage.zarr_adapter:MyAdapter" + + Nonstandard schemes should be vendor-prefixed (`vendor.scheme`) per the + URL pipeline specification. + + An adapter is used in two positions: + + - as an *adapter segment*: `s3://bucket/repo|icechunk://tag.v1` — the + context carries the preceding sub-URLs; + - as a *root scheme*: `gh://org/repo` — `context.preceding` is empty. + """ + + @classmethod + @abstractmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve `segment` (in the context of the pipeline to its left) + into a store and an optional residual path within that store. + + The returned store must already be open and must honor + `context.read_only` (the resolver additionally enforces it by + downgrading — or rejecting — a writable store when the caller + required read-only). + + This coroutine runs on zarr's internal I/O event loop. It must not + block (do I/O through async APIs or a thread executor) and must not + call zarr's synchronous API (`zarr.open`, `Group.open`, or anything + else that uses `zarr.core.sync.sync`) — doing so raises + `SyncError`. To open the preceding pipeline, use + [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + """ + ... diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3bdc254ea5..2ae05bef6f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -108,6 +108,24 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool: return mode in _OVERWRITE_MODES +def _merge_pipeline_zarr_format( + store_path: StorePath, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: + """ + Combine a zarr format selected by a URL pipeline segment (`zarr2:` / + `zarr3:`) with the caller's `zarr_format` argument. Explicitly + conflicting selections raise. + """ + if store_path.zarr_format is None: + return zarr_format + if zarr_format is not None and zarr_format != store_path.zarr_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{store_path.zarr_format}:' " + "segment of the URL pipeline" + ) + return store_path.zarr_format + + def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ Emit a "not yet implemented" warning for each provided keyword argument that is not None. @@ -389,6 +407,7 @@ async def open( else: mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in {"a", "r", "r+", "w"}: @@ -494,13 +513,14 @@ async def save_array( **kwargs Passed through to [`create`][zarr.api.asynchronous.create], e.g., compressor. """ - if zarr_format is None: - zarr_format = _default_zarr_format() if not isinstance(arr, NDArrayLike): raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() if np.isscalar(arr): arr = np.array(arr) shape = arr.shape @@ -550,6 +570,7 @@ async def save_group( """ store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -762,12 +783,12 @@ async def create_group( The new group. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - mode: Literal["a"] = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() return await AsyncGroup.from_store( store=store_path, @@ -857,6 +878,7 @@ async def open_group( ) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) if attributes is None: attributes = {} @@ -1041,9 +1063,6 @@ async def create( z : array The array. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - _warn_unimplemented_kwargs( { "synchronizer": synchronizer, @@ -1063,6 +1082,9 @@ async def create( if mode is None: mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() config_parsed = parse_array_config(config) @@ -1262,6 +1284,7 @@ async def open_array( mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 781bebe534..de04a5bdaa 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -12,6 +12,7 @@ "MetadataValidationError", "NegativeStepError", "NodeTypeValidationError", + "URLPipelineError", "UnstableSpecificationWarning", "VindexInvalidSelectionError", "ZarrDeprecationWarning", @@ -100,6 +101,12 @@ class UnknownCodecError(BaseZarrError): """ +class URLPipelineError(BaseZarrError): + """ + Raised when a URL pipeline cannot be parsed or resolved. + """ + + class NodeTypeValidationError(MetadataValidationError): """ Specialized exception when the node_type of the metadata document is incorrect. diff --git a/src/zarr/registry.py b/src/zarr/registry.py index c2c0eb2921..53875c015c 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -1,5 +1,6 @@ from __future__ import annotations +import threading import warnings from collections import defaultdict from importlib.metadata import entry_points as get_entry_points @@ -7,7 +8,7 @@ from zarr.core.config import BadConfigError, config from zarr.core.dtype import data_type_registry -from zarr.errors import ZarrUserWarning +from zarr.errors import URLPipelineError, ZarrUserWarning if TYPE_CHECKING: from importlib.metadata import EntryPoint @@ -21,6 +22,7 @@ CodecPipeline, ) from zarr.abc.numcodec import Numcodec + from zarr.abc.url_pipeline import URLPipelineAdapter from zarr.core.buffer import Buffer, NDBuffer from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import JSON @@ -32,11 +34,14 @@ "get_codec_class", "get_ndbuffer_class", "get_pipeline_class", + "get_url_adapter", + "list_url_adapter_schemes", "register_buffer", "register_chunk_key_encoding", "register_codec", "register_ndbuffer", "register_pipeline", + "register_url_adapter", ] @@ -62,6 +67,7 @@ def register(self, cls: type[T], qualname: str | None = None) -> None: _buffer_registry: Registry[Buffer] = Registry() _ndbuffer_registry: Registry[NDBuffer] = Registry() _chunk_key_encoding_registry: Registry[ChunkKeyEncoding] = Registry() +_url_adapter_registry: Registry[URLPipelineAdapter] = Registry() """ The registry module is responsible for managing implementations of codecs, @@ -108,6 +114,8 @@ def _collect_entrypoints() -> list[Registry[Any]]: entry_points.select(group="zarr", name="chunk_key_encoding") ) + _url_adapter_registry.lazy_load_list.extend(entry_points.select(group="zarr.url_adapters")) + _pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) _pipeline_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="codec_pipeline") @@ -124,6 +132,7 @@ def _collect_entrypoints() -> list[Registry[Any]]: _buffer_registry, _ndbuffer_registry, _chunk_key_encoding_registry, + _url_adapter_registry, ] @@ -303,6 +312,72 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] +def register_url_adapter(scheme: str, cls: type[URLPipelineAdapter]) -> None: + """ + Register a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + class for a URL scheme. + + Registering a scheme that already has an adapter replaces it and emits a + [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]. + """ + key = scheme.lower() + previous = _url_adapter_registry.get(key) + if previous is not None and previous is not cls: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is being replaced: " + f"{fully_qualified_name(previous)} -> {fully_qualified_name(cls)}", + category=ZarrUserWarning, + stacklevel=2, + ) + _url_adapter_registry.register(cls, key) + + +def list_url_adapter_schemes() -> set[str]: + """ + The set of URL schemes with a registered URL pipeline adapter. + + Includes adapters advertised via not-yet-loaded `zarr.url_adapters` + entry points; consulting this does not import any adapter code. + Schemes are case-insensitive and reported lowercased. + """ + return set(_url_adapter_registry) | { + e.name.lower() for e in _url_adapter_registry.lazy_load_list + } + + +_url_adapter_lock = threading.Lock() + + +def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: + """ + Get the URL pipeline adapter class registered for `scheme`. + + Loads pending `zarr.url_adapters` entry points for this scheme only, so + resolving one scheme never imports other providers' packages. + """ + key = scheme.lower() + # The lock keeps concurrent first-time resolutions of different schemes + # from clobbering each other's rebuild of the pending entry-point list. + with _url_adapter_lock: + if key not in _url_adapter_registry: + remaining = [] + for entry_point in _url_adapter_registry.lazy_load_list: + if entry_point.name.lower() == key: + _url_adapter_registry.register(entry_point.load(), qualname=key) + else: + remaining.append(entry_point) + _url_adapter_registry.lazy_load_list[:] = remaining + try: + return _url_adapter_registry[key] + except KeyError: + registered = sorted(list_url_adapter_schemes()) + raise URLPipelineError( + f"no URL pipeline adapter is registered for scheme {scheme!r}. " + f"Registered schemes: {registered}. Adapters are provided by " + "packages via the 'zarr.url_adapters' entry-point group." + ) from None + + _collect_entrypoints() diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index f1bd1724af..edc01f1a5b 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,6 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore +from zarr.storage._url_pipeline import parse_pipeline, resolve_pipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -25,6 +26,8 @@ "StorePath", "WrapperStore", "ZipStore", + "parse_pipeline", + "resolve_pipeline", ] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 72b5fc8a40..7bf294bfd3 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -21,9 +21,15 @@ AccessModeLiteral, ZarrFormat, ) -from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError +from zarr.errors import ( + ContainsArrayAndGroupError, + ContainsArrayError, + ContainsGroupError, + URLPipelineError, +) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -47,14 +53,22 @@ class StorePath: The store to use. path : str The path within the store. + + Attributes + ---------- + zarr_format : ZarrFormat | None + Zarr format selected by a `zarr2:`/`zarr3:` URL pipeline segment, + or None. Not part of the StorePath's identity (ignored by `__eq__`). """ store: Store path: str + zarr_format: ZarrFormat | None def __init__(self, store: Store, path: str = "") -> None: self.store = store self.path = normalize_path(path) + self.zarr_format = None @property def read_only(self) -> bool: @@ -72,7 +86,10 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No Open StorePath based on the provided mode. * If the mode is None, return an opened version of the store with no changes. - * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError. + * If the mode is 'r+', 'w-', or 'w' and the store is read-only, raise a ValueError. + * If the mode is 'a' (open-or-create) and the store is read-only, serve the + "open" half: the StorePath is opened read-only, and any subsequent write + fails at the store level. * If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True. * If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError. * If the mode is 'w' and the store is not read-only, delete all keys nested within the StorePath. @@ -95,7 +112,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is not "r" and the store is read-only, or + If the mode is "r+", "w-", or "w" and the store is read-only, or """ # fastpath if mode is None @@ -106,8 +123,9 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") if store.read_only: - # Don't allow write operations on a read-only store - if mode != "r": + # mode "a" (open-or-create) on a read-only store means open-only; + # unambiguous write modes are rejected outright. + if mode not in ("r", "a"): raise ValueError( f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode." ) @@ -273,7 +291,9 @@ def delete_sync(self) -> None: def __truediv__(self, other: str) -> StorePath: """Combine this store path with another path""" - return self.__class__(self.store, _join_paths([self.path, other])) + result = self.__class__(self.store, _join_paths([self.path, other])) + result.zarr_format = self.zarr_format + return result def __str__(self) -> str: return _join_paths([str(self.store), self.path]) @@ -348,6 +368,16 @@ async def make_store( """ from zarr.storage._fsspec import FsspecStore # circular import + if isinstance(store_like, str) and is_url_pipeline(store_like): + result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + if normalize_path(result.path): + result.store.close() + raise URLPipelineError( + f"the URL pipeline {store_like!r} resolves to a path inside a store; " + "use zarr.open() or make_store_path() instead of make_store()" + ) + return result.store + # Parse URL early so we can reuse the result for both validation and routing parsed = parse_store_url(store_like) if isinstance(store_like, str) else None @@ -464,6 +494,13 @@ async def make_store_path( """ path_normalized = normalize_path(path) + if isinstance(store_like, str) and is_url_pipeline(store_like): + result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + combined_path = _join_paths([normalize_path(result.path), path_normalized]) + store_path = await StorePath.open(result.store, path=combined_path, mode=mode) + store_path.zarr_format = result.zarr_format + return store_path + if isinstance(store_like, StorePath): # Already a StorePath if storage_options: diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py new file mode 100644 index 0000000000..0161bdee6a --- /dev/null +++ b/src/zarr/storage/_url_pipeline.py @@ -0,0 +1,311 @@ +""" +Parsing and resolution of URL pipelines (https://github.com/jbms/url-pipeline). + +Importing this module is cheap and has no side effects: third-party adapters +(registered through the `zarr.url_adapters` entry-point group) are loaded +only when a pipeline URL naming their scheme is actually resolved. + +The `|` character is reserved as the pipeline delimiter in every string +store specification: a string containing `|` is always routed through the +pipeline machinery, and no percent-escape is decoded. To address a local +file whose *name* contains `|` (or `#`), pass a `pathlib.Path` instead of a +string. + +As a zarr-python extension to the specification (which requires the root +sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be +a schemeless local filesystem path, e.g. `data/example.zip|zip:`. Schemeless +roots are treated as opaque text — no query or fragment splitting is applied +to them. Such pipelines are not portable to other URL pipeline +implementations; portable pipelines should spell the root as a `file:` URL. +""" + +from __future__ import annotations + +import dataclasses +import re +from typing import TYPE_CHECKING, Any + +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, +) +from zarr.errors import URLPipelineError +from zarr.registry import get_url_adapter, list_url_adapter_schemes +from zarr.storage._memory import ManagedMemoryStore +from zarr.storage._utils import parse_store_url + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import AccessModeLiteral + +__all__ = ["is_url_pipeline", "parse_pipeline", "resolve_pipeline"] + +# Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed +# nonstandard schemes (e.g. "earthmover.myscheme"). +_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$") + +# Root scheme at the very start of the sub-URL. The negative lookahead +# excludes fsspec's chained-URL syntax (``zip::file://...``), which is not a +# URL pipeline and keeps flowing through the fsspec machinery. +_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):(?!:)") + +# Schemes zarr resolves natively at the pipeline root. These are never +# dispatched to a registered root adapter, so an installed package cannot +# intercept zarr's own local-path and in-memory routing. +_NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) + +# An absolute Windows drive path (C:\... or C:/...), which counts as an +# absolute path in a `file:` pipeline root. +_WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") + + +def _root_scheme(root_sub_url: str) -> str: + """ + Detect the scheme of the root sub-URL, or `""` for an opaque root. + + Scheme extraction happens on the raw string (so nothing `urlparse` + would strip or reject — whitespace, exotic authorities — can desync the + detected scheme from the body). Single-letter candidates are delegated + to `parse_store_url`, which knows Windows drive letters are not schemes. + """ + match = _ROOT_SCHEME_RE.match(root_sub_url) + if match is None: + return "" + scheme = match.group(1) + if len(scheme) == 1: + try: + return parse_store_url(root_sub_url).scheme.lower() + except ValueError: + return scheme.lower() + return scheme.lower() + + +def _split_query(sub_url: str) -> tuple[str, str | None]: + """Split a sub-URL on the first `?`. Fragments are not supported.""" + if "#" in sub_url: + raise URLPipelineError( + f"URL pipeline sub-URLs do not support fragments: {sub_url!r}. " + "Percent-encode '#' as '%23' if it is part of the path." + ) + body, sep, query = sub_url.partition("?") + return body, query if sep else None + + +def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: + """ + Parse a URL pipeline into its `|`-delimited segments. + + The first segment is the *root* sub-URL; its scheme is detected with the + same rules as ordinary store URLs (Windows drive letters are not + schemes). Subsequent segments are *adapter* sub-URLs of the form + `scheme:body` where the trailing colon is optional when the body is + empty (`zip` is equivalent to `zip:`). + + A schemeless root (a bare local path) is accepted as a zarr-python + extension to the specification and treated as opaque text: no query or + fragment splitting applies, since `?` and `#` are ordinary filename + characters there. See the module docstring. + + Segment text is preserved verbatim (no case or percent-encoding + normalization) except that schemes are lowercased. + """ + parts = url.split("|") + if any(not part for part in parts): + raise URLPipelineError(f"URL pipeline contains an empty sub-URL: {url!r}") + + segments: list[PipelineSegment] = [] + for index, part in enumerate(parts): + if index == 0: + scheme = _root_scheme(part) + if not scheme: + # opaque root (bare local path, fsspec chained URL, ...) + segments.append(PipelineSegment(scheme="", body=part, query=None, raw=part)) + continue + body_and_scheme, query = _split_query(part) + body = body_and_scheme[len(scheme) + 1 :] + segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + else: + body_and_scheme, query = _split_query(part) + scheme, _, body = body_and_scheme.partition(":") + scheme = scheme.lower() + if not _SCHEME_RE.match(scheme): + raise URLPipelineError( + f"invalid adapter scheme {scheme!r} in pipeline segment {part!r}" + ) + segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + return tuple(segments) + + +def _root_routes_to_adapter(scheme: str) -> bool: + """ + Whether a root sub-URL with this scheme is dispatched to a registered + root adapter. Schemes zarr resolves natively (`file:`, `memory:`) and + opaque roots are excluded; the registry check inspects entry-point + names only — no adapter code is imported here. + """ + return ( + bool(scheme) and scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() + ) + + +def is_url_pipeline(url: str) -> bool: + """ + Whether `url` should be routed through the URL pipeline machinery. + + True when the URL contains a `|` separator, or when its scheme has a + registered URL pipeline adapter (a *root adapter* such as `gh:`). + fsspec chained URLs (`zip::file://...`) and zarr's native `file:` / + `memory:` schemes are never routed to a root adapter. + """ + if "|" in url: + return True + return _root_routes_to_adapter(_root_scheme(url)) + + +async def resolve_pipeline( + url: str, + *, + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, +) -> AdapterResolution: + """ + Resolve a URL pipeline into a store and a residual path. + + Parameters + ---------- + url : str + A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`. + mode : AccessModeLiteral | None + The caller's access mode. `"r"` requires a read-only store; the + resolver enforces this on whatever the final adapter returns. + storage_options : dict | None + Options forwarded to the root sub-URL's store (and visible to + adapters via the context). Non-dict forms are reserved for future + per-segment configuration (one mapping per pipeline segment). + """ + segments = parse_pipeline(url) + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): + raise URLPipelineError( + f"{url!r} is not a URL pipeline: it has no '|' separator and no " + f"URL pipeline adapter is registered for scheme {segments[0].scheme!r}" + ) + return await _resolve(segments, mode=mode, storage_options=storage_options) + + +async def _resolve( + segments: tuple[PipelineSegment, ...], + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> AdapterResolution: + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): + return AdapterResolution( + store=await _resolve_root(segments[0], mode=mode, storage_options=storage_options) + ) + + *preceding, last = segments + try: + adapter_cls = get_url_adapter(last.scheme) + except URLPipelineError as exc: + raise URLPipelineError( + f"{exc} Note: '|' is reserved as the URL pipeline delimiter; to " + "address a local file whose name contains '|', pass a " + "pathlib.Path instead of a string." + ) from None + + context = PipelineContext( + preceding=tuple(preceding), + mode=mode, + storage_options=storage_options, + ) + resolution = await adapter_cls.open_pipeline_segment(last, context) + if mode == "r" and not resolution.store.read_only: + # The caller required read-only; enforce it rather than trusting + # the adapter to have honored context.read_only. + try: + read_only_store = resolution.store.with_read_only(True) + except NotImplementedError as exc: + raise URLPipelineError( + f"adapter {last.scheme!r} returned a writable store for mode 'r', " + "and the store does not support read-only conversion via " + ".with_read_only()" + ) from exc + await read_only_store._ensure_open() + resolution = dataclasses.replace(resolution, store=read_only_store) + return resolution + + +async def _resolve_root( + segment: PipelineSegment, + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> Store: + """ + Resolve the root sub-URL of a pipeline into a store. + + `memory:` and `file:` roots are resolved here with the URL pipeline + spec's semantics (spelling equivalences, mandatory absolute `file:` + paths); everything else — bare local paths, fsspec URLs — delegates to + the existing `StoreLike` machinery unchanged. + """ + if segment.scheme == "memory": + return _resolve_memory_root(segment, mode=mode, storage_options=storage_options) + from zarr.storage._common import make_store # circular import + + if segment.scheme == "file": + # Per the spec: file://localhost/p and file:///p are equivalent to + # file:/p; other authorities are unsupported; relative paths are + # forbidden; file: URLs carry no query. + if segment.query is not None: + raise URLPipelineError(f"'file:' pipeline roots do not accept a query: {segment.raw!r}") + body = segment.body + if body.startswith("//"): + authority, sep, rest = body[2:].partition("/") + if authority not in ("", "localhost"): + raise URLPipelineError( + f"unsupported authority {authority!r} in 'file:' pipeline " + f"root {segment.raw!r}; only an empty authority or " + "'localhost' is allowed" + ) + body = f"/{rest}" if sep else "" + # a file URL spells a Windows drive path as /C:/...; strip the + # leading slash so the local-path machinery sees the drive + if body.startswith("/") and _WINDOWS_DRIVE_RE.match(body[1:]): + body = body[1:] + if not (body.startswith("/") or _WINDOWS_DRIVE_RE.match(body)): + raise URLPipelineError( + f"'file:' pipeline roots must carry an absolute path: {segment.raw!r}" + ) + return await make_store(f"file:{body}", mode=mode, storage_options=storage_options) + + try: + return await make_store(segment.raw, mode=mode, storage_options=storage_options) + except ValueError as exc: + raise URLPipelineError( + f"could not resolve the pipeline root {segment.raw!r}: {exc}" + ) from exc + + +def _resolve_memory_root( + segment: PipelineSegment, + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> Store: + """ + Resolve a `memory:` pipeline root per the spec's spelling equivalences: + `memory:` ≡ `memory:/` ≡ `memory://`, and `memory:a` ≡ `memory:/a` ≡ + `memory://a`. The first path component names the managed store; the + remainder is a path within it. + """ + if segment.query is not None: + raise URLPipelineError(f"'memory:' pipeline roots do not accept a query: {segment.raw!r}") + if storage_options: + raise TypeError( + "'storage_options' was provided but unused. " + "'storage_options' is only used when the store is passed as an FSSpec URI string.", + ) + name, _, path = segment.body.lstrip("/").partition("/") + return ManagedMemoryStore(name=name, path=path, read_only=mode == "r") diff --git a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt index 7eb0eb7c86..456c725b4e 100644 --- a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +++ b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -13,4 +13,6 @@ another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer [zarr.codec_pipeline] another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline [zarr.data_type] -new_data_type = package_with_entrypoint:TestDataType \ No newline at end of file +new_data_type = package_with_entrypoint:TestDataType +[zarr.url_adapters] +example-pkg.entrypoint-scheme = package_with_entrypoint:TestEntrypointURLAdapter diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index 23afcf1dc2..cad73d061d 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -7,6 +7,7 @@ import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter from zarr.codecs import BytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.dtype.npy.bool import Bool @@ -16,6 +17,7 @@ from collections.abc import Iterable from typing import Any, ClassVar, Literal, Self + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment from zarr.core.array_spec import ArraySpec from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2 @@ -100,3 +102,16 @@ def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignor if zarr_format == 3: return self._zarr_v3_name raise ValueError("zarr_format must be 2 or 3") + + +class TestEntrypointURLAdapter(URLPipelineAdapter): + """URL pipeline adapter discovered via the zarr.url_adapters entry point.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + from zarr.storage import MemoryStore + + store = await MemoryStore.open(read_only=context.read_only) + return AdapterResolution(store=store, path=segment.body) diff --git a/tests/test_api.py b/tests/test_api.py index 45d0c0dee4..c4e561c180 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -198,6 +198,9 @@ async def test_open_like_creates_array_by_default( async def test_open_like_default_mode_rejects_read_only_store( zarr_format: ZarrFormat, ) -> None: + # mode "a" (open-or-create) on a read-only store serves the "open" half: + # since the node does not exist and creating is impossible, the open + # fails with a not-found error rather than rejecting the mode upfront. ref_arr = zarr.create_array( store={}, shape=(11, 12), @@ -206,7 +209,7 @@ async def test_open_like_default_mode_rejects_read_only_store( zarr_format=zarr_format, ) - with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + with pytest.raises(ValueError, match="No array found"): await zarr.api.asynchronous.open_like( ref_arr, path="foo", diff --git a/tests/test_url_pipeline/__init__.py b/tests/test_url_pipeline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_url_pipeline/conftest.py b/tests/test_url_pipeline/conftest.py new file mode 100644 index 0000000000..13c717e04e --- /dev/null +++ b/tests/test_url_pipeline/conftest.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +import zarr.registry + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture +def clean_url_adapter_registry() -> Generator[None, None, None]: + """Snapshot and restore the URL adapter registry around a test.""" + registry = zarr.registry._url_adapter_registry + saved = dict(registry) + saved_lazy = list(registry.lazy_load_list) + yield + registry.clear() + registry.update(saved) + registry.lazy_load_list[:] = saved_lazy diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py new file mode 100644 index 0000000000..13d7846e85 --- /dev/null +++ b/tests/test_url_pipeline/test_parser.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import sys + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr.errors import URLPipelineError +from zarr.storage._url_pipeline import parse_pipeline + + +def test_single_root_url() -> None: + (segment,) = parse_pipeline("s3://bucket/key") + assert segment.scheme == "s3" + assert segment.body == "//bucket/key" + assert segment.query is None + assert segment.raw == "s3://bucket/key" + assert str(segment) == "s3://bucket/key" + + +def test_schemeless_root() -> None: + (segment,) = parse_pipeline("/local/path") + assert segment.scheme == "" + assert segment.body == "/local/path" + assert segment.raw == "/local/path" + + +def test_adapter_chain() -> None: + segments = parse_pipeline("s3://bucket/data.zip|zip:inner/path|zarr3:") + assert [s.scheme for s in segments] == ["s3", "zip", "zarr3"] + assert segments[1].body == "inner/path" + assert segments[2].body == "" + + +def test_trailing_colon_optional() -> None: + with_colon = parse_pipeline("file:/tmp/x.zip|zip:") + without_colon = parse_pipeline("file:/tmp/x.zip|zip") + assert with_colon[1].scheme == without_colon[1].scheme == "zip" + assert with_colon[1].body == without_colon[1].body == "" + + +def test_scheme_case_insensitive() -> None: + segments = parse_pipeline("FILE:/tmp/x.zip|ZIP:Inner/Path") + assert segments[0].scheme == "file" + assert segments[1].scheme == "zip" + # bodies are case-preserved + assert segments[1].body == "Inner/Path" + + +def test_case_preserved_in_raw() -> None: + # e.g. icechunk snapshot IDs are case-significant + segments = parse_pipeline("file:/tmp/repo|icechunk://ABCDEFGH12345678ABCD/x") + assert segments[1].raw == "icechunk://ABCDEFGH12345678ABCD/x" + assert segments[1].body == "//ABCDEFGH12345678ABCD/x" + + +def test_vendor_prefixed_scheme() -> None: + segments = parse_pipeline("file:/data|vendor-1.custom+adapter:sub/path") + assert segments[1].scheme == "vendor-1.custom+adapter" + assert segments[1].body == "sub/path" + + +def test_query_strings() -> None: + segments = parse_pipeline("https://example.com/d.zip?token=abc|zip:x?opt=1") + assert segments[0].query == "token=abc" + assert segments[0].body == "//example.com/d.zip" + assert segments[1].query == "opt=1" + assert segments[1].body == "x" + + +def test_empty_query() -> None: + (segment,) = parse_pipeline("https://example.com/d?") + assert segment.query == "" + + +def test_windows_drive_path_is_not_a_scheme() -> None: + # On Windows parse_store_url treats C:\... as a local path; elsewhere the + # single-letter scheme is preserved but must not crash the parser. + (segment,) = parse_pipeline(r"C:\data\store") + assert segment.raw == r"C:\data\store" + if sys.platform == "win32": + assert segment.scheme == "" + assert segment.body == r"C:\data\store" + else: + assert segment.scheme == "c" + + +@pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) +def test_empty_sub_url_rejected(url: str) -> None: + with pytest.raises(URLPipelineError, match="empty sub-URL"): + parse_pipeline(url) + + +def test_fragment_rejected() -> None: + with pytest.raises(URLPipelineError, match="fragment"): + parse_pipeline("file:/tmp/x.zip|zip:inner#frag") + + +@pytest.mark.parametrize("segment", ["1zip:", "zi p:x", "zip@:x"]) +def test_invalid_adapter_scheme_rejected(segment: str) -> None: + with pytest.raises(URLPipelineError, match="invalid adapter scheme"): + parse_pipeline(f"file:/tmp/x|{segment}") + + +def test_single_letter_scheme_with_exotic_authority() -> None: + # single-letter candidates are delegated to parse_store_url, which may + # reject an exotic authority; the raw scheme is used as the fallback + segments = parse_pipeline("x://[authority]/path|zip:") + assert segments[0].scheme in ("", "x") + assert segments[0].raw == "x://[authority]/path" + + +def test_fsspec_chained_root_is_opaque() -> None: + # fsspec's ``scheme::`` chaining is not pipeline syntax; such roots are + # schemeless/opaque and flow through the ordinary store machinery + (segment,) = parse_pipeline("zip::file:///tmp/data.zip") + assert segment.scheme == "" + assert segment.body == "zip::file:///tmp/data.zip" + + +def test_whitespace_prefixed_root_is_opaque() -> None: + # urlparse strips leading whitespace when sniffing a scheme; the parser + # must not, or the detected scheme desyncs from the body slice + segments = parse_pipeline("\tfile:/tmp/x|zip:") + assert segments[0].scheme == "" + assert segments[0].body == "\tfile:/tmp/x" + + +def test_schemeless_root_keeps_query_and_fragment_chars() -> None: + # ? and # are ordinary filename characters in a bare local path + segments = parse_pipeline("/tmp/d?v=1|zip:") + assert segments[0].scheme == "" + assert segments[0].body == "/tmp/d?v=1" + assert segments[0].query is None + segments = parse_pipeline("/tmp/d#frag|zip:") + assert segments[0].body == "/tmp/d#frag" + + +def test_round_trip() -> None: + url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:" + segments = parse_pipeline(url) + assert "|".join(s.raw for s in segments) == url + + +# --- property-based tests ----------------------------------------------- +# +# These exercise the parser's *splitting invariants*, not URL pipeline spec +# validity: parse_pipeline is a permissive segment splitter (bodies and +# queries are opaque text to it), and semantic validation is left to the +# resolver and the adapters. Rather than enumerating registered schemes, +# the strategies sample the full scheme grammar the parser accepts +# (RFC 3986 plus "." for vendor prefixes), so every valid root/adapter +# scheme is reachable. Bodies and queries draw from printable ASCII minus +# the characters the parser itself splits on ("|", "#", and "?" for +# bodies); the generated text is not required to be a well-formed URI. + +_ADAPTER_SCHEME = st.from_regex(r"[a-zA-Z][a-zA-Z0-9+.\-]{0,15}", fullmatch=True) +# two or more characters, so Windows drive-letter handling cannot reclassify +# the root scheme as a local path on one platform but not another +_ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True) +_BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?")) +# a root body starting with ":" would spell fsspec's chained-URL syntax +# ("scheme::..."), which the parser deliberately treats as an opaque root +_ROOT_BODY = _BODY.filter(lambda body: not body.startswith(":")) +_QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#")) + + +@st.composite +def pipelines( + draw: st.DrawFn, min_depth: int = 1 +) -> tuple[str, list[tuple[str, str, str | None, str]]]: + """A parseable pipeline URL up to depth 8, with its expected split.""" + depth = draw(st.integers(min_value=min_depth, max_value=8)) + expected = [] + parts = [] + for index in range(depth): + scheme = draw(_ROOT_SCHEME if index == 0 else _ADAPTER_SCHEME) + body = draw(_ROOT_BODY if index == 0 else _BODY) + query = draw(st.none() | _QUERY) + raw = f"{scheme}:{body}" + (f"?{query}" if query is not None else "") + parts.append(raw) + expected.append((scheme.lower(), body, query, raw)) + return "|".join(parts), expected + + +@given(pipelines()) +def test_valid_pipeline_invariants( + case: tuple[str, list[tuple[str, str, str | None, str]]], +) -> None: + url, expected = case + segments = parse_pipeline(url) + # schemes lowercased; bodies and queries preserved verbatim + assert [(s.scheme, s.body, s.query, s.raw) for s in segments] == expected + # lossless round trip + assert "|".join(s.raw for s in segments) == url + + +@given(pipelines(), st.data()) +def test_property_empty_sub_url_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + parts.insert(data.draw(st.integers(0, len(parts))), "") + with pytest.raises(URLPipelineError, match="empty sub-URL"): + parse_pipeline("|".join(parts)) + + +@given(pipelines(min_depth=2), st.data()) +def test_property_invalid_adapter_scheme_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + # corrupt one adapter segment's scheme with a leading character the + # scheme grammar forbids + index = data.draw(st.integers(1, len(parts) - 1)) + parts[index] = data.draw(st.sampled_from(["1", " ", "@", "~"])) + parts[index] + with pytest.raises(URLPipelineError, match="invalid adapter scheme"): + parse_pipeline("|".join(parts)) + + +@given(pipelines(), st.data()) +def test_property_fragment_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + parts[data.draw(st.integers(0, len(parts) - 1))] += "#frag" + with pytest.raises(URLPipelineError, match="fragment"): + parse_pipeline("|".join(parts)) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py new file mode 100644 index 0000000000..198c178764 --- /dev/null +++ b/tests/test_url_pipeline/test_resolver.py @@ -0,0 +1,569 @@ +from __future__ import annotations + +import dataclasses +import sys +from typing import TYPE_CHECKING, ClassVar + +import pytest + +import zarr +import zarr.registry +from zarr.abc.store import Store +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.errors import URLPipelineError, ZarrUserWarning +from zarr.registry import ( + get_url_adapter, + list_url_adapter_schemes, + register_url_adapter, +) +from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore +from zarr.storage._common import make_store, make_store_path +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._utils import _join_paths + +if TYPE_CHECKING: + from pathlib import Path + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + + +class TracingStore(WrapperStore[Store]): + """Wrapper that records the context it was created from.""" + + context: PipelineContext + segment: PipelineSegment + + +class WrapperAdapter(URLPipelineAdapter): + """ + A wrapper-style adapter: resolves the preceding pipeline into a store. + + Follows the wrapper contract: the preceding resolution's residual path + is carried forward (joined with this segment's own path), and unchanged + fields survive via `dataclasses.replace`. + """ + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + store = TracingStore(preceding.store) + store.context = context + store.segment = segment + return dataclasses.replace( + preceding, store=store, path=_join_paths([preceding.path, segment.body]) + ) + + +class NativeAdapter(URLPipelineAdapter): + """A native-style adapter: consumes the preceding URL as a string.""" + + seen_urls: ClassVar[list[str]] = [] + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + cls.seen_urls.append(context.preceding_url) + store = await MemoryStore.open(read_only=context.read_only) + return AdapterResolution(store=store, path=segment.body) + + +class RootAdapter(URLPipelineAdapter): + """A root-scheme adapter (no preceding segments), like al://.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + assert context.preceding == () + if context.mode in ("w", "w-", "r+"): + raise ValueError("read-only scheme") + store = await MemoryStore.open(read_only=True) + return AdapterResolution(store=store, path=segment.body.lstrip("/")) + + +class DisobedientAdapter(URLPipelineAdapter): + """An adapter that ignores `context.read_only` (a contract violation).""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await MemoryStore.open(read_only=False)) + + +class FormatAdapter(URLPipelineAdapter): + """A dummy format-selecting adapter, like the builtin `zarr2:`.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, zarr_format=2) + + +class TestRegistry: + def test_register_and_get(self) -> None: + register_url_adapter("demo", WrapperAdapter) + assert get_url_adapter("demo") is WrapperAdapter + assert get_url_adapter("DEMO") is WrapperAdapter + assert "demo" in list_url_adapter_schemes() + + def test_unknown_scheme(self) -> None: + with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): + get_url_adapter("nonexistent-scheme") + + def test_reregistering_scheme_warns(self) -> None: + register_url_adapter("demo", WrapperAdapter) + with pytest.warns(ZarrUserWarning, match="is being replaced"): + register_url_adapter("demo", NativeAdapter) + assert get_url_adapter("demo") is NativeAdapter + # re-registering the same class is not a collision + register_url_adapter("demo", NativeAdapter) + + @pytest.mark.usefixtures("set_path") + def test_entrypoint_discovery(self) -> None: + assert "example-pkg.entrypoint-scheme" in list_url_adapter_schemes() + cls = get_url_adapter("example-pkg.entrypoint-scheme") + assert cls.__name__ == "TestEntrypointURLAdapter" + + @pytest.mark.usefixtures("set_path") + async def test_entrypoint_end_to_end(self) -> None: + result = await resolve_pipeline("memory://src|example-pkg.entrypoint-scheme:sub/path") + assert result.path == "sub/path" + + @pytest.mark.usefixtures("set_path") + def test_entrypoint_scheme_lookup_is_case_insensitive(self) -> None: + # entry-point names are matched case-insensitively, like schemes + cls = get_url_adapter("Example-PKG.Entrypoint-Scheme") + assert cls.__name__ == "TestEntrypointURLAdapter" + + @pytest.mark.usefixtures("set_path") + def test_loading_one_scheme_leaves_others_pending(self) -> None: + # resolving one scheme must not import other providers' entry points + registry = zarr.registry._url_adapter_registry + assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) + with pytest.raises(URLPipelineError, match="no URL pipeline adapter"): + get_url_adapter("some-other-scheme") + assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) + assert get_url_adapter("example-pkg.entrypoint-scheme").__name__ == ( + "TestEntrypointURLAdapter" + ) + + +class TestIsURLPipeline: + def test_pipe_routes(self) -> None: + assert is_url_pipeline("memory://x|demo:") + + def test_registered_root_scheme_routes(self) -> None: + register_url_adapter("rooty", RootAdapter) + assert is_url_pipeline("rooty://org/repo") + + @pytest.mark.parametrize("url", ["s3://bucket/key", "/local/path", "memory://x", "C:.zarr"]) + def test_plain_urls_do_not_route(self, url: str) -> None: + assert not is_url_pipeline(url) + + def test_fsspec_chained_urls_do_not_route(self) -> None: + # fsspec's ``scheme::`` chaining is not a URL pipeline: even with a + # same-named root adapter registered, these keep flowing to fsspec. + register_url_adapter("zip", RootAdapter) + assert not is_url_pipeline("zip::file:///tmp/data.zip") + assert not is_url_pipeline("simplecache::s3://bucket/key") + + def test_native_schemes_do_not_route_to_adapters(self) -> None: + # an installed package must not be able to intercept zarr's own + # local-path and in-memory routing by registering file:/memory: + register_url_adapter("file", RootAdapter) + register_url_adapter("memory", RootAdapter) + assert not is_url_pipeline("file:/tmp/x") + assert not is_url_pipeline("memory://x") + + def test_exotic_authority_root_scheme(self) -> None: + # The spec's own root-URL example: urlparse rejects the bracketed + # non-IP authority, so scheme detection must work on the raw string + # rather than raising before routing. + url = "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x" + assert not is_url_pipeline(url) + register_url_adapter("vendor1-2.custom-1+proto.ext", RootAdapter) + assert is_url_pipeline(url) + + +class TestResolve: + async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:inner/path") + assert isinstance(result.store, TracingStore) + assert result.path == "inner/path" + assert result.store.context.preceding_url == str(tmp_path) + + async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> None: + # each wrapper joins the preceding residual path with its own, so + # no segment's path is lost in root|wrap:a|wrap:b + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:a|wrap:b") + assert result.path == "a/b" + + async def test_native_adapter_gets_preceding_url(self) -> None: + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("s3://bucket/repo|native:") + assert NativeAdapter.seen_urls == ["s3://bucket/repo"] + + async def test_multi_segment_preceding_url(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("memory://base|wrap:a|native:x") + assert NativeAdapter.seen_urls == ["memory://base|wrap:a"] + + async def test_root_adapter(self) -> None: + register_url_adapter("rooty", RootAdapter) + result = await resolve_pipeline("rooty://org/repo") + assert result.path == "org/repo" + assert result.store.read_only + + async def test_root_adapter_composes_with_chain(self) -> None: + register_url_adapter("rooty", RootAdapter) + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("rooty://org/repo|native:x") + assert NativeAdapter.seen_urls == ["rooty://org/repo"] + + async def test_read_only_flag(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:", mode="r") + assert isinstance(result.store, TracingStore) + assert result.store.context.read_only + assert result.store.context.mode == "r" + result = await resolve_pipeline(f"{tmp_path}|wrap:") + assert isinstance(result.store, TracingStore) + assert not result.store.context.read_only + assert result.store.context.mode is None + + async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: + # the resolver downgrades a writable store returned under mode "r" + # instead of trusting the adapter to have honored context.read_only + register_url_adapter("bad", DisobedientAdapter) + result = await resolve_pipeline("memory://base|bad:", mode="r") + assert result.store.read_only + store = await make_store("memory://base|bad:", mode="r") + assert store.read_only + + async def test_read_only_enforcement_without_conversion_raises(self) -> None: + # a disobedient adapter whose store cannot be converted read-only + class StubbornStore(MemoryStore): + def with_read_only(self, read_only: bool = False) -> MemoryStore: + raise NotImplementedError + + class StubbornAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await StubbornStore.open(read_only=False)) + + register_url_adapter("stubborn", StubbornAdapter) + with pytest.raises(URLPipelineError, match="does not support read-only conversion"): + await resolve_pipeline("memory://base|stubborn:", mode="r") + + async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: + # a wrapper that only reads the preceding resource can open it + # read-only regardless of the caller's mode + class ReadOnlyRootWrapper(WrapperAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding(mode="r") + assert preceding.store.read_only + return dataclasses.replace(preceding, path=segment.body) + + register_url_adapter("rowrap", ReadOnlyRootWrapper) + result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") + assert result.path == "x" + + async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: + # an adapter that consumed its namespaced keys strips them before + # the root store ever sees them + class ConsumingWrapper(WrapperAdapter): + seen: ClassVar[object | None] = None + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + options = dict(context.storage_options or {}) + cls.seen = options.pop("consuming_secret", None) + preceding = await context.resolve_preceding(storage_options=options or None) + return dataclasses.replace(preceding, path=segment.body) + + register_url_adapter("consuming", ConsumingWrapper) + # the local-path root rejects any surviving storage_options, so this + # passing proves the adapter's keys were stripped before resolution + result = await resolve_pipeline( + f"{tmp_path}|consuming:x", storage_options={"consuming_secret": "s3cr3t"} + ) + assert result.path == "x" + assert ConsumingWrapper.seen == "s3cr3t" + + async def test_storage_options_visible_to_adapter(self) -> None: + register_url_adapter("native", NativeAdapter) + + class OptionsProbe(NativeAdapter): + seen_options: dict[str, object] | None = None + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + cls.seen_options = context.storage_options + return await super().open_pipeline_segment(segment, context) + + register_url_adapter("probe", OptionsProbe) + opts = {"anon": True} + await resolve_pipeline("s3://bucket/x|probe:", storage_options=opts) + assert OptionsProbe.seen_options == opts + + async def test_wrapper_at_root_position_raises(self) -> None: + # a wrapper adapter used as the pipeline root has nothing to wrap + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="no preceding sub-URL"): + await resolve_pipeline("wrap:whatever") + + async def test_not_a_pipeline_raises(self) -> None: + with pytest.raises(URLPipelineError, match="is not a URL pipeline"): + await resolve_pipeline("s3://bucket/plain") + + async def test_unknown_adapter_scheme_raises(self) -> None: + with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): + await resolve_pipeline("memory://base|no-such-adapter:") + + async def test_unknown_adapter_error_hints_at_reserved_pipe(self, tmp_path: Path) -> None: + # a filename containing '|' routes here; the error points at the fix + with pytest.raises(URLPipelineError, match="pass a\\s+pathlib.Path"): + await resolve_pipeline(f"{tmp_path}/a|b") + + async def test_base_case_value_error_is_wrapped(self) -> None: + # a root that only the fallback scheme detection accepts must not + # leak urlparse's ValueError out of the base case + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): + await resolve_pipeline("vendor.x://[authority]/path|wrap:") + + async def test_zarr_format_flows_from_adapter(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + result = await resolve_pipeline(f"{tmp_path}|fmt2:") + assert result.zarr_format == 2 + # wrapper adapters carry the format through (dataclasses.replace) + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|fmt2:|wrap:") + assert result.zarr_format == 2 + + +class TestSpecRoots: + """The spec's `memory:` / `file:` root semantics inside pipelines.""" + + @pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"]) + async def test_memory_root_spellings_equivalent(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{root}|wrap:") + assert isinstance(result.store, TracingStore) + inner = result.store._store + assert isinstance(inner, ManagedMemoryStore) + assert inner._name == "" + assert inner.path == "" + + @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) + async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{root}|wrap:") + assert isinstance(result.store, TracingStore) + inner = result.store._store + assert isinstance(inner, ManagedMemoryStore) + assert inner._name == "a" + assert inner.path == "b" + + async def test_memory_root_shares_data_across_spellings(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_group("memory:pipe-share|wrap:", mode="w") + group.create_array("x", shape=(2,), dtype="i4") + reopened = zarr.open_group("memory://pipe-share|wrap:", mode="r") + assert "x" in reopened + + async def test_memory_root_rejects_query(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline("memory:a?opt=1|wrap:") + + async def test_memory_root_rejects_storage_options(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await resolve_pipeline("memory:a|wrap:", storage_options={"anon": True}) + + async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: + # file://localhost/p and file:///p are equivalent to file:/p + register_url_adapter("wrap", WrapperAdapter) + # spell the path URL-style: on Windows C:/... gains a leading slash + posix = tmp_path.as_posix() + url_path = posix if posix.startswith("/") else f"/{posix}" + zarr.open_group(f"file://localhost{url_path}|wrap:", mode="w") + opened = zarr.open_group(f"file:{tmp_path}|wrap:", mode="r") + assert isinstance(opened, zarr.Group) + zarr.open_group(f"file://{url_path}|wrap:", mode="r") + + async def test_file_root_windows_drive_spellings( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Windows drive paths count as absolute, and the URL spelling + # file:///C:/... has its leading slash stripped before the drive + register_url_adapter("wrap", WrapperAdapter) + if sys.platform == "win32": + drive_path = tmp_path.as_posix() # C:/Users/... + else: + # off Windows a drive path is just an odd relative directory; + # chdir into tmp so it is created there + monkeypatch.chdir(tmp_path) + drive_path = "C:/drive/data" + r1 = await resolve_pipeline(f"file:{drive_path}|wrap:") + r2 = await resolve_pipeline(f"file:///{drive_path}|wrap:") + assert isinstance(r1.store, TracingStore) + assert isinstance(r2.store, TracingStore) + + async def test_file_root_rejects_other_authority(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="unsupported authority"): + await resolve_pipeline("file://example.com/tmp/x|wrap:") + + @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) + async def test_file_root_rejects_relative_paths(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="absolute path"): + await resolve_pipeline(f"{root}|wrap:") + + async def test_file_root_rejects_query(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + + +class TestMakeStoreIntegration: + async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + store_path = await make_store_path(f"{tmp_path}|wrap:residual", path="user/sub") + assert store_path.path == "residual/user/sub" + + async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store(f"{tmp_path}|wrap:residual") + + async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: + # an adapter returning path="/" addresses the store root; the raw + # value is normalized before the residual-path rejection + class SlashRootAdapter(WrapperAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, path="/") + + register_url_adapter("slashy", SlashRootAdapter) + store = await make_store(f"{tmp_path}|slashy:") + assert store is not None + + async def test_make_store_closes_store_on_residual_path_error(self) -> None: + closed: list[bool] = [] + + class ClosingStore(MemoryStore): + def close(self) -> None: + closed.append(True) + super().close() + + class LeakProbe(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await ClosingStore.open(), path="residual") + + register_url_adapter("leaky", LeakProbe) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store("memory://base|leaky:") + assert closed == [True] + + async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + store = await make_store(f"{tmp_path}|wrap:") + assert isinstance(store, TracingStore) + + async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") + array = group.create_array("x", shape=(4,), dtype="i4") + array[:] = [1, 2, 3, 4] + assert zarr.open_array(f"{tmp_path}|wrap:x")[2] == 3 + + async def test_pipeline_store_read_only_mode(self) -> None: + register_url_adapter("native", NativeAdapter) + store_path = await make_store_path("memory://base|native:", mode="r") + assert store_path.read_only + + async def test_mode_a_downgrades_to_read_only_open(self) -> None: + # mode "a" (the zarr.open default) is open-or-create: a pipeline + # that resolves to a read-only store serves the "open" half instead + # of failing outright. + register_url_adapter("rooty", RootAdapter) + store_path = await make_store_path("rooty://org/repo", mode="a") + assert store_path.read_only + + async def test_mode_a_downgrade_applies_to_plain_stores_too(self) -> None: + # the open-or-create downgrade lives in StorePath.open, so it is + # uniform across pipeline and non-pipeline stores + store = await MemoryStore.open(read_only=True) + store_path = await make_store_path(store, mode="a") + assert store_path.read_only + + async def test_explicit_write_mode_reaches_adapter(self) -> None: + register_url_adapter("rooty", RootAdapter) + with pytest.raises(ValueError, match="read-only scheme"): + await make_store_path("rooty://org/repo", mode="w") + + async def test_storage_options_forwarded_to_root(self) -> None: + # storage_options reach the root sub-URL via resolve_preceding -> + # make_store. A local-path root does not accept storage_options, so + # forwarding them must raise the same TypeError as a non-pipeline open. + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await make_store("/tmp/some/path|wrap:", storage_options={"anon": True}) + + async def test_store_path_truediv_keeps_format(self, tmp_path: Path) -> None: + # StorePath.__truediv__ constructs a new instance; the pipeline- + # selected format must survive the division + register_url_adapter("fmt2", FormatAdapter) + store_path = await make_store_path(f"{tmp_path}|fmt2:") + assert store_path.zarr_format == 2 + assert (store_path / "sub/node").zarr_format == 2 + + async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w") + assert group.metadata.zarr_format == 2 + + async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=3) + + async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 diff --git a/tests/test_url_pipeline/test_spec_examples.py b/tests/test_url_pipeline/test_spec_examples.py new file mode 100644 index 0000000000..5538074bea --- /dev/null +++ b/tests/test_url_pipeline/test_spec_examples.py @@ -0,0 +1,258 @@ +""" +Conformance tests against the URL pipeline specification. + +Every entry in `SPEC_EXAMPLES` is an example URL from the specification +repository (https://github.com/jbms/url-pipeline @ 1a01ce6), extracted +with the same rule as the spec's own linter (`scripts/lint.py`): backticked +examples on bullet lines, skipping spans without a `:` or `|`. The spec +validates each example (and an uppercased-scheme variant) against its ABNF +grammar, so this corpus is grammar-valid by construction. + +The parser must accept every example — including schemes zarr-python has no +adapter for, since parsing is independent of adapter availability — split it +into the expected sub-URL schemes, and preserve the text losslessly. + +To regenerate after a spec update: extract examples per the rule above and +recompute the expected scheme tuples with `parse_pipeline`, reviewing any +changes against the spec diff. +""" + +from __future__ import annotations + +import re + +import pytest + +from zarr.storage._url_pipeline import parse_pipeline + +SPEC_EXAMPLES: list[tuple[str, tuple[str, ...]]] = [ + # README.md + ("s3://bucket/path/to/archive.zip|zip:path/within/zip.zarr/|zarr3:", ("s3", "zip", "zarr3")), + ( + "file:///tmp/dataset.ocdbt/|ocdbt://2025-01-01T01:23:45.678Z/path/within/database", + ("file", "ocdbt"), + ), + ( + "s3+https://example.com/path/to/database.icechunk/|icechunk://tag.v5/path/to/node/|zarr3:", + ("s3+https", "icechunk", "zarr3"), + ), + ( + "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x", + ("vendor1-2.custom-1+proto.ext",), + ), + ( + "http://example.com/file|vendor1-2.custom+adapter:/path/within/adapter", + ("http", "vendor1-2.custom+adapter"), + ), + ("a.b:?", ("a.b",)), + ("http://somehost/downloads/somefile.zip|zip:", ("http", "zip")), + ("http://example.com/archive.jar|zip:path/to/file.txt", ("http", "zip")), + ("https://host/archive.zip|zip:path/in/outer.zip|zip:path/in/inner", ("https", "zip", "zip")), + ("file:///path/to/archive.zip|zip:path/within/archive", ("file", "zip")), + # avif.md + ("file:/path/to/image.avif|avif:", ("file", "avif")), + ("file:/path/to/image.avif|avif", ("file", "avif")), + # bmp.md + ("file:/path/to/image.bmp|bmp:", ("file", "bmp")), + ("file:/path/to/image.bmp|bmp", ("file", "bmp")), + # byte-range.md + ("file:/path/to/data|byte-range:1000-2000", ("file", "byte-range")), + ("file:/path/to/data|byte-range:0-1", ("file", "byte-range")), + # file.md + ("file:/", ("file",)), + ("file:/a", ("file",)), + ("file:/path/to/file.txt", ("file",)), + ("file://localhost/path/to/file.txt", ("file",)), + ("file://LOCALHOST/path/to/file.txt", ("file",)), + ("file:///path/to/file.txt", ("file",)), + ("file://somehost/sharename/path/to/file.txt", ("file",)), + # gs.md + ("gs://bucket", ("gs",)), + ("gs://bucket/", ("gs",)), + ("gs://bucket/path/within/bucket", ("gs",)), + ("gs://aaa", ("gs",)), + ( + "gs://label1-is-sixty-two-characters-long-xxxxxxxxxxxxxxxxxxxxxxxxxx.label2-is-sixty-two-characters-long-yyyyyyyyyyyyyyyyyyyyyyyyyy.label3-is-sixty-two-characters-long-zzzzzzzzzzzzzzzzzzzzzzzzzz.label4-is-thirty-one-characters", + ("gs",), + ), + # gzip.md + ("file:/path/to/data.gz|gzip:", ("file", "gzip")), + ("file:/path/to/data.gz|gzip", ("file", "gzip")), + # hdf5.md + ("s3://bucket/path/to/file.h5|hdf5:/path/to/dataset", ("s3", "hdf5")), + ("s3://bucket/path/to/file.h5|hdf5:a", ("s3", "hdf5")), + ("file:///path/to/file.h5|hdf5:", ("file", "hdf5")), + ("file:///path/to/file.h5|hdf5", ("file", "hdf5")), + # http.md + ("https://example.com/path/to/resource%20name", ("https",)), + ("https://example.com/path/to/resource?query=value", ("https",)), + ("https://example.com/path/to/?query=value", ("https",)), + ("https://example.com/path/with:colon", ("https",)), + ("https://server.example.com:1234/path/to/array", ("https",)), + ("http://a", ("http",)), + ("http://example.com", ("http",)), + ("http://local%68ost", ("http",)), + ("http://example.com:", ("http",)), + ("http://192.168.10.1:1234", ("http",)), + ("http://[::1]", ("http",)), + ("http://[::ffff:127.0.0.1]", ("http",)), + ("http://example.com/", ("http",)), + # icechunk.md + ("file:///path/to/repo.zarr.icechunk/|icechunk:", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk://branch.main/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:a", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:/path/to/node/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/zarr.json", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/c/0/0/1", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://branch.mybranch/path/to/node/", + ("file", "icechunk"), + ), + ("file:///path/to/repo.zarr.icechunk/|icechunk://branch.a", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://tag.mytag/path/to/node/", + ("file", "icechunk"), + ), + ("file:///path/to/repo.zarr.icechunk/|icechunk://tag.a", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://FWWFQGAW742XMX0F5MF0/path/to/node/", + ("file", "icechunk"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://ABCDEFGHJKMNPQRSTVWX/path/to/node/", + ("file", "icechunk"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk:|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://branch.other/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://tag.v5/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://4N0217AZA4VNPYD0HR0G/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + # jpeg.md + ("file:/path/to/image.jpeg|jpeg:", ("file", "jpeg")), + ("file:/path/to/image.jpeg|jpeg", ("file", "jpeg")), + # json.md + ("file:/path/to/data.json|json:", ("file", "json")), + ("file:/path/to/data.json|json", ("file", "json")), + ("file:/path/to/data.json|json:/path/to/node", ("file", "json")), + ("file:/path/to/data.json|json:/abc~0def", ("file", "json")), + ("file:/path/to/data.json|json:/abc~1def", ("file", "json")), + ("file:/path/to/data.json|json:/", ("file", "json")), + # memory.md + ("memory:", ("memory",)), + ("memory:/", ("memory",)), + ("memory://", ("memory",)), + ("memory:path/to/resource", ("memory",)), + ("memory:a", ("memory",)), + ("memory:another@path+with,lots&of(special;characters)*_!-$'", ("memory",)), + ("memory:/path/to/resource", ("memory",)), + ("memory://path/to/resource", ("memory",)), + # n5.md + ("file:///tmp/data.n5/|n5:path/to/array", ("file", "n5")), + ("file:///tmp/data.n5/|n5:a", ("file", "n5")), + ("file:///tmp/data.n5/|n5:/path/to/array", ("file", "n5")), + ("file:///tmp/data.n5/|n5:", ("file", "n5")), + ("file:///tmp/data.n5/|n5", ("file", "n5")), + # neuroglancer-precomputed.md + ( + "file:///tmp/dataset.precomputed/|neuroglancer-precomputed:", + ("file", "neuroglancer-precomputed"), + ), + ( + "file:///tmp/dataset.precomputed/|neuroglancer-precomputed", + ("file", "neuroglancer-precomputed"), + ), + # ocdbt.md + ("file:///path/to/repo.ocdbt/|ocdbt:", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:path/within/repo/", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:path/within/repo", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:a", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:/path/within/repo", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v123/", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v1", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v123/path/within/repo", ("file", "ocdbt")), + ( + "file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.678Z/path/within/database", + ("file", "ocdbt"), + ), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45Z", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.1Z", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.123456789Z", ("file", "ocdbt")), + # png.md + ("file:/path/to/image.png|png:", ("file", "png")), + ("file:/path/to/image.png|png", ("file", "png")), + # s3+http.md + ("s3+https://endpoint/path/within/bucket", ("s3+https",)), + ("s3+https://endpoint/bucket/path/within/bucket", ("s3+https",)), + ("s3+https://mybucket.s3.amazonaws.com/path/to/file", ("s3+https",)), + ("s3+https://s3.amazonaws.com/mybucket/path/to/file", ("s3+https",)), + ("s3+http://example.com", ("s3+http",)), + ("s3+http://example.com/", ("s3+http",)), + # s3.md + ("s3://bucket", ("s3",)), + ("s3://bucket/", ("s3",)), + ("s3://bucket/path/within/bucket", ("s3",)), + ("s3://aaa", ("s3",)), + ( + "s3://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ("s3",), + ), + # tiff.md + ("file:/path/to/image.tiff|tiff:", ("file", "tiff")), + ("file:/path/to/image.tiff|tiff", ("file", "tiff")), + # webp.md + ("file:/path/to/image.webp|webp:", ("file", "webp")), + ("file:/path/to/image.webp|webp", ("file", "webp")), + # zarr.md + ("file:///path/to/node.zarr/|zarr3:", ("file", "zarr3")), + ("file:///path/to/node.zarr/|zarr3", ("file", "zarr3")), + ("file:///path/to/node.zarr/|zarr2:", ("file", "zarr2")), + ("file:///path/to/node.zarr/|zarr2", ("file", "zarr2")), + ("file:///path/to/node.zarr/|zarr:", ("file", "zarr")), + ("file:///path/to/node.zarr/|zarr", ("file", "zarr")), + # zip.md + ("file:/path/to/archive.zip|zip:path/to/file.txt", ("file", "zip")), + ("file:/path/to/archive.zip|zip", ("file", "zip")), + ("file:/path/to/archive.zip|zip:/path/to/file.txt", ("file", "zip")), + ("file:/path/to/outer.zip|zip:path/to/inner.zip|zip:path/to/file.txt", ("file", "zip", "zip")), + # zstd.md + ("file:/path/to/data.zstd|zstd:", ("file", "zstd")), + ("file:/path/to/data.zstd|zstd", ("file", "zstd")), +] + + +def _uppercase_schemes(example: str) -> str: + """Uppercase every sub-URL scheme, as the spec's linter does.""" + + def repl(match: re.Match[str]) -> str: + return match.group(1) + match.group(2).upper() + + return re.sub(r"((?:^|\|)\s*)([a-zA-Z][a-zA-Z0-9+.-]*)", repl, example) + + +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> None: + segments = parse_pipeline(url) + assert tuple(segment.scheme for segment in segments) == expected_schemes + # lossless round-trip of the original text + assert "|".join(segment.raw for segment in segments) == url + + +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +def test_spec_example_schemes_case_insensitive(url: str, expected_schemes: tuple[str, ...]) -> None: + upper = _uppercase_schemes(url) + segments = parse_pipeline(upper) + assert tuple(segment.scheme for segment in segments) == expected_schemes From 581c5349403610d7062983cd0d54f35269bd68b0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 Sep 2026 19:39:56 -0700 Subject: [PATCH 02/14] polish: validate mode before pipeline resolution, document adapter authoring - make_store validates the access mode before routing a URL pipeline string to adapters, matching every other StoreLike branch - make_store docstring lists URL pipeline strings; StorePath.open documents the invalid-mode ValueError - user guide: StoreLike bullet for pipeline strings and an adapter authoring section in extending.md (runnable example) - changelog: drop PR-relative wording, name the public parser entry points Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 7 ++-- docs/user-guide/extending.md | 48 ++++++++++++++++++++++++ docs/user-guide/storage.md | 3 ++ src/zarr/storage/_common.py | 14 ++++--- tests/test_url_pipeline/test_resolver.py | 7 ++++ 5 files changed, 71 insertions(+), 8 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 757aa13895..30f11a559d 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,11 +1,12 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. This PR adds the parser, the single-method -`zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the +`s3://bucket/data.zip|zip:|zarr3:`. This includes the parser +(`zarr.storage.parse_pipeline` / `zarr.storage.resolve_pipeline`), the +single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the `zarr.url_adapters` entry-point group through which third-party packages (e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme are loaded lazily and individually. Builtin adapters (`zip:`, -`zarr2:`/`zarr3:`) follow in separate pull requests. +`zarr2:`/`zarr3:`) will follow separately. Behavior notes: diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 507afedea7..db5769d348 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -85,6 +85,54 @@ implementation. Custom stores can be created by implementing the [`zarr.abc.store.Store`][] interface. See [developing custom stores](storage.md#developing-custom-stores) for more information. +## Custom URL pipeline adapters + +[URL pipelines](storage.md#user-guide-url-pipelines) are `|`-chained URLs such as +`s3://bucket/data.zip|zip:|zarr3:`. Each sub-URL after the first names an *adapter* +that reinterprets everything to its left. A package provides an adapter for a scheme by +subclassing [`zarr.abc.url_pipeline.URLPipelineAdapter`][] and implementing its single +classmethod, which resolves the adapter's segment into an open store and a residual path: + +```python test="true" session="url-adapter" +import dataclasses + +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.storage import WrapperStore + + +class MyAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + # A *wrapper* adapter opens the resource to its left and wraps it. Carry the + # preceding residual path forward and keep unchanged fields via dataclasses.replace. + preceding = await context.resolve_preceding() + store = WrapperStore(preceding.store) + return dataclasses.replace(preceding, store=store, path=segment.body) +``` + +Register the class under the `zarr.url_adapters` entry-point group, using the URL +scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed +(`vendor.scheme`), per the URL pipeline specification: + +```toml +[project.entry-points."zarr.url_adapters"] +"mypackage.myscheme" = "mypackage.zarr_adapter:MyAdapter" +``` + +Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An +adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in +which case `context.preceding` is empty. The returned store must honor +`context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so +it must not block or call zarr's synchronous API. See +[`zarr.abc.url_pipeline`][] for the full contract. + ## Custom array buffers Zarr-python provides control over where and how arrays are stored in memory through diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 2e84f90ed7..d572c5bc1d 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -100,6 +100,9 @@ print(group) group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r') ``` +- a [URL pipeline](#user-guide-url-pipelines) string containing `|`, such as + `s3://bucket/data.zip|zip:`, which is resolved through registered adapters. + - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 7a93141c60..f6cdfee8d7 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -112,7 +112,8 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is "r+", "w-", or "w" and the store is read-only, or + If the mode is "r+", "w-", or "w" and the store is read-only, or if the + mode is not a valid access mode. """ # fastpath if mode is None @@ -342,6 +343,9 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. + - `str` containing `|` (or whose scheme has a registered root adapter) = the store + resolved by the [URL pipeline][user-guide-url-pipelines]; the pipeline must address + the store root (use `make_store_path` for pipelines that address a path within a store). Parameters ---------- @@ -368,6 +372,10 @@ async def make_store( """ from zarr.storage._fsspec import FsspecStore # circular import + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") + _read_only = mode == "r" + if isinstance(store_like, str) and is_url_pipeline(store_like): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) if normalize_path(result.path): @@ -390,10 +398,6 @@ async def make_store( "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) - if mode is not None and mode not in ANY_ACCESS_MODE: - raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") - _read_only = mode == "r" - if isinstance(store_like, StorePath): # Get underlying store return store_like.store diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 198c178764..75582cdc75 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -505,6 +505,13 @@ async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: store = await make_store(f"{tmp_path}|wrap:") assert isinstance(store, TracingStore) + async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: + # an invalid mode is rejected upfront, like every other StoreLike, + # rather than being handed to adapters + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(ValueError, match="Invalid mode"): + await make_store(f"{tmp_path}|wrap:", mode="invalid") # type: ignore[arg-type] + async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") From 3ed2bb8835aa89488ed5af2a288585f687a5586a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:39 -0700 Subject: [PATCH 03/14] fix: load URL adapter entry points outside the registry lock get_url_adapter took a non-reentrant lock across entry_point.load(), so an adapter module that resolved another scheme at import time deadlocked the process. Matching entry points are now taken off the pending list under the lock and imported outside it; import failures are wrapped in URLPipelineError and leave the entry point discoverable for a retry. A same-named entry point is discarded with a ZarrUserWarning when the scheme is already registered (e.g. by a builtin), instead of staying pending forever; duplicate entry-point names warn and the first wins. Co-Authored-By: Claude Fable 5.1 --- src/zarr/registry.py | 67 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 3487f87268..990ee51572 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -554,27 +554,68 @@ def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: Get the URL pipeline adapter class registered for `scheme`. Loads pending `zarr.url_adapters` entry points for this scheme only, so - resolving one scheme never imports other providers' packages. + resolving one scheme never imports other providers' packages. When the + scheme is already registered (e.g. a builtin adapter), a same-named entry + point is discarded with a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; + when several entry points share the name, the first one wins and a + warning names the ones ignored. + + Raises + ------ + URLPipelineError + If no adapter is registered for the scheme, or if the entry point + providing it fails to import. """ key = scheme.lower() - # The lock keeps concurrent first-time resolutions of different schemes - # from clobbering each other's rebuild of the pending entry-point list. + # Take the matching entry points out of the pending list under the lock, + # so concurrent first-time resolutions of different schemes never clobber + # each other's view of the list. The import itself runs *outside* the lock: + # it may take arbitrarily long and may itself resolve URL adapters. with _url_adapter_lock: - if key not in _url_adapter_registry: - remaining = [] - for entry_point in _url_adapter_registry.lazy_load_list: - if entry_point.name.lower() == key: - _url_adapter_registry.register(entry_point.load(), qualname=key) - else: - remaining.append(entry_point) - _url_adapter_registry.lazy_load_list[:] = remaining + registered = _url_adapter_registry.get(key) + pending = [e for e in _url_adapter_registry.lazy_load_list if e.name.lower() == key] + if pending: + _url_adapter_registry.lazy_load_list[:] = [ + e for e in _url_adapter_registry.lazy_load_list if e.name.lower() != key + ] + if pending and registered is not None: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is already registered " + f"({fully_qualified_name(registered)}); ignoring entry point(s) " + f"{[e.value for e in pending]} of the same name", + category=ZarrUserWarning, + stacklevel=2, + ) + elif pending: + entry_point, *duplicates = pending + if duplicates: + warnings.warn( + f"multiple 'zarr.url_adapters' entry points are named {entry_point.name!r}; " + f"using {entry_point.value!r} and ignoring {[e.value for e in duplicates]}", + category=ZarrUserWarning, + stacklevel=2, + ) + try: + cls = entry_point.load() + except Exception as exc: + # leave the entry points discoverable so a transient import + # failure is not permanent + with _url_adapter_lock: + _url_adapter_registry.lazy_load_list.extend(pending) + raise URLPipelineError( + f"the 'zarr.url_adapters' entry point {entry_point.name!r} " + f"({entry_point.value!r}) could not be loaded: {exc!r}" + ) from exc + with _url_adapter_lock: + if key not in _url_adapter_registry: + _url_adapter_registry.register(cls, qualname=key) try: return _url_adapter_registry[key] except KeyError: - registered = sorted(list_url_adapter_schemes()) + registered_schemes = sorted(list_url_adapter_schemes()) raise URLPipelineError( f"no URL pipeline adapter is registered for scheme {scheme!r}. " - f"Registered schemes: {registered}. Adapters are provided by " + f"Registered schemes: {registered_schemes}. Adapters are provided by " "packages via the 'zarr.url_adapters' entry-point group." ) from None From 9bf86cebb333b837002bd7c77607cfb4ad14e205 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:46 -0700 Subject: [PATCH 04/14] fix: close the adapter's store when read-only conversion fails; document contracts - The resolver closes the store an adapter returned when it cannot be made read-only for mode 'r', instead of leaking it. - resolve_pipeline documents every error type it lets through; the resolve_preceding docstring states the local-file-root limitation. - AdapterResolution and PipelineContext compare and hash by identity, since a Store is not hashable and frozen=True implied otherwise. Co-Authored-By: Claude Fable 5.1 --- src/zarr/abc/url_pipeline.py | 16 +++++++++++++--- src/zarr/storage/_url_pipeline.py | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index 0e0363bcf6..337563685d 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -73,11 +73,14 @@ def __str__(self) -> str: return self.raw -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class AdapterResolution: """ The result of resolving a URL pipeline (or a prefix of one). + Instances are immutable but compare and hash by identity, since a + [`Store`][zarr.abc.store.Store] is not hashable. + Attributes ---------- store : Store @@ -99,12 +102,14 @@ class AdapterResolution: zarr_format: ZarrFormat | None = None -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class PipelineContext: """ Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] describing the pipeline to the left of its segment. + Instances are immutable but compare and hash by identity. + Attributes ---------- preceding : tuple[PipelineSegment, ...] @@ -186,7 +191,12 @@ async def resolve_preceding( Wrapper adapters that only read the preceding resource should pass `mode="r"` so the root is opened read-only and without create-on-open side effects, regardless of the caller's mode. - When omitted, the caller's mode is used. + When omitted, the caller's mode is used. Note that a local + *file* root (e.g. `data.zip|zip:`) can currently only be + resolved in mode `"r"`: every other mode creates a directory at + the root and fails with `FileExistsError`, because there is no + file-resource primitive yet. Writing *through* a container file + is therefore not supported by wrapper adapters at this time. storage_options : dict | None, optional Override the options forwarded to the preceding pipeline. An adapter that consumed adapter-specific keys should pass the diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 0161bdee6a..41a2114daf 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -183,6 +183,19 @@ async def resolve_pipeline( Options forwarded to the root sub-URL's store (and visible to adapters via the context). Non-dict forms are reserved for future per-segment configuration (one mapping per pipeline segment). + + Raises + ------ + URLPipelineError + If the URL cannot be parsed, is not a pipeline, names a scheme with + no registered adapter, names an adapter entry point that fails to + import, or has a root sub-URL that cannot be resolved. + TypeError + If `storage_options` are passed to a root that does not accept them + (local paths and `memory:`), as for non-pipeline stores. + OSError + Errors from opening the root resource (e.g. a missing local directory + in mode `"r"`) propagate unchanged, as for non-pipeline stores. """ segments = parse_pipeline(url) if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): @@ -225,13 +238,17 @@ async def _resolve( # the adapter to have honored context.read_only. try: read_only_store = resolution.store.with_read_only(True) + await read_only_store._ensure_open() except NotImplementedError as exc: + resolution.store.close() raise URLPipelineError( f"adapter {last.scheme!r} returned a writable store for mode 'r', " "and the store does not support read-only conversion via " ".with_read_only()" ) from exc - await read_only_store._ensure_open() + except Exception: + resolution.store.close() + raise resolution = dataclasses.replace(resolution, store=read_only_store) return resolution From c1499610ccdc3720be491e539a02737852cfad3f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:49 -0700 Subject: [PATCH 05/14] fix: honor the pipeline-selected zarr format at every make_store_path caller The zarr2:/zarr3: format carried on StorePath.zarr_format was merged only in zarr.api.asynchronous; create_array, from_array, Group.from_store, Group.open, Array.open and the deprecated AsyncArray._create dropped it and swallowed explicit conflicts. The merge now lives on StorePath.resolve_zarr_format and is applied at each site. To let the pipeline's format apply when the caller does not specify one, create_array (sync and async) and Group.from_store default zarr_format to None, resolving to the configured default (3) as before; Array.open gains a zarr_format parameter mirroring Group.open. Tests: registry deadlock/race/import-failure/shadowing, resolver leak, zarr_format merge at each core site, and pins for the documented memory:/ file: divergences and the local-file-root limitation (xfail, strict). Co-Authored-By: Claude Fable 5.1 --- src/zarr/api/asynchronous.py | 32 +-- src/zarr/api/synchronous.py | 2 +- src/zarr/core/array.py | 14 +- src/zarr/core/group.py | 14 +- src/zarr/storage/_common.py | 33 +++ tests/test_api.py | 2 +- tests/test_url_pipeline/test_resolver.py | 297 ++++++++++++++++++++++- 7 files changed, 359 insertions(+), 35 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 0648769727..c471934b6f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -108,24 +108,6 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool: return mode in _OVERWRITE_MODES -def _merge_pipeline_zarr_format( - store_path: StorePath, zarr_format: ZarrFormat | None -) -> ZarrFormat | None: - """ - Combine a zarr format selected by a URL pipeline segment (`zarr2:` / - `zarr3:`) with the caller's `zarr_format` argument. Explicitly - conflicting selections raise. - """ - if store_path.zarr_format is None: - return zarr_format - if zarr_format is not None and zarr_format != store_path.zarr_format: - raise ValueError( - f"zarr_format={zarr_format} conflicts with the 'zarr{store_path.zarr_format}:' " - "segment of the URL pipeline" - ) - return store_path.zarr_format - - def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ Emit a "not yet implemented" warning for each provided keyword argument that is not None. @@ -407,7 +389,7 @@ async def open( else: mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in {"a", "r", "r+", "w"}: @@ -518,7 +500,7 @@ async def save_array( mode = kwargs.pop("mode", "a") store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() if np.isscalar(arr): @@ -570,7 +552,7 @@ async def save_group( """ store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -786,7 +768,7 @@ async def create_group( mode: Literal["a"] = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -878,7 +860,7 @@ async def open_group( ) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if attributes is None: attributes = {} @@ -1082,7 +1064,7 @@ async def create( if mode is None: mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -1284,7 +1266,7 @@ async def open_array( mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 6975f6d953..4532f4ad54 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -829,7 +829,7 @@ def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 5a8d6bf57e..2ad99791be 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -453,8 +453,9 @@ async def _create( Deprecated in favor of [`zarr.api.asynchronous.create_array`][]. """ - dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) or zarr_format + dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) shape = parse_shapelike(shape) @@ -813,6 +814,7 @@ async def example(): ``` """ store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) @@ -2003,6 +2005,7 @@ def from_dict( def open( cls, store: StoreLike, + zarr_format: ZarrFormat | None = 3, ) -> Self: """Opens an existing Array from a store. @@ -2012,13 +2015,16 @@ def open( Store containing the Array. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. + zarr_format : {2, 3, None}, optional + The zarr format to expect. If None, the format selected by a URL pipeline + segment is used, otherwise the format is inferred from the store contents. Returns ------- Array Array opened from the store. """ - async_array = sync(AsyncArray.open(store)) + async_array = sync(AsyncArray.open(store, zarr_format=zarr_format)) return cls(async_array) @property @@ -4337,6 +4343,7 @@ async def from_array( mode: Literal["a"] = "a" config_parsed = parse_array_config(config) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) ( chunks, @@ -4663,7 +4670,7 @@ async def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, @@ -4826,6 +4833,7 @@ async def create_array( store_path = await make_store_path( store, path=name, mode=mode, storage_options=storage_options ) + zarr_format = store_path.resolve_zarr_format(zarr_format) return await init_array( store_path=store_path, shape=shape_parsed, diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index d734e6b7cd..e46927d678 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -42,6 +42,7 @@ NodeType, ShapeLike, ZarrFormat, + _default_zarr_format, parse_shapelike, ) from zarr.core.config import config @@ -474,9 +475,12 @@ async def from_store( *, attributes: dict[str, Any] | None = None, overwrite: bool = False, - zarr_format: ZarrFormat = 3, + zarr_format: ZarrFormat | None = None, ) -> AsyncGroup: store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() if overwrite: if store_path.store.supports_deletes: @@ -525,6 +529,7 @@ async def open( to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: @@ -1816,7 +1821,7 @@ def from_store( store: StoreLike, *, attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, + zarr_format: ZarrFormat | None = None, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. @@ -1829,8 +1834,9 @@ def from_store( for a description of all valid StoreLike values. attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. - zarr_format : {2, 3}, optional - Zarr storage format version. + zarr_format : {2, 3, None}, optional + Zarr storage format version. If None, the format selected by a URL + pipeline segment is used, falling back to the configured default. overwrite : bool, optional If True, do not raise an error if the group already exists. diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index f6cdfee8d7..fa470e1e3b 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -74,6 +74,36 @@ def __init__(self, store: Store, path: str = "") -> None: def read_only(self) -> bool: return self.store.read_only + def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: + """ + Combine the zarr format selected by a URL pipeline segment (`zarr2:` / + `zarr3:`) with a caller-supplied `zarr_format`. + + Parameters + ---------- + zarr_format : ZarrFormat | None + The format requested by the caller, or None when unspecified. + + Returns + ------- + ZarrFormat | None + The pipeline's format when the caller did not specify one, otherwise the + caller's format. None when neither is set. + + Raises + ------ + ValueError + If the caller's format differs from the one selected by the pipeline. + """ + if self.zarr_format is None: + return zarr_format + if zarr_format is not None and zarr_format != self.zarr_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{self.zarr_format}:' " + "segment of the URL pipeline; pass zarr_format=None to use the pipeline's format" + ) + return self.zarr_format + @classmethod async def _create_open_instance(cls, store: Store, path: str) -> Self: """Helper to create and return a StorePath instance.""" @@ -492,6 +522,9 @@ async def make_store_path( If the StoreLike object is not one of the supported types, or if storage_options is provided but not used. ValueError If path is provided for a store that does not support it. + URLPipelineError + If `store_like` is a [URL pipeline][user-guide-url-pipelines] string that + cannot be resolved. See Also -------- diff --git a/tests/test_api.py b/tests/test_api.py index c4e561c180..294f245e9b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -195,7 +195,7 @@ async def test_open_like_creates_array_by_default( assert np.all(Array(new_arr)[:] == ref_arr.fill_value) -async def test_open_like_default_mode_rejects_read_only_store( +async def test_open_like_default_mode_on_read_only_store_raises_not_found( zarr_format: ZarrFormat, ) -> None: # mode "a" (open-or-create) on a read-only store serves the "open" half: diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 75582cdc75..66b9246a48 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -2,8 +2,12 @@ import dataclasses import sys +import threading +import warnings +from importlib.metadata import EntryPoint from typing import TYPE_CHECKING, ClassVar +import numpy as np import pytest import zarr @@ -22,13 +26,15 @@ register_url_adapter, ) from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore -from zarr.storage._common import make_store, make_store_path +from zarr.storage._common import _has_fsspec, make_store, make_store_path from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import _join_paths if TYPE_CHECKING: from pathlib import Path + from zarr.core.common import AccessModeLiteral + pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") @@ -159,6 +165,152 @@ def test_loading_one_scheme_leaves_others_pending(self) -> None: ) +def _fake_entry_point( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, scheme: str, module: str, source: str +) -> EntryPoint: + """ + Write `source` to an importable module and return a `zarr.url_adapters` + entry point for `scheme` pointing at its `Adapter` attribute. + """ + (tmp_path / f"{module}.py").write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, module, raising=False) + return EntryPoint(name=scheme, value=f"{module}:Adapter", group="zarr.url_adapters") + + +_ADAPTER_SOURCE = """ +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.storage import MemoryStore +{prelude} + +class Adapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment(cls, segment, context): + return AdapterResolution(store=await MemoryStore.open(read_only=context.read_only)) +""" + + +class TestEntryPointLoading: + """Lazy loading of `zarr.url_adapters` entry points.""" + + def test_adapter_import_may_resolve_other_adapters( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # an adapter module that itself resolves a URL adapter at import + # time must not deadlock on the registry lock + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="reentrant", + module="reentrant_adapter", + source=_ADAPTER_SOURCE.format( + prelude=( + "import zarr.registry\n" + "try:\n" + " zarr.registry.get_url_adapter('no-such-scheme-at-import')\n" + "except Exception:\n" + " pass\n" + ) + ), + ) + ) + result: list[object] = [] + thread = threading.Thread(target=lambda: result.append(get_url_adapter("reentrant"))) + thread.start() + thread.join(timeout=10) + assert not thread.is_alive(), "get_url_adapter deadlocked on a re-entrant lookup" + assert getattr(result[0], "__name__", None) == "Adapter" + + def test_concurrent_loads_keep_other_schemes_pending( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # two threads resolving different schemes at once, one with a slow + # import, must not drop a third, unrelated pending entry point + pending = zarr.registry._url_adapter_registry.lazy_load_list + for scheme, prelude in [ + ("slow", "import time\ntime.sleep(0.3)\n"), + ("quick", ""), + ("untouched", ""), + ]: + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme=scheme, + module=f"{scheme}_adapter", + source=_ADAPTER_SOURCE.format(prelude=prelude), + ) + ) + threads = [ + threading.Thread(target=get_url_adapter, args=(scheme,)) for scheme in ("slow", "quick") + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + assert not any(thread.is_alive() for thread in threads) + assert "untouched" in list_url_adapter_schemes() + assert [e.name for e in pending] == ["untouched"] + assert get_url_adapter("untouched").__name__ == "Adapter" + + def test_import_failure_is_wrapped_and_not_permanent(self) -> None: + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + EntryPoint(name="broken", value="no_such_module_xyz:Adapter", group="zarr.url_adapters") + ) + with pytest.raises(URLPipelineError, match="could not be loaded"): + get_url_adapter("broken") + # still advertised, so a transient failure can be retried + assert "broken" in list_url_adapter_schemes() + with pytest.raises(URLPipelineError, match="could not be loaded"): + get_url_adapter("broken") + + def test_registered_adapter_shadows_entry_point_with_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # a builtin registered under a scheme wins over a same-named entry + # point, which is discarded loudly rather than left pending forever + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="shadowed", + module="shadowed_adapter", + source=_ADAPTER_SOURCE.format(prelude="raise RuntimeError('must not import')\n"), + ) + ) + register_url_adapter("shadowed", NativeAdapter) + with pytest.warns(ZarrUserWarning, match="already registered"): + assert get_url_adapter("shadowed") is NativeAdapter + assert not any(e.name == "shadowed" for e in pending) + # a second lookup is silent + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert get_url_adapter("shadowed") is NativeAdapter + + def test_duplicate_entry_points_warn_and_first_wins( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + pending = zarr.registry._url_adapter_registry.lazy_load_list + for module in ("dup_first", "dup_second"): + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="dup", + module=module, + source=_ADAPTER_SOURCE.format(prelude=f"ORIGIN = {module!r}\n"), + ) + ) + with pytest.warns(ZarrUserWarning, match="multiple 'zarr.url_adapters' entry points"): + cls = get_url_adapter("dup") + assert getattr(sys.modules[cls.__module__], "ORIGIN", None) == "dup_first" + assert not any(e.name == "dup" for e in pending) + + class TestIsURLPipeline: def test_pipe_routes(self) -> None: assert is_url_pipeline("memory://x|demo:") @@ -203,6 +355,8 @@ async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: assert isinstance(result.store, TracingStore) assert result.path == "inner/path" assert result.store.context.preceding_url == str(tmp_path) + assert result.store.segment.scheme == "wrap" + assert result.store.segment.body == "inner/path" async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> None: # each wrapper joins the preceding residual path with its own, so @@ -274,6 +428,30 @@ async def open_pipeline_segment( with pytest.raises(URLPipelineError, match="does not support read-only conversion"): await resolve_pipeline("memory://base|stubborn:", mode="r") + async def test_read_only_enforcement_failure_closes_store(self) -> None: + # the adapter's store must not leak when it cannot be made read-only + closed: list[bool] = [] + + class StubbornStore(MemoryStore): + def with_read_only(self, read_only: bool = False) -> MemoryStore: + raise NotImplementedError + + def close(self) -> None: + closed.append(True) + super().close() + + class StubbornAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await StubbornStore.open(read_only=False)) + + register_url_adapter("stubborn", StubbornAdapter) + with pytest.raises(URLPipelineError): + await resolve_pipeline("memory://base|stubborn:", mode="r") + assert closed == [True] + async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: # a wrapper that only reads the preceding resource can open it # read-only regardless of the caller's mode @@ -290,6 +468,34 @@ async def open_pipeline_segment( result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") assert result.path == "x" + async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: + # a wrapper that opens the preceding local *file* read-only gets a + # store rooted at that file (the mechanism a read-only zip: adapter + # can build on) + target = tmp_path / "data.bin" + target.write_bytes(b"payload") + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{target}|wrap:", mode="r") + assert isinstance(result.store, TracingStore) + assert result.store.read_only + + @pytest.mark.parametrize("mode", [None, "a", "w", "r+", "w-"]) + @pytest.mark.xfail( + strict=True, + raises=FileExistsError, + reason=( + "known limitation: there is no file-resource primitive yet, so resolving a " + "local file root in a writable mode hits LocalStore's create-on-open mkdir" + ), + ) + async def test_wrapper_on_local_file_root_writable_modes( + self, tmp_path: Path, mode: AccessModeLiteral | None + ) -> None: + target = tmp_path / "data.bin" + target.write_bytes(b"payload") + register_url_adapter("wrap", WrapperAdapter) + await resolve_pipeline(f"{target}|wrap:", mode=mode) + async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before # the root store ever sees them @@ -453,6 +659,26 @@ async def test_file_root_rejects_query(self, tmp_path: Path) -> None: with pytest.raises(URLPipelineError, match="do not accept a query"): await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + @pytest.mark.skipif(not _has_fsspec, reason="plain memory:// routes to fsspec only with fsspec") + async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: + # documented divergence: a plain memory:// URL is fsspec's in-memory + # filesystem when fsspec is installed, while a pipeline's memory: root + # is zarr's ManagedMemoryStore; the two do not share data + register_url_adapter("wrap", WrapperAdapter) + plain = await make_store("memory://distinct-store") + piped = await make_store("memory://distinct-store|wrap:") + assert not isinstance(plain, ManagedMemoryStore) + assert isinstance(piped, TracingStore) + assert isinstance(piped._store, ManagedMemoryStore) + + async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: + # documented divergence from RFC 8089: consistent with LocalStore, no + # percent-escape is decoded, so %20 names a literal directory + register_url_adapter("wrap", WrapperAdapter) + await resolve_pipeline(f"file:{tmp_path.as_posix()}/a%20b|wrap:") + assert (tmp_path / "a%20b").is_dir() + assert not (tmp_path / "a b").exists() + class TestMakeStoreIntegration: async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: @@ -574,3 +800,72 @@ async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: register_url_adapter("fmt2", FormatAdapter) group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) assert group.metadata.zarr_format == 2 + + +class TestZarrFormatMergeInCore: + """ + Every make_store_path caller honors the pipeline-selected zarr format, not + only the top-level zarr.api functions. + """ + + @pytest.fixture(autouse=True) + def _fmt2(self) -> None: + register_url_adapter("fmt2", FormatAdapter) + + async def test_create_array(self, tmp_path: Path) -> None: + arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(f"{tmp_path}|fmt2:", name="y", shape=(2,), dtype="i4", zarr_format=3) + + async def test_create_array_from_data(self, tmp_path: Path) -> None: + # the data path of create_array goes through from_array + arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(f"{tmp_path}|fmt2:", name="y", data=np.arange(3), zarr_format=3) + + async def test_from_array(self, tmp_path: Path) -> None: + arr = zarr.from_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + assert arr.metadata.zarr_format == 2 + + async def test_group_from_store(self, tmp_path: Path) -> None: + group = zarr.Group.from_store(f"{tmp_path}|fmt2:") + assert group.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.from_store(f"{tmp_path}/other|fmt2:", zarr_format=3) + + async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: + # the default is still the configured format (3) for plain stores + group = zarr.Group.from_store(tmp_path) + assert group.metadata.zarr_format == 3 + + async def test_group_open(self, tmp_path: Path) -> None: + zarr.Group.from_store(f"{tmp_path}|fmt2:") + # explicit None defers to the pipeline; the default (3) conflicts + group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None) + assert group.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.open(f"{tmp_path}|fmt2:") + + async def test_array_open(self, tmp_path: Path) -> None: + zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Array.open(f"{tmp_path}/x|fmt2:") + + async def test_open_like(self, tmp_path: Path) -> None: + # open_like inherits v2 filters/compressor from the reference; the + # pipeline supplies the format, which open_array then creates with + ref = zarr.create_array({}, shape=(2,), dtype="i4", zarr_format=2) + arr = zarr.open_like(ref, store=f"{tmp_path}|fmt2:", path="x") + assert arr.metadata.zarr_format == 2 + + async def test_deprecated_async_array_create(self, tmp_path: Path) -> None: + from zarr.core.array import AsyncArray + + arr = await AsyncArray._create(f"{tmp_path}|fmt2:", shape=(2,), dtype="i4", zarr_format=2) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + await AsyncArray._create(f"{tmp_path}/o|fmt2:", shape=(2,), dtype="i4", zarr_format=3) From 60f14a0989b5d44ed5ed3e129a54df7996b373ab Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:54 -0700 Subject: [PATCH 06/14] docs: fix wrapper example path handling; document review follow-ups - The extending guide's wrapper example dropped the preceding residual path while its comment claimed the opposite; it now joins the two and a tested snippet shows root|a|b resolving to a/b. - Changelog and storage guide state the memory:// routing split with fsspec, the undecoded percent-escapes in file: roots, the local-file-root limitation, the entry-point collision warnings, and the zarr_format default changes. Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 24 ++++++++++++++++++++++-- docs/user-guide/extending.md | 22 +++++++++++++++++++--- docs/user-guide/storage.md | 6 ++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 30f11a559d..3ea14b30d6 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -17,8 +17,28 @@ Behavior notes: registered adapters cannot intercept zarr's native `file:`/`memory:` routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. - Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline - spec's semantics (spelling equivalences; `file:` must be absolute, with at - most a `localhost` authority). + spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must + carry an absolute path with at most a `localhost` authority). Percent-escapes + are *not* decoded in `file:` roots, matching `LocalStore`; this is a + documented divergence from RFC 8089. When fsspec is installed, a plain + `memory://name` URL is still routed to fsspec's in-memory filesystem, whereas + a pipeline's `memory:` root is zarr's `ManagedMemoryStore`; the two do not + share data. +- Wrapper adapters can open a local *file* root only in mode `"r"` (via + `PipelineContext.resolve_preceding(mode="r")`); other modes create a + directory at the root and fail. Writing through a container file is not yet + supported. +- A `zarr.url_adapters` entry point whose scheme is already registered (e.g. by + a builtin adapter) is ignored with a `ZarrUserWarning`, as are duplicate + entry-point names; an entry point that fails to import raises + `URLPipelineError` naming it, and stays discoverable for a retry. +- `zarr.create_array` and `Group.from_store` now default `zarr_format` to + `None`, resolving to the configured default (3) as before unless a URL + pipeline selects a format; `Array.open` gains a `zarr_format` parameter + mirroring `Group.open`. `Group.open` and `Array.open` keep their default of + 3, so pass `zarr_format=None` to defer to a pipeline's format. A format + selected by a pipeline that conflicts with an explicit `zarr_format` raises + `ValueError` in every `open*`/`create*`/`save*` entry point. - Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store now serves the "open" half instead of raising upfront, for all stores; unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index db5769d348..d5fe5726ca 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -110,11 +110,27 @@ class MyAdapter(URLPipelineAdapter): async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext ) -> AdapterResolution: - # A *wrapper* adapter opens the resource to its left and wraps it. Carry the - # preceding residual path forward and keep unchanged fields via dataclasses.replace. + # A *wrapper* adapter opens the resource to its left and wraps it. Join the + # preceding residual path with this segment's own path, and keep every other + # field (e.g. zarr_format) via dataclasses.replace. preceding = await context.resolve_preceding() store = WrapperStore(preceding.store) - return dataclasses.replace(preceding, store=store, path=segment.body) + path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part) + return dataclasses.replace(preceding, store=store, path=path) +``` + +Wrapper adapters compose: in `memory://x|mypackage.myscheme:a|mypackage.myscheme:b` the +inner adapter sees the outer one's residual path `a` and the pipeline resolves to `a/b`. + +```python test="true" session="url-adapter" +import asyncio + +from zarr.registry import register_url_adapter +from zarr.storage import resolve_pipeline + +register_url_adapter("mypackage.myscheme", MyAdapter) +resolution = asyncio.run(resolve_pipeline("memory://x|mypackage.myscheme:a|mypackage.myscheme:b")) +assert resolution.path == "a/b" ``` Register the class under the `zarr.url_adapters` entry-point group, using the URL diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index d572c5bc1d..010abc08a4 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -131,6 +131,12 @@ Registered adapters cannot intercept zarr's native `file:` and `memory:` root schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to fsspec. +Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, +`memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). +When fsspec is installed, a plain `memory://name` URL *without* a `|` is still routed to +fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an +absolute path; percent-escapes are not decoded, matching the [local store](#local-store). + ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers From 03ab3b05612591aad8248ddc4de3df7a6c677185 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 13:54:40 -0700 Subject: [PATCH 07/14] test: skip fsspec memory:// divergence test when fsspec lacks AsyncFileSystemWrapper fsspec < 2024.12.0 (the min_deps job) cannot open a plain memory:// URL at all, so the routing comparison only applies with fsspec>=2024.12.0. Co-Authored-By: Claude Opus 5.5 --- tests/test_url_pipeline/test_resolver.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 66b9246a48..f9941b5101 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -4,11 +4,12 @@ import sys import threading import warnings -from importlib.metadata import EntryPoint +from importlib.metadata import EntryPoint, version from typing import TYPE_CHECKING, ClassVar import numpy as np import pytest +from packaging.version import parse as parse_version import zarr import zarr.registry @@ -30,6 +31,12 @@ from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import _join_paths +# fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL +# (a sync filesystem) cannot be opened at all there +_has_async_fsspec_wrapper = _has_fsspec and parse_version(version("fsspec")) >= parse_version( + "2024.12.0" +) + if TYPE_CHECKING: from pathlib import Path @@ -659,7 +666,10 @@ async def test_file_root_rejects_query(self, tmp_path: Path) -> None: with pytest.raises(URLPipelineError, match="do not accept a query"): await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") - @pytest.mark.skipif(not _has_fsspec, reason="plain memory:// routes to fsspec only with fsspec") + @pytest.mark.skipif( + not _has_async_fsspec_wrapper, + reason="plain memory:// routes to fsspec only with fsspec>=2024.12.0", + ) async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: # documented divergence: a plain memory:// URL is fsspec's in-memory # filesystem when fsspec is installed, while a pipeline's memory: root From e467a0a616b109165618beb7bb52b9958fe7c153 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:20:27 -0700 Subject: [PATCH 08/14] feat: let URL pipelines pick the format in Group.open / Array.open `Group.open`, `Array.open` and their async variants defaulted to `zarr_format=3`, so a pipeline with a `zarr2:` segment raised a conflict unless the caller passed `zarr_format=None`. The default is now a private "unspecified" sentinel. When the caller does not pass a format, a URL pipeline decides: its format segment if any, otherwise auto-detection. Every other store keeps the default of 3, so the metadata probing of non-pipeline callers is unchanged. An explicit format (including None) behaves as before. Co-Authored-By: Claude Opus 5.5 --- src/zarr/core/array.py | 21 ++++++++---- src/zarr/core/common.py | 18 ++++++++++ src/zarr/core/group.py | 23 +++++++++---- src/zarr/storage/_common.py | 43 ++++++++++++++++++++++++ tests/test_url_pipeline/test_resolver.py | 33 ++++++++++++++++-- 5 files changed, 122 insertions(+), 16 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index f32ee7a94b..c5ffff7654 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -57,6 +57,7 @@ parse_chunk_key_encoding, ) from zarr.core.common import ( + _UNSPECIFIED, JSON, ZARR_JSON, ZARRAY_JSON, @@ -67,6 +68,7 @@ ShapeLike, ZarrFormat, _default_zarr_format, + _UnspecifiedType, _warn_order_kwarg, ceildiv_int, concurrent_map, @@ -154,7 +156,7 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage._common import StorePath, make_store_path +from zarr.storage._common import StorePath, _resolve_open_zarr_format, make_store_path from zarr.storage._utils import _relativize_path if TYPE_CHECKING: @@ -779,7 +781,7 @@ def from_dict( async def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, ) -> AnyAsyncArray: """ Async method to open an existing Zarr array from a given store. @@ -791,7 +793,10 @@ async def open( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : ZarrFormat | None, optional - The Zarr format version (default is 3). + The Zarr format version; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. Returns ------- @@ -820,7 +825,7 @@ async def example(): ``` """ store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) + zarr_format = _resolve_open_zarr_format(store, store_path, zarr_format, default=3) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) @@ -2055,7 +2060,7 @@ def from_dict( def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, ) -> Self: """Opens an existing Array from a store. @@ -2066,8 +2071,10 @@ def open( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : {2, 3, None}, optional - The zarr format to expect. If None, the format selected by a URL pipeline - segment is used, otherwise the format is inferred from the store contents. + The zarr format to expect; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. Returns ------- diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index de009d0b50..e6b13d8ea2 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -277,6 +277,24 @@ def _warn_order_kwarg() -> None: warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) +class _UnspecifiedType(Enum): + """ + Type of the `_UNSPECIFIED` sentinel. + + Used as a parameter default where "not passed" must be told apart from an + explicit `None` (for example, `zarr_format` in `Group.open`, where `None` + means "auto-detect"). + """ + + UNSPECIFIED = "unspecified" + + def __repr__(self) -> str: + return "" + + +_UNSPECIFIED: Final = _UnspecifiedType.UNSPECIFIED + + def _default_zarr_format() -> ZarrFormat: """Return the default zarr_format.""" return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index b541aff437..80dfa24aee 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -32,6 +32,7 @@ from zarr.core.attributes import Attributes from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ( + _UNSPECIFIED, JSON, ZARR_JSON, ZARRAY_JSON, @@ -44,6 +45,7 @@ ShapeLike, ZarrFormat, _default_zarr_format, + _UnspecifiedType, parse_shapelike, ) from zarr.core.config import config @@ -68,7 +70,7 @@ ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import _resolve_open_zarr_format, make_store_path from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: @@ -551,7 +553,7 @@ async def from_store( async def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, use_consolidated: bool | str | None = None, ) -> AsyncGroup: """Open a new AsyncGroup @@ -559,7 +561,13 @@ async def open( Parameters ---------- store : StoreLike - zarr_format : {2, 3}, optional + zarr_format : {2, 3, None}, optional + The Zarr format to expect; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. An explicit + value that conflicts with a pipeline's format segment raises + `ValueError`. use_consolidated : bool or str, default None Whether to use consolidated metadata. @@ -579,7 +587,7 @@ async def open( to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) + zarr_format = _resolve_open_zarr_format(store, store_path, zarr_format, default=3) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: @@ -2003,7 +2011,7 @@ def from_store( def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, ) -> Group: """Open a group from an initialized store. @@ -2014,7 +2022,10 @@ def open( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : {2, 3, None}, optional - Zarr storage format version. + Zarr storage format version; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. Returns ------- diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index f649a7eea6..0e3f34afbd 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -20,6 +20,7 @@ ZGROUP_JSON, AccessModeLiteral, ZarrFormat, + _UnspecifiedType, ) from zarr.errors import ( ContainsArrayAndGroupError, @@ -556,6 +557,48 @@ async def make_store_path( return await StorePath.open(store, path=path_normalized, mode=mode) +def _resolve_open_zarr_format( + store_like: StoreLike, + store_path: StorePath, + zarr_format: ZarrFormat | _UnspecifiedType | None, + *, + default: ZarrFormat | None, +) -> ZarrFormat | None: + """ + Resolve the `zarr_format` of an `open` classmethod whose default is unspecified. + + An explicit `zarr_format` (including None, meaning auto-detect) is merged with + the format selected by a URL pipeline as in + [`StorePath.resolve_zarr_format`][zarr.storage.StorePath.resolve_zarr_format]. + When the caller did not pass one, a URL pipeline decides: its format segment + (`zarr2:`/`zarr3:`) if any, otherwise auto-detection. Every other store keeps + `default`, so non-pipeline callers see no change. + + Parameters + ---------- + store_like : StoreLike + The store argument the caller passed. + store_path : StorePath + The store path resolved from `store_like`. + zarr_format : ZarrFormat | None | _UnspecifiedType + The caller's format, or the unspecified sentinel. + default : ZarrFormat | None + The format to use when neither the caller nor a URL pipeline selects one. + + Returns + ------- + ZarrFormat | None + The format to open with, or None to auto-detect. + """ + if not isinstance(zarr_format, _UnspecifiedType): + return store_path.resolve_zarr_format(zarr_format) + if store_path.zarr_format is not None: + return store_path.zarr_format + if isinstance(store_like, str) and is_url_pipeline(store_like): + return None + return default + + async def ensure_no_existing_node( store_path: StorePath, zarr_format: ZarrFormat, diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index f9941b5101..c90be09d38 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -852,18 +852,45 @@ async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) - async def test_group_open(self, tmp_path: Path) -> None: zarr.Group.from_store(f"{tmp_path}|fmt2:") - # explicit None defers to the pipeline; the default (3) conflicts + # the unspecified default defers to the pipeline, as does explicit + # None; only an explicit, different format conflicts + assert zarr.Group.open(f"{tmp_path}|fmt2:").metadata.zarr_format == 2 group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.open(f"{tmp_path}|fmt2:") + zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=3) async def test_array_open(self, tmp_path: Path) -> None: zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + assert zarr.Array.open(f"{tmp_path}/x|fmt2:").metadata.zarr_format == 2 arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Array.open(f"{tmp_path}/x|fmt2:") + zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=3) + + async def test_async_open_classmethods(self, tmp_path: Path) -> None: + from zarr.core.array import AsyncArray + from zarr.core.group import AsyncGroup + + zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + group = await AsyncGroup.open(f"{tmp_path}|fmt2:") + assert group.metadata.zarr_format == 2 + arr = await AsyncArray.open(f"{tmp_path}/x|fmt2:") + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + await AsyncArray.open(f"{tmp_path}/x|fmt2:", zarr_format=3) + + async def test_open_classmethod_default_without_pipeline(self, tmp_path: Path) -> None: + # non-pipeline callers keep the historical default of 3: a v2 node is + # not found unless the caller asks for 2 or None + zarr.create_array(tmp_path / "x", shape=(2,), dtype="i4", zarr_format=2) + zarr.create_group(tmp_path / "g", zarr_format=2) + with pytest.raises(FileNotFoundError): + zarr.Array.open(tmp_path / "x") + with pytest.raises(FileNotFoundError): + zarr.Group.open(tmp_path / "g") + assert zarr.Array.open(tmp_path / "x", zarr_format=None).metadata.zarr_format == 2 + assert zarr.Group.open(tmp_path / "g", zarr_format=None).metadata.zarr_format == 2 async def test_open_like(self, tmp_path: Path) -> None: # open_like inherits v2 filters/compressor from the reference; the From 9f3d9b3255ee8a2279e98900b45aa27672ab9064 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:20:46 -0700 Subject: [PATCH 09/14] feat: add builtin zarr:, zarr2: and zarr3: URL pipeline adapters The format adapters from the URL pipeline spec (schemes/zarr.md). The segment body is a path within the preceding resource and is joined onto the preceding residual path; `zarr2:` / `zarr3:` record the Zarr format and `zarr:` leaves it to auto-detection (keeping a format pinned by an earlier segment). Every other field of the preceding resolution is carried forward with `dataclasses.replace`, so a format segment also works as an intermediate segment. The builtins are registered as lazily loaded entry points ahead of the third-party ones in `_collect_entrypoints`, so importing zarr does not import them, and a third-party `zarr.url_adapters` entry point with the same name is ignored with a warning. They are adapter-only schemes: a string without `|` such as `zarr3:foo` is never routed to them as a pipeline root, so its meaning is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/zarr/registry.py | 76 ++++- src/zarr/storage/_url_adapters/__init__.py | 7 + src/zarr/storage/_url_adapters/_format.py | 117 +++++++ src/zarr/storage/_url_pipeline.py | 19 +- .../test_url_pipeline/test_format_adapters.py | 317 ++++++++++++++++++ tests/test_url_pipeline/test_resolver.py | 4 +- 6 files changed, 521 insertions(+), 19 deletions(-) create mode 100644 src/zarr/storage/_url_adapters/__init__.py create mode 100644 src/zarr/storage/_url_adapters/_format.py create mode 100644 tests/test_url_pipeline/test_format_adapters.py diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 990ee51572..703d7c246b 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -309,6 +309,18 @@ def _collect_entrypoints() -> list[Registry[Any]]: entry_points.select(group="zarr", name="chunk_key_encoding") ) + # Builtin adapters go first, as lazily loaded entry points, so that they + # are imported only when a pipeline naming them is resolved and so that a + # third-party entry point with the same name is reported as shadowed. + _url_adapter_registry.lazy_load_list.extend( + _builtin_url_adapter_entry_point(scheme) + for scheme in _BUILTIN_URL_ADAPTERS + if scheme not in _url_adapter_registry + and not any( + e.name == scheme and _is_builtin_url_adapter_entry_point(e) + for e in _url_adapter_registry.lazy_load_list + ) + ) _url_adapter_registry.lazy_load_list.extend(entry_points.select(group="zarr.url_adapters")) _pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) @@ -513,6 +525,27 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] +# URL pipeline adapters shipped with zarr-python, as `scheme -> "module:attr"`. +# They are registered as lazily loaded entry points (see `_collect_entrypoints`), +# so importing zarr never imports them. These are adapter-only schemes: they are +# never dispatched to as a pipeline *root* (see `zarr.storage._url_pipeline`). +_BUILTIN_URL_ADAPTERS: dict[str, str] = { + "zarr": "zarr.storage._url_adapters._format:ZarrAdapter", + "zarr2": "zarr.storage._url_adapters._format:Zarr2Adapter", + "zarr3": "zarr.storage._url_adapters._format:Zarr3Adapter", +} + + +def _builtin_url_adapter_entry_point(scheme: str) -> EntryPoint: + from importlib.metadata import EntryPoint + + return EntryPoint(name=scheme, value=_BUILTIN_URL_ADAPTERS[scheme], group="zarr.url_adapters") + + +def _is_builtin_url_adapter_entry_point(entry_point: EntryPoint) -> bool: + return _BUILTIN_URL_ADAPTERS.get(entry_point.name.lower()) == entry_point.value + + def register_url_adapter(scheme: str, cls: type[URLPipelineAdapter]) -> None: """ Register a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] @@ -554,11 +587,15 @@ def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: Get the URL pipeline adapter class registered for `scheme`. Loads pending `zarr.url_adapters` entry points for this scheme only, so - resolving one scheme never imports other providers' packages. When the - scheme is already registered (e.g. a builtin adapter), a same-named entry - point is discarded with a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; - when several entry points share the name, the first one wins and a - warning names the ones ignored. + resolving one scheme never imports other providers' packages. zarr's + builtin adapters (`zarr:`, `zarr2:`, `zarr3:`, ...) are pending entry points + too, and take precedence over a third-party entry point of the same name. + When the scheme is already registered, or a builtin provides it, a + same-named third-party entry point is discarded with a + [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; when several third-party + entry points share the name, the first one wins and a warning names the + ones ignored. An explicit `register_url_adapter` call overrides a builtin + silently. Raises ------ @@ -578,17 +615,30 @@ def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: _url_adapter_registry.lazy_load_list[:] = [ e for e in _url_adapter_registry.lazy_load_list if e.name.lower() != key ] + # A pending builtin entry point is redundant once the scheme is registered + # explicitly (an intentional override), so only third-party entry points + # are reported as shadowed. + third_party = [e for e in pending if not _is_builtin_url_adapter_entry_point(e)] if pending and registered is not None: - warnings.warn( - f"URL pipeline adapter for scheme {scheme!r} is already registered " - f"({fully_qualified_name(registered)}); ignoring entry point(s) " - f"{[e.value for e in pending]} of the same name", - category=ZarrUserWarning, - stacklevel=2, - ) + if third_party: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is already registered " + f"({fully_qualified_name(registered)}); ignoring entry point(s) " + f"{[e.value for e in third_party]} of the same name", + category=ZarrUserWarning, + stacklevel=2, + ) elif pending: entry_point, *duplicates = pending - if duplicates: + if duplicates and _is_builtin_url_adapter_entry_point(entry_point): + warnings.warn( + f"the builtin URL pipeline adapter for scheme {entry_point.name!r} takes " + f"precedence; ignoring 'zarr.url_adapters' entry point(s) " + f"{[e.value for e in duplicates]} of the same name", + category=ZarrUserWarning, + stacklevel=2, + ) + elif duplicates: warnings.warn( f"multiple 'zarr.url_adapters' entry points are named {entry_point.name!r}; " f"using {entry_point.value!r} and ignoring {[e.value for e in duplicates]}", diff --git a/src/zarr/storage/_url_adapters/__init__.py b/src/zarr/storage/_url_adapters/__init__.py new file mode 100644 index 0000000000..04f042c1ae --- /dev/null +++ b/src/zarr/storage/_url_adapters/__init__.py @@ -0,0 +1,7 @@ +""" +URL pipeline adapters shipped with zarr-python. + +These are registered as lazily loaded builtin entry points by +`zarr.registry` (see `_BUILTIN_URL_ADAPTERS`), so nothing in this package is +imported until a pipeline naming one of its schemes is resolved. +""" diff --git a/src/zarr/storage/_url_adapters/_format.py b/src/zarr/storage/_url_adapters/_format.py new file mode 100644 index 0000000000..99b44ebff8 --- /dev/null +++ b/src/zarr/storage/_url_adapters/_format.py @@ -0,0 +1,117 @@ +""" +The `zarr:`, `zarr2:` and `zarr3:` URL pipeline adapters. + +Per the URL pipeline specification (`schemes/zarr.md`), these address a Zarr +array or group within the `directory` resource to their left: + + file:///path/to/data.zarr/|zarr3:path/to/node + +The segment body is a path *within* the preceding resource. `zarr2:` and +`zarr3:` select the Zarr format; `zarr:` leaves it to auto-detection. +""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING, ClassVar + +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.errors import URLPipelineError +from zarr.storage._utils import _join_paths, normalize_path + +if TYPE_CHECKING: + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment + from zarr.core.common import ZarrFormat + +__all__ = ["Zarr2Adapter", "Zarr3Adapter", "ZarrAdapter"] + + +class _ZarrFormatAdapter(URLPipelineAdapter): + """ + Shared implementation of the format adapters. + + Resolves the preceding pipeline with the caller's mode and options, joins + the segment body onto the preceding residual path, and records the Zarr + format. Every other field of the preceding resolution is carried forward + unchanged via `dataclasses.replace`. + """ + + zarr_format: ClassVar[ZarrFormat | None] + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve a `zarr:`/`zarr2:`/`zarr3:` segment. + + Parameters + ---------- + segment : PipelineSegment + The format segment. Its body is a path within the preceding resource. + context : PipelineContext + The pipeline to the left of the segment. + + Returns + ------- + AdapterResolution + The preceding resolution with the segment path appended and the + Zarr format recorded. + + Raises + ------ + URLPipelineError + If the segment carries a query, has an invalid path, is the pipeline + root, or selects a format that conflicts with an earlier format segment. + """ + if segment.query is not None: + raise URLPipelineError( + f"'{segment.scheme}:' pipeline segments do not accept a query: {segment.raw!r}" + ) + try: + subpath = normalize_path(segment.body) + except ValueError as exc: + raise URLPipelineError( + f"invalid path in pipeline segment {segment.raw!r}: {exc}" + ) from exc + + preceding = await context.resolve_preceding() + + zarr_format = cls.zarr_format + if preceding.zarr_format is not None: + if zarr_format is None: + # `zarr:` (auto-detect) never discards a format pinned earlier + zarr_format = preceding.zarr_format + elif zarr_format != preceding.zarr_format: + preceding.store.close() + raise URLPipelineError( + f"pipeline segment {segment.raw!r} selects Zarr format {zarr_format}, " + f"but an earlier segment selected Zarr format {preceding.zarr_format}" + ) + + path = _join_paths([normalize_path(preceding.path), subpath]) + return dataclasses.replace(preceding, path=path, zarr_format=zarr_format) + + +class ZarrAdapter(_ZarrFormatAdapter): + """ + The `zarr:` adapter: a Zarr node whose format is auto-detected. + """ + + zarr_format = None + + +class Zarr2Adapter(_ZarrFormatAdapter): + """ + The `zarr2:` adapter: a Zarr format 2 node. + """ + + zarr_format = 2 + + +class Zarr3Adapter(_ZarrFormatAdapter): + """ + The `zarr3:` adapter: a Zarr format 3 node. + """ + + zarr_format = 3 diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 41a2114daf..b649244ebc 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -31,7 +31,7 @@ PipelineSegment, ) from zarr.errors import URLPipelineError -from zarr.registry import get_url_adapter, list_url_adapter_schemes +from zarr.registry import _BUILTIN_URL_ADAPTERS, get_url_adapter, list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore from zarr.storage._utils import parse_store_url @@ -55,6 +55,11 @@ # intercept zarr's own local-path and in-memory routing. _NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) +# Adapter schemes zarr ships (e.g. `zarr3:`). The spec defines these as adapter +# schemes only, so they are never dispatched to as a pipeline root: a string +# without `|` such as `zarr3:foo` keeps its pre-pipeline meaning. +_BUILTIN_ADAPTER_SCHEMES = frozenset(_BUILTIN_URL_ADAPTERS) + # An absolute Windows drive path (C:\... or C:/...), which counts as an # absolute path in a `file:` pipeline root. _WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") @@ -140,12 +145,16 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: def _root_routes_to_adapter(scheme: str) -> bool: """ Whether a root sub-URL with this scheme is dispatched to a registered - root adapter. Schemes zarr resolves natively (`file:`, `memory:`) and - opaque roots are excluded; the registry check inspects entry-point - names only — no adapter code is imported here. + root adapter. Schemes zarr resolves natively (`file:`, `memory:`), the + builtin adapter-only schemes (`zarr:`, `zarr3:`, ...) and opaque roots are + excluded; the registry check inspects entry-point names only — no adapter + code is imported here. """ return ( - bool(scheme) and scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() + bool(scheme) + and scheme not in _NATIVE_ROOT_SCHEMES + and scheme not in _BUILTIN_ADAPTER_SCHEMES + and scheme in list_url_adapter_schemes() ) diff --git a/tests/test_url_pipeline/test_format_adapters.py b/tests/test_url_pipeline/test_format_adapters.py new file mode 100644 index 0000000000..fa6deef29d --- /dev/null +++ b/tests/test_url_pipeline/test_format_adapters.py @@ -0,0 +1,317 @@ +"""Tests for the builtin `zarr:`, `zarr2:` and `zarr3:` URL pipeline adapters.""" + +from __future__ import annotations + +import dataclasses +import subprocess +import sys +import uuid +from importlib.metadata import EntryPoint +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +import zarr +import zarr.registry +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.errors import URLPipelineError, ZarrUserWarning +from zarr.registry import get_url_adapter, list_url_adapter_schemes, register_url_adapter +from zarr.storage import LocalStore, ManagedMemoryStore, WrapperStore +from zarr.storage._common import make_store_path +from zarr.storage._url_adapters._format import Zarr2Adapter, Zarr3Adapter, ZarrAdapter +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._utils import _join_paths + +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + + from zarr.core.common import ZarrFormat + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + +FORMAT_SEGMENTS: list[tuple[str, ZarrFormat | None]] = [ + ("zarr2:", 2), + ("zarr3:", 3), + ("zarr:", None), +] + + +@pytest.fixture(params=["local", "memory"]) +def root(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[str, None, None]: + """A pipeline root: a local directory, or a fresh named in-memory store.""" + if request.param == "local": + yield str(tmp_path / "root") + return + # a named managed memory store lives only while something references it, + # so hold one for the duration of the test + name = uuid.uuid4().hex + keepalive = ManagedMemoryStore(name=name) + yield f"memory:{name}" + del keepalive + + +class _WrapAdapter(URLPipelineAdapter): + """A wrapper adapter that follows the documented wrapper contract.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace( + preceding, + store=WrapperStore(preceding.store), + path=_join_paths([preceding.path, segment.body]), + ) + + +def _expected(fmt: ZarrFormat | None) -> ZarrFormat: + """The format a node created through a segment ends up with.""" + return 3 if fmt is None else fmt + + +class TestRegistration: + def test_builtin_schemes_are_listed(self) -> None: + assert {"zarr", "zarr2", "zarr3"} <= list_url_adapter_schemes() + + @pytest.mark.parametrize( + ("scheme", "cls"), [("zarr", ZarrAdapter), ("zarr2", Zarr2Adapter), ("zarr3", Zarr3Adapter)] + ) + def test_builtin_lookup(self, scheme: str, cls: type[URLPipelineAdapter]) -> None: + assert get_url_adapter(scheme) is cls + assert get_url_adapter(scheme.upper()) is cls + + def test_import_zarr_does_not_import_adapters(self) -> None: + # builtins are lazily loaded entry points, like third-party adapters + code = ( + "import sys, zarr, zarr.storage\n" + "assert 'zarr.storage._url_adapters._format' not in sys.modules\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + def test_third_party_entry_point_is_shadowed_with_warning(self) -> None: + registry = zarr.registry._url_adapter_registry + registry.pop("zarr3", None) + registry.lazy_load_list[:] = [ + zarr.registry._builtin_url_adapter_entry_point("zarr3"), + *(e for e in registry.lazy_load_list if e.name != "zarr3"), + EntryPoint(name="zarr3", value="other_pkg:Adapter", group="zarr.url_adapters"), + ] + with pytest.warns(ZarrUserWarning, match="builtin URL pipeline adapter.*takes precedence"): + assert get_url_adapter("zarr3") is Zarr3Adapter + assert not any(e.name == "zarr3" for e in registry.lazy_load_list) + + def test_explicit_registration_overrides_builtin_silently(self) -> None: + # an explicit register_url_adapter call is deliberate: the pending + # builtin entry point is dropped without a warning + registry = zarr.registry._url_adapter_registry + registry.pop("zarr3", None) + registry.lazy_load_list[:] = [ + zarr.registry._builtin_url_adapter_entry_point("zarr3"), + *(e for e in registry.lazy_load_list if e.name != "zarr3"), + ] + register_url_adapter("zarr3", ZarrAdapter) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert get_url_adapter("zarr3") is ZarrAdapter + + def test_collect_entrypoints_does_not_duplicate_builtins(self) -> None: + zarr.registry._collect_entrypoints() + zarr.registry._collect_entrypoints() + pending = [ + e + for e in zarr.registry._url_adapter_registry.lazy_load_list + if zarr.registry._is_builtin_url_adapter_entry_point(e) + ] + assert len(pending) == len({e.name for e in pending}) + + @pytest.mark.parametrize("url", ["zarr:foo", "zarr2:foo", "zarr3://foo", "ZARR3:foo"]) + def test_builtin_schemes_are_not_root_schemes(self, url: str) -> None: + # a string without '|' keeps its pre-pipeline meaning + assert not is_url_pipeline(url) + + +class TestResolution: + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + async def test_sets_format(self, tmp_path: Path, segment: str, fmt: ZarrFormat | None) -> None: + result = await resolve_pipeline(f"{tmp_path}|{segment}") + assert isinstance(result.store, LocalStore) + assert result.path == "" + assert result.zarr_format == fmt + + @pytest.mark.parametrize("scheme", ["zarr", "zarr2", "zarr3"]) + async def test_colon_is_optional(self, tmp_path: Path, scheme: str) -> None: + with_colon = await resolve_pipeline(f"{tmp_path}|{scheme}:") + without = await resolve_pipeline(f"{tmp_path}|{scheme}") + assert with_colon.zarr_format == without.zarr_format + + async def test_body_is_a_path_within_the_preceding_resource(self, tmp_path: Path) -> None: + result = await resolve_pipeline(f"{tmp_path}|zarr3:a/b") + assert result.path == "a/b" + assert result.zarr_format == 3 + + @pytest.mark.parametrize("body", ["/a/b", "a/b/", "/a//b/"]) + async def test_body_is_normalized(self, tmp_path: Path, body: str) -> None: + result = await resolve_pipeline(f"{tmp_path}|zarr3:{body}") + assert result.path == "a/b" + + async def test_body_joins_onto_preceding_residual_path(self) -> None: + # memory:name/sub roots carry their path inside the store; a wrapper + # carries a residual path; both are kept + register_url_adapter("wrap", _WrapAdapter) + result = await resolve_pipeline("memory:fmt-join/sub|wrap:w|zarr2:a/b") + assert result.path == "w/a/b" + assert result.zarr_format == 2 + + async def test_intermediate_segment(self) -> None: + # a format segment may precede another adapter, which carries the + # format and the residual path forward + register_url_adapter("wrap", _WrapAdapter) + result = await resolve_pipeline("memory:fmt-mid|zarr2:a|wrap:b") + assert isinstance(result.store, WrapperStore) + assert result.path == "a/b" + assert result.zarr_format == 2 + + async def test_keeps_other_fields(self) -> None: + # the adapter uses dataclasses.replace, so the preceding store is kept + result = await resolve_pipeline("memory:fmt-keep|zarr3:") + assert isinstance(result.store, ManagedMemoryStore) + + async def test_auto_keeps_an_earlier_format(self, tmp_path: Path) -> None: + result = await resolve_pipeline(f"{tmp_path}|zarr2:a|zarr:b") + assert result.zarr_format == 2 + assert result.path == "a/b" + + async def test_conflicting_format_segments_raise(self, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match="earlier segment selected Zarr format 2"): + await resolve_pipeline(f"{tmp_path}|zarr2:|zarr3:") + + async def test_query_is_rejected(self, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline(f"{tmp_path}|zarr3:a?x=1") + + @pytest.mark.parametrize("body", ["..", "a/../b", "./a"]) + async def test_invalid_path_is_rejected(self, tmp_path: Path, body: str) -> None: + with pytest.raises(URLPipelineError, match="invalid path"): + await resolve_pipeline(f"{tmp_path}|zarr3:{body}") + + async def test_mode_is_forwarded(self, tmp_path: Path) -> None: + zarr.create_group(tmp_path) + store_path = await make_store_path(f"{tmp_path}|zarr3:", mode="r") + assert store_path.read_only + assert store_path.zarr_format == 3 + + +class TestEndToEnd: + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_open_group(self, root: str, segment: str, fmt: ZarrFormat | None) -> None: + group = zarr.open_group(f"{root}|{segment}", mode="w") + assert group.metadata.zarr_format == _expected(fmt) + group.create_array("x", shape=(3,), dtype="i4")[:] = [1, 2, 3] + reopened = zarr.open_group(f"{root}|{segment}", mode="r") + assert reopened.metadata.zarr_format == _expected(fmt) + arr = reopened["x"] + assert isinstance(arr, zarr.Array) + np.testing.assert_array_equal(arr[:], [1, 2, 3]) + + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_open(self, root: str, segment: str, fmt: ZarrFormat | None) -> None: + group = zarr.open(f"{root}|{segment}", mode="w") + assert isinstance(group, zarr.Group) + assert group.metadata.zarr_format == _expected(fmt) + arr = zarr.open(f"{root}|{segment}x", mode="w", shape=(2,), dtype="i4") + assert isinstance(arr, zarr.Array) + assert arr.metadata.zarr_format == _expected(fmt) + reopened = zarr.open(f"{root}|{segment}x", mode="r") + assert isinstance(reopened, zarr.Array) + assert reopened.metadata.zarr_format == _expected(fmt) + + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_create_and_open_array(self, root: str, segment: str, fmt: ZarrFormat | None) -> None: + arr = zarr.create_array(f"{root}|{segment}", name="a/b", shape=(4,), dtype="i4") + assert arr.metadata.zarr_format == _expected(fmt) + arr[:] = np.arange(4) + opened = zarr.open_array(f"{root}|{segment}a/b", mode="r") + assert opened.metadata.zarr_format == _expected(fmt) + np.testing.assert_array_equal(opened[:], np.arange(4)) + # the user-supplied path joins onto the segment path + via_path = zarr.open_array(f"{root}|{segment}a", path="b", mode="r") + np.testing.assert_array_equal(via_path[:], np.arange(4)) + + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_group_and_array_open_classmethods( + self, root: str, segment: str, fmt: ZarrFormat | None + ) -> None: + zarr.create_array(f"{root}|{segment}", name="x", shape=(2,), dtype="i4") + group = zarr.Group.open(f"{root}|{segment}") + assert group.metadata.zarr_format == _expected(fmt) + arr = zarr.Array.open(f"{root}|{segment}x") + assert arr.metadata.zarr_format == _expected(fmt) + + @pytest.mark.parametrize("stored", [2, 3]) + def test_auto_detects_existing_format(self, root: str, stored: ZarrFormat) -> None: + zarr.create_array(f"{root}|zarr{stored}:", name="x", shape=(2,), dtype="i4") + assert zarr.open_group(f"{root}|zarr:", mode="r").metadata.zarr_format == stored + assert zarr.open_array(f"{root}|zarr:x", mode="r").metadata.zarr_format == stored + assert zarr.Group.open(f"{root}|zarr:").metadata.zarr_format == stored + assert zarr.Array.open(f"{root}|zarr:x").metadata.zarr_format == stored + + def test_stacked_segment_paths(self, root: str) -> None: + zarr.create_array(f"{root}|zarr3:a/b", name="c", shape=(2,), dtype="i4") + assert zarr.open_array(f"{root}|zarr3:a/b/c", mode="r").shape == (2,) + group = zarr.open_group(f"{root}|zarr3:a", mode="r") + assert "b" in group + + def test_wrong_format_is_not_found(self, root: str) -> None: + zarr.create_group(f"{root}|zarr2:") + with pytest.raises(FileNotFoundError): + zarr.open_group(f"{root}|zarr3:", mode="r") + + @pytest.mark.parametrize( + ("segment", "explicit"), [("zarr2:", 3), ("zarr3:", 2)], ids=["zarr2-vs-3", "zarr3-vs-2"] + ) + def test_explicit_format_conflict(self, root: str, segment: str, explicit: ZarrFormat) -> None: + url = f"{root}|{segment}" + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_group(url, mode="w", zarr_format=explicit) + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(url, name="x", shape=(1,), dtype="i4", zarr_format=explicit) + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.open(url, zarr_format=explicit) + + def test_explicit_format_with_auto_segment(self, root: str) -> None: + # zarr: selects no format, so an explicit format is not a conflict + group = zarr.open_group(f"{root}|zarr:", mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 + + +ZARR_SPEC_EXAMPLES: list[tuple[str, ZarrFormat | None]] = [ + ("file:///path/to/node.zarr/|zarr3:", 3), + ("file:///path/to/node.zarr/|zarr3", 3), + ("file:///path/to/node.zarr/|zarr2:", 2), + ("file:///path/to/node.zarr/|zarr2", 2), + ("file:///path/to/node.zarr/|zarr:", None), + ("file:///path/to/node.zarr/|zarr", None), +] + + +@pytest.mark.parametrize(("example", "fmt"), ZARR_SPEC_EXAMPLES) +def test_zarr_scheme_spec_examples(tmp_path: Path, example: str, fmt: ZarrFormat | None) -> None: + """Every example in the spec's `schemes/zarr.md` resolves and opens.""" + posix = tmp_path.as_posix() + url_path = posix if posix.startswith("/") else f"/{posix}" + url = example.replace("/path/to", url_path, 1) + stored: ZarrFormat = 3 if fmt is None else fmt + zarr.create_group(tmp_path / "node.zarr", zarr_format=stored) + group = zarr.open_group(url, mode="r") + assert group.metadata.zarr_format == stored + assert zarr.Group.open(url).metadata.zarr_format == stored diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index c90be09d38..531fd70e9b 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -259,7 +259,9 @@ def test_concurrent_loads_keep_other_schemes_pending( thread.join(timeout=10) assert not any(thread.is_alive() for thread in threads) assert "untouched" in list_url_adapter_schemes() - assert [e.name for e in pending] == ["untouched"] + assert [ + e.name for e in pending if not zarr.registry._is_builtin_url_adapter_entry_point(e) + ] == ["untouched"] assert get_url_adapter("untouched").__name__ == "Adapter" def test_import_failure_is_wrapped_and_not_permanent(self) -> None: From 131ea0b41e75ff63e141062f9660c09b3ac57fcf Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:20:52 -0700 Subject: [PATCH 10/14] docs: document the zarr:, zarr2: and zarr3: adapters in the storage guide Co-Authored-By: Claude Opus 5.5 --- docs/user-guide/storage.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 010abc08a4..eeeb53e0f8 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -114,10 +114,28 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name *adapter* that reinterprets everything to its left (e.g. `s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the `zarr.url_adapters` entry-point group — see -[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. Builtin -adapters (`zip:`, `zarr2:`/`zarr3:`) are under development and will expand this -section. URLs without a `|` (and without a registered root scheme) are handled -exactly as before. +[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. URLs +without a `|` (and without a registered root scheme) are handled exactly as before. + +Zarr ships the `zarr:`, `zarr2:` and `zarr3:` format adapters. The text after the +colon is a path within the resource to its left, and `zarr2:` / `zarr3:` select the +Zarr format (`zarr:` auto-detects it). The colon is optional when the path is empty: + +```python exec="true" session="storage-url-pipeline" source="above" result="ansi" +import tempfile + +import zarr + +root = tempfile.mkdtemp() +group = zarr.open_group(f"{root}|zarr2:", mode="w") +group.create_array("data", shape=(4,), dtype="int32") +print(zarr.open_array(f"{root}|zarr2:data", mode="r").metadata.zarr_format) +print(zarr.open_group(f"file:{root}|zarr", mode="r").metadata.zarr_format) +``` + +A `zarr_format` argument that conflicts with a `zarr2:` / `zarr3:` segment raises +`ValueError`. `Group.open` and `Array.open` take their format from the pipeline when +`zarr_format` is not passed. `storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); adapters may consume adapter-specific, namespaced keys. From 37b2f49009551c6e1fdef9067205ae538a6b4dbd Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:27:32 -0700 Subject: [PATCH 11/14] docs: add changelog entry for #4476 Co-Authored-By: Claude Opus 5.5 --- changes/4476.feature.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changes/4476.feature.md diff --git a/changes/4476.feature.md b/changes/4476.feature.md new file mode 100644 index 0000000000..413ef01a7d --- /dev/null +++ b/changes/4476.feature.md @@ -0,0 +1,11 @@ +Added the builtin `zarr:`, `zarr2:` and `zarr3:` URL pipeline adapters, so a pipeline +can address a Zarr node within a resource and select its format, e.g. +`zarr.open_group("file:///data/archive|zarr2:path/to/group")`. The text after the colon +is a path within the preceding resource; `zarr2:` / `zarr3:` select the Zarr format and +`zarr:` auto-detects it. These schemes are adapter-only, so strings without `|` behave as +before. + +`Group.open`, `Array.open`, `AsyncGroup.open` and `AsyncArray.open` no longer default to +`zarr_format=3` when given a URL pipeline: if `zarr_format` is not passed, the pipeline's +format segment decides (auto-detection when there is none). For every other store the +default is still 3. From 1fed45670b9520aa660b8f9a9a4741903b7e1e1e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:30:53 -0700 Subject: [PATCH 12/14] feat: add a read-only builtin zip: URL pipeline adapter `zip:` (schemes/zip.md) addresses an entry or directory inside the ZIP archive that the preceding pipeline resolves to. The preceding pipeline is resolved with mode "r", and the archive is read through that store with async byte-range requests only: the central directory is fetched from the end of the archive (parsed in a worker thread), and each member is fetched and decompressed on demand, with true range reads for stored members. A thin read-only reader over the preceding store's bytes is used instead of `ZipStore`, because `ZipStore` does blocking file I/O and needs a local path or a sync file object. The reader works the same way for local files, fsspec objects, `memory:` entries and nested `zip:` archives, and never blocks the IO event loop. Reading a root that *is* the archive relies on `get("")` addressing the root object (follow-up F2); that is isolated in `_read_resource_range` / `_resource_size` so the file-resource primitive can replace it. Writable modes ("w", "w-", "r+") raise `URLPipelineError`; "a" and None open read-only. One leading "/" in the body is ignored and more than one is invalid. The preceding store is closed when the archive cannot be opened, and the reader closes it on `close()`. Co-Authored-By: Claude Opus 5.5 --- src/zarr/registry.py | 1 + src/zarr/storage/_url_adapters/_zip.py | 447 ++++++++++++++++++++ tests/test_url_pipeline/test_zip_adapter.py | 358 ++++++++++++++++ 3 files changed, 806 insertions(+) create mode 100644 src/zarr/storage/_url_adapters/_zip.py create mode 100644 tests/test_url_pipeline/test_zip_adapter.py diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 703d7c246b..260d5c4ddb 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -533,6 +533,7 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: "zarr": "zarr.storage._url_adapters._format:ZarrAdapter", "zarr2": "zarr.storage._url_adapters._format:Zarr2Adapter", "zarr3": "zarr.storage._url_adapters._format:Zarr3Adapter", + "zip": "zarr.storage._url_adapters._zip:ZipAdapter", } diff --git a/src/zarr/storage/_url_adapters/_zip.py b/src/zarr/storage/_url_adapters/_zip.py new file mode 100644 index 0000000000..fab07ddcae --- /dev/null +++ b/src/zarr/storage/_url_adapters/_zip.py @@ -0,0 +1,447 @@ +""" +The read-only `zip:` URL pipeline adapter. + +Per the URL pipeline specification (`schemes/zip.md`), `zip:` addresses an +entry or directory within the ZIP archive that the preceding pipeline +resolves to: + + s3://bucket/archive.zip|zip:path/within/archive|zarr3: + +The archive is read through the preceding store with async byte-range +requests: the central directory is fetched from the end of the archive once, +and each member is fetched (and decompressed) on demand. This works the same +way for local, fsspec, in-memory and nested (`zip:` inside `zip:`) archives, +and never blocks zarr's I/O event loop on file I/O. +""" + +from __future__ import annotations + +import asyncio +import bz2 +import dataclasses +import io +import struct +import zipfile +import zlib +from typing import TYPE_CHECKING + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.core.buffer import default_buffer_prototype +from zarr.errors import URLPipelineError +from zarr.storage._utils import normalize_path + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment + from zarr.core.buffer import Buffer, BufferPrototype + +__all__ = ["ZipAdapter", "ZipReaderStore"] + +# The end-of-central-directory record is 22 bytes plus a comment of at most +# 65535 bytes, so this suffix always contains it. +_EOCD_SEARCH_BYTES = 22 + 0xFFFF +# Smallest range fetched when the central directory extends past the suffix. +_MIN_FETCH_BYTES = 1 << 16 +# Members whose compressed size exceeds this are decompressed in a thread. +_THREAD_DECOMPRESS_BYTES = 1 << 20 + +_LOCAL_HEADER = struct.Struct("<4s2B4HL2L2H") +_LOCAL_HEADER_SIGNATURE = b"PK\x03\x04" +_WRITE_MODES = frozenset({"w", "w-", "r+"}) + + +async def _read_resource_range( + store: Store, key: str, byte_range: ByteRequest | None = None +) -> bytes: + """ + Read bytes of the single-object resource at `key` in `store`. + + This is the one place that relies on a `file`-kind resource being readable + as a store key: an archive that is an entry of another store (e.g. a nested + `zip:`) is the value at `key`; an archive that *is* the preceding root (a + local file, an fsspec object) is read with the empty key, which `LocalStore` + and `FsspecStore` happen to resolve to the root object itself. + """ + value = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) + if value is None: + raise FileNotFoundError(f"{key!r} in {store}" if key else str(store)) + return value.to_bytes() + + +async def _resource_size(store: Store, key: str) -> int: + """The size in bytes of the single-object resource at `key` in `store`.""" + try: + return await store.getsize(key) + except FileNotFoundError as exc: + raise FileNotFoundError(f"{key!r} in {store}" if key else str(store)) from exc + + +class _MissingRangeError(Exception): + def __init__(self, start: int, stop: int) -> None: + super().__init__(start, stop) + self.start = start + self.stop = stop + + +class _SparseFile(io.RawIOBase): + """ + A read-only, seekable view of a file of known size of which only some + byte ranges have been fetched. Reading outside them raises + `_MissingRangeError`, so the caller can fetch the range and retry. + """ + + def __init__(self, size: int, segments: dict[int, bytes]) -> None: + self._size = size + self._segments = segments + self._pos = 0 + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self._pos + + def seek(self, pos: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + self._pos = pos + elif whence == io.SEEK_CUR: + self._pos += pos + else: + self._pos = self._size + pos + return self._pos + + def readinto(self, b: bytearray | memoryview) -> int: # type: ignore[override] + n = min(len(b), self._size - self._pos) + if n <= 0: + return 0 + for start, data in self._segments.items(): + if start <= self._pos and self._pos + n <= start + len(data): + offset = self._pos - start + b[:n] = data[offset : offset + n] + self._pos += n + return n + raise _MissingRangeError(self._pos, self._pos + n) + + +def _parse_central_directory(size: int, segments: dict[int, bytes]) -> list[zipfile.ZipInfo]: + with zipfile.ZipFile(_SparseFile(size, segments)) as zf: + return zf.infolist() + + +def _decompress(info: zipfile.ZipInfo, data: bytes) -> bytes: + if info.compress_type == zipfile.ZIP_STORED: + result = data + elif info.compress_type == zipfile.ZIP_DEFLATED: + result = zlib.decompress(data, -15) + elif info.compress_type == zipfile.ZIP_BZIP2: + result = bz2.decompress(data) + else: + raise NotImplementedError( + f"ZIP member {info.filename!r} uses compression method {info.compress_type}, " + "which zip: does not support (supported: stored, deflate, bzip2)" + ) + if zlib.crc32(result) != info.CRC: + raise OSError(f"bad CRC-32 for ZIP member {info.filename!r}") + return result + + +def _slice(data: bytes, byte_range: ByteRequest | None) -> bytes: + if byte_range is None: + return data + if isinstance(byte_range, RangeByteRequest): + return data[byte_range.start : byte_range.end] + if isinstance(byte_range, OffsetByteRequest): + return data[byte_range.offset :] + if isinstance(byte_range, SuffixByteRequest): + return data[max(0, len(data) - byte_range.suffix) :] + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + + +class ZipReaderStore(Store): + """ + A read-only store over a ZIP archive that is itself a resource of another store. + + The archive is the value at `key` in `source` (the empty key addresses a + store rooted at a single object, such as a local file). It is read with + async range requests only. Use `open_archive` to create one; the store owns + `source` and closes it on `close`. + + Parameters + ---------- + source : Store + The store holding the archive. + key : str + The key of the archive in `source`. + size : int + The archive size in bytes. + infos : Iterable[zipfile.ZipInfo] + The archive's central directory entries. + """ + + supports_writes: bool = False + supports_deletes: bool = False + supports_listing: bool = True + + def __init__( + self, source: Store, key: str, size: int, infos: Iterable[zipfile.ZipInfo] + ) -> None: + super().__init__(read_only=True) + self._source = source + self._key = key + self._size = size + # zarr keys are files: directory entries carry no data + self._infos = {info.filename: info for info in infos if not info.is_dir()} + self._data_offsets: dict[str, int] = {} + self._is_open = True + + @classmethod + async def open_archive(cls, source: Store, key: str = "") -> ZipReaderStore: + """ + Read the central directory of the archive at `key` in `source`. + + Parameters + ---------- + source : Store + The store holding the archive. + key : str, optional + The key of the archive in `source`; empty for a store rooted at + the archive itself. + + Returns + ------- + ZipReaderStore + An open, read-only store over the archive. + + Raises + ------ + zipfile.BadZipFile + If the resource is not a ZIP archive. + FileNotFoundError + If there is no resource at `key`. + """ + size = await _resource_size(source, key) + tail = min(size, _EOCD_SEARCH_BYTES) + segments = { + size - tail: await _read_resource_range( + source, key, RangeByteRequest(size - tail, size) + ) + } + while True: + try: + # parsing a large central directory is CPU-bound + infos = await asyncio.to_thread(_parse_central_directory, size, segments) + except _MissingRangeError as missing: + stop = min(size, max(missing.stop, missing.start + _MIN_FETCH_BYTES)) + segments[missing.start] = await _read_resource_range( + source, key, RangeByteRequest(missing.start, stop) + ) + continue + return cls(source, key, size, infos) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self._source == other._source + and self._key == other._key + ) + + def __str__(self) -> str: + return f"{self._source}|zip:" if not self._key else f"{self._source}/{self._key}|zip:" + + def __repr__(self) -> str: + return f"ZipReaderStore('{self}')" + + def close(self) -> None: + # docstring inherited + super().close() + self._source.close() + + async def _data_offset(self, info: zipfile.ZipInfo) -> int: + """The archive offset of a member's data, after its local header.""" + offset = self._data_offsets.get(info.filename) + if offset is None: + header = await _read_resource_range( + self._source, + self._key, + RangeByteRequest(info.header_offset, info.header_offset + _LOCAL_HEADER.size), + ) + fields = _LOCAL_HEADER.unpack(header) + if fields[0] != _LOCAL_HEADER_SIGNATURE: + raise zipfile.BadZipFile(f"bad local file header for {info.filename!r}") + name_length, extra_length = fields[-2], fields[-1] + offset = info.header_offset + _LOCAL_HEADER.size + name_length + extra_length + self._data_offsets[info.filename] = offset + return offset + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + info = self._infos.get(key) + if info is None: + return None + if info.flag_bits & 0x1: + raise NotImplementedError(f"ZIP member {key!r} is encrypted") + start = await self._data_offset(info) + if info.compress_type == zipfile.ZIP_STORED and byte_range is not None: + # uncompressed members support true range reads + length = info.file_size + if isinstance(byte_range, RangeByteRequest): + lo, hi = min(byte_range.start, length), min(byte_range.end, length) + elif isinstance(byte_range, OffsetByteRequest): + lo, hi = min(byte_range.offset, length), length + elif isinstance(byte_range, SuffixByteRequest): + lo, hi = max(0, length - byte_range.suffix), length + else: + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + if hi <= lo: + return prototype.buffer.from_bytes(b"") + data = await _read_resource_range( + self._source, self._key, RangeByteRequest(start + lo, start + hi) + ) + return prototype.buffer.from_bytes(data) + raw = await _read_resource_range( + self._source, self._key, RangeByteRequest(start, start + info.compress_size) + ) + if info.compress_size > _THREAD_DECOMPRESS_BYTES: + data = await asyncio.to_thread(_decompress, info, raw) + else: + data = _decompress(info, raw) + return prototype.buffer.from_bytes(_slice(data, byte_range)) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + return list( + await asyncio.gather( + *(self.get(key, prototype, byte_range) for key, byte_range in key_ranges) + ) + ) + + async def getsize(self, key: str) -> int: + # docstring inherited + info = self._infos.get(key) + if info is None: + raise FileNotFoundError(key) + return info.file_size + + async def exists(self, key: str) -> bool: + # docstring inherited + return key in self._infos + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + self._check_writable() + + async def delete(self, key: str) -> None: + # docstring inherited + self._check_writable() + + async def list(self) -> AsyncIterator[str]: + # docstring inherited + for key in self._infos: + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + for key in self._infos: + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + prefix = prefix.rstrip("/") + prefix = f"{prefix}/" if prefix else "" + seen: set[str] = set() + for key in self._infos: + if key.startswith(prefix): + child = key[len(prefix) :].split("/", 1)[0] + if child and child not in seen: + seen.add(child) + yield child + + +def _parse_zip_path(segment: PipelineSegment) -> str: + """The path within the archive addressed by a `zip:` segment.""" + if segment.query is not None: + raise URLPipelineError(f"'zip:' pipeline segments do not accept a query: {segment.raw!r}") + body = segment.body + if body.startswith("//"): + raise URLPipelineError( + f"invalid 'zip:' path in {segment.raw!r}: more than one leading '/' is not allowed" + ) + try: + return normalize_path(body.removeprefix("/")) + except ValueError as exc: + raise URLPipelineError(f"invalid path in pipeline segment {segment.raw!r}: {exc}") from exc + + +class ZipAdapter(URLPipelineAdapter): + """ + The `zip:` adapter: an entry or directory within a ZIP archive (read-only). + """ + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve a `zip:` segment. + + Parameters + ---------- + segment : PipelineSegment + The `zip:` segment. Its body is a path within the archive. + context : PipelineContext + The pipeline to the left of the segment, which must resolve to the archive. + + Returns + ------- + AdapterResolution + A read-only store over the archive, with the segment path as the residual path. + + Raises + ------ + URLPipelineError + If the mode requires writing, the segment is malformed, or the preceding + resource is not a readable ZIP archive. + """ + if context.mode in _WRITE_MODES: + raise URLPipelineError( + f"zip: is read-only until writable ZIP support lands; cannot open " + f"{segment.raw!r} with mode={context.mode!r}. Use mode='r' (or 'a' to open " + "an existing archive), or write the archive with zarr.storage.ZipStore." + ) + path = _parse_zip_path(segment) + preceding = await context.resolve_preceding(mode="r") + try: + store = await ZipReaderStore.open_archive( + preceding.store, normalize_path(preceding.path) + ) + except (OSError, zipfile.BadZipFile, ValueError) as exc: + preceding.store.close() + raise URLPipelineError( + f"could not open the ZIP archive {context.preceding_url!r} for " + f"{segment.raw!r}: {exc}" + ) from exc + except BaseException: + preceding.store.close() + raise + return dataclasses.replace(preceding, store=store, path=path) diff --git a/tests/test_url_pipeline/test_zip_adapter.py b/tests/test_url_pipeline/test_zip_adapter.py new file mode 100644 index 0000000000..4c34cc5a01 --- /dev/null +++ b/tests/test_url_pipeline/test_zip_adapter.py @@ -0,0 +1,358 @@ +"""Tests for the builtin, read-only `zip:` URL pipeline adapter.""" + +from __future__ import annotations + +import dataclasses +import io +import zipfile +from typing import TYPE_CHECKING, ClassVar + +import numpy as np +import pytest + +import zarr +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.core.buffer import default_buffer_prototype +from zarr.errors import URLPipelineError +from zarr.registry import get_url_adapter, register_url_adapter +from zarr.storage import LocalStore, WrapperStore, ZipStore +from zarr.storage._common import _has_fsspec +from zarr.storage._url_adapters._zip import ZipAdapter, ZipReaderStore +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline + +if TYPE_CHECKING: + from pathlib import Path + + from zarr.core.common import AccessModeLiteral + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + + +def _write_archive(path: Path, *, compression: int = zipfile.ZIP_STORED) -> Path: + """An archive with a v3 hierarchy at `inner-dir/subgroup`, written by ZipStore.""" + with ZipStore(path, mode="w", compression=compression) as store: + root = zarr.open_group(store, mode="w", path="inner-dir") + sub = root.create_group("subgroup", attributes={"title": "headline"}) + arr = sub.create_array("x", shape=(6,), chunks=(2,), dtype="i4") + arr[:] = np.arange(6) + return path + + +def _url_path(path: Path) -> str: + """A path spelled for a `file:` URL (a Windows drive gains a leading slash).""" + posix = path.as_posix() + return posix if posix.startswith("/") else f"/{posix}" + + +@pytest.fixture +def archive(tmp_path: Path) -> Path: + return _write_archive(tmp_path / "my-archive.zip") + + +class _CloseSpyStore(WrapperStore[Store]): + closed: ClassVar[list[str]] = [] + + def close(self) -> None: + type(self).closed.append(str(self)) + super().close() + + +class _CloseSpyAdapter(URLPipelineAdapter): + """Wraps the preceding store so tests can observe it being closed.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, store=_CloseSpyStore(preceding.store)) + + +def test_registered_as_builtin() -> None: + assert get_url_adapter("zip") is ZipAdapter + + +@pytest.mark.parametrize("url", ["zip:/tmp/a.zip", "zip:///tmp/a.zip", "zip::file:///tmp/a.zip"]) +def test_zip_is_not_a_root_scheme(url: str) -> None: + # strings without '|' keep their pre-pipeline meaning (fsspec's zip://) + assert not is_url_pipeline(url) + + +class TestRead: + def test_headline_example(self, archive: Path) -> None: + # zarr-developers/zarr-python#2943's headline example, with a local root + group = zarr.open_group(f"{archive}|zip:inner-dir|zarr3:subgroup", mode="r") + assert group.attrs["title"] == "headline" + x = group["x"] + assert isinstance(x, zarr.Array) + np.testing.assert_array_equal(x[:], np.arange(6)) + + @pytest.mark.parametrize( + "template", + [ + "file:{url}|zip:", + "file://{url}|zip:", + "file://localhost{url}|zip", + "{path}|zip:", + ], + ) + def test_root_spellings(self, archive: Path, template: str) -> None: + url = template.format(url=_url_path(archive), path=archive) + group = zarr.open_group(f"{url}|zarr3:inner-dir/subgroup", mode="r") + assert group.attrs["title"] == "headline" + + @pytest.mark.parametrize("body", ["inner-dir/subgroup", "/inner-dir/subgroup", "inner-dir/"]) + def test_leading_slash_is_ignored(self, archive: Path, body: str) -> None: + result = zarr.open_group(f"file:{_url_path(archive)}|zip:{body}", mode="r") + assert isinstance(result, zarr.Group) + + def test_double_leading_slash_is_invalid(self, archive: Path) -> None: + with pytest.raises(URLPipelineError, match="more than one leading '/'"): + zarr.open_group(f"{archive}|zip://inner-dir", mode="r") + + @pytest.mark.parametrize("body", ["..", "a/../b"]) + def test_invalid_path(self, archive: Path, body: str) -> None: + with pytest.raises(URLPipelineError, match="invalid path"): + zarr.open_group(f"{archive}|zip:{body}", mode="r") + + def test_query_is_rejected(self, archive: Path) -> None: + with pytest.raises(URLPipelineError, match="do not accept a query"): + zarr.open_group(f"{archive}|zip:a?x=1", mode="r") + + def test_open_array_through_paths(self, archive: Path) -> None: + arr = zarr.open_array(f"{archive}|zip:inner-dir/subgroup/x", mode="r") + np.testing.assert_array_equal(arr[1:4], [1, 2, 3]) + via_path = zarr.open_array(f"{archive}|zip:inner-dir", path="subgroup/x", mode="r") + np.testing.assert_array_equal(via_path[:], np.arange(6)) + + @pytest.mark.parametrize("mode", [None, "a", "r"]) + def test_default_and_open_modes_are_read_only( + self, archive: Path, mode: AccessModeLiteral | None + ) -> None: + # "a" (the zarr.open default) is open-or-create: zip: serves the open half + url = f"{archive}|zip:inner-dir/subgroup" + node = zarr.open(url) if mode is None else zarr.open(url, mode=mode) + assert isinstance(node, zarr.Group) + assert node.store.read_only + with pytest.raises(ValueError, match="read-only"): + node.create_group("new") + + @pytest.mark.parametrize("mode", ["w", "w-", "r+"]) + def test_write_modes_raise(self, archive: Path, mode: AccessModeLiteral) -> None: + with pytest.raises(URLPipelineError, match="zip: is read-only until"): + zarr.open_group(f"{archive}|zip:", mode=mode) + + @pytest.mark.parametrize( + "compression", [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED, zipfile.ZIP_BZIP2] + ) + def test_compression_methods(self, tmp_path: Path, compression: int) -> None: + path = _write_archive(tmp_path / "c.zip", compression=compression) + arr = zarr.open_array(f"{path}|zip:inner-dir/subgroup/x", mode="r") + np.testing.assert_array_equal(arr[:], np.arange(6)) + + def test_unsupported_compression(self, tmp_path: Path) -> None: + path = tmp_path / "lzma.zip" + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_LZMA) as zf: + zf.writestr("zarr.json", b"{}") + group_url = f"{path}|zip:" + with pytest.raises(NotImplementedError, match="compression method"): + zarr.open_group(group_url, mode="r") + + def test_archive_with_directory_entries(self, tmp_path: Path, archive: Path) -> None: + # archives made by `zip -r` carry directory entries; they are not keys + rewritten = tmp_path / "with-dirs.zip" + with zipfile.ZipFile(archive) as src, zipfile.ZipFile(rewritten, "w") as dst: + dst.writestr("inner-dir/", b"") + dst.writestr("inner-dir/subgroup/", b"") + for info in src.infolist(): + dst.writestr(info, src.read(info)) + group = zarr.open_group(f"{rewritten}|zip:inner-dir", mode="r") + assert sorted(group.group_keys()) == ["subgroup"] + + def test_large_central_directory(self, tmp_path: Path) -> None: + # a central directory larger than the initial suffix read is fetched + # in further range requests + path = tmp_path / "many.zip" + with zipfile.ZipFile(path, "w") as zf: + for i in range(3000): + zf.writestr(f"entries/padding-name-to-grow-the-directory-{i:05d}", b"x") + zf.writestr("zarr.json", b'{"zarr_format": 3, "node_type": "group"}') + group = zarr.open_group(f"{path}|zip:", mode="r") + assert isinstance(group, zarr.Group) + + def test_nested_zip(self, tmp_path: Path, archive: Path) -> None: + outer = tmp_path / "outer.zip" + with zipfile.ZipFile(outer, "w") as zf: + zf.write(archive, "nested/inner.zip") + group = zarr.open_group( + f"file:{_url_path(outer)}|zip:nested/inner.zip|zip:inner-dir|zarr3:subgroup", + mode="r", + ) + assert group.attrs["title"] == "headline" + + def test_nested_zip_compressed_outer(self, tmp_path: Path, archive: Path) -> None: + outer = tmp_path / "outer.zip" + with zipfile.ZipFile(outer, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.write(archive, "inner.zip") + arr = zarr.open_array(f"{outer}|zip:inner.zip|zip:inner-dir/subgroup/x", mode="r") + np.testing.assert_array_equal(arr[:], np.arange(6)) + + @pytest.mark.skipif(not _has_fsspec, reason="requires fsspec") + def test_fsspec_root(self, archive: Path) -> None: + # fsspec's `local://` protocol: an FsspecStore root, read by range requests + from zarr.storage import FsspecStore + + result = zarr.open_group(f"local://{archive.as_posix()}|zip:inner-dir|zarr3:subgroup") + assert isinstance(result.store, ZipReaderStore) + assert isinstance(result.store._source, FsspecStore) + assert result.attrs["title"] == "headline" + + @pytest.mark.skipif(not _has_fsspec, reason="requires fsspec") + def test_fsspec_memory_filesystem_root(self, archive: Path) -> None: + # an fsspec in-memory filesystem stands in for a remote object store + fsspec = pytest.importorskip("fsspec") + from fsspec.implementations.memory import MemoryFileSystem + + class RemoteLikeFS(MemoryFileSystem): # type: ignore[misc] + protocol = ("zarr-test-remote",) + store: ClassVar[dict[str, object]] = {} + pseudo_dirs: ClassVar[list[str]] = [""] + + @classmethod + def _strip_protocol(cls, path: str) -> str: + return "/" + path.removeprefix("zarr-test-remote://").lstrip("/") + + fsspec.register_implementation("zarr-test-remote", RemoteLikeFS, clobber=True) + RemoteLikeFS().pipe("/bucket/my-archive.zip", archive.read_bytes()) + group = zarr.open_group("zarr-test-remote://bucket/my-archive.zip|zip:inner-dir/subgroup") + assert group.attrs["title"] == "headline" + + def test_memory_root(self, archive: Path) -> None: + from zarr.core.buffer import cpu + from zarr.storage import ManagedMemoryStore + + holder = ManagedMemoryStore(name="zip-memory-root") + zarr.core.sync.sync(holder.set("dir/a.zip", cpu.Buffer.from_bytes(archive.read_bytes()))) + group = zarr.open_group("memory:zip-memory-root/dir/a.zip|zip:inner-dir/subgroup") + assert group.attrs["title"] == "headline" + + +class TestErrors: + def test_not_a_zip_file(self, tmp_path: Path) -> None: + path = tmp_path / "not.zip" + path.write_bytes(b"definitely not a zip archive") + with pytest.raises(URLPipelineError, match="could not open the ZIP archive"): + zarr.open_group(f"{path}|zip:", mode="r") + + def test_missing_archive(self, tmp_path: Path) -> None: + with pytest.raises((URLPipelineError, FileNotFoundError)): + zarr.open_group(f"{tmp_path / 'missing.zip'}|zip:", mode="r") + + def test_directory_root(self, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match="could not open the ZIP archive"): + zarr.open_group(f"{tmp_path}|zip:", mode="r") + + def test_preceding_store_closed_on_error(self, tmp_path: Path) -> None: + register_url_adapter("spy", _CloseSpyAdapter) + _CloseSpyStore.closed.clear() + path = tmp_path / "not.zip" + path.write_bytes(b"definitely not a zip archive") + with pytest.raises(URLPipelineError): + zarr.open_group(f"{path}|spy:|zip:", mode="r") + assert len(_CloseSpyStore.closed) == 1 + + async def test_close_closes_source(self, archive: Path) -> None: + register_url_adapter("spy", _CloseSpyAdapter) + _CloseSpyStore.closed.clear() + result = await resolve_pipeline(f"{archive}|spy:|zip:", mode="r") + assert _CloseSpyStore.closed == [] + result.store.close() + assert len(_CloseSpyStore.closed) == 1 + + +class TestZipReaderStore: + @pytest.fixture + async def store(self, tmp_path: Path) -> ZipReaderStore: + path = tmp_path / "s.zip" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("a/b", b"0123456789") + zf.writestr("a/c/d", b"x") + zf.writestr("e", b"y") + with zipfile.ZipFile(path, "a", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr("z", b"0123456789" * 10) + source = await LocalStore.open(path, read_only=True) + return await ZipReaderStore.open_archive(source) + + async def test_get_and_ranges(self, store: ZipReaderStore) -> None: + proto = default_buffer_prototype() + for key in ("a/b", "z"): + full = await store.get(key, proto) + assert full is not None + data = full.to_bytes() + cases: list[tuple[ByteRequest, bytes]] = [ + (RangeByteRequest(2, 5), data[2:5]), + (RangeByteRequest(8, 50), data[8:50]), + (OffsetByteRequest(7), data[7:]), + (SuffixByteRequest(3), data[-3:]), + (RangeByteRequest(200, 300), b""), + ] + for request, expected in cases: + got = await store.get(key, proto, request) + assert got is not None + assert got.to_bytes() == expected + assert await store.get("missing", proto) is None + values = await store.get_partial_values( + proto, [("e", None), ("a/b", RangeByteRequest(0, 1))] + ) + assert [v.to_bytes() if v else None for v in values] == [b"y", b"0"] + + async def test_listing(self, store: ZipReaderStore) -> None: + assert sorted([k async for k in store.list()]) == ["a/b", "a/c/d", "e", "z"] + assert sorted([k async for k in store.list_prefix("a/")]) == ["a/b", "a/c/d"] + assert sorted([k async for k in store.list_dir("")]) == ["a", "e", "z"] + assert sorted([k async for k in store.list_dir("a")]) == ["b", "c"] + assert await store.exists("e") + assert not await store.exists("a") + assert await store.getsize("z") == 100 + with pytest.raises(FileNotFoundError): + await store.getsize("missing") + + async def test_read_only(self, store: ZipReaderStore) -> None: + from zarr.core.buffer import cpu + + assert store.read_only + with pytest.raises(ValueError, match="read-only"): + await store.set("k", cpu.Buffer.from_bytes(b"")) + with pytest.raises(ValueError, match="read-only"): + await store.delete("e") + + async def test_eq_and_str(self, store: ZipReaderStore) -> None: + same = store + assert store == same + assert store != object() + assert str(store).endswith("|zip:") + assert "ZipReaderStore" in repr(store) + + async def test_bad_crc(self, tmp_path: Path) -> None: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("k", b"payload") + raw = bytearray(buf.getvalue()) + raw[raw.index(b"payload")] ^= 0xFF + path = tmp_path / "bad.zip" + path.write_bytes(bytes(raw)) + store = await ZipReaderStore.open_archive(await LocalStore.open(path, read_only=True)) + with pytest.raises(OSError, match="bad CRC-32"): + await store.get("k", default_buffer_prototype()) From 5a23da9918c9b3076224b437410b1dc025d4d220 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:30:55 -0700 Subject: [PATCH 13/14] docs: document the read-only zip: adapter in the storage guide Co-Authored-By: Claude Opus 5.5 --- docs/user-guide/storage.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index eeeb53e0f8..d8391cc632 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -137,6 +137,24 @@ A `zarr_format` argument that conflicts with a `zarr2:` / `zarr3:` segment raise `ValueError`. `Group.open` and `Array.open` take their format from the pipeline when `zarr_format` is not passed. +The builtin `zip:` adapter reads a ZIP archive addressed by the sub-URLs to its left, +which may be a local file, an fsspec URL (e.g. `s3://bucket/data.zip`), or an entry of +another archive (`outer.zip|zip:inner.zip|zip:`). The text after the colon is a path +within the archive (one leading `/` is ignored). `zip:` is currently read-only: it opens +archives in modes `"r"` and `"a"` (open-or-create serves the "open" half) and raises +`URLPipelineError` for `"w"`, `"w-"` and `"r+"`; write archives with +[`zarr.storage.ZipStore`][]. + +```python exec="true" session="storage-url-pipeline" source="above" result="ansi" +import os + +archive = os.path.join(root, "example.zip") +with zarr.storage.ZipStore(archive, mode="w") as store: + zarr.create_array(store, name="inner/data", shape=(3,), dtype="int32")[:] = [1, 2, 3] + +print(zarr.open_array(f"{archive}|zip:inner|zarr3:data")[:]) +``` + `storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); adapters may consume adapter-specific, namespaced keys. Non-dict forms of `storage_options` are reserved for future per-segment From e543de1fa3f032276c5755d106d071518f9f42d0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 14:35:51 -0700 Subject: [PATCH 14/14] docs: add changelog entry for #4477 Co-Authored-By: Claude Opus 5.5 --- changes/4477.feature.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/4477.feature.md diff --git a/changes/4477.feature.md b/changes/4477.feature.md new file mode 100644 index 0000000000..01d102d848 --- /dev/null +++ b/changes/4477.feature.md @@ -0,0 +1,6 @@ +Added a builtin, read-only `zip:` URL pipeline adapter for addressing data inside a ZIP +archive, e.g. `zarr.open_group("s3://bucket/data.zip|zip:inner-dir|zarr3:subgroup")`. The +archive can be a local file, an fsspec object, or an entry of another archive +(`outer.zip|zip:inner.zip|zip:`), and is read with async byte-range requests. Writable +modes (`"w"`, `"w-"`, `"r+"`) raise `URLPipelineError` for now; mode `"a"` opens the +archive read-only.