obstore-based Store implementation

.pre-commit-config.yaml

@@ -8,9 +8,9 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.9.9
     hooks:
-    - id: ruff
-      args: ["--fix", "--show-fixes"]
-    - id: ruff-format
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
     rev: v2.4.1
     hooks:
@@ -19,8 +19,8 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
-    - id: check-yaml
-    - id: trailing-whitespace
+      - id: check-yaml
+      - id: trailing-whitespace
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.15.0
     hooks:
@@ -31,9 +31,10 @@ repos:
           - packaging
           - donfig
           - numcodecs[crc32c]
-          - numpy==2.1  # until https://github.com/numpy/numpy/issues/28034 is resolved
+          - numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved
           - typing_extensions
           - universal-pathlib
+          - obstore==0.5.1
           # Tests
           - pytest
   - repo: https://github.com/scientific-python/cookie

changes/1661.feature.rst

@@ -0,0 +1 @@
+Add experimental ObjectStore storage class based on obstore.

docs/user-guide/storage.rst

@@ -99,6 +99,44 @@ Zarr data (metadata and chunks) to a dictionary.:
    >>> zarr.create_array(store=store, shape=(2,), dtype='float64')
    <Array memory://... shape=(2,) dtype=float64>
 
+Object Store
+~~~~~~~~~~~~
+
+
+:class:`zarr.storage.ObjectStore` stores the contents of the Zarr hierarchy using any ObjectStore
+`storage implementation <https://developmentseed.org/obstore/latest/api/store/>`_, such as
+AWS S3, Google Cloud Storage, and Azure Blob Storage. This store is backed by `obstore <https://developmentseed.org/obstore/latest/>`_, which
+builds on the production quality Rust library `object_store <https://docs.rs/object_store/latest/object_store/>`_.
+
+
+   >>> from zarr.storage import ObjectStore
+   >>> from obstore.store import MemoryStore
+   >>>
+   >>> store = ObjectStore(MemoryStore())
+   >>> zarr.create_array(store=store, shape=(2,), dtype='float64')
+   <Array object://... shape=(2,) dtype=float64>
+
+Here's an example of using ObjectStore for accessing remote data:
+
+   >>> from zarr.storage import ObjectStore
+   >>> from obstore.store import S3Store
+   >>>
+   >>> s3_store = S3Store('noaa-nwm-retro-v2-zarr-pds', skip_signature=True, region="us-west-2")
+   >>> store = zarr.storage.ObjectStore(store=s3_store, read_only=True)
+   >>> group = zarr.open_group(store=store, mode='r')
+   >>> group.info
+   Name        :
+   Type        : Group
+   Zarr format : 2
+   Read-only   : True
+   Store type  : ObjectStore
+   No. members : 12
+   No. arrays  : 12
+   No. groups  : 0
+
+.. warning::
+   The :class:`zarr.storage.ObjectStore` class is experimental.
+
 .. _user-guide-custom-stores:
 
 Developing custom stores

pyproject.toml

@@ -64,6 +64,7 @@ keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
 # User extras
 remote = [
     "fsspec>=2023.10.0",
+    "obstore==0.5.1",
 ]
 gpu = [
     "cupy-cuda12x",
@@ -165,7 +166,7 @@ run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tes
 list-env = "pip list"
 
 [tool.hatch.envs.doctest]
-features = ["test", "optional"]
+features = ["test", "optional", "remote"]
 description = "Test environment for doctests"
 
 [tool.hatch.envs.doctest.scripts]
@@ -211,6 +212,7 @@ dependencies = [
     'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib',
     'typing_extensions @ git+https://github.com/python/typing_extensions',
     'donfig @ git+https://github.com/pytroll/donfig',
+    'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore',
     # test deps
     'zarr[test]',
 ]

src/zarr/storage/__init__.py

@@ -8,6 +8,7 @@
 from zarr.storage._local import LocalStore
 from zarr.storage._logging import LoggingStore
 from zarr.storage._memory import GpuMemoryStore, MemoryStore
+from zarr.storage._obstore import ObjectStore
 from zarr.storage._wrapper import WrapperStore
 from zarr.storage._zip import ZipStore
 
@@ -17,6 +18,7 @@
     "LocalStore",
     "LoggingStore",
     "MemoryStore",
+    "ObjectStore",
     "StoreLike",
     "StorePath",
     "WrapperStore",