Skip to content

add numcodecs.zarr3.to_zarr3 method #741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docs/release.rst
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ Release notes
Unreleased
----------


.. _unreleased:

Unreleased
Expand All @@ -20,6 +19,9 @@ Unreleased
Enhancements
~~~~~~~~~~~~

* implement ``to_zarr3`` in ``numcodecs.zarr3`` to enable conversion of a codec to its zarr3-compatible equivalent.
By :user:`Hannes Spitz <brokkoli71>`

Improvements
~~~~~~~~~~~~

Expand Down Expand Up @@ -63,7 +65,7 @@ Enhancements
Removals
~~~~~~~~

The following ``blosc`` funcitons are removed, with no replacement.
The following ``blosc`` functions are removed, with no replacement.
This is because they were not intended to be public API.

- ``numcodecs.blosc.init``
Expand Down
64 changes: 64 additions & 0 deletions numcodecs/tests/test_zarr3.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

import numpy as np
import pytest
from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec
from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike

if TYPE_CHECKING: # pragma: no cover
import zarr
Expand Down Expand Up @@ -277,3 +279,65 @@
def test_to_dict():
codec = numcodecs.zarr3.LZ4(level=5)
assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}}


@pytest.mark.parametrize(
("codec_v2", "expected_v3_cls"),
[
(numcodecs.BZ2(), numcodecs.zarr3.BZ2),
(numcodecs.CRC32(), numcodecs.zarr3.CRC32),
(numcodecs.CRC32C(), numcodecs.zarr3.CRC32C),
(numcodecs.LZ4(), numcodecs.zarr3.LZ4),
(numcodecs.LZMA(), numcodecs.zarr3.LZMA),
(numcodecs.ZFPY(), numcodecs.zarr3.ZFPY),
(numcodecs.Adler32(), numcodecs.zarr3.Adler32),
(
numcodecs.AsType(encode_dtype=np.float64, decode_dtype=np.float32),
numcodecs.zarr3.AsType,
),
(numcodecs.BitRound(keepbits=10), numcodecs.zarr3.BitRound),
(numcodecs.Blosc(), numcodecs.zarr3.Blosc),
(numcodecs.Delta(dtype=np.float64), numcodecs.zarr3.Delta),
(
numcodecs.FixedScaleOffset(offset=1000, scale=10, dtype='f8', astype='u1'),
numcodecs.zarr3.FixedScaleOffset,
),
(numcodecs.Fletcher32(), numcodecs.zarr3.Fletcher32),
(numcodecs.GZip(), numcodecs.zarr3.GZip),
(numcodecs.JenkinsLookup3(), numcodecs.zarr3.JenkinsLookup3),
(numcodecs.PCodec(), numcodecs.zarr3.PCodec),
(numcodecs.PackBits(), numcodecs.zarr3.PackBits),
(numcodecs.Quantize(digits=1, dtype='f8'), numcodecs.zarr3.Quantize),
(numcodecs.Shuffle(), numcodecs.zarr3.Shuffle),
(numcodecs.Zlib(), numcodecs.zarr3.Zlib),
(numcodecs.Zstd(), numcodecs.zarr3.Zstd),
],
)
def test_cast_numcodecs_to_v3(store: Store, codec_v2, expected_v3_cls) -> None:
result_v3 = numcodecs.zarr3.to_zarr3(codec_v2)

assert result_v3.__class__ == expected_v3_cls
assert result_v3.codec_config == codec_v2.get_config()

filters: FiltersLike = "auto"
serializer: SerializerLike = "auto"
compressors: CompressorsLike = "auto"
if isinstance(result_v3, ArrayArrayCodec):
filters = [result_v3]
elif isinstance(result_v3, ArrayBytesCodec):
serializer = result_v3
elif isinstance(result_v3, BytesBytesCodec):
compressors = [result_v3]
else:
raise TypeError(f"unsupported type: {result_v3.__class__}")

Check warning on line 332 in numcodecs/tests/test_zarr3.py

View check run for this annotation

Codecov / codecov/patch

numcodecs/tests/test_zarr3.py#L332

Added line #L332 was not covered by tests

zarr.create_array(
store,
shape=(64,),
chunks=(64,),
dtype=np.bool,
fill_value=0,
filters=filters,
compressors=compressors,
serializer=serializer,
)
18 changes: 18 additions & 0 deletions numcodecs/zarr3.py
Original file line number Diff line number Diff line change
Expand Up @@ -399,3 +399,21 @@
"Zlib",
"Zstd",
]


def to_zarr3(
codec: numcodecs.abc.Codec,
) -> _NumcodecsBytesBytesCodec | _NumcodecsArrayBytesCodec | _NumcodecsArrayArrayCodec:
"""Convert a numcodecs codec to its zarr3-compatible equivalent."""
codec_name = codec.__class__.__name__
zarr3_module = numcodecs.zarr3

if not hasattr(zarr3_module, codec_name):
raise ValueError(f"No Zarr3 wrapper found for codec: {codec_name}")

Check warning on line 412 in numcodecs/zarr3.py

View check run for this annotation

Codecov / codecov/patch

numcodecs/zarr3.py#L412

Added line #L412 was not covered by tests

zarr3_codec_class = getattr(zarr3_module, codec_name)

config = codec.get_config()
config.pop("id", None)

return zarr3_codec_class(**config)
Loading