Merge zarr-developers/main into jakirkham/fix_pcodec_soft

.github/workflows/ci.yaml

@@ -102,5 +102,6 @@ jobs:
 
       - uses: codecov/codecov-action@v5
         with:
+          fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true

.github/workflows/wheel.yaml

@@ -26,7 +26,7 @@ jobs:
         with:
           submodules: true
 
-      - uses: pypa/cibuildwheel@v2.21.3
+      - uses: pypa/cibuildwheel@v2.22.0
 
       - uses: actions/upload-artifact@v4
         with:

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     - id: debug-statements
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.4
+    rev: v0.8.1
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]

docs/release.rst

@@ -13,6 +13,38 @@ Release notes
 
 .. _unreleased:
 
+Unreleased
+----------
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+* All arguments to the ``PCodec`` constructor except for ``level``
+  are now keyword only, to support the updated API.
+  By :user:`Sam Levang <slevang>`, :issue:`623`
+
+
+Fixes
+~~~~~
+* Fixes issue with ``Delta`` Zarr 3 codec not working with ``astype``.
+  By :user:`Norman Rzepka <normanrz>`, :issue:`664`
+
+
+Improvements
+~~~~~~~~~~~~
+* Add support for ``pcodec`` 0.3. This exposes the new ``delta_spec``
+  and ``paging_spec`` arguments, but maintains full backwards
+  compatibility for data written with older package versions.
+  By :user:`Sam Levang <slevang>`, :issue:`623`
+* If an import error is raised when trying to define a codec that is *not*
+  an optional dependency, it is no longer silently caught. Instead it will
+  be propagated to the user, as this indicates an issue with the installed
+  package.
+
+  Import errors caused by optional dependencies (ZFPY, MsgPack, CRC32C, and PCodec)
+  are still silently caught.
+  By :user:`David Stansby <dstansby>`, :issue:`550`.
+
+
 0.14.1
 ------
 

docs/zarr3.rst

@@ -1,3 +1,5 @@
+.. _Zarr 3 codecs:
+
 Zarr 3 codecs
 =============
 .. automodule:: numcodecs.zarr3

fixture/pcodec/codec.06/config.json

@@ -0,0 +1,9 @@
+{
+    "delta_encoding_order": 2,
+    "delta_spec": "auto",
+    "equal_pages_up_to": 262144,
+    "id": "pcodec",
+    "level": 8,
+    "mode_spec": "auto",
+    "paging_spec": "equal_pages_up_to"
+}

fixture/pcodec/codec.07/config.json

@@ -0,0 +1,9 @@
+{
+    "delta_encoding_order": null,
+    "delta_spec": "try_lookback",
+    "equal_pages_up_to": 262144,
+    "id": "pcodec",
+    "level": 8,
+    "mode_spec": "auto",
+    "paging_spec": "equal_pages_up_to"
+}

fixture/pcodec/codec.08/config.json

@@ -0,0 +1,9 @@
+{
+    "delta_encoding_order": null,
+    "delta_spec": "none",
+    "equal_pages_up_to": 262144,
+    "id": "pcodec",
+    "level": 8,
+    "mode_spec": "auto",
+    "paging_spec": "equal_pages_up_to"
+}

fixture/pcodec/codec.09/config.json

@@ -0,0 +1,9 @@
+{
+    "delta_encoding_order": 1,
+    "delta_spec": "try_consecutive",
+    "equal_pages_up_to": 262144,
+    "id": "pcodec",
+    "level": 8,
+    "mode_spec": "auto",
+    "paging_spec": "equal_pages_up_to"
+}

numcodecs/__init__.py

@@ -36,41 +36,32 @@
 
 register_codec(BZ2)
 
-with suppress(ImportError):
-    from numcodecs.lzma import LZMA
+from numcodecs.lzma import LZMA
 
-    register_codec(LZMA)
+register_codec(LZMA)
 
-with suppress(ImportError):
-    from numcodecs import blosc
-    from numcodecs.blosc import Blosc
-
-    register_codec(Blosc)
-    # initialize blosc
-    try:
-        ncores = multiprocessing.cpu_count()
-    except OSError:  # pragma: no cover
-        ncores = 1
-    blosc.init()
-    blosc.set_nthreads(min(8, ncores))
-    atexit.register(blosc.destroy)
+from numcodecs import blosc
+from numcodecs.blosc import Blosc
 
-with suppress(ImportError):
-    from numcodecs import zstd as zstd
-    from numcodecs.zstd import Zstd
+register_codec(Blosc)
+# initialize blosc
+try:
+    ncores = multiprocessing.cpu_count()
+except OSError:  # pragma: no cover
+    ncores = 1
+blosc.init()
+blosc.set_nthreads(min(8, ncores))
+atexit.register(blosc.destroy)
 
-    register_codec(Zstd)
+from numcodecs import zstd as zstd
+from numcodecs.zstd import Zstd
 
-with suppress(ImportError):
-    from numcodecs import lz4 as lz4
-    from numcodecs.lz4 import LZ4
+register_codec(Zstd)
 
-    register_codec(LZ4)
+from numcodecs import lz4 as lz4
+from numcodecs.lz4 import LZ4
 
-with suppress(ImportError):
-    from numcodecs.zfpy import ZFPY
-
-    register_codec(ZFPY)
+register_codec(LZ4)
 
 from numcodecs.astype import AsType
 
@@ -112,38 +103,43 @@
 
 register_codec(BitRound)
 
-with suppress(ImportError):
-    from numcodecs.msgpacks import MsgPack
-
-    register_codec(MsgPack)
-
 from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3
 
 register_codec(CRC32)
 register_codec(Adler32)
 register_codec(JenkinsLookup3)
 
-with suppress(ImportError):
-    from numcodecs.checksum32 import CRC32C
-
-    register_codec(CRC32C)
-
 from numcodecs.json import JSON
 
 register_codec(JSON)
 
-with suppress(ImportError):
-    from numcodecs import vlen as vlen
-    from numcodecs.vlen import VLenArray, VLenBytes, VLenUTF8
+from numcodecs import vlen as vlen
+from numcodecs.vlen import VLenArray, VLenBytes, VLenUTF8
 
-    register_codec(VLenUTF8)
-    register_codec(VLenBytes)
-    register_codec(VLenArray)
+register_codec(VLenUTF8)
+register_codec(VLenBytes)
+register_codec(VLenArray)
 
 from numcodecs.fletcher32 import Fletcher32
 
 register_codec(Fletcher32)
 
+# Optional depenedencies
+with suppress(ImportError):
+    from numcodecs.zfpy import ZFPY
+
+    register_codec(ZFPY)
+
+with suppress(ImportError):
+    from numcodecs.msgpacks import MsgPack
+
+    register_codec(MsgPack)
+
+with suppress(ImportError):
+    from numcodecs.checksum32 import CRC32C
+
+    register_codec(CRC32C)
+
 with suppress(ImportError):
     from numcodecs.pcodec import PCodec
 

numcodecs/abc.py

@@ -84,7 +84,7 @@ def get_config(self):
         # override in sub-class if need special encoding of config values
 
         # setup config object
-        config = dict(id=self.codec_id)
+        config = {'id': self.codec_id}
 
         # by default, assume all non-private members are configuration
         # parameters - override this in sub-class if not the case

numcodecs/categorize.py

@@ -85,12 +85,12 @@ def decode(self, buf, out=None):
         return dec
 
     def get_config(self):
-        config = dict(
-            id=self.codec_id,
-            labels=self.labels,
-            dtype=self.dtype.str,
-            astype=self.astype.str,
-        )
+        config = {
+            'id': self.codec_id,
+            'labels': self.labels,
+            'dtype': self.dtype.str,
+            'astype': self.astype.str,
+        }
         return config
 
     def __repr__(self):

numcodecs/delta.py

@@ -91,7 +91,7 @@ def decode(self, buf, out=None):
 
     def get_config(self):
         # override to handle encoding dtypes
-        return dict(id=self.codec_id, dtype=self.dtype.str, astype=self.astype.str)
+        return {'id': self.codec_id, 'dtype': self.dtype.str, 'astype': self.astype.str}
 
     def __repr__(self):
         r = f'{type(self).__name__}(dtype={self.dtype.str!r}'

numcodecs/fixedscaleoffset.py

@@ -116,13 +116,13 @@ def decode(self, buf, out=None):
 
     def get_config(self):
         # override to handle encoding dtypes
-        return dict(
-            id=self.codec_id,
-            scale=self.scale,
-            offset=self.offset,
-            dtype=self.dtype.str,
-            astype=self.astype.str,
-        )
+        return {
+            'id': self.codec_id,
+            'scale': self.scale,
+            'offset': self.offset,
+            'dtype': self.dtype.str,
+            'astype': self.astype.str,
+        }
 
     def __repr__(self):
         r = f'{type(self).__name__}(scale={self.scale}, offset={self.offset}, dtype={self.dtype.str!r}'

numcodecs/json.py

@@ -52,17 +52,17 @@ def __init__(
             else:
                 separators = ', ', ': '
         separators = tuple(separators)
-        self._encoder_config = dict(
-            skipkeys=skipkeys,
-            ensure_ascii=ensure_ascii,
-            check_circular=check_circular,
-            allow_nan=allow_nan,
-            indent=indent,
-            separators=separators,
-            sort_keys=sort_keys,
-        )
+        self._encoder_config = {
+            'skipkeys': skipkeys,
+            'ensure_ascii': ensure_ascii,
+            'check_circular': check_circular,
+            'allow_nan': allow_nan,
+            'indent': indent,
+            'separators': separators,
+            'sort_keys': sort_keys,
+        }
         self._encoder = _json.JSONEncoder(**self._encoder_config)
-        self._decoder_config = dict(strict=strict)
+        self._decoder_config = {'strict': strict}
         self._decoder = _json.JSONDecoder(**self._decoder_config)
 
     def encode(self, buf):
@@ -89,7 +89,7 @@ def decode(self, buf, out=None):
             return dec
 
     def get_config(self):
-        config = dict(id=self.codec_id, encoding=self._text_encoding)
+        config = {'id': self.codec_id, 'encoding': self._text_encoding}
         config.update(self._encoder_config)
         config.update(self._decoder_config)
         return config

numcodecs/msgpacks.py

@@ -75,12 +75,12 @@ def decode(self, buf, out=None):
             return dec
 
     def get_config(self):
-        return dict(
-            id=self.codec_id,
-            raw=self.raw,
-            use_single_float=self.use_single_float,
-            use_bin_type=self.use_bin_type,
-        )
+        return {
+            'id': self.codec_id,
+            'raw': self.raw,
+            'use_single_float': self.use_single_float,
+            'use_bin_type': self.use_bin_type,
+        }
 
     def __repr__(self):
         return f'MsgPack(raw={self.raw!r}, use_bin_type={self.use_bin_type!r}, use_single_float={self.use_single_float!r})'