fix(dtype): accept any byte order for byte-order-irrelevant v2 dtype names

changes/351.bugfix.md

@@ -0,0 +1 @@
+Zarr format 2 arrays whose data type is a single-byte boolean or integer, or raw bytes, written with a `<` or `>` byte order (e.g. `<i1`, `>b1`, `<V4`, as written by netCDF-C, Zarr.jl and jzarr) can be read again, including as fields of structured data types. The byte order of these data types is not relevant, and metadata is written back with the canonical `|` spelling. Also, `datetime64` and `timedelta64` data types no longer accept an object codec in Zarr format 2, and a structured data type with an unrecognized field data type is rejected with `DataTypeValidationError`.

examples/custom_dtype/custom_dtype.py

@@ -22,7 +22,7 @@
 import pytest
 
 import zarr
-from zarr.dtype import ZDType, check_dtype_spec_v2, data_type_registry
+from zarr.dtype import ZDType, check_dtype_spec_no_object_codec_v2, data_type_registry
 from zarr.errors import DataTypeValidationError
 from zarr.types import JSON, DTypeConfig_V2, DTypeJSON, ZarrFormat
 
@@ -79,9 +79,7 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[Literal["|b
 
         See the Zarr docs for more information about the JSON encoding for data types.
         """
-        return (
-            check_dtype_spec_v2(data) and data["name"] == "int2" and data["object_codec_id"] is None
-        )
+        return check_dtype_spec_no_object_codec_v2(data) and data["name"] == "int2"
 
     @classmethod
     def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["int2"]]:

src/zarr/core/dtype/common.py

@@ -115,6 +115,19 @@ def check_dtype_spec_v2(data: object) -> TypeGuard[DTypeSpec_V2]:
     return isinstance(data["object_codec_id"], str | None)
 
 
+def check_dtype_spec_no_object_codec_v2(
+    data: object,
+) -> TypeGuard[DTypeConfig_V2[DTypeName_V2, None]]:
+    """
+    Type guard for narrowing a python object to a Zarr V2 data type without an object codec.
+
+    Only the data types stored with the NumPy "O" data type have an object codec, so every other
+    data type should check its Zarr V2 JSON with this function rather than
+    [`check_dtype_spec_v2`][zarr.dtype.check_dtype_spec_v2].
+    """
+    return check_dtype_spec_v2(data) and data["object_codec_id"] is None
+
+
 # By comparison, The JSON representation of a dtype in zarr v3 is much simpler.
 # It's either a string, or a structured dict
 DTypeSpec_V3 = str | NamedConfig[str, Mapping[str, object]]

src/zarr/core/dtype/npy/bool.py

@@ -9,7 +9,7 @@
     DTypeConfig_V2,
     DTypeJSON,
     HasItemSize,
-    check_dtype_spec_v2,
+    check_dtype_spec_no_object_codec_v2,
 )
 from zarr.core.dtype.wrapper import TBaseDType, ZDType
 from zarr.errors import DataTypeValidationError
@@ -103,11 +103,7 @@ def _check_json_v2(
         ``TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]``
             True if the input is a valid JSON representation, False otherwise.
         """
-        return (
-            check_dtype_spec_v2(data)
-            and data["name"] == cls._zarr_v2_name
-            and data["object_codec_id"] is None
-        )
+        return check_dtype_spec_no_object_codec_v2(data) and data["name"] == cls._zarr_v2_name
 
     @classmethod
     def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["bool"]]:

src/zarr/core/dtype/npy/bytes.py

@@ -14,6 +14,7 @@
     HasItemSize,
     HasLength,
     HasObjectCodec,
+    check_dtype_spec_no_object_codec_v2,
     check_dtype_spec_v2,
     v3_unstable_dtype_warning,
 )
@@ -267,10 +268,9 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[NullterminatedBytesJSON_V2
         """
 
         return (
-            check_dtype_spec_v2(data)
+            check_dtype_spec_no_object_codec_v2(data)
             and isinstance(data["name"], str)
             and re.match(r"^\|S\d+$", data["name"]) is not None
-            and data["object_codec_id"] is None
         )
 
     @classmethod
@@ -669,10 +669,9 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[RawBytesJSON_V2]:
 
         """
         return (
-            check_dtype_spec_v2(data)
+            check_dtype_spec_no_object_codec_v2(data)
             and isinstance(data["name"], str)
             and re.match(r"^\|V\d+$", data["name"]) is not None
-            and data["object_codec_id"] is None
         )
 
     @classmethod

src/zarr/core/dtype/npy/complex.py

@@ -17,7 +17,7 @@
     DTypeJSON,
     HasEndianness,
     HasItemSize,
-    check_dtype_spec_v2,
+    check_dtype_spec_no_object_codec_v2,
 )
 from zarr.core.dtype.npy.common import (
     ComplexLike,
@@ -106,11 +106,7 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]
         bool
             True if the input is a valid JSON representation, False otherwise.
         """
-        return (
-            check_dtype_spec_v2(data)
-            and data["name"] in cls._zarr_v2_names
-            and data["object_codec_id"] is None
-        )
+        return check_dtype_spec_no_object_codec_v2(data) and data["name"] in cls._zarr_v2_names
 
     @classmethod
     def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]:

src/zarr/core/dtype/npy/float.py

@@ -10,7 +10,7 @@
     DTypeJSON,
     HasEndianness,
     HasItemSize,
-    check_dtype_spec_v2,
+    check_dtype_spec_no_object_codec_v2,
 )
 from zarr.core.dtype.npy.common import (
     FloatLike,
@@ -91,11 +91,7 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]
         TypeGuard[DTypeConfig_V2[str, None]]
             True if the input is a valid JSON representation of this data type, False otherwise.
         """
-        return (
-            check_dtype_spec_v2(data)
-            and data["name"] in cls._zarr_v2_names
-            and data["object_codec_id"] is None
-        )
+        return check_dtype_spec_no_object_codec_v2(data) and data["name"] in cls._zarr_v2_names
 
     @classmethod
     def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]:

src/zarr/core/dtype/npy/int.py

@@ -19,7 +19,7 @@
     DTypeJSON,
     HasEndianness,
     HasItemSize,
-    check_dtype_spec_v2,
+    check_dtype_spec_no_object_codec_v2,
 )
 from zarr.core.dtype.npy.common import (
     check_json_int,
@@ -86,11 +86,7 @@ def _check_json_v2(cls, data: object) -> TypeGuard[DTypeConfig_V2[str, None]]:
             False otherwise.
         """
 
-        return (
-            check_dtype_spec_v2(data)
-            and data["name"] in cls._zarr_v2_names
-            and data["object_codec_id"] is None
-        )
+        return check_dtype_spec_no_object_codec_v2(data) and data["name"] in cls._zarr_v2_names
 
     @classmethod
     def _check_json_v3(cls, data: object) -> TypeGuard[str]:
@@ -415,11 +411,7 @@ class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]):
 
     dtype_cls = np.dtypes.UInt8DType
     _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8"
-    _zarr_v2_names: ClassVar[tuple[Literal["|u1"], Literal["<u1"], Literal[">u1"]]] = (
-        "|u1",
-        "<u1",
-        ">u1",
-    )
+    _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",)
 
     @classmethod
     def from_native_dtype(cls, dtype: TBaseDType) -> Self:

src/zarr/core/dtype/npy/string.py

@@ -24,6 +24,7 @@
     HasItemSize,
     HasLength,
     HasObjectCodec,
+    check_dtype_spec_no_object_codec_v2,
     check_dtype_spec_v2,
 )
 from zarr.core.dtype.npy.common import (
@@ -191,10 +192,9 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[FixedLengthUTF32JSON_V2]:
             Whether the input is a valid JSON representation of a NumPy U dtype.
         """
         return (
-            check_dtype_spec_v2(data)
+            check_dtype_spec_no_object_codec_v2(data)
             and isinstance(data["name"], str)
             and re.match(r"^[><]U\d+$", data["name"]) is not None
-            and data["object_codec_id"] is None
         )
 
     @classmethod

src/zarr/core/dtype/npy/structured.py

@@ -12,7 +12,7 @@
     DTypeJSON,
     HasItemSize,
     StructuredName_V2,
-    check_dtype_spec_v2,
+    check_dtype_spec_no_object_codec_v2,
     check_structured_dtype_name_v2,
     v3_unstable_dtype_warning,
 )
@@ -30,6 +30,22 @@
 StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int
 
 
+def _field_from_json(
+    data: DTypeJSON, *, zarr_format: ZarrFormat
+) -> ZDType[TBaseDType, TBaseScalar]:
+    """
+    Parse the data type of a structured field, reporting an unrecognized field data type as an
+    invalid structured data type.
+    """
+    # avoid circular import
+    from zarr.core.dtype import get_data_type_from_json
+
+    try:
+        return get_data_type_from_json(data, zarr_format=zarr_format)
+    except ValueError as e:
+        raise DataTypeValidationError(f"Invalid structured field data type: {e}") from e
+
+
 class StructuredJSON_V2(DTypeConfig_V2[StructuredName_V2, None]):
     """
     A wrapper around the JSON representation of the ``Structured`` data type in Zarr V2.
@@ -239,10 +255,9 @@ def _check_json_v2(
             for Zarr V2, False otherwise.
         """
         return (
-            check_dtype_spec_v2(data)
+            check_dtype_spec_no_object_codec_v2(data)
             and not isinstance(data["name"], str)
             and check_structured_dtype_name_v2(data["name"])
-            and data["object_codec_id"] is None
         )
 
     @classmethod
@@ -271,9 +286,6 @@ def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[StructuredJSON_V3]:
 
     @classmethod
     def _from_json_v2(cls, data: DTypeJSON) -> Self:
-        # avoid circular import
-        from zarr.core.dtype import get_data_type_from_json
-
         if cls._check_json_v2(data):
             # structured dtypes are constructed directly from a list of lists
             # note that we do not handle the object codec here! this will prevent structured
@@ -283,9 +295,7 @@ def _from_json_v2(cls, data: DTypeJSON) -> Self:
                 fields=tuple(  # type: ignore[str-unpack]
                     (  # type: ignore[misc]
                         f_name,
-                        get_data_type_from_json(
-                            {"name": f_dtype, "object_codec_id": None}, zarr_format=2
-                        ),
+                        _field_from_json({"name": f_dtype, "object_codec_id": None}, zarr_format=2),
                     )
                     for f_name, f_dtype in name
                 )
@@ -295,14 +305,12 @@ def _from_json_v2(cls, data: DTypeJSON) -> Self:
 
     @classmethod
     def _from_json_v3(cls, data: DTypeJSON) -> Self:
-        from zarr.core.dtype import get_data_type_from_json
-
         if cls._check_json_v3(data):
             config = data["configuration"]
             meta_fields = config["fields"]
             return cls(
                 fields=tuple(
-                    (f_name, get_data_type_from_json(f_dtype, zarr_format=3))  # type: ignore[misc]
+                    (f_name, _field_from_json(f_dtype, zarr_format=3))  # type: ignore[misc]
                     for f_name, f_dtype in meta_fields
                 )
             )
@@ -557,8 +565,6 @@ def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[StructJSON_V3]:  # type: i
 
     @classmethod
     def _from_json_v3(cls, data: DTypeJSON) -> Self:
-        from zarr.core.dtype import get_data_type_from_json
-
         if cls._check_json_v3(data):
             config = data["configuration"]
             meta_fields = config["fields"]
@@ -570,7 +576,7 @@ def _from_json_v3(cls, data: DTypeJSON) -> Self:
                 else:
                     # Legacy tuple-style field format from "structured" dtype
                     f_name, f_dtype = field  # type: ignore[unreachable]
-                parsed_fields.append((f_name, get_data_type_from_json(f_dtype, zarr_format=3)))  # type: ignore[arg-type]
+                parsed_fields.append((f_name, _field_from_json(f_dtype, zarr_format=3)))  # type: ignore[arg-type]
             return cls(fields=tuple(parsed_fields))
         msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON object with the key {cls._zarr_v3_name!r}"
         raise DataTypeValidationError(msg)

src/zarr/core/dtype/npy/time.py

@@ -23,7 +23,7 @@
     DTypeJSON,
     HasEndianness,
     HasItemSize,
-    check_dtype_spec_v2,
+    check_dtype_spec_no_object_codec_v2,
 )
 from zarr.core.dtype.npy.common import (
     DATETIME_UNIT,
@@ -369,7 +369,7 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[TimeDelta64JSON_V2]:
             True if the JSON input is a valid representation of this class,
             otherwise False.
         """
-        if not check_dtype_spec_v2(data):
+        if not check_dtype_spec_no_object_codec_v2(data):
             return False
         name = data["name"]
         # match <m[ns], >m[M], etc
@@ -644,7 +644,7 @@ def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSON_V2]:
             True if the input is a valid JSON representation of a NumPy datetime64 data type,
             otherwise False.
         """
-        if not check_dtype_spec_v2(data):
+        if not check_dtype_spec_no_object_codec_v2(data):
             return False
         name = data["name"]
         if not isinstance(name, str):