From 1f740da54bfbe754e27b169cdc770f85cf1e31e7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 16:53:26 +0200 Subject: [PATCH 1/5] fix(zarr-metadata): from_key_value reads what to_key_value writes `to_key_value` keys its result by the store-key literals, and a mapping's key type is invariant, so passing that result to `from_key_value`, which took `Mapping[str, bytes]`, did not type-check. `from_key_value` now takes a mapping keyed by `str` or by any narrower key type. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/4421.bugfix.md | 4 +++ .../src/zarr_metadata/model/_array.py | 5 ++-- .../src/zarr_metadata/model/_group.py | 7 +++-- .../src/zarr_metadata/model/_validation.py | 15 +++++++--- .../zarr-metadata/tests/model/test_array.py | 28 ++++++------------- 5 files changed, 30 insertions(+), 29 deletions(-) create mode 100644 packages/zarr-metadata/changes/4421.bugfix.md diff --git a/packages/zarr-metadata/changes/4421.bugfix.md b/packages/zarr-metadata/changes/4421.bugfix.md new file mode 100644 index 0000000000..cb9d53847a --- /dev/null +++ b/packages/zarr-metadata/changes/4421.bugfix.md @@ -0,0 +1,4 @@ +`from_key_value` takes a mapping keyed by `str` or by any narrower key +type, so reading back what `to_key_value` wrote type-checks. `to_key_value` +keys its result by the store-key literals, and since a mapping's key type is +invariant, a `Mapping[str, bytes]` parameter refused it. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 26a69a304a..bfee447956 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -14,6 +14,7 @@ from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, + StoreKey, ValidationProblem, arrays_to_tuples, dump_store_json, @@ -322,7 +323,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: return must_understand_subset(self.extra_fields) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: + def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY)) def to_key_value( @@ -468,7 +469,7 @@ def from_json(cls, data: object) -> ZarrV2ArrayMetadata: ) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: + def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2ArrayMetadata: zarray_raw = load_store_json(mapping, ZARR_V2_ARRAY_METADATA_STORE_KEY) if not isinstance(zarray_raw, Mapping): return cls.from_json(zarray_raw) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 5519e6fbb9..d71badfd1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -18,6 +18,7 @@ from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, + StoreKey, ValidationProblem, arrays_to_tuples, dump_store_json, @@ -176,7 +177,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: return must_understand_subset(self.extra_fields) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: + def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY)) def to_key_value( @@ -314,7 +315,7 @@ def from_json(cls, data: object) -> ZarrV2GroupMetadata: return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: + def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2GroupMetadata: zgroup_raw = load_store_json(mapping, ZARR_V2_GROUP_METADATA_STORE_KEY) if not isinstance(zgroup_raw, Mapping): return cls.from_json(zgroup_raw) @@ -426,7 +427,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: return cls(metadata=entries_tupled) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: + def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2ConsolidatedMetadata: return cls.from_json(load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY)) def to_key_value( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index f927851e4c..2051d39884 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -17,7 +17,7 @@ import math from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal, NoReturn, cast +from typing import Final, Literal, NoReturn, TypeVar, cast from typing_extensions import TypeIs @@ -836,7 +836,11 @@ def _reject_json_constant(constant: str) -> NoReturn: raise ValueError(f"non-standard JSON constant {constant!r}") -def load_store_json(mapping: Mapping[str, bytes], key: str) -> object: +StoreKey = TypeVar("StoreKey", bound=str) +"""The key type of a mapping of store keys to bytes: `str`, or the literal keys one document names.""" + + +def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: """Decode the JSON document stored at `key` in `mapping`. Returns `object`, not `Any`: what a store holds is unknown until a @@ -848,12 +852,15 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> object: `invalid_json` problem, rather than leaking `KeyError` / `json.JSONDecodeError` to callers. """ - if key not in mapping: + # Read by a `str` key whatever narrower key type the mapping declares: + # a key it does not hold is only absent. + stored = cast("Mapping[str, bytes]", mapping) + if key not in stored: raise MetadataValidationError( [ValidationProblem((key,), "missing store key", "missing_key")] ) try: - return json.loads(mapping[key], parse_constant=_reject_json_constant) + return json.loads(stored[key], parse_constant=_reject_json_constant) except (UnicodeDecodeError, ValueError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 69a3823b6f..85ec3258a9 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -672,29 +672,17 @@ def test_roundtrip_model_json_model( assert model_cls.from_json(model.to_json()) == model -# --- Round-trips (model → key_value → model, parametrized) ----------------- - -ROUNDTRIP_KEY_VALUE_PARAMS = [ - pytest.param( - ZarrV3ArrayMetadata, - ZarrV3ArrayMetadata.create_default(attributes={"a": 1}), - id="v3", - ), - pytest.param( - ZarrV2ArrayMetadata, - ZarrV2ArrayMetadata.create_default(attributes={"a": 1}), - id="v2", - ), -] +# --- Round-trips (model → key_value → model) -------------------------------- -@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) -def test_roundtrip_via_key_value( - model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], - model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, -) -> None: +def test_roundtrip_via_key_value() -> None: """A model round-trips through to_key_value/from_key_value back to an equal model.""" - assert model_cls.from_key_value(model.to_key_value()) == model + # Not parametrized over the two classes: a mapping's key type is + # invariant, so a reader cannot take the union of what they write. + v3 = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}) + v2 = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}) + assert ZarrV3ArrayMetadata.from_key_value(v3.to_key_value()) == v3 + assert ZarrV2ArrayMetadata.from_key_value(v2.to_key_value()) == v2 # --- Round-trips (json → model → json, direction distinct — kept direct) --- From 71b8d113e004b376da5217209f1d9c0efc37d78d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 16:53:29 +0200 Subject: [PATCH 2/5] build(zarr-metadata): type-check the tests, and test both ends of the version range Pyright checks `tests` as well as `src`, relaxed only for the inferred types of JSON literals, and `# type: ignore` is switched off: a suppression names its rule. The tests' own type errors are fixed without casts. `just check` runs the suite on the oldest and newest interpreters the package declares, and pyright is pinned at 1.1.414. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/justfile | 49 +++++++++++++++---- packages/zarr-metadata/pyproject.toml | 35 +++++++++++-- .../zarr-metadata/tests/model/test_array.py | 5 +- .../zarr-metadata/tests/model/test_group.py | 2 +- .../tests/model/test_pydantic.py | 10 ++-- .../tests/model/test_pydantic_module.py | 6 ++- 6 files changed, 86 insertions(+), 21 deletions(-) diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index 843286b89f..b7734c885a 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -8,26 +8,55 @@ set positional-arguments default: @just --list -# Run the test suite; extra args are passed to pytest +# Run the test suite on the project interpreter; extra args are passed to pytest test *args: uv run --group test pytest tests "$@" +# The oldest and newest interpreters the package is declared for: the floor is +# `requires-python`, the ceiling the newest classifier, both read from +# pyproject.toml. CI's matrix is written out in its workflow, so a change to +# the declared range is made there too. +# uv fetches an interpreter it does not have. A difference in how a version +# materialises annotations or resolves a stub shows up here, not only in CI. +# Run the test suite on the oldest and newest supported interpreters +test-versions *args: + #!/usr/bin/env bash + set -euo pipefail + read -r floor ceiling < <(uv run python - <<'EOF' + import re + import tomllib + + project = tomllib.load(open("pyproject.toml", "rb"))["project"] + floor = re.fullmatch(r">=\s*(\d+\.\d+)", project["requires-python"]).group(1) + declared = [ + classifier.rsplit(" ", 1)[1] + for classifier in project["classifiers"] + if classifier.startswith("Programming Language :: Python :: 3.") + ] + print(floor, max(declared, key=lambda version: tuple(map(int, version.split("."))))) + EOF + ) + for version in "$floor" "$ceiling"; do + echo "== python $version ==" + uv run --python "$version" --group test pytest tests "$@" + done + # Lint with the same invocation CI uses lint: uvx ruff check . -# Pinned to the last pyright that types PEP 661 sentinels in class attributes -# correctly; 1.1.405+ regressed (microsoft/pyright#11115). Unpin when fixed. -pyright_version := "1.1.404" +# Pinned so a pyright release cannot turn CI red on its own schedule. Bump it +# deliberately: run the new version, read what it found, then move the pin. +pyright_version := "1.1.414" -# CI runs pyright on python 3.11; the pinned pyright predates 3.14, whose -# stdlib it cannot parse, so pin the interpreter to match CI. -# Type-check the package sources +# Run under the interpreter CI runs pyright under, so a version-dependent +# stdlib or stub difference shows up here rather than only in CI. +# Type-check the package, sources and tests alike typecheck: - uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright src + uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright -# Run everything CI runs for this package -check: lint typecheck test docs-check +# Run everything CI runs for this package, on both ends of the version range +check: lint typecheck test-versions docs-check # Preview the changelog that the next release would generate changelog-draft: diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index f0a09b16a6..daa6cbd7e2 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -135,15 +135,42 @@ checks = [ "PR06", ] -# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in -# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel -# relies on. Use the same pin locally; unpin when the fix lands. +# The pyright version lives in the justfile, which is what CI runs; pinning it +# keeps a pyright release from turning CI red on its own schedule. [tool.pyright] -include = ["src"] +include = ["src", "tests"] +# Pyright is the only checker that runs on this package (the root mypy +# config covers the root package alone), and it honours `# type: ignore` +# as a blanket suppression of every diagnostic on the line. So that +# spelling is switched off: a suppression is `# pyright: ignore[rule]`, +# names the rule, and is an error once it stops suppressing anything. +enableTypeIgnoreComments = false +reportUnnecessaryTypeIgnoreComment = "error" +# `tests` is a package imported as `tests.*`, which pytest resolves from the +# rootdir; pyright needs the same root on its search path. +extraPaths = ["."] enableExperimentalFeatures = true typeCheckingMode = "strict" pythonVersion = "3.11" +# Tests are checked, but not for the inference strictness the sources are held +# to: test bodies are full of untyped JSON literals whose inferred types are of +# no interest, and strict mode reports about ninety of those. What is kept is +# everything a rename or a signature change breaks -- a dangling import, a call +# that no longer type-checks, an argument of the wrong type -- which is the +# reason for checking tests at all. Tests also reach into private modules and +# define functions purely to be refused at registration, both deliberately. +[[tool.pyright.executionEnvironments]] +root = "tests" +reportMissingTypeArgument = false +reportPrivateUsage = false +reportUnknownArgumentType = false +reportUnknownLambdaType = false +reportUnknownMemberType = false +reportUnknownParameterType = false +reportUnknownVariableType = false +reportUnusedFunction = false + [tool.towncrier] # Fragments for this package live alongside the package source, separate # from the parent zarr-python `changes/` directory, so a PR touching only diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 85ec3258a9..c33e249072 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -965,6 +965,7 @@ def test_parse_metadata_field_materializes_abstract_containers() -> None: assert isinstance(parsed, dict) assert parsed == {"name": "example", "configuration": {"values": (0, 1)}} + assert "configuration" in parsed assert type(parsed["configuration"]) is dict @@ -1547,7 +1548,7 @@ def test_configuration_values_must_be_json() -> None: def test_v3_extension_keys_must_be_strings() -> None: """A non-string top-level key cannot be represented by a v3 document type.""" - doc: dict[object, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc: dict[object, object] = {**ZarrV3ArrayMetadata.create_default().to_json()} doc[1] = {"must_understand": False} assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ ((), "invalid_type") @@ -1746,7 +1747,7 @@ def test_v2_absent_dimension_separator_means_dot() -> None: del doc["dimension_separator"] model = ZarrV2ArrayMetadata.from_json(doc) assert model.dimension_separator == "." - assert model.to_json()["dimension_separator"] == "." + assert model.to_json().get("dimension_separator") == "." def test_v2_from_key_value_without_separator_means_dot() -> None: diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index b0e451b0a0..c62e0fd32a 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -163,7 +163,7 @@ def test_group_guards_reject_noncanonical_nested_json() -> None: assert not is_group_metadata_v3(v3) assert not is_group_metadata_v2(v2) assert parse_group_metadata_v3(v3)["extension"] == (0, 1) - assert parse_group_metadata_v2(v2)["attributes"] == {"values": (0, 1)} + assert parse_group_metadata_v2(v2).get("attributes") == {"values": (0, 1)} def test_group_v3_extension_fields_are_validated() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index e5714bb8fb..7951cac5c3 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -229,13 +229,17 @@ def _canonicalize(cls, data: object) -> object: """Route every input document through the library's validation and normalization; pydantic then parses only canonical documents.""" if isinstance(data, Mapping): - doc = dict(ZarrV3ArrayMetadata.from_json(data).to_json()) + metadata = ZarrV3ArrayMetadata.from_json(data).to_json() + doc: dict[str, object] = dict(metadata) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if isinstance(doc[key], str): doc[key] = {"name": doc[key]} - for key in ("codecs", "storage_transformers"): + for key, entries in ( + ("codecs", metadata["codecs"]), + ("storage_transformers", metadata.get("storage_transformers", ())), + ): doc[key] = tuple( - {"name": item} if isinstance(item, str) else item for item in doc.get(key, ()) + {"name": item} if isinstance(item, str) else item for item in entries ) doc.setdefault("attributes", {}) return doc diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 80067e81ae..65c7c09f00 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -7,12 +7,14 @@ import json import warnings +from collections.abc import Mapping import pytest from jsonschema import Draft202012Validator from pydantic import BaseModel, TypeAdapter, ValidationError import zarr_metadata.pydantic as zmp +from zarr_metadata._common import JSONValue from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, @@ -156,7 +158,9 @@ def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: assert Draft202012Validator(adapter.json_schema()).is_valid(doc) -def _assert_runtime_and_schema_reject(field_type: object, document: dict[str, object]) -> None: +def _assert_runtime_and_schema_reject( + field_type: object, document: Mapping[str, JSONValue] +) -> None: adapter = TypeAdapter(field_type) with pytest.raises(ValidationError): adapter.validate_python(document) From e2a17ff27e393a60d149faceae29b229182eb633 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 16:53:30 +0200 Subject: [PATCH 3/5] fix(zarr-metadata): MetadataValidationError pickles and copies as its problems An exception pickles and copies as its class called with its `args`, which for this error are its message, so a copy was built from the message and held its characters as problems. It is now rebuilt from its problems, and keeps the rest of its state, notes included. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/4422.bugfix.md | 3 +++ .../src/zarr_metadata/model/_validation.py | 10 ++++++++++ packages/zarr-metadata/tests/model/test_array.py | 11 +++++++++++ 3 files changed, 24 insertions(+) create mode 100644 packages/zarr-metadata/changes/4422.bugfix.md diff --git a/packages/zarr-metadata/changes/4422.bugfix.md b/packages/zarr-metadata/changes/4422.bugfix.md new file mode 100644 index 0000000000..7bf24c6d2b --- /dev/null +++ b/packages/zarr-metadata/changes/4422.bugfix.md @@ -0,0 +1,3 @@ +`MetadataValidationError` pickles and copies with its message. A copy was +built from the error's message, and its message came out as the +original's characters, one per line. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 2051d39884..cda02c0631 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -73,6 +73,16 @@ def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) super().__init__("\n".join(str(problem) for problem in self.problems)) + def __reduce__( + self, + ) -> tuple[ + type[MetadataValidationError], tuple[tuple[ValidationProblem, ...]], dict[str, object] + ]: + # An exception pickles and copies as its class called with its + # `args`, which here are the message; it is built from its problems, + # and the rest of its state -- its notes among it -- follows. + return (type(self), (self.problems,), self.__dict__) + def _prefix( loc_head: str | int, problems: Sequence[ValidationProblem] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index c33e249072..60ae480239 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -3,6 +3,7 @@ import copy import dataclasses import json +import pickle from collections import UserDict from collections.abc import Callable from typing import TYPE_CHECKING, get_args @@ -1252,6 +1253,16 @@ def test_metadata_validation_error_holds_problems() -> None: assert "data_type: expected a metadata field" in str(err) +def test_the_error_pickles_and_copies_as_its_problems() -> None: + error = MetadataValidationError([ValidationProblem(("a",), "bad a", "invalid_value")]) + error.add_note("while reading a") + for again in (pickle.loads(pickle.dumps(error)), copy.copy(error), copy.deepcopy(error)): + assert type(again) is MetadataValidationError + assert again.problems == error.problems + assert str(again) == str(error) + assert again.__notes__ == ["while reading a"] + + def test_prefix_prepends_loc_head() -> None: """_prefix prepends a loc head to each problem's loc.""" problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] From 307a4c30390e5c7fb19d471bb00cd7da4dfad3f1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 16:53:33 +0200 Subject: [PATCH 4/5] feat(zarr-metadata)!: problems and errors check their values at run time `ValidationProblem` checks that its `loc` is a tuple of strings and integers, its `message` a string and its `kind` one of the kinds, and `MetadataValidationError` that it holds problems; each raises `TypeError` otherwise. A caller that is not type-checked fails where it builds the value, not where a location is read. The v2 consolidated reader reported a non-string document key as a member at a location made of the key, which the check refuses; it now reports it at the document, as every other reader does. BREAKING CHANGE: a `loc` that is a list, or holds a `bool`, is refused. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/4422.feature.md | 7 +++ .../src/zarr_metadata/model/_group.py | 7 ++- .../src/zarr_metadata/model/_validation.py | 32 +++++++++++- .../zarr-metadata/tests/model/test_array.py | 49 +++++++++++++++++++ .../zarr-metadata/tests/model/test_group.py | 13 +++++ 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 packages/zarr-metadata/changes/4422.feature.md diff --git a/packages/zarr-metadata/changes/4422.feature.md b/packages/zarr-metadata/changes/4422.feature.md new file mode 100644 index 0000000000..360b0b1c35 --- /dev/null +++ b/packages/zarr-metadata/changes/4422.feature.md @@ -0,0 +1,7 @@ +**Breaking:** `ValidationProblem` and `MetadataValidationError` check at run +time what their annotations declare, and raise `TypeError` otherwise. A +problem's `loc` is a tuple of strings and integers, so a list, which was +accepted before, is now refused, as is a `bool`; its `message` is a string +and its `kind` one of the kinds; an error holds problems. A caller that is +not type-checked fails where it builds the value, not later, where a +location is read. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index d71badfd1c..628aa3ab7f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -378,7 +378,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: raise MetadataValidationError( [ValidationProblem((), "expected a mapping", "invalid_type")] ) - doc = cast("Mapping[str, object]", normalized) + doc = cast("Mapping[object, object]", normalized) problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") @@ -386,7 +386,10 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: ] problems.extend( ValidationProblem((key,), "unexpected document member", "invalid_value") - for key in doc.keys() - {"zarr_consolidated_format", "metadata"} + if isinstance(key, str) + else ValidationProblem((), f"non-string document key {key!r}", "invalid_type") + for key in doc + if key not in {"zarr_consolidated_format", "metadata"} ) if "zarr_consolidated_format" in doc and ( not isinstance(doc["zarr_consolidated_format"], int) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index cda02c0631..8832b8ec4f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -17,7 +17,7 @@ import math from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal, NoReturn, TypeVar, cast +from typing import Final, Literal, NoReturn, TypeVar, cast, get_args from typing_extensions import TypeIs @@ -54,6 +54,26 @@ class ValidationProblem: message: str kind: ProblemKind + def __post_init__(self) -> None: + # The runtime half of the annotations: a rule written without a type + # checker, as an extension's may be, fails where it builds a problem + # rather than reporting one at a location that is not one. + loc = cast("object", self.loc) + if not isinstance(loc, tuple) or not all( + isinstance(part, str) or (isinstance(part, int) and not isinstance(part, bool)) + for part in cast("tuple[object, ...]", loc) + ): + msg = f"a ValidationProblem's loc is a tuple of keys and indices, got {loc!r}" + raise TypeError(msg) + message = cast("object", self.message) + if not isinstance(message, str): + msg = f"a ValidationProblem's message is a string, got {message!r}" + raise TypeError(msg) + kind = cast("object", self.kind) + if not isinstance(kind, str) or kind not in get_args(ProblemKind): + msg = f"a ValidationProblem's kind is one of {get_args(ProblemKind)!r}, got {kind!r}" + raise TypeError(msg) + def __str__(self) -> str: location = ".".join(str(part) for part in self.loc) if self.loc else "" return f"{location}: {self.message}" @@ -71,6 +91,16 @@ class MetadataValidationError(ValueError): def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) + for entry in cast("tuple[object, ...]", self.problems): + # The runtime half of the annotation: a caller that is not + # type-checked, and hands over anything else, fails here + # rather than far away, where a `loc` is read off it. + if not isinstance(entry, ValidationProblem): + msg = ( + "MetadataValidationError takes ValidationProblem values, " + f"got {type(entry).__name__}" + ) + raise TypeError(msg) super().__init__("\n".join(str(problem) for problem in self.problems)) def __reduce__( diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 60ae480239..41386edcdd 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1263,6 +1263,55 @@ def test_the_error_pickles_and_copies_as_its_problems() -> None: assert again.__notes__ == ["while reading a"] +def test_error_a_problem_refuses_a_loc_that_is_not_a_tuple() -> None: + # The missing comma: `("level")` is a string, and read as a location + # it would be the path through each of its characters. + with pytest.raises(TypeError, match="loc is a tuple of keys and indices"): + ValidationProblem(("level"), "bad level", "invalid_value") # pyright: ignore[reportArgumentType] + + +@pytest.mark.parametrize("part", [True, 1.5], ids=["bool", "float"]) +def test_error_a_problem_refuses_a_loc_part_that_is_not_a_key_or_an_index(part: object) -> None: + # `True` passes as an `int`, and would index a sequence as 1. + with pytest.raises(TypeError, match="loc is a tuple of keys and indices"): + ValidationProblem(("a", part), "bad a", "invalid_value") # pyright: ignore[reportArgumentType] + + +def test_error_a_problem_refuses_a_message_that_is_not_a_string() -> None: + with pytest.raises(TypeError, match="message is a string"): + ValidationProblem(("level",), 7, "invalid_value") # pyright: ignore[reportArgumentType] + + +def test_error_a_problem_refuses_a_kind_that_is_not_one() -> None: + # A kind outside the set is one a consumer that dispatches on kinds + # never sees. + with pytest.raises(TypeError, match="kind is one of"): + ValidationProblem(("level",), "bad level", "invalid") # pyright: ignore[reportArgumentType] + + +class _EqualToEveryKind: + """Not a kind, though it compares equal to each.""" + + def __eq__(self, other: object) -> bool: + return True + + def __hash__(self) -> int: + return 0 + + +def test_error_a_problem_refuses_a_kind_that_only_compares_equal_to_one() -> None: + # `in` tests equality, which any object can claim. + with pytest.raises(TypeError, match="kind is one of"): + ValidationProblem(("level",), "bad level", _EqualToEveryKind()) # pyright: ignore[reportArgumentType] + + +def test_error_the_error_refuses_what_is_not_a_problem() -> None: + # A list of one-element tuples of problems is the likely slip: it + # would otherwise fail far away, where a `loc` is read off an entry. + with pytest.raises(TypeError, match="takes ValidationProblem values, got tuple"): + MetadataValidationError([(ValidationProblem(("a",), "bad a", "invalid_value"),)]) # pyright: ignore[reportArgumentType] + + def test_prefix_prepends_loc_head() -> None: """_prefix prepends a loc head to each problem's loc.""" problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index c62e0fd32a..2d8dd74534 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -498,6 +498,19 @@ def test_v2_consolidated_rejects_unknown_document_member() -> None: ZarrV2ConsolidatedMetadata.from_json(doc) +@pytest.mark.parametrize("key", [1, None], ids=["int", "none"]) +def test_v2_consolidated_rejects_non_string_document_key(key: object) -> None: + """A non-string key is a problem at the document: not a member at a + location that reads as an index, nor a `TypeError`.""" + doc = {"zarr_consolidated_format": 1, "metadata": {}, key: "x"} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ConsolidatedMetadata.from_json(doc) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + # --- must_understand partition ------------------------------------------------ From 91e1b68c20047b7d5e970b373fb05736580d0a5c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 17:03:01 +0200 Subject: [PATCH 5/5] fix(zarr-metadata): attributes may hold NaN and the infinities zarr-python writes attributes with the defaults of Python's `json` module, so an xarray `_FillValue` of NaN is stored as a bare `NaN`, and the model refused to read such a store. Attributes are user data, which the spec asks only to be JSON values and never interprets, so a non-finite number in them is now read, validated and written back as Python's `json` writes it. Anywhere the spec interprets a value one is still refused, and the store reader now says where instead of failing to decode the document. `refine_json` walks a value once, normalizing and judging it, and `refine_user_data` is the same walk where a non-finite number is the float it is. Only the validators say where user data is: they walk a node's attributes with `refine_user_data`, the store reader decodes and leaves the judging to them, and `to_key_value` validates what it writes with the `parse_*` its reader uses, so it no longer writes a document the reader refuses. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/4420.bugfix.md | 33 ++ .../src/zarr_metadata/model/_array.py | 18 +- .../src/zarr_metadata/model/_group.py | 49 ++- .../src/zarr_metadata/model/_validation.py | 143 +++++--- .../src/zarr_metadata/pydantic.py | 7 + .../zarr-metadata/tests/model/test_array.py | 6 +- .../tests/model/test_pydantic_module.py | 18 +- .../tests/model/test_refine_json.py | 103 ++++++ .../tests/model/test_store_json.py | 328 ++++++++++++++++++ 9 files changed, 636 insertions(+), 69 deletions(-) create mode 100644 packages/zarr-metadata/changes/4420.bugfix.md create mode 100644 packages/zarr-metadata/tests/model/test_refine_json.py create mode 100644 packages/zarr-metadata/tests/model/test_store_json.py diff --git a/packages/zarr-metadata/changes/4420.bugfix.md b/packages/zarr-metadata/changes/4420.bugfix.md new file mode 100644 index 0000000000..a77d08aa32 --- /dev/null +++ b/packages/zarr-metadata/changes/4420.bugfix.md @@ -0,0 +1,33 @@ +Attributes are user data, and a non-finite number in them is read, +validated and written back. zarr-python writes attributes with the +defaults of Python's `json` module, so an xarray `_FillValue` or a CF +`missing_value` of NaN is stored as a bare `NaN`, which RFC 8259 lacks; +the model refused such a document and could not read the store at all. +A node's `attributes`, a v2 `.zattrs`, the `.zattrs` entries of a v2 +`.zmetadata`, and the attributes of the nodes a v3 group's inline +`consolidated_metadata` holds may now hold `NaN`, `Infinity` and +`-Infinity`, in every document `validate_*`, `parse_*` and `is_*`, and in +`from_key_value` and `to_key_value`. Wherever the spec interprets a value +it spells those numbers as strings, so anywhere else a non-finite number +is still refused, and the store reader now says where one is instead of +failing to decode the document. `is_json`, `validate_json` and +`parse_json` judge by RFC 8259 alone, so a document the node guards +accept can hold a value `is_json` refuses. + +`to_key_value` validates the document it writes as `from_key_value` +validates what it reads, and raises `MetadataValidationError` with every +problem instead of writing a document the reader refuses. A model built +by hand is not validated, so the writer is where these are caught: + +- a non-finite number outside attributes, which was refused before too; +- a non-string attribute key, which was written as a string; +- a value that is not JSON, which raised `TypeError`; +- anything else the reader refuses, such as `dimension_names` longer than + `shape`, which was written and then refused on read. + +Any sequence is written as an array. A number too large for a float, such +as `1e400`, is read as infinity, so in attributes it is now accepted and +written back as `Infinity`. + +Through `zarr_metadata.pydantic`, pydantic writes a non-finite attribute +as `null` unless the enclosing model sets `ser_json_inf_nan="constants"`. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index bfee447956..a89885130f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -329,7 +329,10 @@ def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV3ArrayMetadat def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: - return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)} + # A model built by hand is not validated: its document is written only + # if it reads as `from_json` reads one, and every problem is raised. + document = parse_array_metadata_v3(self.to_json()) + return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(document, indent=indent)} class ZarrV2ArrayMetadataPartial(TypedDict, total=False): @@ -494,11 +497,16 @@ def to_key_value( ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` # document must exclude them. The `.zattrs` key is present exactly - # when attributes are set (even empty) — UNSET emits no file. - zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + # when attributes are set (even empty) — UNSET emits no file. A model + # built by hand is not validated: its document is written only if it + # reads as `from_json` reads one, and every problem is raised. + document = parse_array_metadata_v2(self.to_json()) + zarray = {k: v for k, v in document.items() if k != "attributes"} out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent) } - if self.attributes is not UNSET: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(self.attributes, indent=indent) + if "attributes" in document: + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( + document["attributes"], indent=indent + ) return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 628aa3ab7f..9ba5393106 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -25,8 +25,9 @@ load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, + refine_json, + refine_user_data, validate_consolidated_metadata_v3, - validate_json, ) from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY @@ -183,7 +184,10 @@ def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV3GroupMetadat def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: - return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)} + # A model built by hand is not validated: its document is written only + # if it reads as `from_json` reads one, and every problem is raised. + document = parse_group_metadata_v3(self.to_json()) + return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(document, indent=indent)} @dataclass(frozen=True, slots=True, kw_only=True) @@ -340,13 +344,18 @@ def to_key_value( ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` # document must exclude them. The `.zattrs` key is present exactly - # when attributes are set (even empty) — UNSET emits no file. - zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + # when attributes are set (even empty) — UNSET emits no file. A model + # built by hand is not validated: its document is written only if it + # reads as `from_json` reads one, and every problem is raised. + document = parse_group_metadata_v2(self.to_json()) + zgroup = {k: v for k, v in document.items() if k != "attributes"} out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent) } - if self.attributes is not UNSET: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(self.attributes, indent=indent) + if "attributes" in document: + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( + document["attributes"], indent=indent + ) return out @@ -403,6 +412,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: "invalid_value", ) ) + refined: dict[str, JSONValue] = {} if "metadata" in doc: entries = doc["metadata"] if not isinstance(entries, Mapping) or not all( @@ -414,20 +424,20 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: ) ) else: + # Each entry is the document its key names: a `.zattrs` is + # user data, and any other is JSON by RFC 8259. for key, value in cast("Mapping[str, object]", entries).items(): - problems.extend( - ValidationProblem( - ("metadata", key, *problem.loc), problem.message, problem.kind - ) - for problem in validate_json(value) + refine = ( + refine_user_data + if key.rsplit("/", 1)[-1] == ZARR_V2_ATTRIBUTES_STORE_KEY + else refine_json ) + entry, found = refine(value, ("metadata", key)) + problems.extend(found) + refined[key] = entry if len(problems) != 0: raise MetadataValidationError(problems) - entries_tupled = cast( - "dict[str, JSONValue]", - arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), - ) - return cls(metadata=entries_tupled) + return cls(metadata=refined) @classmethod def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2ConsolidatedMetadata: @@ -436,6 +446,7 @@ def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2Consolidated def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: - return { - ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent) - } + # A model built by hand is not validated: it is written only as + # `from_json` reads it, and every problem is raised. + document = ZarrV2ConsolidatedMetadata.from_json(self.to_json()).to_json() + return {ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(document, indent=indent)} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 8832b8ec4f..177ae81a27 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -17,7 +17,7 @@ import math from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal, NoReturn, TypeVar, cast, get_args +from typing import Final, Literal, TypeVar, cast, get_args from typing_extensions import TypeIs @@ -44,8 +44,9 @@ class ValidationProblem: """A single structural problem found while validating a metadata document. - `loc` is the path from the document root to the offending value, e.g. - `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `loc` is the path from the root of what was judged to the offending + value, e.g. `("codecs", 0, "name")`, and an empty `loc` refers to that + root: for the validators here, the document. `kind` classifies the failure mode for programmatic dispatch; `message` is the human-readable description. """ @@ -122,43 +123,101 @@ def _prefix( def validate_json(value: object) -> tuple[ValidationProblem, ...]: - """Return every reason `value` is not JSON-serializable (recursively).""" + """Return every reason `value` is not JSON-serializable (recursively): `refine_json`'s problems.""" + return refine_json(value)[1] + + +def refine_json( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """`value` as JSON with arrays as tuples, or None with every reason it is not JSON. + + One walk normalizes and judges: a mapping becomes a `dict` with + string keys, a sequence a tuple, and a float must be finite. A value + that is not JSON is None, with the problems located at the leaves + that are not. JSON's `null` is None too, with no problem, so whether + `value` is JSON is whether there are problems. + """ + return _refine(value, loc, finite=True) + + +def refine_user_data( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """User data refined as `refine_json` refines JSON, a non-finite number being the float it is. + + A node's attributes are user data. The spec asks only that each be a + JSON value and interprets none, and zarr-python writes them with the + defaults of Python's `json` module, so an attribute can hold `NaN`, + `Infinity` or `-Infinity` -- xarray's `_FillValue`, a CF + `missing_value` -- which RFC 8259 lacks. Wherever the spec interprets + a value it spells those numbers as strings, so a validator walks what + it interprets with `refine_json`, and only user data with this. + """ + return _refine(value, loc, finite=False) + + +_Refined = tuple[JSONValue | None, tuple[ValidationProblem, ...]] + + +def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refined: + """`refine_json`, a non-finite number being JSON unless `finite`.""" if isinstance(value, float): - if math.isfinite(value): - return () - return (ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value"),) + if not finite or math.isfinite(value): + return value, () + return None, ( + ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value"), + ) if isinstance(value, (str, int, bool)) or value is None: - return () - problems: list[ValidationProblem] = [] + return value, () if isinstance(value, Mapping): + # Walked here rather than through `_refine_members`, so that each + # level of nesting costs one frame, as deep as the interpreter goes. + members: dict[str, JSONValue] = {} + found_in_members: list[ValidationProblem] = [] for key, item in cast("Mapping[object, object]", value).items(): if not isinstance(key, str): - problems.append( - ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + found_in_members.append( + ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") ) continue - problems.extend(_prefix(key, validate_json(item))) - return tuple(problems) + member, found = _refine(item, (*loc, key), finite=finite) + found_in_members.extend(found) + if len(found) == 0: + members[key] = member + return (members if len(found_in_members) == 0 else None), tuple(found_in_members) if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + entries: list[JSONValue] = [] + found_in_entries: list[ValidationProblem] = [] for index, item in enumerate(cast("Sequence[object]", value)): - problems.extend(_prefix(index, validate_json(item))) - return tuple(problems) - return (ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type"),) + entry, found = _refine(item, (*loc, index), finite=finite) + found_in_entries.extend(found) + if len(found) == 0: + entries.append(entry) + return (tuple(entries) if len(found_in_entries) == 0 else None), tuple(found_in_entries) + return None, ( + ValidationProblem(loc, f"not a JSON-serializable value: {value!r}", "invalid_type"), + ) + +def _is_canonical_json(value: object, *, finite: bool = True) -> TypeIs[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`. -def _is_canonical_json(value: object) -> TypeIs[JSONValue]: - """Whether `value` already uses the concrete containers in `JSONValue`.""" + A non-finite number counts only when `finite` is false, as a document's + guard passes it: where one may be is the document's validator's to say. + """ if isinstance(value, float): - return math.isfinite(value) + return not finite or math.isfinite(value) if isinstance(value, (str, int, bool)) or value is None: return True if isinstance(value, (list, tuple)): sequence = cast("list[object] | tuple[object, ...]", value) - return all(_is_canonical_json(item) for item in sequence) + return all(_is_canonical_json(item, finite=finite) for item in sequence) if isinstance(value, dict): mapping = cast("dict[object, object]", value) return all( - isinstance(key, str) and _is_canonical_json(item) for key, item in mapping.items() + isinstance(key, str) and _is_canonical_json(item, finite=finite) + for key, item in mapping.items() ) return False @@ -170,11 +229,10 @@ def is_json(value: object) -> TypeIs[JSONValue]: def parse_json(value: object) -> JSONValue: """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" - normalized = arrays_to_tuples(value) - problems = validate_json(normalized) + refined, problems = refine_json(value) if len(problems) != 0: raise MetadataValidationError(problems) - return cast(JSONValue, normalized) + return refined # The standard top-level keys of a v3 array metadata document. Anything outside @@ -525,7 +583,7 @@ def _validate_attributes(value: object) -> tuple[ValidationProblem, ...]: ) problems: list[ValidationProblem] = [] for key, item in cast("Mapping[str, object]", value).items(): - problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + problems.extend(refine_user_data(item, ("attributes", key))[1]) return tuple(problems) @@ -606,7 +664,7 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: """Whether `value` is a structurally-valid v3 array metadata document.""" return ( - _is_canonical_json(value) + _is_canonical_json(value, finite=False) and not validate_array_metadata_v3(value) and _is_canonical_array_metadata_v3(value) ) @@ -714,7 +772,7 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: """Whether `value` is a structurally-valid v2 array metadata document.""" return ( - _is_canonical_json(value) + _is_canonical_json(value, finite=False) and not validate_array_metadata_v2(value) and _is_canonical_array_metadata_v2(value) ) @@ -826,7 +884,7 @@ def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" - return _is_canonical_json(value) and not validate_group_metadata_v3(value) + return _is_canonical_json(value, finite=False) and not validate_group_metadata_v3(value) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: @@ -859,7 +917,7 @@ def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" - return _is_canonical_json(value) and not validate_group_metadata_v2(value) + return _is_canonical_json(value, finite=False) and not validate_group_metadata_v2(value) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: @@ -871,11 +929,6 @@ def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: return cast(ZarrV2GroupMetadataJSON, normalized) -def _reject_json_constant(constant: str) -> NoReturn: - """Reject the JavaScript constants accepted by Python's JSON decoder.""" - raise ValueError(f"non-standard JSON constant {constant!r}") - - StoreKey = TypeVar("StoreKey", bound=str) """The key type of a mapping of store keys to bytes: `str`, or the literal keys one document names.""" @@ -887,10 +940,13 @@ def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: validator says otherwise, and `Any` would let unchecked values flow into typed positions silently. Narrow the result with a `parse_*`. - Every ingestion failure surfaces as `MetadataValidationError`: a missing - store key is a `missing_key` problem and undecodable bytes are an - `invalid_json` problem, rather than leaking `KeyError` / - `json.JSONDecodeError` to callers. + Decoding is Python's, so `NaN`, `Infinity` and `-Infinity` are read as + the floats they spell, as zarr-python writes attributes; where one may + be is the document's validator's to say. Every ingestion failure here + surfaces as `MetadataValidationError`: a missing store key is a + `missing_key` problem and undecodable bytes are an `invalid_json` + problem, rather than leaking `KeyError` / `json.JSONDecodeError` to + callers. """ # Read by a `str` key whatever narrower key type the mapping declares: # a key it does not hold is only absent. @@ -900,7 +956,7 @@ def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: [ValidationProblem((key,), "missing store key", "missing_key")] ) try: - return json.loads(stored[key], parse_constant=_reject_json_constant) + return json.loads(stored[key]) except (UnicodeDecodeError, ValueError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] @@ -908,8 +964,13 @@ def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: - """Encode a metadata document as strict RFC 8259 JSON bytes.""" - return json.dumps(value, indent=indent, allow_nan=False).encode("utf-8") + """Encode a document its validator has passed as JSON bytes. + + A non-finite number is written as Python's `json` writes it (`NaN`, + `Infinity`, `-Infinity`), as zarr-python writes attributes; the + validator is what keeps one out of anywhere else. + """ + return json.dumps(value, indent=indent, allow_nan=True).encode("utf-8") def arrays_to_tuples(obj: object) -> object: diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 5b0c9e5b57..d333f629be 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -14,6 +14,13 @@ so a failed parse surfaces as a pydantic `ValidationError` carrying the loc-annotated problem messages. +A node's attributes may hold `NaN`, `Infinity` or `-Infinity`, which the +models read and write as zarr-python does. Pydantic writes JSON by its own +rules, and by default writes such a number as `null`. A `BaseModel` holding +one of these fields keeps it with `ser_json_inf_nan="constants"` in its +`model_config`; a `TypeAdapter` over a field type takes no `config`, so +write its value with the model's `to_key_value` instead. + Usage: import zarr_metadata.pydantic as zmp diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 41386edcdd..b7d5ff46fb 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1464,12 +1464,12 @@ def test_array_v3_from_json_materializes_abstract_containers() -> None: def test_from_key_value_rejects_non_standard_json_constant() -> None: - """Store JSON decoding rejects JavaScript NaN/Infinity constants.""" + """A JavaScript NaN constant outside the attributes is located, not decoded as a fill value.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc["fill_value"] = float("nan") raw = json.dumps(doc) - with pytest.raises(MetadataValidationError, match="invalid JSON"): + with pytest.raises(MetadataValidationError, match="fill_value: non-finite float nan"): ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) @@ -1477,7 +1477,7 @@ def test_to_key_value_rejects_non_finite_model_value() -> None: """Strict encoding prevents directly-constructed models from writing invalid JSON.""" model = ZarrV3ArrayMetadata.create_default(fill_value=float("nan")) - with pytest.raises(ValueError, match="JSON compliant"): + with pytest.raises(MetadataValidationError, match="fill_value: non-finite float nan"): model.to_key_value() diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 65c7c09f00..c1976d59f2 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -6,12 +6,13 @@ """ import json +import math import warnings from collections.abc import Mapping import pytest from jsonschema import Draft202012Validator -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import zarr_metadata.pydantic as zmp from zarr_metadata._common import JSONValue @@ -277,3 +278,18 @@ def test_core_package_does_not_import_pydantic() -> None: code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" subprocess.run([sys.executable, "-c", code], check=True) + + +def test_a_non_finite_attribute_is_kept_by_constants_serialization() -> None: + """Pydantic writes a non-finite number as `null` unless the enclosing + model spells it as the store does; the module docstring says how.""" + + class Manifest(BaseModel): + model_config = ConfigDict(ser_json_inf_nan="constants") + metadata: zmp.ZarrV3GroupMetadata + + document = b'{"metadata": {"zarr_format": 3, "node_type": "group", "attributes": {"x": NaN}}}' + written = Manifest.model_validate_json(document).model_dump_json() + held = Manifest.model_validate_json(written).metadata.attributes["x"] + assert isinstance(held, float) + assert math.isnan(held) diff --git a/packages/zarr-metadata/tests/model/test_refine_json.py b/packages/zarr-metadata/tests/model/test_refine_json.py new file mode 100644 index 0000000000..2440cf95c2 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_refine_json.py @@ -0,0 +1,103 @@ +"""A value refined to JSON, or the reasons it is not.""" + +from __future__ import annotations + +import math +from collections import OrderedDict +from typing import TYPE_CHECKING + +import pytest + +from zarr_metadata.model._validation import refine_json, refine_user_data + +if TYPE_CHECKING: + from collections.abc import Callable + + from zarr_metadata._common import JSONValue + from zarr_metadata.model import ValidationProblem + + _Refiner = Callable[[object], tuple[JSONValue | None, tuple[ValidationProblem, ...]]] + + +@pytest.mark.parametrize( + ("value", "refined"), + [ + (1, 1), + (2.5, 2.5), + (True, True), + ("x", "x"), + (None, None), + ([1, [2, 3]], (1, (2, 3))), + ((1, 2), (1, 2)), + ({"a": [1], "b": {"c": None}}, {"a": (1,), "b": {"c": None}}), + (OrderedDict(k=[0]), {"k": (0,)}), + ], + ids=["int", "float", "bool", "str", "null", "nested-lists", "tuple", "object", "any-mapping"], +) +def test_json_is_refined_to_tuples_and_dicts(value: object, refined: object) -> None: + # One walk normalizes and judges. + assert refine_json(value) == (refined, ()) + + +def test_user_data_holds_non_finite_numbers() -> None: + # User data is JSON as Python's `json` module reads and writes it: the + # walk is `refine_json`'s, and a non-finite number is the float it is. + assert refine_user_data({"range": [-math.inf, math.inf]}) == ( + {"range": (-math.inf, math.inf)}, + (), + ) + refined, problems = refine_user_data(math.nan) + assert problems == () + assert isinstance(refined, float) + assert math.isnan(refined) + + +# User data is refused, as JSON is, wherever it is not JSON. +@pytest.mark.parametrize("refine", [refine_json, refine_user_data], ids=["json", "user-data"]) +@pytest.mark.parametrize( + ("value", "loc"), + # `bytes` is a sequence of integers, and still not a JSON array. + [({"a": object()}, ("a",)), (b"bytes", ())], + ids=["object", "bytes"], +) +def test_error_a_leaf_that_is_not_json_is_located( + refine: _Refiner, value: object, loc: tuple[str | int, ...] +) -> None: + refined, problems = refine(value) + assert refined is None + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, "invalid_type")] + + +@pytest.mark.parametrize("refine", [refine_json, refine_user_data], ids=["json", "user-data"]) +def test_error_a_non_string_key_is_located_at_its_object(refine: _Refiner) -> None: + refined, problems = refine([1, [2, {3: 4}]]) + assert refined is None + assert [(problem.loc, problem.kind) for problem in problems] == [((1, 1), "invalid_type")] + + +@pytest.mark.parametrize( + ("value", "loc"), + [({"x": math.inf}, ("x",)), ({"x": [math.nan]}, ("x", 0))], + ids=["infinite", "nan-in-array"], +) +def test_error_a_non_finite_number_is_not_json(value: object, loc: tuple[str | int, ...]) -> None: + refined, problems = refine_json(value) + assert refined is None + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, "invalid_value")] + + +def test_error_every_leaf_that_is_not_json_is_reported() -> None: + refined, problems = refine_json({"a": object(), "b": [1, object()]}) + assert refined is None + assert [problem.loc for problem in problems] == [("a",), ("b", 1)] + + +def test_a_value_nested_hundreds_deep_is_read() -> None: + # One frame per level of nesting: as deep as the interpreter goes, less + # what the test runner's own frames take. + deep: dict[str, object] = {} + for _ in range(600): + deep = {"k": deep} + refined, problems = refine_json(deep) + assert problems == () + assert refined is not None diff --git a/packages/zarr-metadata/tests/model/test_store_json.py b/packages/zarr-metadata/tests/model/test_store_json.py new file mode 100644 index 0000000000..76bfd252b4 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_store_json.py @@ -0,0 +1,328 @@ +"""Where a non-finite number may be: in a node's attributes, as user data. + +zarr-python writes attributes with the defaults of Python's `json` module, +so an attribute may hold `NaN`, `Infinity` or `-Infinity`. The model reads +such a store, validates it, and writes it back the same way. Wherever the +spec interprets a value, a non-finite number is refused when read, and a +document the reader would refuse is not written. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Protocol + +import pytest + +from zarr_metadata.model import ( + MetadataValidationError, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + is_array_metadata_v2, + is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + + +class _Written(Protocol): + """What a model writes: store keys, each with the bytes it holds.""" + + def items(self) -> Iterable[tuple[str, bytes]]: ... + + +class _Stored(Protocol): + """A model that writes itself to a store.""" + + def to_json(self) -> Mapping[str, object]: ... + + def to_key_value(self, *, indent: int | str | None = None) -> _Written: ... + + +ATTRIBUTES: dict[str, object] = { + "_FillValue": math.nan, + "valid_range": [-math.inf, math.inf], + "cf": {"missing_value": -math.inf}, +} + + +def _held(attributes: object) -> tuple[bool, bool, bool]: + """Whether each non-finite number in `ATTRIBUTES` is still there, arrays as tuples.""" + assert isinstance(attributes, Mapping) + fill = attributes["_FillValue"] + cf = attributes["cf"] + assert isinstance(cf, Mapping) + return ( + isinstance(fill, float) and math.isnan(fill), + attributes["valid_range"] == (-math.inf, math.inf), + cf["missing_value"] == -math.inf, + ) + + +_ARRAY: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) +_INLINE: dict[str, object] = {"kind": "inline", "must_understand": False} + + +def _stored(key: str, document: object) -> Mapping[str, bytes]: + return {key: json.dumps(document).encode()} + + +def _v3_array() -> Mapping[str, bytes]: + return _stored("zarr.json", {**_ARRAY, "attributes": ATTRIBUTES}) + + +def _v3_group() -> Mapping[str, bytes]: + return _stored("zarr.json", {"zarr_format": 3, "node_type": "group", "attributes": ATTRIBUTES}) + + +def _v3_consolidated() -> Mapping[str, bytes]: + node = {**_ARRAY, "attributes": ATTRIBUTES} + document = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": {**_INLINE, "metadata": {"a": node}}, + } + return _stored("zarr.json", document) + + +def _v2_array() -> Mapping[str, bytes]: + zarray = dict(ZarrV2ArrayMetadata.create_default().to_json()) + return {".zarray": json.dumps(zarray).encode(), ".zattrs": json.dumps(ATTRIBUTES).encode()} + + +def _v2_group() -> Mapping[str, bytes]: + return { + ".zgroup": json.dumps({"zarr_format": 2}).encode(), + ".zattrs": json.dumps(ATTRIBUTES).encode(), + } + + +def _v2_consolidated() -> Mapping[str, bytes]: + document = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": ATTRIBUTES}, + } + return {".zmetadata": json.dumps(document).encode()} + + +def _consolidated_node_attributes(store: Mapping[str, bytes]) -> object: + consolidated = ZarrV3GroupMetadata.from_key_value(store).consolidated_metadata + assert isinstance(consolidated, ZarrV3ConsolidatedMetadata) + return consolidated.metadata["a"].attributes + + +# Each case: the store, its reader, the attributes read from a store, and +# the guard the model's document passes. Declared, so that each reader and +# lambda is checked against its model. +_Case = tuple[ + Callable[[], Mapping[str, bytes]], + Callable[[Mapping[str, bytes]], _Stored], + Callable[[Mapping[str, bytes]], object], + Callable[[object], bool] | None, +] +_CASES: dict[str, _Case] = { + "v3-array": ( + _v3_array, + ZarrV3ArrayMetadata.from_key_value, + lambda store: ZarrV3ArrayMetadata.from_key_value(store).attributes, + is_array_metadata_v3, + ), + "v3-group": ( + _v3_group, + ZarrV3GroupMetadata.from_key_value, + lambda store: ZarrV3GroupMetadata.from_key_value(store).attributes, + is_group_metadata_v3, + ), + "v3-consolidated-node": ( + _v3_consolidated, + ZarrV3GroupMetadata.from_key_value, + _consolidated_node_attributes, + is_group_metadata_v3, + ), + "v2-array": ( + _v2_array, + ZarrV2ArrayMetadata.from_key_value, + lambda store: ZarrV2ArrayMetadata.from_key_value(store).attributes, + is_array_metadata_v2, + ), + "v2-group": ( + _v2_group, + ZarrV2GroupMetadata.from_key_value, + lambda store: ZarrV2GroupMetadata.from_key_value(store).attributes, + is_group_metadata_v2, + ), + "v2-consolidated": ( + _v2_consolidated, + ZarrV2ConsolidatedMetadata.from_key_value, + lambda store: ZarrV2ConsolidatedMetadata.from_key_value(store).metadata[".zattrs"], + None, + ), +} + + +@pytest.mark.parametrize( + ("store", "read", "attributes_in", "guard"), list(_CASES.values()), ids=list(_CASES) +) +def test_attributes_holding_non_finite_numbers_round_trip_through_the_store( + store: Callable[[], Mapping[str, bytes]], + read: Callable[[Mapping[str, bytes]], _Stored], + attributes_in: Callable[[Mapping[str, bytes]], object], + guard: Callable[[object], bool] | None, +) -> None: + stored = store() + assert _held(attributes_in(stored)) == (True, True, True) + model = read(stored) + if guard is not None: + assert guard(model.to_json()) + written = dict(model.to_key_value().items()) + assert b"NaN" in b"".join(written.values()) + assert _held(attributes_in(written)) == (True, True, True) + + +def test_a_sequence_is_written_as_an_array() -> None: + # The writer writes the document the reader's validation passed, arrays + # normalized as the reader normalizes them. + model = ZarrV3GroupMetadata.create_default(attributes={"r": range(3)}) # pyright: ignore[reportArgumentType] + written = json.loads(model.to_key_value()["zarr.json"]) + assert written["attributes"] == {"r": [0, 1, 2]} + + +_V2_ARRAY: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + +# Each case: a reader, a store it refuses, and where the non-finite number is. +_Refused = tuple[ + Callable[[Mapping[str, bytes]], object], Mapping[str, bytes], tuple[str | int, ...] +] +_REFUSED: dict[str, _Refused] = { + "fill-value": ( + ZarrV3ArrayMetadata.from_key_value, + _stored("zarr.json", {**_ARRAY, "fill_value": math.nan}), + ("fill_value",), + ), + "codec-configuration": ( + ZarrV3ArrayMetadata.from_key_value, + _stored( + "zarr.json", + { + **_ARRAY, + "codecs": [ + {"name": "bytes", "configuration": {"endian": "little", "scale": math.inf}} + ], + }, + ), + ("codecs", 0, "configuration", "scale"), + ), + "extension-field": ( + ZarrV3ArrayMetadata.from_key_value, + _stored("zarr.json", {**_ARRAY, "extension": {"must_understand": False, "x": math.nan}}), + ("extension", "x"), + ), + "consolidated-node": ( + ZarrV3GroupMetadata.from_key_value, + _stored( + "zarr.json", + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + **_INLINE, + "metadata": {"a": {**_ARRAY, "fill_value": math.nan}}, + }, + }, + ), + ("consolidated_metadata", "metadata", "a", "fill_value"), + ), + # Only a v3 group holds node documents inline; in an array, a member of + # that name is an extension the spec does not declare. + "consolidated-in-an-array": ( + ZarrV3ArrayMetadata.from_key_value, + _stored( + "zarr.json", + { + **_ARRAY, + "consolidated_metadata": { + "must_understand": False, + "metadata": {"a": {"attributes": {"x": math.nan}}}, + }, + }, + ), + ("consolidated_metadata", "metadata", "a", "attributes", "x"), + ), + # A v2 array's attributes live in `.zattrs`; its `.zarray` is RFC 8259 + # throughout -- which is where zarr-python 3.0 once wrote a bare `NaN`. + "zarray": ( + ZarrV2ArrayMetadata.from_key_value, + _stored(".zarray", {**_V2_ARRAY, "fill_value": math.nan}), + ("fill_value",), + ), + # Each `.zmetadata` entry is the document its key names. + "zmetadata-entry": ( + ZarrV2ConsolidatedMetadata.from_key_value, + _stored( + ".zmetadata", + { + "zarr_consolidated_format": 1, + "metadata": {"a/.zattrs": {"x": math.nan}, "a/.zarray": {"fill_value": math.nan}}, + }, + ), + ("metadata", "a/.zarray", "fill_value"), + ), +} + + +@pytest.mark.parametrize(("read", "store", "loc"), list(_REFUSED.values()), ids=list(_REFUSED)) +def test_error_a_non_finite_number_outside_attributes_is_located_when_read( + read: Callable[[Mapping[str, bytes]], object], + store: Mapping[str, bytes], + loc: tuple[str | int, ...], +) -> None: + # Python's decoder reads a bare `NaN` as a float; where the spec + # interprets a value, the document's validator says where it is. + with pytest.raises(MetadataValidationError) as raised: + read(store) + assert [(problem.loc, problem.kind) for problem in raised.value.problems] == [ + (loc, "invalid_value") + ] + + +@pytest.mark.parametrize( + ("model", "problems"), + [ + ( + ZarrV3ArrayMetadata.create_default(fill_value=math.nan, attributes={"x": math.nan}), + [(("fill_value",), "invalid_value")], + ), + ( + ZarrV3GroupMetadata.create_default(attributes={"s": {1, 2}}), # pyright: ignore[reportArgumentType] + [(("attributes", "s"), "invalid_type")], + ), + ( + ZarrV3GroupMetadata.create_default(attributes={1: "a"}), # pyright: ignore[reportArgumentType] + [(("attributes",), "invalid_type")], + ), + ( + ZarrV3ArrayMetadata.create_default(shape=(2,), dimension_names=("x", "y")), + [(("dimension_names",), "invalid_value")], + ), + ], + ids=["non-finite-fill-value", "not-json", "non-string-key", "dimension-names-past-shape"], +) +def test_error_a_document_the_reader_refuses_is_not_written( + model: _Stored, problems: list[tuple[tuple[str | int, ...], str]] +) -> None: + # A model built by hand is not validated; the writer validates what it + # writes as the reader does. A non-string key was written as a string, + # a value that is not JSON raised `TypeError`, and the last was written + # and then refused on read. + with pytest.raises(MetadataValidationError) as raised: + model.to_key_value() + assert [(problem.loc, problem.kind) for problem in raised.value.problems] == problems