From cdf3ccfe6c3abe5ad63f41c7cef366fd1036c9f8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 11:17:00 +0200 Subject: [PATCH 1/5] feat(zarr-metadata): must_understand false refused at every extension point Ignoring a codec gives wrong bytes as surely as ignoring a data type gives wrong values, so no extension point may be declared ignorable: `validate_array_metadata_v3` now refuses `must_understand: false` on a codec or a storage transformer, as it already did on a data type, a chunk grid and a chunk key encoding. The spec names only those three, which this package reads as an oversight rather than a licence. `must_understand: false` keeps its meaning where it has one: an unknown top-level extension field, which a reader really can skip. It changes verdicts and nothing else depends on it, so it stands on its own as a reading of the spec. The definition layer above reads every metadata field under the same policy. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../src/zarr_metadata/model/_validation.py | 19 ++++++++++++++++++- .../zarr-metadata/tests/model/test_array.py | 14 +++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index eca0d0ae3c..59c9de3249 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -410,6 +410,13 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: problems.extend(_validate_dim_sequence(doc, "shape")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + # Every extension *point* must be understood: ignoring a codec gives + # wrong bytes just as surely as ignoring a data type gives wrong + # values. The spec names only the first three + # (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), + # which this package reads as an oversight rather than a licence. + # `must_understand: false` keeps its meaning where it has one: an + # unknown top-level extension *field*, which a reader really can skip. for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if key in doc: problems.extend( @@ -431,7 +438,17 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: ) ) for index, entry in enumerate(entries): - problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + problems.extend( + _prefix( + key, + _prefix( + index, + validate_metadata_field_v3( + entry, allow_must_understand_false=False + ), + ), + ) + ) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "dimension_names" in doc: diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 5fab5d9b9b..4586e9c26c 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -785,11 +785,19 @@ def test_metadata_field_rejects_unknown_envelope_member() -> None: @pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) -def test_optional_extension_points_allow_must_understand_false(field: str) -> None: - """Codecs and storage transformers may be explicitly ignorable.""" +def test_every_extension_point_rejects_must_understand_false(field: str) -> None: + """No extension point may be declared ignorable. + + Ignoring a codec gives wrong bytes as surely as ignoring a data type + gives wrong values, so `must_understand` is a property of the kind of + metadata rather than a per-occurrence choice. The spec names only the + three required points; this package reads that as an oversight. + """ doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc[field] = ({"name": "optional", "must_understand": False},) - assert validate_array_metadata_v3(doc) == () + assert [problem.loc for problem in validate_array_metadata_v3(doc)] == [ + (field, 0, "must_understand") + ] @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) From f2c905bc54320602757d2c683db28c47d89efed6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 11:18:07 +0200 Subject: [PATCH 2/5] docs(zarr-metadata): the fragment for must_understand false refused everywhere Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/353.feature.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 packages/zarr-metadata/changes/353.feature.md diff --git a/packages/zarr-metadata/changes/353.feature.md b/packages/zarr-metadata/changes/353.feature.md new file mode 100644 index 0000000000..89528b9b23 --- /dev/null +++ b/packages/zarr-metadata/changes/353.feature.md @@ -0,0 +1,4 @@ +`must_understand: false` is refused at every extension point, codecs and +storage transformers included: ignoring a codec gives wrong bytes as +surely as ignoring a data type gives wrong values, and the spec naming +only three points is read as an oversight. From c23abd3648096dc2159b5c7604152e19e918bfff Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 11:46:20 +0200 Subject: [PATCH 3/5] fix(zarr-metadata): the array schema refuses must_understand false where the validator does The JSON schema the pydantic integration generates typed a codec and a storage transformer with the envelope that allows `must_understand: false`, so it accepted an array document the validator refuses. Every extension point of an array document now has the mandatory envelope, and the test that holds schema and runtime together covers all five. A metadata field read on its own keeps the general envelope, as `parse_metadata_field_v3` does. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/353.feature.md | 3 ++- .../src/zarr_metadata/_pydantic_schema.py | 10 ++++++---- .../zarr-metadata/tests/model/test_pydantic_module.py | 11 +++++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/zarr-metadata/changes/353.feature.md b/packages/zarr-metadata/changes/353.feature.md index 89528b9b23..198d5c6050 100644 --- a/packages/zarr-metadata/changes/353.feature.md +++ b/packages/zarr-metadata/changes/353.feature.md @@ -1,4 +1,5 @@ `must_understand: false` is refused at every extension point, codecs and storage transformers included: ignoring a codec gives wrong bytes as surely as ignoring a data type gives wrong values, and the spec naming -only three points is read as an oversight. +only three points is read as an oversight. The JSON schema +`zarr_metadata.pydantic` generates for an array document says the same. diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index 51ec633b82..8b0e354bc4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -20,7 +20,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): - """Closed v3 named configuration accepted at optional extension points.""" + """Closed v3 named configuration read on its own, outside a document, where `must_understand` may be `false`.""" name: str configuration: NotRequired[Mapping[str, JSONValue]] @@ -28,7 +28,7 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): - """Closed named configuration accepted where understanding is mandatory.""" + """Closed named configuration at an extension point of a document, where understanding is mandatory.""" name: str configuration: NotRequired[Mapping[str, JSONValue]] @@ -37,7 +37,9 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON -ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] +ZarrV3CodecPipelineJSON = Annotated[ + tuple[ZarrV3MandatoryMetadataFieldJSON, ...], Field(min_length=1) +] ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] @@ -53,7 +55,7 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): fill_value: JSONValue codecs: ZarrV3CodecPipelineJSON attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MandatoryMetadataFieldJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index c1976d59f2..cb45310759 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -197,11 +197,14 @@ def test_v2_array_schema_allows_empty_filters() -> None: assert Draft202012Validator(adapter.json_schema()).is_valid(doc) -@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) -def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: - """Mandatory v3 extension points cannot opt out of understanding.""" +@pytest.mark.parametrize( + "field", ["data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers"] +) +def test_v3_array_schema_rejects_false_at_every_extension_point(field: str) -> None: + """No extension point of an array document opts out of understanding, in the schema as at runtime.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) - doc[field] = {"name": "example", "must_understand": False} + entry = {"name": "example", "must_understand": False} + doc[field] = [entry] if field in ("codecs", "storage_transformers") else entry _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) From 0c2da51c8cec2e280e21e95a34c63e94c61cd1ff Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 27 Sep 2026 12:02:09 +0200 Subject: [PATCH 4/5] docs(zarr-metadata): cite the spec lines that name the three points The comment linked the section's opening rules, not the sentence that limits `must_understand=False` to three extension points. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/src/zarr_metadata/model/_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 59c9de3249..0d2c1eeeb3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -413,7 +413,7 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: # Every extension *point* must be understood: ignoring a codec gives # wrong bytes just as surely as ignoring a data type gives wrong # values. The spec names only the first three - # (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), + # (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581), # which this package reads as an oversight rather than a licence. # `must_understand: false` keeps its meaning where it has one: an # unknown top-level extension *field*, which a reader really can skip. From b510d0af3cd9a47ee689466872982ee635a98644 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 27 Sep 2026 12:02:40 +0200 Subject: [PATCH 5/5] docs(zarr-metadata): the fragment takes this PR's number and says what breaks Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/353.feature.md | 5 ----- packages/zarr-metadata/changes/4433.feature.md | 11 +++++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) delete mode 100644 packages/zarr-metadata/changes/353.feature.md create mode 100644 packages/zarr-metadata/changes/4433.feature.md diff --git a/packages/zarr-metadata/changes/353.feature.md b/packages/zarr-metadata/changes/353.feature.md deleted file mode 100644 index 198d5c6050..0000000000 --- a/packages/zarr-metadata/changes/353.feature.md +++ /dev/null @@ -1,5 +0,0 @@ -`must_understand: false` is refused at every extension point, codecs and -storage transformers included: ignoring a codec gives wrong bytes as -surely as ignoring a data type gives wrong values, and the spec naming -only three points is read as an oversight. The JSON schema -`zarr_metadata.pydantic` generates for an array document says the same. diff --git a/packages/zarr-metadata/changes/4433.feature.md b/packages/zarr-metadata/changes/4433.feature.md new file mode 100644 index 0000000000..3b7eec2471 --- /dev/null +++ b/packages/zarr-metadata/changes/4433.feature.md @@ -0,0 +1,11 @@ +**Breaking:** `must_understand: false` is refused at every extension point +of an array document, codecs and storage transformers included, not only +at `data_type`, `chunk_grid` and `chunk_key_encoding`: ignoring a codec +gives wrong bytes as surely as ignoring a data type gives wrong values, +and the spec naming only three points is read as an oversight. A document +that declares a codec or a storage transformer ignorable now has a problem +at `codecs.N.must_understand` or `storage_transformers.N.must_understand`, +so the readers refuse it, and `to_key_value` refuses to write a model that +holds one. The JSON schema `zarr_metadata.pydantic` generates for an array +document says the same. A metadata field read on its own still takes +`must_understand: false`.