diff --git a/packages/zarr-metadata/changes/349.feature.md b/packages/zarr-metadata/changes/349.feature.md new file mode 100644 index 0000000000..80dba3d2d5 --- /dev/null +++ b/packages/zarr-metadata/changes/349.feature.md @@ -0,0 +1,36 @@ +Each extension's metadata is a definition: a value holding the name the +metadata carries, the TypedDict its configuration is -- the one +declaration of that JSON -- and plain functions over that TypedDict for +the rules. `zarr_metadata.v3.definition` is the door, and its module +docstring the guide. + +Reading a field is three steps, each usable on its own by a caller +holding nothing but JSON. `check(value, SomeTypedDict)`, from +`zarr_metadata.typed_json`, type-checks JSON against a TypedDict and needs +nothing else; `definition.judge(configuration)` is the check, each nested +field's envelope judged, and then the rules; `resolve(field, CodecDefinition, scope)` +reads a whole field in a scope -- its envelope judged, its name related to +a definition, its configuration judged, each nested field read the same +way -- and returns what the scope made of it, `Resolved`, with every +problem located. A name nothing in scope claims is `out_of_scope` and left +unjudged. A field is read as one of the five kinds, with or without type +arguments, and `configuration_of(resolved, GZIP_CODEC)` is its +configuration typed as that definition's TypedDict. + +The TypedDict reads as the typing spec defines it -- `total`, `Required`, +`NotRequired`, `closed` and `extra_items` mean what they mean to a type +checker, whether or not its module postpones annotations -- and a +configuration comes back holding what its TypedDict admits and nothing +else: a key a closed TypedDict does not declare is reported, as +`unknown_key`, and left out, so a rule never meets one. A definition +refuses a TypedDict that says nothing of the keys it does not declare, +and a member typed `ZarrV3MetadataFieldJSON`, which would check as plain +JSON; a member holding another metadata field is typed with the field +alias of its kind, `CodecField`. + +`gzip`, `bytes`, `crc32c` and the regular chunk grid are defined +(`GZIP_CODEC`, `BYTES_CODEC`, `CRC32C_CODEC`, `REGULAR_CHUNK_GRID`), and +`CORE` and `CORE_AND_EXTENSIONS` are the scopes that hold them. A third +party writes a TypedDict, a rule function and a definition, and extends a +scope with it; a definition checks itself when built, and nothing happens +at class creation. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index e4e27e6d6e..d52da16fa4 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -19,6 +19,10 @@ The package is organized to mirror the structure of the Zarr specifications: documents, with subpackages for [chunk grids](v3/chunk_grid.md), [chunk key encodings](v3/chunk_key_encoding.md), [codecs](v3/codec.md), and [data types](v3/data_type.md) +- [`zarr_metadata.v3.definition`](v3/definition.md) — each extension's + metadata as a definition: the TypedDict its configuration is, and the + rules on it; check JSON against a TypedDict, judge a configuration, or + read a whole field in a scope. Its module docstring is the guide The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so diff --git a/packages/zarr-metadata/docs/api/v3/definition.md b/packages/zarr-metadata/docs/api/v3/definition.md new file mode 100644 index 0000000000..f08965c684 --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/definition.md @@ -0,0 +1,8 @@ +--- +title: definition +--- + +::: zarr_metadata.v3.definition + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 10990e3218..7708513d24 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -24,6 +24,7 @@ nav: - ' zarr_metadata.v3.chunk_key_encoding': api/v3/chunk_key_encoding.md - ' zarr_metadata.v3.codec': api/v3/codec.md - ' zarr_metadata.v3.data_type': api/v3/data_type.md + - ' zarr_metadata.v3.definition': api/v3/definition.md - Release notes: release-notes.md # This site is a Read the Docs subproject of zarr-python; give readers a way # back to the parent docs, which list every companion package. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 2f58f9b8f2..b7c62205ff 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -49,17 +49,14 @@ is_array_metadata_v3, is_group_metadata_v2, is_group_metadata_v3, - is_metadata_field_v3, parse_array_metadata_v2, parse_array_metadata_v3, parse_group_metadata_v2, parse_group_metadata_v3, - parse_metadata_field_v3, validate_array_metadata_v2, validate_array_metadata_v3, validate_group_metadata_v2, validate_group_metadata_v3, - validate_metadata_field_v3, ) # Store keys are facts about the on-disk specs, so they are defined in the @@ -81,6 +78,11 @@ ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataStoreKey, ) +from zarr_metadata.v3._common import ( + is_metadata_field_v3, + parse_metadata_field_v3, + validate_metadata_field_v3, +) from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataStoreKey, diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index df68471532..b899937bd9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -22,10 +22,10 @@ load_store_json, parse_array_metadata_v2, parse_array_metadata_v3, - parse_metadata_field_v3, ) from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY +from zarr_metadata.v3._common import parse_metadata_field_v3 from zarr_metadata.v3.array import ZARR_V3_ARRAY_METADATA_STORE_KEY if TYPE_CHECKING: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 0d2c1eeeb3..12410814f0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -31,7 +31,9 @@ from zarr_metadata._json import prefixed as _prefix from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3._common import ( + validate_metadata_field_v3, +) from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON @@ -144,87 +146,6 @@ def _validate_other_members( return tuple(problems) -def validate_metadata_field_v3( - value: object, *, allow_must_understand_false: bool = True -) -> tuple[ValidationProblem, ...]: - """Return every reason `value` is not a v3 metadata field. - - A metadata field is a bare name string or a mapping containing `name` and - optional `configuration` and `must_understand` members. - """ - if isinstance(value, str): - return () - if not isinstance(value, Mapping): - return ( - ValidationProblem( - (), - "expected a metadata field (string or extension object)", - "invalid_type", - ), - ) - field = cast("Mapping[object, object]", value) - problems: list[ValidationProblem] = [] - allowed_keys = frozenset({"name", "configuration", "must_understand"}) - for key in field: - if not isinstance(key, str): - problems.append( - ValidationProblem((), f"non-string metadata field key {key!r}", "invalid_type") - ) - elif key not in allowed_keys: - problems.append( - ValidationProblem((key,), "unexpected metadata field member", "invalid_value") - ) - if not isinstance(field.get("name"), str): - problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) - if "configuration" in field: - configuration = field["configuration"] - if not isinstance(configuration, Mapping): - problems.append( - ValidationProblem(("configuration",), "expected a mapping", "invalid_type") - ) - elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): - problems.append( - ValidationProblem(("configuration",), "expected string keys", "invalid_type") - ) - else: - for key, item in cast("Mapping[str, object]", configuration).items(): - problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) - if "must_understand" in field: - must_understand = field["must_understand"] - if not isinstance(must_understand, bool): - problems.append( - ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") - ) - elif not allow_must_understand_false and not must_understand: - problems.append( - ValidationProblem( - ("must_understand",), - "false is not supported at this extension point", - "invalid_value", - ) - ) - return tuple(problems) - - -def is_metadata_field_v3(value: object) -> TypeGuard[ZarrV3MetadataFieldJSON]: - """Whether `value` is a v3 metadata field: a bare name or a named config.""" - if isinstance(value, str): - return True - if not isinstance(value, dict): - return False - field = cast("dict[object, object]", value) - return _is_canonical_json(field) and not validate_metadata_field_v3(field) - - -def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: - """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" - normalized = arrays_to_tuples(value) - problems = validate_metadata_field_v3(normalized) - if len(problems) != 0: - raise MetadataValidationError(problems) - return cast(ZarrV3MetadataFieldJSON, normalized) - - def _is_array(value: object) -> TypeGuard[Sequence[object]]: """Whether `value` reads as a JSON array: a sequence that is not a string or bytes. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py index 406b76b723..4f60ad0ff2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py @@ -1,11 +1,23 @@ -"""Internal cross-cutting aliases for Zarr v3 metadata. +"""The v3 metadata field: its JSON, and the validators that judge one on its own. -This module is private (underscore-prefixed) and exists to avoid circular -imports between leaf modules and sub-package `__init__.py` re-exports. -Public consumers should import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. +Private, and below both readers of a field: the model, which judges the +fields of a document, and the definitions, which read a field's +configuration. Public consumers import `ZarrV3MetadataFieldJSON` from +`zarr_metadata.v3`, and the validators from `zarr_metadata.model`. """ +from collections.abc import Mapping +from typing import TypeGuard, cast + from zarr_metadata._common import ZarrV3NamedConfigJSON +from zarr_metadata._json import ( + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + is_canonical_json, + prefixed, + validate_json, +) ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON """The JSON shape of any v3 metadata extension-point entry: either a bare @@ -18,6 +30,90 @@ """ +def validate_metadata_field_v3( + value: object, *, allow_must_understand_false: bool = True +) -> tuple[ValidationProblem, ...]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a mapping containing `name` and + optional `configuration` and `must_understand` members. + """ + if isinstance(value, str): + return () + if not isinstance(value, Mapping): + return ( + ValidationProblem( + (), + "expected a metadata field (string or extension object)", + "invalid_type", + ), + ) + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + allowed_keys = frozenset({"name", "configuration", "must_understand"}) + for key in field: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string metadata field key {key!r}", "invalid_type") + ) + elif key not in allowed_keys: + problems.append( + ValidationProblem((key,), "unexpected metadata field member", "invalid_value") + ) + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(prefixed("configuration", prefixed(key, validate_json(item)))) + if "must_understand" in field: + must_understand = field["must_understand"] + if not isinstance(must_understand, bool): + problems.append( + ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") + ) + elif not allow_must_understand_false and not must_understand: + problems.append( + ValidationProblem( + ("must_understand",), + "false is not supported at this extension point", + "invalid_value", + ) + ) + return tuple(problems) + + +def is_metadata_field_v3(value: object) -> TypeGuard[ZarrV3MetadataFieldJSON]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + if isinstance(value, str): + return True + if not isinstance(value, dict): + return False + field = cast("dict[object, object]", value) + return is_canonical_json(field) and not validate_metadata_field_v3(field) + + +def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: + """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_metadata_field_v3(normalized) + if len(problems) != 0: + raise MetadataValidationError(problems) + return cast(ZarrV3MetadataFieldJSON, normalized) + + __all__ = [ "ZarrV3MetadataFieldJSON", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "validate_metadata_field_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py new file mode 100644 index 0000000000..3a78c5bc88 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -0,0 +1,611 @@ +"""One metadata field, read against its definition. + +Three steps, each feeding the next and needing more than the one before: + +1. The type check needs the value and a TypedDict. `check` runs the + checker compiled from the TypedDict's annotations over the JSON: + its shape, member by member, every problem located, and a value of + the TypedDict back -- a key it does not declare is reported and left + out. A member holding another metadata field -- a shard's codecs, a + cast's data type -- is annotated with a field alias, `CodecField`, + and checked as an envelope: a name, or a named configuration. Which + definition the name denotes is a scope's question, so the check + needs none. +2. The rules need the definition: plain functions over the checked + TypedDict, for everything finer than a type -- a bound, members read + together. `Definition.judge` is the check and then the rules, for a + caller holding one configuration. +3. The reading needs a scope. `resolve` relates the field's name to a + definition through a `Context`, judges the configuration, and reads + each nested field the check found, in the same scope. + +A definition is a value, not a class to subclass. It holds the name the +metadata carries, the TypedDict that is the one declaration of the +configuration's JSON, and its rules, and it checks itself when it is +built. What kind of metadata it defines is its type -- `CodecDefinition`, +`ChunkGridDefinition` -- which is how a scope files it. Nothing happens +at class creation. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable, Iterable, Iterator, Mapping +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, + Final, + Generic, + Literal, + TypeAlias, + cast, + get_args, + get_origin, +) + +from typing_extensions import TypeAliasType, TypeVar, is_typeddict + +from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON +from zarr_metadata._json import ValidationProblem, refine_json +from zarr_metadata._typed_json import ( + Loc, + Parsed, + Parser, + parser, + problem, + typeddict_keys, + unread_in, +) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON, validate_metadata_field_v3 + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr_metadata.v3._registry import Context + +C = TypeVar("C", default=Mapping[str, JSONValue]) +"""A configuration: the TypedDict a definition declares its JSON as. + +Any configuration is a `Mapping[str, JSONValue]`, which is what a kind +written without one stands for: `resolve(field, CodecDefinition, scope)` +reads as `CodecDefinition[Mapping[str, JSONValue]]`, nothing unknown. +""" + +T = TypeVar("T") + +D = TypeVar("D", bound="Definition[Any]") + +Problems: TypeAlias = tuple[ValidationProblem, ...] + + +def no_rules(configuration: object) -> Iterator[ValidationProblem]: + """The rules of a definition with none: every well-typed configuration is allowed.""" + yield from () + + +def no_name_rules(name: str) -> Iterator[ValidationProblem]: + """The name rules of a definition that claims one name: that name is allowed.""" + yield from () + + +@dataclass(frozen=True, kw_only=True, slots=True) +class Definition(Generic[C]): + """One extension's metadata, as JSON: its name, the TypedDict its configuration is, its rules. + + `configuration` is the TypedDict, and so the one declaration of the + JSON: the checker is compiled from it, the static type of a checked + configuration is it, and a document's author writes to it. It reads + as the typing spec defines it -- `total`, `Required`, `NotRequired`, + `closed` and `extra_items` mean what they mean to a type checker -- + and it says what a key it does not declare is: with `closed=True`, a + problem, reported and left out; with `extra_items=`, a key of that + type; with `closed=False`, anything. `rules` yields what the spec + disallows in a configuration of that type, as it finds each; it is + handed only a configuration that has passed the check, holding what + the TypedDict admits and nothing else -- `judge` is the two, for a + caller holding JSON. + + A family claims many names -- every `r`, one data type -- through + `names`, and says which of them are allowed through `name_rules`, + whose problems land on the field, since a name is not configuration. + Every other definition claims `name` alone. + + Built by hand, a definition refuses what it could not read with: a + `configuration` that is not a TypedDict, says nothing of the keys it + does not declare, or has a member no checker reads, which is named; + and a `name`, `names` or rules that are not what they say. + """ + + name: str + """The name the metadata carries; for a family, the one it is filed under in a scope.""" + configuration: type[C] + """The TypedDict the configuration is.""" + rules: Callable[[C], Iterable[ValidationProblem]] = no_rules + """What the spec disallows in a well-typed configuration, located in it.""" + names: Callable[[str], bool] | None = None + """Which names a family claims; None for a definition that claims `name` alone.""" + name_rules: Callable[[str], Iterable[ValidationProblem]] = no_name_rules + """What the spec disallows in a claimed name.""" + + def __post_init__(self) -> None: + refusal = _malformed(self) or self._refusal() + if refusal is not None: + raise TypeError(refusal) + try: + _vet(self.configuration) + except TypeError as error: + msg = f"{self.name!r}: {error}" + raise TypeError(msg) from error + + def _refusal(self) -> str | None: + """What is wrong with the members a kind adds; None when nothing is, or it adds none.""" + return None + + @property + def requires_configuration(self) -> bool: + """Whether a document must write a configuration: whether the TypedDict has a required key.""" + return len(typeddict_keys(self.configuration).required) != 0 + + def claims(self, name: str) -> bool: + """Whether `name` denotes this definition.""" + return name == self.name if self.names is None else self.names(name) + + def check(self, value: object, loc: Loc = ()) -> tuple[C | None, Problems]: + """`value` type-checked as this definition's configuration, each nested field's envelope judged. + + `zarr_metadata.typed_json.check` is the type check alone; this also + judges the envelope of each metadata field a member holds. + """ + return _configuration_checked(value, self.configuration, loc) + + def judge(self, value: object, loc: Loc = ()) -> tuple[C | None, Problems]: + """`value` type-checked, then judged by the rules: the configuration if it holds, and every problem. + + The rules are asked only of a configuration that type-checked and + whose nested fields are well formed, holding what its TypedDict + admits and nothing else, so a caller holding JSON never reaches a + rule with a member of the wrong type, or one the type says cannot + be there. + """ + configuration, problems = self.check(value, loc) + if configuration is None: + return None, problems + refused = _located(loc, tuple(self.rules(configuration))) + return (configuration if len(refused) == 0 else None), (*problems, *refused) + + +def _malformed(definition: Definition[Any]) -> str | None: + """What makes a hand-built definition unusable before its configuration is read; None if nothing does.""" + name = cast("object", definition.name) + if not isinstance(name, str): + return f"a definition's name is a string, got {name!r}" + functions: dict[str, object] = {"rules": definition.rules, "name_rules": definition.name_rules} + if definition.names is not None: + functions["names"] = definition.names + return next( + ( + f"{name!r}: {member} is a function, got {value!r}" + for member, value in functions.items() + if not callable(value) + ), + None, + ) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DataTypeDefinition(Definition[C]): + """A data type.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkGridDefinition(Definition[C]): + """A chunk grid.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkKeyEncodingDefinition(Definition[C]): + """A chunk key encoding.""" + + +CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] +"""What a codec does to what it is handed: the three positions a pipeline orders.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CodecDefinition(Definition[C]): + """A codec, and what it does to what it is handed.""" + + kind: CodecKind + + def _refusal(self) -> str | None: + kind: object = self.kind + if kind not in get_args(CodecKind): + return f"{self.name!r}: kind is one of {get_args(CodecKind)!r}, got {kind!r}" + return None + + +@dataclass(frozen=True, kw_only=True, slots=True) +class StorageTransformerDefinition(Definition[C]): + """A storage transformer.""" + + +KINDS: Final[tuple[type[Definition[Any]], ...]] = ( + DataTypeDefinition, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + StorageTransformerDefinition, +) +"""The kinds of metadata a document holds, each filed apart in a scope.""" + + +def kind_of(definition: Definition[Any]) -> type[Definition[Any]] | None: + """The kind `definition` is; None for a definition of no kind, which no scope files.""" + return next((kind for kind in KINDS if isinstance(definition, kind)), None) + + +def as_kind(kind: object) -> type[Definition[Any]]: + """The kind of metadata `kind` names, type arguments dropped; `TypeError` if it names none. + + A scope files definitions by kind, so a field is read as one of + `KINDS` -- `CodecDefinition`, or `CodecDefinition[Any]` -- and never + as the base `Definition` or a class of the caller's own, under which + nothing is filed: a field read as one would go unjudged. + """ + origin = get_origin(kind) or kind + found = next((known for known in KINDS if origin is known), None) + if found is None: + names = ", ".join(known.__name__ for known in KINDS) + msg = f"{kind!r} is not a kind of metadata; read a field as one of {names}" + raise TypeError(msg) + return found + + +DataTypeField = TypeAliasType("DataTypeField", ZarrV3MetadataFieldJSON) +"""A configuration member holding a data type, read in the scope the member's field is read in.""" +ChunkGridField = TypeAliasType("ChunkGridField", ZarrV3MetadataFieldJSON) +"""A configuration member holding a chunk grid.""" +ChunkKeyEncodingField = TypeAliasType("ChunkKeyEncodingField", ZarrV3MetadataFieldJSON) +"""A configuration member holding a chunk key encoding.""" +CodecField = TypeAliasType("CodecField", ZarrV3MetadataFieldJSON) +"""A configuration member holding a codec: a shard's `codecs` is `tuple[CodecField, ...]`.""" +StorageTransformerField = TypeAliasType("StorageTransformerField", ZarrV3MetadataFieldJSON) +"""A configuration member holding a storage transformer.""" + +_FIELD_KINDS: Final[Mapping[object, type[Definition[Any]]]] = { + DataTypeField: DataTypeDefinition, + ChunkGridField: ChunkGridDefinition, + ChunkKeyEncodingField: ChunkKeyEncodingDefinition, + CodecField: CodecDefinition, + StorageTransformerField: StorageTransformerDefinition, +} + + +@dataclass(frozen=True, slots=True) +class _NestedField: + """A metadata field the check met inside a configuration: where it sits, its kind, its JSON. + + What the checker hands back where a field alias is, in place of the + JSON: `_checked` collects each one and puts its JSON back. Carried in + the value rather than recorded on the side, so a branch of a union + that did not match leaves none behind. + """ + + loc: Loc + kind: type[Definition[Any]] + json: JSONValue + + +def _field(annotation: object) -> Parser | None: + """The checker's one shape of this module's own: a member holding a metadata field. + + Checked as a bare name or an object, and handed on: the envelope is + judged, and the name related to a definition, by whoever reads the + field -- `check` without a scope, `resolve` in one. + """ + try: + kind = _FIELD_KINDS.get(annotation) + except TypeError: # an unhashable annotation is no field alias + return None + if kind is None: + return None + + def parse(value: object, loc: Loc) -> Parsed: + if not isinstance(value, (str, Mapping)): + return value, problem(loc, f"expected a metadata field, got {value!r}") + # Refined JSON, which the checker only knows as `object`. + return _NestedField(loc, kind, cast("JSONValue", value)), () + + return parse + + +def _vetting(annotation: object) -> Parser | None: + """`_field`, refusing what a definition's configuration cannot hold and still be read. + + A member typed with the envelope's own TypedDict -- + `ZarrV3MetadataFieldJSON` -- checks as plain JSON: its name would never + be related to a definition, nor its configuration judged. And a + TypedDict that says nothing of the keys it does not declare is open, + so a key it does not declare would never be reported. + """ + if annotation is ZarrV3NamedConfigJSON: + msg = ( + "a member typed ZarrV3MetadataFieldJSON checks as plain JSON, and is never read as " + "a field; annotate it with the field alias of its kind: " + + ", ".join(cast("TypeAliasType", alias).__name__ for alias in _FIELD_KINDS) + ) + raise TypeError(msg) + if ( + isinstance(annotation, type) + and is_typeddict(annotation) + and not typeddict_keys(annotation).declared + ): + msg = ( + f"{annotation.__name__} says nothing of the keys it does not declare, so it is " + "open, and such a key would go unreported; declare it closed=True, or " + "extra_items= for what such a key holds, or closed=False to take any" + ) + raise TypeError(msg) + return _field(annotation) + + +@functools.cache +def _vet(configuration: type) -> None: + """Refuse a configuration no definition could read with, saying what is wrong with it.""" + if not is_typeddict(configuration): + msg = ( + f"configuration is {configuration!r}; give the TypedDict the configuration's JSON takes" + ) + raise TypeError(msg) + _vetting(configuration) + unread = unread_in(configuration, _vetting) + if unread is not None: + msg = f"{unread}; a finer rule goes in the definition's `rules`" + raise TypeError(msg) + + +@dataclass(frozen=True, slots=True) +class _Checker: + """A TypedDict's checker, compiled once, and whether a value of it can hold a nested field.""" + + parse: Parser + nests: bool + + +@functools.cache +def _checker(shape: type) -> _Checker: + """The checker for a TypedDict, compiled once; `TypeError` naming what no checker reads.""" + met: list[object] = [] + + def leaf(annotation: object) -> Parser | None: + found = _field(annotation) + if found is not None: + met.append(annotation) + return found + + return _Checker(parser(shape, leaf), len(met) != 0) + + +def _checked( + shape: type, value: object, loc: Loc +) -> tuple[object, Problems, tuple[_NestedField, ...]]: + """`value` checked as `shape`: the typed value, every problem, and the fields nested in it, in order.""" + checker = _checker(shape) + typed, found = checker.parse(value, loc) + if not checker.nests: + return typed, found, () + nested: list[_NestedField] = [] + return _put_back(typed, nested), found, tuple(nested) + + +def _put_back(value: object, nested: list[_NestedField]) -> object: + """`value` with each nested field the checker handed back put back as its JSON, collected in order.""" + if isinstance(value, _NestedField): + nested.append(value) + return value.json + if isinstance(value, tuple): + return tuple(_put_back(entry, nested) for entry in cast("tuple[object, ...]", value)) + if isinstance(value, dict): + entries = cast("dict[str, object]", value) + return {key: _put_back(entry, nested) for key, entry in entries.items()} + return value + + +def _usable(problems: Sequence[ValidationProblem]) -> bool: + """Whether a value with these problems still reads: an unknown key is survivable, nothing else is.""" + return all(found.kind == "unknown_key" for found in problems) + + +def _located(prefix: Loc, problems: Iterable[ValidationProblem]) -> Problems: + return tuple( + ValidationProblem((*prefix, *found.loc), found.message, found.kind) for found in problems + ) + + +def _envelope(field: _NestedField) -> Problems: + """What is wrong with a nested field's envelope, at the field.""" + return _located( + field.loc, validate_metadata_field_v3(field.json, allow_must_understand_false=False) + ) + + +def _configuration_checked( + value: object, shape: type[T], loc: Loc = () +) -> tuple[T | None, Problems]: + """`value` checked as `shape`, as `typed_json.check` checks it, and each nested field's envelope judged. + + The step a definition's `judge` starts from. A member typed with a + field alias holds a metadata field, whose envelope is judged as a + document's is: a stray member, or a `must_understand` of `false`, is a + problem of the configuration, and the value does not come back. + """ + if not is_typeddict(shape): + msg = f"{shape!r} is not a TypedDict" + raise TypeError(msg) + refined, problems = refine_json(value, loc) + if refined is None: + return None, problems + typed, found, nested = _checked(shape, refined, loc) + problems = (*found, *(problem for field in nested for problem in _envelope(field))) + return (cast("T", typed) if _usable(problems) else None), problems + + +def named_configuration( + value: object, +) -> tuple[str | None, Mapping[str, object] | None, Problems]: + """Split a metadata field into `(name, configuration, problems)`. + + A bare name, or an object carrying one. A `None` name means the value + is not a metadata field at all; a `None` configuration means the bare + spelling was used, or the key was left out. A configuration that is + present and not an object is the one problem reported, at + `("configuration",)`. + """ + if isinstance(value, str): + return value, None, () + if not isinstance(value, Mapping): + return None, None, () + entry = cast("Mapping[str, object]", value) + name = entry.get("name") + if not isinstance(name, str): + return None, None, () + if "configuration" not in entry: + return name, None, () + configuration = entry["configuration"] + if not isinstance(configuration, Mapping): + return name, None, problem(("configuration",), f"expected an object, got {configuration!r}") + return name, cast("Mapping[str, object]", configuration), () + + +Unread = Literal["out_of_scope", "invalid"] +"""A field no definition read: nothing in scope claims its name, or it could not be read.""" + +Resolution = Literal["read"] | Unread +"""What a scope made of a field: read by the definition that claims it, or unread, and why.""" + + +@dataclass(frozen=True, slots=True) +class Resolved(Generic[D]): + """One metadata field, as read in a scope: its JSON, and what the scope made of it. + + `resolution` is what became of the configuration: read by the + definition that claims the name, claimed by nothing, or not readable. + A problem with the envelope around it -- a stray member, a + `must_understand` of `false` -- is reported with the field, and leaves + the resolution as it is. + """ + + json: JSONValue + """The field as written, refined: arrays as tuples. `None` for a value that was not JSON.""" + resolution: Resolution + definition: D | None + """The definition that claims the field's name; None when nothing in scope does, or it names none.""" + configuration: Mapping[str, JSONValue] | None + """The configuration, type-checked and allowed by the rules, when the field was read; None otherwise.""" + + +def configuration_of(resolved: Resolved[Any], definition: Definition[C]) -> C | None: + """The configuration `resolved` holds, typed as `definition` declares it, if `definition` read it. + + `Resolved` holds a configuration as the mapping every one is; asked + with the definition that read the field, this is the same mapping, as + its TypedDict. None when another definition read it, or none did. + """ + if resolved.definition is not definition or resolved.configuration is None: + return None + return cast("C", resolved.configuration) + + +def resolve( + data: object, kind: type[D], context: Context, loc: Loc = () +) -> tuple[Resolved[D], Problems]: + """`data`, one metadata field, read as a `kind` in `context`: what the scope made of it, and every problem. + + All three steps for one field. `data` is refined to JSON and its + envelope judged -- an extra member, a `configuration` that is not an + object, a `must_understand` that is not a boolean or is `false`, each + a problem. The name is related to a definition in `context`; the + configuration is checked against its TypedDict and judged by its + rules; each nested field the check met is read the same way, in the + same scope. A name nothing claims is `out_of_scope`: an unmodelled + extension, left unjudged, which is what keeps the format open. `loc` + prefixes every problem. `kind` is one of `KINDS`, with or without + type arguments; anything else is a `TypeError`. + """ + asked = as_kind(kind) + refined, problems = refine_json(data, loc) + if refined is None: + return Resolved(None, "invalid", None, None), problems + resolved, found = _resolve_field(refined, asked, context, loc) + return cast("Resolved[D]", resolved), found + + +def _resolve_field( + data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc +) -> tuple[Resolved[Definition[Any]], Problems]: + """A refined field with its envelope judged, then read. + + The resolution is what became of the configuration. A stray member or + a `must_understand` of `false` says nothing about it, so it is reported + beside the field that was read, which later layers can still judge. + """ + envelope = _located(loc, validate_metadata_field_v3(data, allow_must_understand_false=False)) + resolved, found = _read(data, kind, context, loc) + return resolved, (*envelope, *found) + + +def _read( + data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc +) -> tuple[Resolved[Definition[Any]], Problems]: + name, given, malformed = named_configuration(data) + if name is None or len(malformed) != 0: + return Resolved(data, "invalid", None, None), () + definition = context.claimant(kind, name) + if definition is None: + return Resolved(data, "out_of_scope", None, None), () + at = (*loc, "configuration") + problems = list(_located(loc, definition.name_rules(name))) + if given is None and definition.requires_configuration: + problems.extend(problem(at, f"{name!r} requires a configuration", "missing_key")) + return Resolved(data, "invalid", definition, None), tuple(problems) + typed, found, nested = _checked(definition.configuration, {} if given is None else given, at) + problems.extend(found) + envelopes = [_envelope(field) for field in nested] + sound = _usable(found) and all(_usable(envelope) for envelope in envelopes) + configuration = cast("Mapping[str, JSONValue]", typed) if sound else None + if configuration is not None: + problems.extend(_located(at, definition.rules(configuration))) + for field, envelope in zip(nested, envelopes, strict=True): + problems.extend(envelope) + problems.extend(_read(field.json, field.kind, context, field.loc)[1]) + if not _usable(problems): + return Resolved(data, "invalid", definition, None), tuple(problems) + return Resolved(data, "read", definition, configuration), tuple(problems) + + +__all__ = [ + "KINDS", + "ChunkGridDefinition", + "ChunkGridField", + "ChunkKeyEncodingDefinition", + "ChunkKeyEncodingField", + "CodecDefinition", + "CodecField", + "CodecKind", + "DataTypeDefinition", + "DataTypeField", + "Definition", + "Resolution", + "Resolved", + "StorageTransformerDefinition", + "StorageTransformerField", + "Unread", + "as_kind", + "configuration_of", + "kind_of", + "named_configuration", + "no_name_rules", + "no_rules", + "resolve", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py new file mode 100644 index 0000000000..7fdf0d6089 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -0,0 +1,117 @@ +"""Which definitions are in scope while a document is read. + +The only registry the package needs. Reading a field means deciding which +definition its name denotes, and that decision is the *scope* the field +is read in, not a property of the definitions. A scope is a value, built +from definitions: each is filed under its kind, which is its type, and +its name, which it carries. + +Two scopes, because "is this valid?" has two useful answers. `CORE` is +what the Zarr v3 specification itself defines, so a document read in it +uses nothing an implementation could refuse for being optional. +`CORE_AND_EXTENSIONS` adds what `zarr-extensions` registers and this +package defines. A name in neither is not refused -- that is what keeps +the format open -- it is read as `out_of_scope` and left unjudged. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, cast + +from zarr_metadata.v3._definition import KINDS, Definition, as_kind, kind_of +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID +from zarr_metadata.v3.codec.bytes import BYTES_CODEC +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC + +if TYPE_CHECKING: + from zarr_metadata.v3._definition import D + +Tables = Mapping[type[Definition[Any]], Mapping[str, Definition[Any]]] +"""By kind, then by the name each definition is filed under.""" + + +@dataclass(frozen=True, slots=True) +class Context: + """The definitions in scope while metadata is read. + + A value with no reading of its own: `resolve` reads a field in it, + and `claimant` is the one question it answers, which definition a + name belongs to. Built from definitions with `Context.of`, extended + with more by `extended_with`. + """ + + tables: Tables + + @classmethod + def of(cls, *definitions: Definition[Any]) -> Context: + """A scope of exactly these definitions; a later one takes a name over from an earlier. + + `TypeError` for a definition of no kind, which no position in a + document could hold. + """ + tables: dict[type[Definition[Any]], dict[str, Definition[Any]]] = { + kind: {} for kind in KINDS + } + for definition in definitions: + kind = kind_of(definition) + if kind is None: + msg = ( + f"{definition.name!r} is a definition of no kind; build it as a " + "CodecDefinition, DataTypeDefinition, ChunkGridDefinition, " + "ChunkKeyEncodingDefinition or StorageTransformerDefinition" + ) + raise TypeError(msg) + tables[kind][definition.name] = definition + return cls( + MappingProxyType({kind: MappingProxyType(table) for kind, table in tables.items()}) + ) + + def extended_with(self, *definitions: Definition[Any]) -> Context: + """This scope, plus definitions of your own. + + A name already filed under the same kind is taken over by what is + passed here, which is how a reader substitutes its own reading of + a codec the package already defines. + """ + return Context.of(*self.definitions(), *definitions) + + def definitions(self) -> tuple[Definition[Any], ...]: + """Every definition in scope, kind by kind.""" + return tuple(entry for table in self.tables.values() for entry in table.values()) + + def claimant(self, kind: type[D], name: str) -> D | None: + """The definition of `kind` in scope that claims `name`; None if none does. + + Asks each definition filed under the kind whether the name is its + own -- a family claims every `r` -- rather than looking a key + up, so the names a scope files under exist for `extended_with` to + take one over. + """ + table = self.tables.get(as_kind(kind), {}) + return cast( + "D | None", next((entry for entry in table.values() if entry.claims(name)), None) + ) + + +_CORE: Final[tuple[Definition[Any], ...]] = ( + BYTES_CODEC, + CRC32C_CODEC, + GZIP_CODEC, + REGULAR_CHUNK_GRID, +) +"""What the Zarr v3 specification itself defines.""" + +_EXTENSIONS: Final[tuple[Definition[Any], ...]] = () +"""What `zarr-extensions` registers and this package defines.""" + +CORE: Final = Context.of(*_CORE) +"""Only what the Zarr v3 specification defines.""" + +CORE_AND_EXTENSIONS: Final = Context.of(*_CORE, *_EXTENSIONS) +"""What the specification defines, plus what `zarr-extensions` registers.""" + +__all__ = ["CORE", "CORE_AND_EXTENSIONS", "Context", "Tables"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index bdcd9e06c5..a329da80d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -4,10 +4,14 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids """ +from collections.abc import Iterator from typing import Final, Literal, NotRequired from typing_extensions import TypedDict +from zarr_metadata._json import ValidationProblem +from zarr_metadata.v3._definition import ChunkGridDefinition + REGULAR_CHUNK_GRID_NAME: Final = "regular" """The `name` field value of the regular chunk grid.""" @@ -38,7 +42,23 @@ class RegularChunkGridObject(TypedDict, closed=True): https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ + +def _rules(configuration: RegularChunkGridConfiguration) -> Iterator[ValidationProblem]: + """Every chunk extent is at least 1.""" + for index, extent in enumerate(configuration["chunk_shape"]): + if extent < 1: + yield ValidationProblem( + ("chunk_shape", index), f"expected an integer >= 1, got {extent}", "invalid_value" + ) + + +REGULAR_CHUNK_GRID: Final = ChunkGridDefinition( + name=REGULAR_CHUNK_GRID_NAME, configuration=RegularChunkGridConfiguration, rules=_rules +) +"""The `regular` chunk grid.""" + __all__ = [ + "REGULAR_CHUNK_GRID", "REGULAR_CHUNK_GRID_NAME", "RegularChunkGridConfiguration", "RegularChunkGridMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index c8a9a150fc..734fc02bbf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -14,6 +14,10 @@ `codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. +Each codec's pipeline position (`array -> array`, `array -> bytes`, +`bytes -> bytes`) is its definition's `kind`, in +`zarr_metadata.v3.definition`. + See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 4feb6b8c1c..79674827e7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -8,6 +8,8 @@ from typing_extensions import TypedDict +from zarr_metadata.v3._definition import CodecDefinition + BYTES_CODEC_NAME: Final = "bytes" """The `name` field value of the `bytes` codec.""" @@ -56,7 +58,17 @@ class BytesCodecObject(TypedDict, closed=True): https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ +BYTES_CODEC: Final = CodecDefinition( + name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, kind="array_bytes" +) +"""The `bytes` codec. + +No rule of its own: whether `endian` is required depends on the data type +the codec is handed, which is a question about the array, not the field. +""" + __all__ = [ + "BYTES_CODEC", "BYTES_CODEC_NAME", "ENDIANNESS", "BytesCodecConfiguration", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 05661d0b59..3078b074cc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -11,6 +11,8 @@ from typing_extensions import TypedDict +from zarr_metadata.v3._definition import CodecDefinition + CRC32C_CODEC_NAME: Final = "crc32c" """The `name` field value of the `crc32c` codec.""" @@ -45,7 +47,13 @@ class Crc32cCodecObject(TypedDict, closed=True): """ +CRC32C_CODEC: Final = CodecDefinition( + name=CRC32C_CODEC_NAME, configuration=Empty, kind="bytes_bytes" +) +"""The `crc32c` codec: a checksum has nothing to configure.""" + __all__ = [ + "CRC32C_CODEC", "CRC32C_CODEC_NAME", "Crc32cCodecMetadata", "Crc32cCodecName", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index d516088f99..28381c76cc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -4,10 +4,14 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html """ +from collections.abc import Iterator from typing import Final, Literal, NotRequired from typing_extensions import TypedDict +from zarr_metadata._json import ValidationProblem +from zarr_metadata.v3._definition import CodecDefinition + GZIP_CODEC_NAME: Final = "gzip" """The `name` field value of the `gzip` codec.""" @@ -48,7 +52,23 @@ class GzipCodecObject(TypedDict, closed=True): only the object form is valid; the short-hand-name form is not permitted. """ + +def _rules(configuration: GzipCodecConfiguration) -> Iterator[ValidationProblem]: + """`level` is an integer from 0 to 9.""" + level = configuration["level"] + if not 0 <= level <= 9: + yield ValidationProblem( + ("level",), f"expected an integer in [0, 9], got {level}", "invalid_value" + ) + + +GZIP_CODEC: Final = CodecDefinition( + name=GZIP_CODEC_NAME, configuration=GzipCodecConfiguration, kind="bytes_bytes", rules=_rules +) +"""The `gzip` codec.""" + __all__ = [ + "GZIP_CODEC", "GZIP_CODEC_NAME", "GzipCodecConfiguration", "GzipCodecMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py new file mode 100644 index 0000000000..ef2248d0ff --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -0,0 +1,180 @@ +"""Metadata fields read against their definitions: the public door. + +Every Zarr v3 extension point -- codecs, data types, chunk grids, chunk +key encodings, storage transformers -- is a metadata field: a name, and +a configuration whose JSON the extension defines. A **definition** says +what that JSON is and what is allowed in it. It is a value, not a class +to subclass: + +- `name`, the name the metadata carries; +- `configuration`, the TypedDict the configuration's JSON is, which is + the one declaration of it: the type checker is compiled from it, and a + checked configuration has it as its static type. It reads as the + typing spec defines a TypedDict -- `total`, `Required`, `NotRequired`, + `closed` and `extra_items` mean what they mean to a type checker, + whether or not its module postpones annotations; +- `rules`, a function over that TypedDict yielding what the spec + disallows -- a bound, members read together -- located in the + configuration. + +What kind of metadata a definition defines is its type: `CodecDefinition` +(with the codec's `kind`), `DataTypeDefinition`, `ChunkGridDefinition`, +`ChunkKeyEncodingDefinition`, `StorageTransformerDefinition`. + +**Reading JSON.** Three steps, each feeding the next, and each usable on +its own by a caller that holds nothing but JSON: + +1. `check(value, SomeTypedDict)`, from `zarr_metadata.typed_json` and + here too, type-checks JSON against a TypedDict and needs nothing else: + a value of the TypedDict or None, and every problem, each located. The + value holds what the TypedDict admits and nothing else. A member typed + with a field alias is checked as the JSON a metadata field is. +2. `definition.judge(configuration)` is the check, each nested field's + envelope judged -- a stray member, a `must_understand` of `false` -- + and then the rules, for one configuration: + `GZIP_CODEC.judge({"level": 12})`. +3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole + field in a scope: its envelope judged, its name related to a + definition, its configuration judged, and each nested field read the + same way. It returns `Resolved` -- the field's JSON, its + `resolution`, the definition and the checked configuration -- and + every problem. A name nothing in scope claims is `out_of_scope`: + left unjudged, which is what keeps the format open. A field is read + as one of the five kinds, with or without type arguments; + `resolve(field, Definition, scope)` is a `TypeError`, since nothing + is filed under it. `configuration_of(resolved, GZIP_CODEC)` is the + configuration typed as that definition's TypedDict, when it read it. + + from zarr_metadata.v3.codec.gzip import GZIP_CODEC + from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS, + CodecDefinition, + configuration_of, + resolve, + ) + + resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, + CodecDefinition, CORE_AND_EXTENSIONS) + resolved.resolution # 'invalid' + problems[0].loc # ('configuration', 'level') + + resolved, problems = resolve({"name": "gzip", "configuration": {"level": 5}}, + CodecDefinition, CORE_AND_EXTENSIONS) + configuration_of(resolved, GZIP_CODEC) # {'level': 5}, a GzipCodecConfiguration + +Problems are values, not exceptions: `ValidationProblem(loc, message, +kind)`, with `kind` one of `invalid_type`, `invalid_value`, +`missing_key`, `unknown_key` and `invalid_json`. An unknown key is +survivable -- the field is still read, the key reported, and the +configuration comes back without it. + +**Writing an extension.** A TypedDict, a function for its rules, and a +definition; then a scope that holds it: + + from collections.abc import Iterator + + from typing_extensions import TypedDict + + from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS, + CodecDefinition, + ValidationProblem, + ) + + + class AcmeLz4Configuration(TypedDict, closed=True): + acceleration: int + + + def acme_lz4_rules(configuration: AcmeLz4Configuration) -> Iterator[ValidationProblem]: + if configuration["acceleration"] < 1: + yield ValidationProblem(("acceleration",), "expected an integer >= 1", "invalid_value") + + + ACME_LZ4 = CodecDefinition( + name="acme.lz4", + configuration=AcmeLz4Configuration, + kind="bytes_bytes", + rules=acme_lz4_rules, + ) + SCOPE = CORE_AND_EXTENSIONS.extended_with(ACME_LZ4) + +The TypedDict says what a key it does not declare is: with +`closed=True`, a problem, as above; with `extra_items=`, a key holding +that type; with `closed=False`, anything at all. One that says none of +these is open by default, and would take a misspelled key without a +word, so a definition refuses it. Its members are the shapes JSON takes: +`int`, `float`, `bool`, `str`, `None`, `JSONValue`, a `Literal`, +`tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, +`Mapping[str, V]`, a `NewType` and a type alias. + +A member holding another metadata field is annotated with the field alias +of its kind -- a container codec's `codecs: tuple[CodecField, ...]` -- and +read in the scope its field is read in. `ZarrV3MetadataFieldJSON` is the +same JSON, but checks as JSON and nothing more, so a definition refuses a +member typed with it. A family, one definition for many names, claims +them through `names` and says which are allowed through `name_rules`. + +A definition checks itself when it is built, and each of these is a +`TypeError` saying what is wrong: a `configuration` that is not a +TypedDict, says nothing of the keys it does not declare, or has a member +no checker reads, named down to the TypedDict that holds it; a `name` +that is not a string; `rules`, `name_rules` or `names` that are not +functions; a codec `kind` that is not one of the three. A scope refuses +a definition of no kind. Nothing happens at class creation. +""" + +from zarr_metadata._common import JSONValue +from zarr_metadata._json import MetadataValidationError, ProblemKind, ValidationProblem +from zarr_metadata._typed_json import Loc, check +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3._definition import ( + ChunkGridDefinition, + ChunkGridField, + ChunkKeyEncodingDefinition, + ChunkKeyEncodingField, + CodecDefinition, + CodecField, + CodecKind, + DataTypeDefinition, + DataTypeField, + Definition, + Resolution, + Resolved, + StorageTransformerDefinition, + StorageTransformerField, + Unread, + configuration_of, + resolve, +) +from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context + +__all__ = [ + "CORE", + "CORE_AND_EXTENSIONS", + "ChunkGridDefinition", + "ChunkGridField", + "ChunkKeyEncodingDefinition", + "ChunkKeyEncodingField", + "CodecDefinition", + "CodecField", + "CodecKind", + "Context", + "DataTypeDefinition", + "DataTypeField", + "Definition", + "JSONValue", + "Loc", + "MetadataValidationError", + "ProblemKind", + "Resolution", + "Resolved", + "StorageTransformerDefinition", + "StorageTransformerField", + "Unread", + "ValidationProblem", + "ZarrV3MetadataFieldJSON", + "check", + "configuration_of", + "resolve", +] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 026eeeb553..23eb6d8fe2 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -267,6 +267,12 @@ def test_all_is_grouped_and_unique() -> None: "FillValue", "Configuration", "Component", + # What one kind of extension's metadata is, as JSON: `CodecDefinition`, + # the value holding a codec's name, configuration TypedDict and rules. + "Definition", + # A configuration member holding a metadata field of that kind: + # `CodecField`, read in the scope its field is read in. + "Field", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") @@ -279,6 +285,13 @@ def test_all_is_grouped_and_unique() -> None: "Base64Bytes", "BloscCName", "BloscShuffle", + "CodecKind", + "Context", + "Definition", + "Loc", + "Resolution", + "Resolved", + "Unread", "CastOutOfRangeMode", "CastRoundingMode", "Endianness", @@ -286,7 +299,6 @@ def test_all_is_grouped_and_unique() -> None: "HexFloat32", "HexFloat64", "JSONValue", - "Loc", "MetadataValidationError", "NumpyDatetime64", "NumpyTimeUnit", @@ -411,7 +423,14 @@ def _literal_backed_constants() -> list[tuple[str, str, str]]: for const_name, value in vars(module).items(): if const_name.startswith("_") or not const_name.isupper(): continue - members = frozenset(value) if isinstance(value, tuple) else frozenset({value}) + if isinstance(value, tuple): + members = frozenset(value) + elif isinstance(value, str): + members = frozenset({value}) + else: + # A value that is not a name -- a definition, a scope -- + # backs no Literal type and carries no signal here. + continue if not all(isinstance(m, str) for m in members): continue matches = [t for t, args in literals.items() if args == members] @@ -443,7 +462,14 @@ def _value_tied_constants() -> set[str]: for const_name, value in vars(module).items(): if const_name.startswith("_") or not const_name.isupper(): continue - members = frozenset(value) if isinstance(value, tuple) else frozenset({value}) + if isinstance(value, tuple): + members = frozenset(value) + elif isinstance(value, str): + members = frozenset({value}) + else: + # A value that is not a name -- a definition, a scope -- + # backs no Literal type and carries no signal here. + continue if not all(isinstance(m, str) for m in members): continue if sum(1 for args in literals if args == members) > 1: diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py new file mode 100644 index 0000000000..a6ec226927 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -0,0 +1,600 @@ +"""One metadata field, read against its definition: checked, judged, and read in a scope.""" + +from __future__ import annotations + +import math +import re +from collections.abc import ( + Mapping, # noqa: TC003 - a TypedDict's annotations are evaluated at run time +) +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Final, NotRequired, cast + +import pytest +from typing_extensions import TypedDict + +from zarr_metadata.model._array import ZarrV3ArrayMetadata +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID +from zarr_metadata.v3.codec.crc32c import Empty +from zarr_metadata.v3.codec.gzip import GZIP_CODEC, GzipCodecConfiguration +from zarr_metadata.v3.definition import ( + CORE, + CORE_AND_EXTENSIONS, + ChunkGridDefinition, + CodecDefinition, + CodecField, + Context, + DataTypeDefinition, + Definition, + JSONValue, + ValidationProblem, + ZarrV3MetadataFieldJSON, + check, + configuration_of, + resolve, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + from decimal import Decimal + + +class AcmeStackConfiguration(TypedDict, closed=True): + """A third party's container: a codec holding a pipeline of codecs.""" + + codecs: tuple[CodecField, ...] + + +def acme_stack_rules(configuration: AcmeStackConfiguration) -> Iterator[ValidationProblem]: + # A rule may read a nested field's name: it is asked only of a + # configuration whose nested envelopes are sound. + for index, codec in enumerate(configuration["codecs"]): + if (codec if isinstance(codec, str) else codec["name"]) == "acme.stack": + yield ValidationProblem( + ("codecs", index), "a stack does not hold itself", "invalid_value" + ) + + +ACME_STACK = CodecDefinition( + name="acme.stack", + configuration=AcmeStackConfiguration, + kind="bytes_bytes", + rules=acme_stack_rules, +) + +ACME_FIXED = re.compile(r"acme\.fixed(\d+)") + + +def acme_fixed_name_rules(name: str) -> Iterator[ValidationProblem]: + match = ACME_FIXED.fullmatch(name) + if match is not None and int(match.group(1)) % 8 != 0: + yield ValidationProblem((), "expected a width that is a multiple of 8", "invalid_value") + + +ACME_FIXED_TYPE = DataTypeDefinition( + name="acme.fixed", + configuration=Empty, + names=lambda name: ACME_FIXED.fullmatch(name) is not None, + name_rules=acme_fixed_name_rules, +) + + +class AcmeLevelConfiguration(TypedDict, closed=True): + """Under postponed annotations, as in this module, `NotRequired` is written as a string.""" + + level: NotRequired[int] + + +class AcmeTotalConfiguration(TypedDict, total=False, closed=True): + level: int + + +class AcmeTreeConfiguration(TypedDict, closed=True): + """A configuration that holds itself, as JSON can to any depth.""" + + label: str + children: tuple[AcmeTreeConfiguration, ...] + + +class ByCodec(TypedDict, closed=True): + codec: CodecField + weight: int + + +class Raw(TypedDict, closed=True): + codec: Mapping[str, JSONValue] + note: str + + +class AcmeFallbackConfiguration(TypedDict, closed=True): + """A union whose branches disagree on whether `codec` is a metadata field.""" + + fallback: ByCodec | Raw + + +class AcmeRoutesConfiguration(TypedDict, extra_items=CodecField): + """Every key a route, holding the codec that takes it.""" + + +ACME_BOUNDS: Final = {"level": (0, 9), "window": (9, 15)} + + +class AcmeBoundedConfiguration(TypedDict, closed=True): + level: int + window: NotRequired[int] + + +def acme_bounded_rules(configuration: AcmeBoundedConfiguration) -> Iterator[ValidationProblem]: + # Every key it meets is one its TypedDict declares. + for key, value in cast("Mapping[str, int]", configuration).items(): + low, high = ACME_BOUNDS[key] + if not low <= value <= high: + yield ValidationProblem((key,), f"expected {low} to {high}", "invalid_value") + + +class AcmePairedConfiguration(TypedDict, closed=True): + first: NotRequired[int] + second: NotRequired[int] + + +def acme_paired_rules(configuration: AcmePairedConfiguration) -> Iterator[ValidationProblem]: + if ("first" in configuration) != ("second" in configuration): + yield ValidationProblem((), "expected first and second, or neither", "invalid_value") + + +ACME_LEVEL = CodecDefinition( + name="acme.level", configuration=AcmeLevelConfiguration, kind="bytes_bytes" +) +ACME_TOTAL = CodecDefinition( + name="acme.total", configuration=AcmeTotalConfiguration, kind="bytes_bytes" +) +ACME_TREE = CodecDefinition( + name="acme.tree", configuration=AcmeTreeConfiguration, kind="bytes_bytes" +) +ACME_FALLBACK = CodecDefinition( + name="acme.fallback", configuration=AcmeFallbackConfiguration, kind="bytes_bytes" +) +ACME_ROUTES = CodecDefinition( + name="acme.routes", configuration=AcmeRoutesConfiguration, kind="bytes_bytes" +) +ACME_BOUNDED = CodecDefinition( + name="acme.bounded", + configuration=AcmeBoundedConfiguration, + kind="bytes_bytes", + rules=acme_bounded_rules, +) +ACME_PAIRED = CodecDefinition( + name="acme.paired", + configuration=AcmePairedConfiguration, + kind="bytes_bytes", + rules=acme_paired_rules, +) + +SCOPE = CORE_AND_EXTENSIONS.extended_with( + ACME_STACK, + ACME_FIXED_TYPE, + ACME_LEVEL, + ACME_TOTAL, + ACME_TREE, + ACME_FALLBACK, + ACME_ROUTES, + ACME_BOUNDED, + ACME_PAIRED, +) + + +def _locs(problems: tuple[ValidationProblem, ...]) -> list[tuple[tuple[str | int, ...], str]]: + return [(found.loc, found.kind) for found in problems] + + +def test_core_is_a_subset_of_core_and_extensions() -> None: + assert set(CORE.definitions()) <= set(CORE_AND_EXTENSIONS.definitions()) + + +@pytest.mark.parametrize( + ("field", "kind", "resolution", "configuration", "problems"), + [ + ( + {"name": "gzip", "configuration": {"level": 5}}, + CodecDefinition, + "read", + {"level": 5}, + [], + ), + ("crc32c", CodecDefinition, "read", {}, []), + ({"name": "crc32c"}, CodecDefinition, "read", {}, []), + ({"name": "crc32c", "configuration": {}}, CodecDefinition, "read", {}, []), + ({"name": "crc32c", "must_understand": True}, CodecDefinition, "read", {}, []), + ("bytes", CodecDefinition, "read", {}, []), + ( + {"name": "bytes", "configuration": {"endian": "big"}}, + CodecDefinition, + "read", + {"endian": "big"}, + [], + ), + ( + {"name": "regular", "configuration": {"chunk_shape": [2, 3]}}, + ChunkGridDefinition, + "read", + {"chunk_shape": (2, 3)}, + [], + ), + # An unknown key is survivable: reported, left out of the + # configuration, and the field still read. + ( + {"name": "gzip", "configuration": {"level": 5, "extra": 1}}, + CodecDefinition, + "read", + {"level": 5}, + [(("configuration", "extra"), "unknown_key")], + ), + # Nothing the rules could meet but what the TypedDict declares. + ( + {"name": "acme.bounded", "configuration": {"level": 5, "windw": 10}}, + CodecDefinition, + "read", + {"level": 5}, + [(("configuration", "windw"), "unknown_key")], + ), + # A key that is not required, written as a string under postponed + # annotations, and every key of a `total=False` TypedDict. + ("acme.level", CodecDefinition, "read", {}, []), + ("acme.total", CodecDefinition, "read", {}, []), + # A kind with type arguments is that kind. + ( + {"name": "gzip", "configuration": {"level": 5}}, + CodecDefinition[Any], + "read", + {"level": 5}, + [], + ), + ( + { + "name": "acme.tree", + "configuration": {"label": "a", "children": [{"label": "b", "children": []}]}, + }, + CodecDefinition, + "read", + {"label": "a", "children": ({"label": "b", "children": ()},)}, + [], + ), + # Only the union branch that read has its nested fields read: here + # `codec` is plain JSON, not a gzip missing its configuration. + ( + { + "name": "acme.fallback", + "configuration": {"fallback": {"codec": {"name": "gzip"}, "note": "x"}}, + }, + CodecDefinition, + "read", + {"fallback": {"codec": {"name": "gzip"}, "note": "x"}}, + [], + ), + # `extra_items` of a field alias: every other key holds a codec. + ( + {"name": "acme.routes", "configuration": {"fast": "crc32c"}}, + CodecDefinition, + "read", + {"fast": "crc32c"}, + [], + ), + # Nothing in scope claims it: left unjudged, not refused. + ({"name": "zfpy", "configuration": {"x": 1}}, CodecDefinition, "out_of_scope", None, []), + # A nested field is read in the same scope; one out of scope is left be. + ( + {"name": "acme.stack", "configuration": {"codecs": ["crc32c", "zfpy"]}}, + CodecDefinition, + "read", + {"codecs": ("crc32c", "zfpy")}, + [], + ), + # A family claims many names. + ("acme.fixed16", DataTypeDefinition, "read", {}, []), + ], + ids=[ + "gzip", + "bare-name", + "object-without-configuration", + "empty-configuration", + "must-understand", + "bytes-bare", + "bytes-endian", + "regular-grid", + "unknown-key", + "unknown-key-before-the-rules", + "not-required-postponed", + "total-false", + "kind-with-type-arguments", + "recursive", + "union-branch-that-read", + "extra-items-of-fields", + "out-of-scope", + "nested", + "family", + ], +) +def test_a_field_is_read_in_scope( + field: object, + kind: type[Definition[Any]], + resolution: str, + configuration: dict[str, object] | None, + problems: list[tuple[tuple[str | int, ...], str]], +) -> None: + resolved, found = resolve(field, kind, SCOPE) + assert resolved.resolution == resolution + assert resolved.configuration == configuration + assert _locs(found) == problems + + +def test_error_a_rule_refuses_a_value() -> None: + resolved, found = resolve( + {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, SCOPE + ) + assert resolved.resolution == "invalid" + assert resolved.configuration is None + assert _locs(found) == [(("configuration", "level"), "invalid_value")] + + +def test_error_a_member_of_the_wrong_type_is_not_asked_of_the_rules() -> None: + resolved, found = resolve( + {"name": "gzip", "configuration": {"level": "5"}}, CodecDefinition, SCOPE + ) + assert resolved.resolution == "invalid" + assert _locs(found) == [(("configuration", "level"), "invalid_type")] + + +def test_error_a_required_configuration_is_missing() -> None: + resolved, found = resolve("gzip", CodecDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert _locs(found) == [(("configuration",), "missing_key")] + + +def test_error_the_configuration_is_not_an_object() -> None: + resolved, found = resolve({"name": "gzip", "configuration": 5}, CodecDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert [found.loc for found in found] == [("configuration",)] + + +def test_error_must_understand_false_is_refused() -> None: + # The envelope's problem, reported with the field; the configuration + # was read, so a later layer can still judge the codec. + resolved, found = resolve({"name": "crc32c", "must_understand": False}, CodecDefinition, SCOPE) + assert resolved.resolution == "read" + assert _locs(found) == [(("must_understand",), "invalid_value")] + + +def test_error_a_value_that_is_not_json() -> None: + resolved, found = resolve( + {"name": "gzip", "configuration": {"level": math.nan}}, CodecDefinition, SCOPE + ) + assert (resolved.json, resolved.resolution) == (None, "invalid") + assert _locs(found) == [(("configuration", "level"), "invalid_value")] + + +def test_error_a_value_that_is_not_a_field() -> None: + resolved, found = resolve(5, CodecDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert len(found) == 1 + + +def test_error_a_regular_grid_extent_is_zero() -> None: + resolved, found = resolve( + {"name": "regular", "configuration": {"chunk_shape": [2, 0]}}, ChunkGridDefinition, SCOPE + ) + assert resolved.resolution == "invalid" + assert _locs(found) == [(("configuration", "chunk_shape", 1), "invalid_value")] + + +def test_error_a_nested_field_is_judged_where_it_sits() -> None: + field = { + "name": "acme.stack", + "configuration": {"codecs": ["crc32c", {"name": "gzip", "configuration": {"level": 12}}]}, + } + resolved, found = resolve(field, CodecDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert _locs(found) == [ + (("configuration", "codecs", 1, "configuration", "level"), "invalid_value") + ] + + +def test_error_a_nested_field_in_extra_items_is_judged_where_it_sits() -> None: + field = { + "name": "acme.routes", + "configuration": {"slow": {"name": "gzip", "configuration": {"level": 12}}}, + } + resolved, found = resolve(field, CodecDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert _locs(found) == [(("configuration", "slow", "configuration", "level"), "invalid_value")] + + +def test_error_a_container_rule_is_not_asked_of_a_malformed_nested_field() -> None: + # The stack's rule reads each nested field's name; one with no name is + # reported where it sits, and the rule is not asked, as `judge` would not. + field = {"name": "acme.stack", "configuration": {"codecs": [{"configuration": {}}]}} + resolved, found = resolve(field, CodecDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert _locs(found) == [(("configuration", "codecs", 0, "name"), "invalid_type")] + assert ACME_STACK.judge(field["configuration"])[0] is None + + +def test_error_a_rule_about_the_whole_configuration_lands_on_it() -> None: + # A rule reports relative to the configuration: an empty location is + # the configuration, judged alone or read in a field. + _, judged = ACME_PAIRED.judge({"first": 1}) + _, read = resolve( + {"name": "acme.paired", "configuration": {"first": 1}}, CodecDefinition, SCOPE + ) + assert _locs(judged) == [((), "invalid_value")] + assert _locs(read) == [(("configuration",), "invalid_value")] + + +def test_error_a_nested_member_is_not_a_field() -> None: + resolved, found = resolve( + {"name": "acme.stack", "configuration": {"codecs": [5]}}, CodecDefinition, SCOPE + ) + assert resolved.resolution == "invalid" + assert _locs(found) == [(("configuration", "codecs", 0), "invalid_type")] + + +def test_error_a_family_judges_the_name_it_claims() -> None: + resolved, found = resolve("acme.fixed12", DataTypeDefinition, SCOPE) + assert resolved.resolution == "invalid" + assert _locs(found) == [((), "invalid_value")] + + +def test_check_needs_nothing_but_the_value_and_a_typeddict() -> None: + # The first step: JSON against a TypedDict. A nested field is an + # envelope here, so no scope is needed and no name is judged. + assert check({"level": 5}, GzipCodecConfiguration) == ({"level": 5}, ()) + stack, problems = check({"codecs": ["gzip", {"name": "anything"}]}, AcmeStackConfiguration) + assert (stack, problems) == ({"codecs": ("gzip", {"name": "anything"})}, ()) + # The check and the reading agree on which union branch a value is. + fallback = {"fallback": {"codec": {"name": "gzip"}, "note": "x"}} + assert check(fallback, AcmeFallbackConfiguration) == (fallback, ()) + + +def test_check_reads_a_whole_array_document() -> None: + # A null in `dimension_names`, and a top-level key the document does + # not declare, typed by its `extra_items`. + document = { + **ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), + "dimension_names": [None], + "acme": {"must_understand": False}, + } + typed, problems = check(document, ZarrV3ArrayMetadataJSON) + assert problems == () + assert typed is not None + assert typed.get("dimension_names") == (None,) + + +def test_configuration_of_types_what_its_definition_read() -> None: + resolved, _ = resolve({"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, SCOPE) + assert configuration_of(resolved, GZIP_CODEC) == {"level": 5} + assert configuration_of(resolved, ACME_LEVEL) is None + + +def test_error_check_locates_what_is_not_the_typeddict() -> None: + typed, problems = check({"level": "5", "extra": 1}, GzipCodecConfiguration) + assert typed is None + assert sorted(_locs(problems)) == [(("extra",), "unknown_key"), (("level",), "invalid_type")] + + +def test_error_check_judges_a_nested_envelope() -> None: + typed, problems = check( + {"codecs": [{"name": "gzip", "configuration": 3}]}, AcmeStackConfiguration + ) + assert typed is None + assert [found.loc for found in problems] == [("codecs", 0, "configuration")] + + +def test_judge_is_the_check_and_then_the_rules() -> None: + # A caller holding one configuration: the rules are asked only of a + # configuration that type-checked, so they never meet a wrong type. + assert GZIP_CODEC.judge({"level": 5}) == ({"level": 5}, ()) + refused, problems = GZIP_CODEC.judge({"level": 12}) + assert (refused, _locs(problems)) == (None, [(("level",), "invalid_value")]) + mistyped, problems = GZIP_CODEC.judge({"level": "x"}) + assert (mistyped, _locs(problems)) == (None, [(("level",), "invalid_type")]) + + +def test_a_scope_takes_a_name_over() -> None: + strict = CodecDefinition( + name="gzip", + configuration=GzipCodecConfiguration, + kind="bytes_bytes", + rules=lambda configuration: ( + [ValidationProblem(("level",), "level 0 stores uncompressed", "invalid_value")] + if configuration["level"] == 0 + else [] + ), + ) + scope = CORE_AND_EXTENSIONS.extended_with(strict) + _, found = resolve({"name": "gzip", "configuration": {"level": 0}}, CodecDefinition, scope) + assert _locs(found) == [(("configuration", "level"), "invalid_value")] + assert scope.claimant(ChunkGridDefinition, "regular") is REGULAR_CHUNK_GRID + + +class Unreadable(TypedDict, closed=True): + members: set[int] + + +class Loose(TypedDict): + level: int + + +class AcmePlainConfiguration(TypedDict, closed=True): + inner: ZarrV3MetadataFieldJSON + + +class AcmeDecimal(TypedDict, closed=True): + value: Decimal # a name the type checker sees, and the running module does not + + +class AcmeUnresolvedConfiguration(TypedDict, closed=True): + inner: AcmeDecimal + + +@dataclass(frozen=True, kw_only=True, slots=True) +class AcmeCodecDefinition(CodecDefinition[Any]): + """A reader's own kind of codec, under which no scope files anything.""" + + +def test_error_a_definition_configuration_is_a_typeddict() -> None: + with pytest.raises(TypeError, match="give the TypedDict"): + CodecDefinition(name="acme.bad", configuration=dict, kind="bytes_bytes") + + +def test_error_a_definition_member_is_a_shape_json_takes() -> None: + with pytest.raises(TypeError, match="Unreadable: members is not a shape JSON takes"): + CodecDefinition(name="acme.bad", configuration=Unreadable, kind="bytes_bytes") + + +def test_error_a_configuration_says_what_its_other_keys_are() -> None: + # Open by default, it would take a misspelled key without a word. + with pytest.raises(TypeError, match="Loose says nothing of the keys it does not declare"): + CodecDefinition(name="acme.loose", configuration=Loose, kind="bytes_bytes") + + +def test_error_a_member_typed_as_plain_field_json_is_refused() -> None: + # It checks as JSON, and its name would never be related to a definition. + with pytest.raises(TypeError, match="annotate it with the field alias of its kind"): + CodecDefinition(name="acme.plain", configuration=AcmePlainConfiguration, kind="bytes_bytes") + + +def test_error_a_configuration_whose_annotations_do_not_resolve() -> None: + # Named down to the TypedDict that holds the annotation. + with pytest.raises(TypeError, match="AcmeDecimal: name 'Decimal' is not defined"): + CodecDefinition( + name="acme.unresolved", + configuration=AcmeUnresolvedConfiguration, + kind="bytes_bytes", + ) + + +def test_error_a_definition_name_is_a_string() -> None: + with pytest.raises(TypeError, match="a definition's name is a string"): + CodecDefinition(name=5, configuration=Empty, kind="bytes_bytes") # pyright: ignore[reportArgumentType] + + +def test_error_a_codec_kind_is_one_of_three() -> None: + with pytest.raises(TypeError, match="kind is one of"): + CodecDefinition(name="acme.k", configuration=Empty, kind="bytes_to_array") # pyright: ignore[reportArgumentType] + + +def test_error_a_family_names_its_names_with_a_function() -> None: + with pytest.raises(TypeError, match="names is a function"): + CodecDefinition(name="acme.n", configuration=Empty, kind="bytes_bytes", names="acme.n") # pyright: ignore[reportArgumentType] + + +@pytest.mark.parametrize("kind", [Definition, AcmeCodecDefinition]) +def test_error_a_field_is_read_as_a_kind_of_metadata(kind: type[Definition[Any]]) -> None: + # Nothing is filed under either, so the field would go unjudged. + with pytest.raises(TypeError, match="is not a kind of metadata"): + resolve({"name": "gzip", "configuration": {"level": 99}}, kind, SCOPE) + + +def test_error_a_scope_refuses_a_definition_of_no_kind() -> None: + with pytest.raises(TypeError, match="a definition of no kind"): + Context.of(Definition(name="acme.kindless", configuration=Empty))