From 200fb7de052ad73369fca60a103aae1dda54cded Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 13:17:17 +0200 Subject: [PATCH 1/9] feat(zarr-metadata): the checker: parsers and writers compiled from annotations The second of the stack, and a module on its own: `v3/_typed_json` compiles, once per annotation, a parser for each shape JSON takes -- `int`, `float`, `bool`, `str`, `JSONValue`, a `Literal`, `tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, a record dataclass, `Mapping[str, V]`, a `NewType`, and `| UNSET` for a member a document may leave out -- and the writer that is its inverse. A parser is `(value, loc, state) -> (typed value, problems)`, generic in a state it hands through untouched, so a caller's leaf can read its own shapes at any depth: the entity layer's leaf, in the next PR, reads a field that holds another entity in a scope. The module knows nothing of entities. Read once per class, cached: `field_hints` resolves a dataclass's annotations where each class was defined, under PEP 649 as before it, and skips class variables, which `declared_class_vars` reports so a base can owe one. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/model/_validation.py | 13 +- .../src/zarr_metadata/v3/_typed_json.py | 885 ++++++++++++++++++ .../zarr-metadata/tests/v3/test_typed_json.py | 298 ++++++ 3 files changed, 1193 insertions(+), 3 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py create mode 100644 packages/zarr-metadata/tests/v3/test_typed_json.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 57ad33c33f..0b03b82c16 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -10,8 +10,8 @@ Every `ValidationProblem` carries a machine-readable `kind` alongside its human-readable `message`, so consumers can dispatch on the failure mode -(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without -string-matching messages. +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`, +`unknown_key`) without string-matching messages. """ from __future__ import annotations @@ -29,7 +29,7 @@ from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] """Machine-readable classification of a `ValidationProblem`. - `missing_key`: a required key (document key or store key) is absent. @@ -38,6 +38,13 @@ - `invalid_value`: a value has an acceptable type but an invalid content (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). - `invalid_json`: bytes that do not decode as JSON. +- `unknown_key`: a key an object's type does not declare, where the type + says it is closed, as a closed TypedDict does. Whether a Zarr + configuration is closed is rarely said (zarr-developers/zarr-specs#270 + has been open since 2023), and many readers refuse such a key. It gets + a kind of its own so that a caller who tolerates it can tell it from a + wrong value, and so that it never masks the other findings about the + same object. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py new file mode 100644 index 0000000000..95fb8668e9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -0,0 +1,885 @@ +"""JSON values parsed by type annotation. + +A dataclass's fields are its schema, and this module reads that schema +in both directions. `parser_for` turns a field annotation into a parser: +a function of a JSON value and its location that returns the typed +value and every problem found with it. `writer_for` turns the same +annotation into the parser's inverse: a function of the typed value that +returns the JSON a document writes for it. The annotations they read are the shapes JSON +takes and no others -- `int`, `float` for any number, `bool`, `str`, +`JSONValue`, a `Literal` of names, `tuple[T, ...]` and `tuple[T1, T2]`, +a union of those, an object described by a TypedDict or a record +dataclass, `Mapping[str, V]`, a `NewType` as the type it names -- which +is what keeps it small. `UNSET` in a union says the member may be +absent; `is_optional` reads that, and a parser only ever sees a present +value. + +Nothing here knows what a metadata entity is. A caller with a shape of +its own -- a field that holds another entity, read through a scope -- +passes a `leaf`, which is asked first for every annotation at every +depth; the parser it returns is used as it is. A parser is compiled +once per annotation and takes the reading it runs in as an argument, +so what varies between reads never has to be compiled into it. +""" + +from __future__ import annotations + +import functools +import sys +import types +from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy +from dataclasses import is_dataclass +from typing import ( + TYPE_CHECKING, + Annotated, + ClassVar, + Literal, + NewType, + NotRequired, + Required, + TypeAlias, + TypeVar, + Union, + cast, + get_args, + get_origin, + get_type_hints, +) + +from typing_extensions import ReadOnly, TypeIs, is_typeddict + +from zarr_metadata._common import JSONValue +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ValidationProblem, is_json + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ProblemKind + + +S = TypeVar("S") +"""The reading a parser runs in: whatever the caller hands through, untouched here.""" + +Loc: TypeAlias = tuple[str | int, ...] +"""Where in a document a value sits: the keys and indices down to it.""" + +Parsed: TypeAlias = tuple[object, tuple[ValidationProblem, ...]] +"""What a parser returns: the typed value, and every problem found with it.""" + +Parser: TypeAlias = Callable[[object, Loc, S], Parsed] +"""One value against one annotation, located at `loc`, in a reading `S`. + +A parser is compiled once from the annotation and run many times; what +varies between runs -- for a caller whose leaf reads a nested entity, +the scope to read it in -- is the reading, an argument every parser +hands down to the parsers it is built from and reads nothing of itself. +""" + +Leaf: TypeAlias = Callable[[object], "Parser[S] | None"] +"""A caller's own shapes: asked first for every annotation, None to decline.""" + +Writer: TypeAlias = Callable[[object], JSONValue] +"""A typed value as the JSON a document writes for it: a parser's inverse over the same annotation.""" + +WriterLeaf: TypeAlias = Callable[[object], "Writer | None"] +"""A caller's own shapes, for writing: asked first for every annotation, None to decline.""" + +RecordWriter: TypeAlias = Callable[[object], dict[str, JSONValue]] +"""A record dataclass as the JSON object a document writes for it.""" + + +def problem( + loc: Loc, message: str, kind: ProblemKind = "invalid_type" +) -> tuple[ValidationProblem, ...]: + """One problem, as the one-element tuple every parser returns. + + A tuple so that a parser can return it directly and a rule can + `found.extend(problem(...))` and raise `MetadataValidationError(found)` + once. The default `kind` names a type mismatch; a value rule passes + `"invalid_value"`. + """ + return (ValidationProblem(loc, message, kind),) + + +def is_integer(value: object) -> TypeIs[int]: + """A JSON integer: an `int`, and not a `bool`. + + `True` is an `int` in Python and `true` is not a number in JSON, so + the two have to be told apart everywhere a number is expected. + """ + return not isinstance(value, bool) and isinstance(value, int) + + +# --- annotations --------------------------------------------------------- + + +def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: + """An annotation's type, and the metadata `Annotated` wrapped it in. + + `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict + key, not part of the value's type; peeled with the `Annotated` layers, + in whatever order they were written. + """ + metadata: list[object] = [] + while True: + origin = get_origin(annotation) + if origin is Annotated: + inner, *extras = get_args(annotation) + metadata.extend(extras) + annotation = inner + elif origin in (NotRequired, Required, ReadOnly): + (annotation,) = get_args(annotation) + else: + return annotation, tuple(metadata) + + +def is_not_required(annotation: object) -> bool: + """Whether a TypedDict key is marked `NotRequired`, under whatever qualifiers.""" + while True: + origin = get_origin(annotation) + if origin is NotRequired: + return True + if origin is Annotated: + annotation = get_args(annotation)[0] + elif origin in (Required, ReadOnly): + (annotation,) = get_args(annotation) + else: + return False + + +def is_union(annotation: object) -> bool: + return get_origin(annotation) in (Union, types.UnionType) + + +def is_optional(annotation: object) -> bool: + """Whether a field may be absent: its type admits `UNSET`.""" + inner, _ = strip_annotation(annotation) + return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) + + +def without_unset(inner: object) -> object: + """The type of a present value: the union less `UNSET`.""" + if not is_union(inner): + return inner + arguments = get_args(inner) + present = [argument for argument in arguments if argument is not UNSET] + if len(present) == len(arguments): + return inner + if len(present) == 1: + return present[0] + return Union[tuple(present)] # noqa: UP007 - built from a tuple, which `|` cannot be + + +def own_annotations(klass: type) -> dict[str, object]: + """A class's own annotations, unevaluated. + + From 3.14 a class does not carry an `__annotations__` dict until it is + asked for one, and asking evaluates every annotation at once -- so a + `ClassVar` naming something imported only for the type checker would + fail the whole class. `annotationlib` can hand them back as the text + they were written as, which is what the callers here want anyway: + class variables are skipped by text before anything is evaluated. + Earlier versions leave the dict on the class, strings or values as + the module chose. + """ + if sys.version_info >= (3, 14): + import annotationlib + + return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) + return dict(vars(klass).get("__annotations__", {})) + + +def is_class_var(annotation: object) -> bool: + """Whether an annotation says `ClassVar`. + + `from __future__ import annotations` leaves them as strings, so this + reads the text when it gets one -- the same thing `dataclasses` does, + and for the same reason: resolving the name needs a module namespace + that is not available while the class is still being built. + """ + if isinstance(annotation, str): + head = annotation.strip().split("[", 1)[0].strip() + return head.rsplit(".", 1)[-1] == "ClassVar" + return annotation is ClassVar or get_origin(annotation) is ClassVar + + +@functools.cache +def field_hints(cls: type) -> Mapping[str, object]: + """The dataclass fields of `cls`, resolved, base first. + + Each class's own annotations are resolved in that class's module, + and class variables are skipped *before* resolving, by text -- so a + `ClassVar` whose annotation names something imported only for the + type checker cannot fail registration. `@dataclass` sees the same + set, in the same order. + + Cached per class: a class's annotations are fixed once it exists, + and resolving them costs a third of a read. A name that does not + resolve raises, and a raise is not cached. Read-only, since every + caller shares the one mapping. + """ + hints: dict[str, object] = {} + for ancestor in reversed(cls.__mro__): + raw = { + name: annotation + for name, annotation in own_annotations(ancestor).items() + if not is_class_var(annotation) + } + if len(raw) == 0: + continue + shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) + hints.update(get_type_hints(shell, include_extras=True)) + return types.MappingProxyType(hints) + + +def declared_class_vars(cls: type) -> dict[str, type]: + """Every class variable annotated anywhere in `cls`'s ancestry. + + Mapped to the class that annotated it, so a message can say where the + requirement comes from. Base first, so a redeclaration names the + nearest ancestor. + """ + found: dict[str, type] = {} + for ancestor in reversed(cls.__mro__): + for name, annotation in own_annotations(ancestor).items(): + if is_class_var(annotation): + found[name] = ancestor + return found + + +# --- what a message calls a shape, and which shape a value has ------------- + + +def describe(annotation: object) -> str: + """The annotation as a message would name it: "an integer", "an object".""" + inner = without_unset(strip_annotation(annotation)[0]) + if inner is int: + return "an integer" + if inner is float: + return "a number" + if inner is bool: + return "a boolean" + if inner is str: + return "a string" + if inner is JSONValue: + return "a JSON value" + origin = get_origin(inner) + if origin is Literal: + return f"one of {tuple(sorted(get_args(inner), key=repr))!r}" + if is_union(inner): + return " or ".join(describe(branch) for branch in get_args(inner)) + if origin is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + return f"an array of {describe(arguments[0])} elements" + if len(arguments) == 2: + return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" + return f"an array of {len(arguments)} elements" + if origin in (Mapping, dict): + return "an object" + if isinstance(inner, NewType): + return describe(inner.__supertype__) + if is_typeddict(inner) or is_dataclass(inner): + return "an object" + return "a value" + + +def shape_of(annotation: object) -> str | None: + """The top-level JSON shape an annotation admits, for choosing a union branch. + + None means any shape -- a JSON value, a union that mixes them, or a + shape the caller's `leaf` reads. + """ + inner = without_unset(strip_annotation(annotation)[0]) + if inner is int: + return "int" + if inner is float: + return "number" + if inner is bool: + return "bool" + if inner is str: + return "str" + origin = get_origin(inner) + if origin is Literal: + # `True` is a bool before it is an int, as `is_integer` says. + values: tuple[object, ...] = get_args(inner) + shapes = {shape_of(type(value)) for value in values} + return shapes.pop() if len(shapes) == 1 else None + if origin is tuple: + return "tuple" + if origin in (Mapping, dict): + return "mapping" + if isinstance(inner, NewType): + return shape_of(inner.__supertype__) + if is_typeddict(inner) or is_dataclass(inner): + return "mapping" + return None + + +def has_shape(shape: str | None, value: object) -> bool: + if shape is None: + return True + if shape == "int": + return is_integer(value) + if shape == "number": + return not isinstance(value, bool) and isinstance(value, (int, float)) + if shape == "bool": + return isinstance(value, bool) + if shape == "str": + return isinstance(value, str) + if shape == "tuple": + return isinstance(value, tuple) + return isinstance(value, Mapping) # "mapping" + + +# --- the parsers --------------------------------------------------------- + + +def _scalar(description: str, admits: Callable[[object], bool]) -> Parser[object]: + def parse(value: object, loc: Loc, state: object) -> Parsed: + if admits(value): + return value, () + return value, problem(loc, f"expected {description}, got {value!r}") + + return parse + + +_INTEGER: Parser[object] = _scalar("an integer", is_integer) +_NUMBER: Parser[object] = _scalar( + "a number", lambda value: not isinstance(value, bool) and isinstance(value, (int, float)) +) +_BOOLEAN: Parser[object] = _scalar("a boolean", lambda value: isinstance(value, bool)) +_STRING: Parser[object] = _scalar("a string", lambda value: isinstance(value, str)) +_JSON: Parser[object] = _scalar("a JSON value", is_json) + + +def one_of(allowed: tuple[object, ...]) -> Parser[object]: + """A member whose type is a closed set of values. + + Equal and of the same type: JSON `true` is not the integer 1, though + Python says `True == 1`. + """ + + def parse(value: object, loc: Loc, state: object) -> Parsed: + if not any(value == entry and type(value) is type(entry) for entry in allowed): + return value, problem( + loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value" + ) + return value, () + + return parse + + +def sequence_of(element: Parser[S]) -> Parser[S]: + """A member whose type is an array of one element type, parsed element by element.""" + + def parse(value: object, loc: Loc, state: S) -> Parsed: + if not isinstance(value, (list, tuple)): + return value, problem(loc, f"expected a sequence, got {value!r}") + entries = cast("list[object] | tuple[object, ...]", value) + parsed: list[object] = [] + found: list[ValidationProblem] = [] + for index, entry in enumerate(entries): + item, problems = element(entry, (*loc, index), state) + parsed.append(item) + found.extend(problems) + return tuple(parsed), tuple(found) + + return parse + + +def fixed_tuple(elements: Sequence[Parser[S]], description: str) -> Parser[S]: + """A member whose type is an array of a fixed length, parsed position by position.""" + + def parse(value: object, loc: Loc, state: S) -> Parsed: + if not isinstance(value, (list, tuple)): + return value, problem(loc, f"expected {description}, got {value!r}") + entries = tuple(cast("list[object] | tuple[object, ...]", value)) + if len(entries) != len(elements): + return entries, problem(loc, f"expected {description}, got {entries!r}") + parsed: list[object] = [] + found: list[ValidationProblem] = [] + for position, (element, entry) in enumerate(zip(elements, entries, strict=True)): + item, problems = element(entry, (*loc, position), state) + parsed.append(item) + found.extend(problems) + return tuple(parsed), tuple(found) + + return parse + + +def any_of(branches: Sequence[tuple[object, Parser[S]]], description: str) -> Parser[S]: + """A member whose type is a union of shapes, parsed by the branch it fits. + + The branch whose top-level shape the value has is the one that + reports -- so an element inside a malformed array is located inside + the array, rather than the whole array being called wrong. A value + fitting no branch's shape is reported once, by what was expected; + one fitting several is parsed by the first that accepts it, and + reported by the first that does not. + """ + + def parse(value: object, loc: Loc, state: S) -> Parsed: + first: Parsed | None = None + for annotation, branch in branches: + if not has_shape(shape_of(annotation), value): + continue + result = branch(value, loc, state) + if len(result[1]) == 0: + return result + if first is None: + first = result + if first is None: + return value, problem(loc, f"expected {description}, got {value!r}") + return first + + return parse + + +Members: TypeAlias = Mapping[str, tuple[bool, "Parser[S]"]] +"""An object's declared keys: whether each is required, and its parser.""" + + +def _keys( + members: Members[S], entries: Mapping[str, object], loc: Loc, state: S +) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """The declared keys of one object, each parsed at its own key. + + Closed, like every configuration in this package: a key the type does + not declare is `unknown_key`, a required one missing is `missing_key`, + both located at the key. An optional key left out is parsed as + `UNSET`, so a record never depends on a default for it. + """ + parsed: dict[str, object] = {} + found: list[ValidationProblem] = [] + for key in entries: + if key not in members: + found.extend(problem((*loc, key), f"unexpected key {key!r}", "unknown_key")) + for key, (required, member) in members.items(): + if key not in entries: + if required: + found.extend(problem((*loc, key), f"missing required key {key!r}", "missing_key")) + else: + parsed[key] = UNSET + continue + item, problems = member(entries[key], (*loc, key), state) + parsed[key] = item + found.extend(problems) + return parsed, tuple(found) + + +def object_of(members: Members[S]) -> Parser[S]: + """A member that is itself an object with declared keys, kept as the mapping it came as. + + A key the type does not declare is reported and kept: the member + still says what the document said. + """ + + def parse(value: object, loc: Loc, state: S) -> Parsed: + if not isinstance(value, Mapping): + return value, problem(loc, f"expected an object, got {value!r}") + entries = cast("Mapping[str, object]", value) + parsed, found = _keys(members, entries, loc, state) + return {**entries, **{key: item for key, item in parsed.items() if key in entries}}, found + + return parse + + +def record_of(record: Callable[..., object], members: Members[S]) -> Parser[S]: + """A member that is itself an object with declared keys, built as a dataclass. + + Built only from an object whose every key read; otherwise the value + comes back as it came, with the reasons. A record has no rules of + its own, and every member it is built from has just been checked, + so building it cannot fail. + """ + + def parse(value: object, loc: Loc, state: S) -> Parsed: + if not isinstance(value, Mapping): + return value, problem(loc, f"expected an object, got {value!r}") + entries = cast("Mapping[str, object]", value) + parsed, found = _keys(members, entries, loc, state) + if any(entry.kind != "unknown_key" for entry in found): + return entries, found + return record(**parsed), found + + return parse + + +def mapping_of(value: Parser[S]) -> Parser[S]: + """A member whose type is an object with any keys, parsed value by value. + + The open counterpart of `object_of`: a `Mapping[str, V]` says nothing + about which keys there are, only what each value must be. + """ + + def parse(candidate: object, loc: Loc, state: S) -> Parsed: + if not isinstance(candidate, Mapping): + return candidate, problem(loc, f"expected an object, got {candidate!r}") + entries = cast("Mapping[str, object]", candidate) + parsed: dict[str, object] = {} + found: list[ValidationProblem] = [] + for key, entry in entries.items(): + item, problems = value(entry, (*loc, key), state) + parsed[key] = item + found.extend(problems) + return parsed, tuple(found) + + return parse + + +# --- the compiler -------------------------------------------------------- + + +def members_of(annotations: Mapping[str, object], leaf: Leaf[S]) -> Members[S] | None: + """A member table for an object's keys; None if any key's type has no parser. + + For a caller with a record class of its own to hand to `record_of`; + `parser_for` builds the same table for a record it meets as an + annotation. + """ + members: dict[str, tuple[bool, Parser[S]]] = {} + for key, annotation in annotations.items(): + member = parser_for(annotation, leaf) + if member is None: + return None + required = not is_not_required(annotation) and not is_optional(annotation) + members[key] = (required, member) + return members + + +def _union(inner: object, leaf: Leaf[S]) -> Parser[S] | None: + compiled = [(branch, parser_for(branch, leaf)) for branch in get_args(inner)] + branches = [(branch, member) for branch, member in compiled if member is not None] + if len(branches) != len(compiled): + return None + return any_of(branches, describe(inner)) + + +def _tuple(inner: object, leaf: Leaf[S]) -> Parser[S] | None: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = parser_for(arguments[0], leaf) + return None if element is None else sequence_of(element) + compiled = [parser_for(argument, leaf) for argument in arguments] + elements = [element for element in compiled if element is not None] + if len(elements) != len(compiled): + return None + return fixed_tuple(elements, describe(inner)) + + +def _mapping(inner: object, leaf: Leaf[S]) -> Parser[S] | None: + arguments = get_args(inner) + if len(arguments) != 2 or arguments[0] is not str: + return None + value = parser_for(arguments[1], leaf) + return None if value is None else mapping_of(value) + + +def no_leaf(annotation: object) -> Parser[object] | None: + """The leaf of a caller with no shapes of its own.""" + return None + + +def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: + """The parser a field annotation implies, or None if it implies none. + + A small compiler over the shapes JSON takes and no others, listed in + the module docstring. `leaf` is asked first, here and at every depth + -- inside a union, an array, an object -- and what it returns is + used as it is. Closed: an annotation outside these implies no + parser, and an entity declaring one is refused at registration. The + field is written as one of these shapes instead, with any finer + rule in the configuration's `problems`. + """ + inner = without_unset(strip_annotation(annotation)[0]) + found = leaf(inner) + if found is not None: + return found + if inner is int: + return _INTEGER + if inner is float: + return _NUMBER + if inner is bool: + return _BOOLEAN + if inner is str: + return _STRING + if inner is JSONValue: + return _JSON + if get_origin(inner) is Literal: + # Sorted, because the order `get_args` reports is not the order + # the `Literal` was written in: two `Literal`s over the same + # values compare and hash equal, so the first one built anywhere + # in the process is the one every later one resolves to. The + # parse is a membership test either way; this is so the message + # listing the values does not depend on import order. + return one_of(tuple(sorted(get_args(inner), key=repr))) + if is_union(inner): + return _union(inner, leaf) + if get_origin(inner) is tuple: + return _tuple(inner, leaf) + if is_typeddict(inner): + members = members_of(get_type_hints(inner, include_extras=True), leaf) + return None if members is None else object_of(members) + if get_origin(inner) in (Mapping, dict): + return _mapping(inner, leaf) + if isinstance(inner, NewType): + # A `NewType` is its supertype to a document; the distinction is + # the code's, for a value it has vouched for. + return parser_for(inner.__supertype__, leaf) + # Last, because `is_dataclass` narrows what pyright knows of `inner` + # for every line after it. + if isinstance(inner, type) and is_dataclass(inner): + if "__post_init__" in vars(inner): + # A record is plain data, built whenever its keys read; a + # rule about it belongs in the configuration's `problems`, + # which is asked for every problem rather than the first. + msg = ( + f"{inner.__name__} defines __post_init__; a record is plain data, and a rule " + "about it belongs in the configuration's `problems`" + ) + raise TypeError(msg) + members = members_of(field_hints(inner), leaf) + return None if members is None else record_of(inner, members) + return None + + +def parser(annotation: object, leaf: Leaf[S]) -> Parser[S]: + """The parser a field annotation implies; `TypeError` if it implies none.""" + found = parser_for(annotation, leaf) + if found is None: + msg = f"{annotation!r} is not a shape JSON takes" + raise TypeError(msg) + return found + + +# --- the writers --------------------------------------------------------- +# +# The inverse of each parser, over the same annotation. A writer is asked +# for a value the entity holds, which its field's type vouches for; the +# checks here are what stand between a hand-built entity holding +# something that is not JSON and a document that is not JSON. + + +def _not_json(value: object) -> TypeError: + return TypeError( + f"{value!r} is not a JSON value; an entity's members are the JSON the document writes" + ) + + +def _as_json(value: object) -> JSONValue: + """A scalar or a JSON value as it is.""" + if is_json(value): + return value + raise _not_json(value) + + +def _copied(value: object) -> JSONValue: + """A JSON value, copied: the document handed out is not a handle on a frozen entity.""" + if is_json(value): + return deepcopy(value) + raise _not_json(value) + + +def each_of(element: Writer) -> Writer: + """An array, each element written by its type.""" + + def write(value: object) -> JSONValue: + if not isinstance(value, (list, tuple)): + raise _not_json(value) + return tuple(element(entry) for entry in cast("list[object] | tuple[object, ...]", value)) + + return write + + +def positions_of(elements: Sequence[Writer]) -> Writer: + """An array of a fixed length, each position written by its type.""" + + def write(value: object) -> JSONValue: + if not isinstance(value, (list, tuple)): + raise _not_json(value) + entries = cast("list[object] | tuple[object, ...]", value) + if len(entries) != len(elements): + raise _not_json(entries) + return tuple(element(entry) for element, entry in zip(elements, entries, strict=True)) + + return write + + +def one_of_writers(branches: Sequence[tuple[object, Writer]]) -> Writer: + """A union, written by the branch whose shape the value has, as the parser chose it.""" + + def write(value: object) -> JSONValue: + for annotation, branch in branches: + if has_shape(shape_of(annotation), value): + return branch(value) + return _as_json(value) + + return write + + +def keys_of(members: Mapping[str, Writer]) -> Writer: + """An object kept as a mapping: declared keys written by their types, the rest copied.""" + + def write(value: object) -> JSONValue: + if not isinstance(value, Mapping): + raise _not_json(value) + entries = cast("Mapping[str, object]", value) + return { + key: members[key](entry) if key in members else _copied(entry) + for key, entry in entries.items() + } + + return write + + +def fields_of(members: Mapping[str, Writer]) -> RecordWriter: + """A record dataclass as an object: each field written by its type, an absent optional one left out.""" + + def write(value: object) -> dict[str, JSONValue]: + written: dict[str, JSONValue] = {} + for key, member in members.items(): + entry = getattr(value, key) + if entry is not UNSET: + written[key] = member(entry) + return written + + return write + + +def values_of(value: Writer) -> Writer: + """An object of any keys, each value written by its type.""" + + def write(candidate: object) -> JSONValue: + if not isinstance(candidate, Mapping): + raise _not_json(candidate) + entries = cast("Mapping[str, object]", candidate) + return {key: value(entry) for key, entry in entries.items()} + + return write + + +def _writers_of(annotations: Mapping[str, object], leaf: WriterLeaf) -> dict[str, Writer] | None: + members: dict[str, Writer] = {} + for key, annotation in annotations.items(): + member = writer_for(annotation, leaf) + if member is None: + return None + members[key] = member + return members + + +def no_writer_leaf(annotation: object) -> Writer | None: + """The leaf of a caller with no shapes of its own.""" + return None + + +def writer_for(annotation: object, leaf: WriterLeaf) -> Writer | None: + """The writer a field annotation implies, or None if it implies none. + + The inverse of `parser_for` over the same shapes: what the parser + reads from a document, the writer puts back. `leaf` is asked first, + here and at every depth, as the parser's is. + """ + inner = without_unset(strip_annotation(annotation)[0]) + found = leaf(inner) + if found is not None: + return found + if inner is int or inner is float or inner is bool or inner is str: + return _as_json + if get_origin(inner) is Literal: + return _as_json + if inner is JSONValue: + return _copied + if is_union(inner): + compiled = [(branch, writer_for(branch, leaf)) for branch in get_args(inner)] + branches = [(branch, member) for branch, member in compiled if member is not None] + return one_of_writers(branches) if len(branches) == len(compiled) else None + if get_origin(inner) is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = writer_for(arguments[0], leaf) + return None if element is None else each_of(element) + compiled = [writer_for(argument, leaf) for argument in arguments] + elements = [element for element in compiled if element is not None] + return positions_of(elements) if len(elements) == len(compiled) else None + if is_typeddict(inner): + members = _writers_of(get_type_hints(inner, include_extras=True), leaf) + return None if members is None else keys_of(members) + if get_origin(inner) in (Mapping, dict): + arguments = get_args(inner) + if len(arguments) != 2 or arguments[0] is not str: + return None + value = writer_for(arguments[1], leaf) + return None if value is None else values_of(value) + if isinstance(inner, NewType): + return writer_for(inner.__supertype__, leaf) + if isinstance(inner, type) and is_dataclass(inner): + members = _writers_of(field_hints(inner), leaf) + return None if members is None else fields_of(members) + return None + + +def writer(annotation: object, leaf: WriterLeaf) -> Writer: + """The writer a field annotation implies; `TypeError` if it implies none.""" + found = writer_for(annotation, leaf) + if found is None: + msg = f"{annotation!r} is not a shape JSON takes" + raise TypeError(msg) + return found + + +def record_writer(record: type, leaf: WriterLeaf) -> RecordWriter: + """The writer of a record dataclass, as the object it writes; `TypeError` for a field no writer reads.""" + members = _writers_of(field_hints(record), leaf) + if members is None: + msg = f"{record.__name__} has a field that is not a shape JSON takes" + raise TypeError(msg) + return fields_of(members) + + +__all__ = [ + "Leaf", + "Loc", + "Members", + "Parsed", + "Parser", + "RecordWriter", + "Writer", + "WriterLeaf", + "any_of", + "declared_class_vars", + "describe", + "each_of", + "field_hints", + "fields_of", + "fixed_tuple", + "has_shape", + "is_class_var", + "is_integer", + "is_not_required", + "is_optional", + "is_union", + "keys_of", + "mapping_of", + "members_of", + "no_leaf", + "no_writer_leaf", + "object_of", + "one_of", + "one_of_writers", + "own_annotations", + "parser", + "parser_for", + "positions_of", + "problem", + "record_of", + "record_writer", + "sequence_of", + "shape_of", + "strip_annotation", + "values_of", + "without_unset", + "writer", + "writer_for", +] diff --git a/packages/zarr-metadata/tests/v3/test_typed_json.py b/packages/zarr-metadata/tests/v3/test_typed_json.py new file mode 100644 index 0000000000..0a8484d066 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_typed_json.py @@ -0,0 +1,298 @@ +"""The checker: parsers and writers compiled from annotations for the shapes JSON takes. + +Tested on its own, with no entity in sight. A leaf stands in for +whatever a caller adds -- the entity layer adds the shape of a field +holding another entity -- and `no_leaf` adds nothing. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import ClassVar, Literal, NewType, NotRequired, cast + +import pytest +from typing_extensions import ReadOnly, TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata.model import UNSET +from zarr_metadata.v3._typed_json import ( + Loc, + Parsed, + Parser, + declared_class_vars, + describe, + field_hints, + is_class_var, + is_not_required, + is_optional, + no_leaf, + no_writer_leaf, + parser, + parser_for, + record_writer, + shape_of, + writer, + writer_for, +) + +Level = NewType("Level", int) + + +class Options(TypedDict, closed=True): + level: int + note: NotRequired[str] + tag: ReadOnly[NotRequired[str]] + + +@dataclass(frozen=True) +class Record: + level: int + note: str | UNSET = UNSET + + +@dataclass(frozen=True) +class Checked: + level: int + + def __post_init__(self) -> None: + return None + + +@dataclass(frozen=True) +class Declared: + level: int + identifier: ClassVar[str] = "declared" + owed: ClassVar[int] + + +def _read(annotation: object, value: object) -> Parsed: + return parser(annotation, no_leaf)(value, (), None) + + +@pytest.mark.parametrize( + ("annotation", "value", "parsed"), + [ + (int, 3, 3), + (float, 3, 3), + (float, 2.5, 2.5), + (bool, True, True), + (str, "x", "x"), + (Literal["a", "b"], "a", "a"), + (Literal[1, 2], 2, 2), + (tuple[int, ...], [1, 2], (1, 2)), + (tuple[int, str], (1, "a"), (1, "a")), + (int | str, "a", "a"), + (Mapping[str, int], {"k": 1}, {"k": 1}), + (JSONValue, {"any": [1, None]}, {"any": [1, None]}), + (Level, 5, 5), + (int | UNSET, 4, 4), + ], + ids=[ + "int", + "number-from-int", + "number", + "bool", + "str", + "literal-str", + "literal-int", + "sequence", + "fixed-tuple", + "union", + "mapping", + "json", + "newtype", + "optional-present", + ], +) +def test_a_value_of_the_shape_reads_as_itself( + annotation: object, value: object, parsed: object +) -> None: + # The shapes JSON takes and no others, each compiled once from its + # annotation; an array comes back as a tuple, the rest as it came. + assert _read(annotation, value) == (parsed, ()) + + +@pytest.mark.parametrize( + ("annotation", "value", "loc", "kind"), + [ + (int, True, (), "invalid_type"), + (int, 2.0, (), "invalid_type"), + (float, "2", (), "invalid_type"), + (str, 1, (), "invalid_type"), + (Literal["a"], "b", (), "invalid_value"), + (Literal[1], True, (), "invalid_value"), + (tuple[int, ...], (1, "x"), (1,), "invalid_type"), + (tuple[int, ...], 5, (), "invalid_type"), + (int | str, 2.5, (), "invalid_type"), + (Mapping[str, int], {"k": "v"}, ("k",), "invalid_type"), + (JSONValue, {"k": object()}, (), "invalid_type"), + ], + ids=[ + "bool-is-not-int", + "float-is-not-int", + "str-is-not-number", + "int-is-not-str", + "literal-other", + "literal-bool-is-not-int", + "element", + "not-a-sequence", + "no-branch", + "mapping-value", + "not-json", + ], +) +def test_error_a_value_of_another_shape_is_located( + annotation: object, value: object, loc: Loc, kind: str +) -> None: + parsed, problems = _read(annotation, value) + assert parsed == value + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] + + +def test_a_typed_dict_keeps_what_the_document_said() -> None: + # A closed TypedDict member reports a key it does not declare and + # keeps it: the member still says what the document said. An + # optional key left out stays out. + read = parser(Options, no_leaf) + assert read({"level": 1}, (), None) == ({"level": 1}, ()) + value, problems = read({"level": 1, "extra": 2}, (), None) + assert value == {"level": 1, "extra": 2} + assert [(problem.loc, problem.kind) for problem in problems] == [(("extra",), "unknown_key")] + + +def test_error_a_typed_dict_misses_a_required_key() -> None: + value, problems = parser(Options, no_leaf)({"note": "n"}, (), None) + assert value == {"note": "n"} + assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "missing_key")] + + +def test_a_record_is_built_when_every_member_reads() -> None: + # An optional member left out is `UNSET` in the record, so no + # default of the record's own decides what a document said; an + # unknown key is reported and survives. + read = parser(Record, no_leaf) + assert read({"level": 1}, (), None) == (Record(level=1, note=UNSET), ()) + record, problems = read({"level": 1, "typo": 0}, (), None) + assert record == Record(level=1) + assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "unknown_key")] + + +def test_error_a_record_is_not_built_around_a_member_that_did_not_read() -> None: + value, problems = parser(Record, no_leaf)({"level": "high"}, (), None) + assert value == {"level": "high"} + assert [(problem.loc, problem.kind, problem.message) for problem in problems] == [ + (("level",), "invalid_type", "expected an integer, got 'high'") + ] + + +def test_error_a_record_may_not_define_post_init() -> None: + # A record is plain data, built whenever its keys read; a rule about + # it belongs where the caller asks for rules. + with pytest.raises(TypeError, match="Checked defines __post_init__; a record is plain data"): + parser(Checked, no_leaf) + + +def test_error_an_annotation_outside_the_shapes_has_no_parser() -> None: + assert parser_for(set[int], no_leaf) is None + with pytest.raises(TypeError, match="is not a shape JSON takes"): + parser(set[int], no_leaf) + + +def test_a_leaf_is_asked_first_at_every_depth() -> None: + # The one hook: a caller's own shapes, consulted before the built-in + # ones, inside an array or an object as at the top. + class Marker: + pass + + def leaf(annotation: object) -> Parser[None] | None: + if annotation is not Marker: + return None + + def parse(value: object, loc: Loc, state: None) -> Parsed: + if isinstance(value, str) and value.startswith("m:"): + return value[2:], () + return value, ((loc, "expected a marker"),) # pyright: ignore[reportReturnType] + + return parse + + read = parser(tuple[Marker, ...], leaf) + assert read(["m:a", "m:b"], (), None) == (("a", "b"), ()) + + +def test_writers_are_the_parsers_inverse() -> None: + # What a parser reads from a document, a writer puts back: a record + # as the object of its present members, an absent optional member + # left out, a JSON-valued member copied rather than shared. + assert record_writer(Record, no_writer_leaf)(Record(level=1)) == {"level": 1} + assert record_writer(Record, no_writer_leaf)(Record(level=1, note="n")) == { + "level": 1, + "note": "n", + } + assert writer(tuple[int, str], no_writer_leaf)((1, "a")) == (1, "a") + held = {"k": [1, 2]} + written = writer(JSONValue, no_writer_leaf)(held) + assert written == held + assert written is not held + + +def test_error_a_fixed_tuple_of_the_wrong_length_is_not_written() -> None: + write = writer_for(tuple[int, str], no_writer_leaf) + assert write is not None + with pytest.raises(TypeError, match="is not a JSON value"): + write((1,)) + + +def test_field_hints_resolve_per_class_and_skip_class_variables() -> None: + # Under PEP 649 the annotations are strings, resolved where the class + # is; a class variable is not a field, and one a base annotates and + # nothing sets is owed. + assert dict(field_hints(Declared)) == {"level": int} + assert declared_class_vars(Declared) == {"identifier": Declared, "owed": Declared} + hints = field_hints(Declared) + assert field_hints(Declared) is hints + with pytest.raises(TypeError, match="does not support item assignment"): + cast("dict[str, object]", hints)["level"] = str + + +@pytest.mark.parametrize( + ("annotation", "expected"), + [ + ("ClassVar[str]", True), + ("ClassVar", True), + ("typing.ClassVar[str]", True), + ("t.ClassVar[str]", True), + ("str", False), + ("ClassVarLike[str]", False), + ("Final[ClassVar[str]]", False), + ], +) +def test_a_class_var_is_read_from_any_spelling(annotation: str, expected: bool) -> None: + assert is_class_var(annotation) is expected + + +@pytest.mark.parametrize( + ("annotation", "shape"), + [ + (Literal[True], "bool"), + (Literal[1, 2], "int"), + (Literal["a", "b"], "str"), + (Literal[0, "auto"], None), + (tuple[int, ...], "tuple"), + (Options, "mapping"), + (JSONValue, None), + ], +) +def test_a_shape_is_what_a_union_dispatches_on(annotation: object, shape: str | None) -> None: + assert shape_of(annotation) == shape + + +def test_a_mixed_literal_is_described() -> None: + assert describe(Literal[0, "auto"]) == "one of ('auto', 0)" + + +def test_optional_and_not_required_are_read_off_the_annotation() -> None: + assert is_optional(int | UNSET) + assert not is_optional(int) + assert is_not_required(NotRequired[int]) + assert not is_not_required(int) From 0f1488a8c8ea8108c40c5bc93ea782877fe09b7e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 13:17:38 +0200 Subject: [PATCH 2/9] docs(zarr-metadata): the fragment for the checker Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/347.misc.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 packages/zarr-metadata/changes/347.misc.md diff --git a/packages/zarr-metadata/changes/347.misc.md b/packages/zarr-metadata/changes/347.misc.md new file mode 100644 index 0000000000..0c83eb4147 --- /dev/null +++ b/packages/zarr-metadata/changes/347.misc.md @@ -0,0 +1,12 @@ +`v3/_typed_json` is the type checker the entity layer will read +metadata with, on its own: for each shape JSON takes -- `int`, `float`, +`bool`, `str`, `JSONValue`, a `Literal`, `tuple[T, ...]` and +`tuple[T1, T2]`, a union, a TypedDict, a record dataclass, +`Mapping[str, V]`, a `NewType`, and `| UNSET` for a member a document may +leave out -- a parser compiled once from the annotation, and the writer +that is its inverse. A parser is `(value, loc, state) -> (typed value, +problems)`, generic in a state it hands through, and a caller's leaf is +asked first at every depth for shapes of its own. `field_hints` resolves +a dataclass's annotations where each class was defined and skips class +variables; `declared_class_vars` reports those. The module knows nothing +of entities. From 572ae7938e4137144b85c487d6cc58522c12cb4e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 09:21:24 +0200 Subject: [PATCH 3/9] fix(zarr-metadata): the checker reads a TypedDict as the typing spec defines it The definition layer makes a configuration's TypedDict the one declaration of its JSON, so the checker has to read a TypedDict the way the typing spec does, and it did not: - Whether a key is required came from `NotRequired` wrappers alone, so `total=False` was ignored, and callers that asked `__required_keys__` instead got a different answer: the runtime works that out before the annotations are evaluated, and a qualifier written as a string, as `from __future__ import annotations` writes every one, is invisible to it. `typeddict_keys` now reads `Required` and `NotRequired` off the evaluated annotations and falls back on the declaring class's `total` for a key with neither. - Every TypedDict was treated as closed. `closed`, `extra_items` and openness are now read as the spec says, inherited from the bases when a class says nothing, which the runtime does not record; bases that disagree are a `TypeError`. - The typed value kept a closed TypedDict's unknown keys, so a value typed as the TypedDict held keys the type says cannot exist. It is now a new dict of the keys the type admits, in the order they came. - `None` is JSON null, a type alias reads as what it stands for, and a TypedDict or alias that holds itself compiles instead of recursing without end. An annotation that does not resolve is a `TypeError` naming its TypedDict at any depth, not a `NameError` from inside. Parsers are pure, `(value, loc) -> (typed value, problems)`. The state argument was for the entity layer's scoped reads; the definition layer finds nested fields in the typed value instead, so the branch of a union that did not match leaves nothing behind. The record dataclasses, the writers, `| UNSET` members, `field_hints` and `declared_class_vars` were the entity layer's too, and have no caller in this stack. `test_typed_json_properties` holds the checker to a reference predicate worked out from a drawn spec -- TypedDicts with bases, per-class `total`, qualifiers in any order, every kind of openness, classes that hold themselves -- built with evaluated and with postponed annotations, over JSON values of capped depth that conform, nearly conform, or are drawn at large. Each of the bugs above, put back, fails it. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/347.misc.md | 25 +- .../src/zarr_metadata/v3/_typed_json.py | 903 ++++++++---------- .../zarr-metadata/tests/v3/test_typed_json.py | 505 +++++++--- .../tests/v3/test_typed_json_properties.py | 611 ++++++++++++ 4 files changed, 1384 insertions(+), 660 deletions(-) create mode 100644 packages/zarr-metadata/tests/v3/test_typed_json_properties.py diff --git a/packages/zarr-metadata/changes/347.misc.md b/packages/zarr-metadata/changes/347.misc.md index 0c83eb4147..46dd1b3999 100644 --- a/packages/zarr-metadata/changes/347.misc.md +++ b/packages/zarr-metadata/changes/347.misc.md @@ -1,12 +1,15 @@ -`v3/_typed_json` is the type checker the entity layer will read +`v3/_typed_json` is the type checker the definition layer reads metadata with, on its own: for each shape JSON takes -- `int`, `float`, -`bool`, `str`, `JSONValue`, a `Literal`, `tuple[T, ...]` and -`tuple[T1, T2]`, a union, a TypedDict, a record dataclass, -`Mapping[str, V]`, a `NewType`, and `| UNSET` for a member a document may -leave out -- a parser compiled once from the annotation, and the writer -that is its inverse. A parser is `(value, loc, state) -> (typed value, -problems)`, generic in a state it hands through, and a caller's leaf is -asked first at every depth for shapes of its own. `field_hints` resolves -a dataclass's annotations where each class was defined and skips class -variables; `declared_class_vars` reports those. The module knows nothing -of entities. +`bool`, `str`, `None`, `JSONValue`, a `Literal`, `tuple[T, ...]` and +`tuple[T1, T2]`, a union, a TypedDict, `Mapping[str, V]`, a `NewType` +and a type alias -- a parser compiled once from the annotation. A +parser is a pure function, `(value, loc) -> (typed value, problems)`, +and a caller's leaf is asked first at every depth for shapes of its +own. A TypedDict is read as the typing spec defines it: `typeddict_keys` +takes each key's `Required` or `NotRequired` from the evaluated +annotations, so a module written with `from __future__ import +annotations` reads like any other, falls back on the declaring class's +`total`, and takes `closed` and `extra_items` from the class or, when it +says neither, from its bases. What comes back is a value of the type: a +key a closed TypedDict does not declare is reported and left out. The +module knows nothing of metadata fields. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index 95fb8668e9..f39f215a96 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -1,91 +1,91 @@ -"""JSON values parsed by type annotation. - -A dataclass's fields are its schema, and this module reads that schema -in both directions. `parser_for` turns a field annotation into a parser: -a function of a JSON value and its location that returns the typed -value and every problem found with it. `writer_for` turns the same -annotation into the parser's inverse: a function of the typed value that -returns the JSON a document writes for it. The annotations they read are the shapes JSON -takes and no others -- `int`, `float` for any number, `bool`, `str`, -`JSONValue`, a `Literal` of names, `tuple[T, ...]` and `tuple[T1, T2]`, -a union of those, an object described by a TypedDict or a record -dataclass, `Mapping[str, V]`, a `NewType` as the type it names -- which -is what keeps it small. `UNSET` in a union says the member may be -absent; `is_optional` reads that, and a parser only ever sees a present -value. - -Nothing here knows what a metadata entity is. A caller with a shape of -its own -- a field that holds another entity, read through a scope -- -passes a `leaf`, which is asked first for every annotation at every -depth; the parser it returns is used as it is. A parser is compiled -once per annotation and takes the reading it runs in as an argument, -so what varies between reads never has to be compiled into it. +"""JSON values checked against type annotations. + +`parser_for` turns an annotation into a parser: a function of a JSON +value and its location that returns the typed value and every problem +found with it. The annotations it reads are the shapes JSON takes and no +others -- `int`, `float` for any number, `bool`, `str`, `None` for null, +`JSONValue`, a `Literal` of JSON scalars, `tuple[T, ...]` and +`tuple[T1, T2]`, a union of those, an object described by a TypedDict, +`Mapping[str, V]`, a `NewType` as the type it names, and a type alias as +the type it stands for -- which is what keeps it small. An annotation +outside these implies no parser. + +A TypedDict is read as the typing spec defines it, which is not always +what its runtime attributes say: `typeddict_keys` reads which keys it +declares, what each holds and whether it is required, and what every +other key may hold. What a parser returns is a value of its annotation: +an array comes back as a tuple, and an object as a new dict of the keys +its type admits, so a key a closed TypedDict does not declare is +reported, as `unknown_key`, and left out. + +Nothing here knows what a metadata field is. A caller with a shape of its +own passes a `leaf`, which is asked first for every annotation at every +depth; the parser it returns is used as it is. Parsers are compiled once +per annotation and are pure functions of the value, so the branch of a +union that did not match leaves nothing behind. """ from __future__ import annotations import functools -import sys import types +import typing from collections.abc import Callable, Mapping, Sequence -from copy import deepcopy -from dataclasses import is_dataclass +from dataclasses import dataclass from typing import ( TYPE_CHECKING, Annotated, - ClassVar, + Final, + ForwardRef, Literal, + Never, NewType, - NotRequired, - Required, + NoReturn, TypeAlias, - TypeVar, - Union, cast, get_args, get_origin, get_type_hints, ) -from typing_extensions import ReadOnly, TypeIs, is_typeddict +import typing_extensions +from typing_extensions import NoExtraItems, TypeIs, is_typeddict from zarr_metadata._common import JSONValue -from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem, is_json if TYPE_CHECKING: from zarr_metadata.model._validation import ProblemKind -S = TypeVar("S") -"""The reading a parser runs in: whatever the caller hands through, untouched here.""" - Loc: TypeAlias = tuple[str | int, ...] """Where in a document a value sits: the keys and indices down to it.""" Parsed: TypeAlias = tuple[object, tuple[ValidationProblem, ...]] """What a parser returns: the typed value, and every problem found with it.""" -Parser: TypeAlias = Callable[[object, Loc, S], Parsed] -"""One value against one annotation, located at `loc`, in a reading `S`. - -A parser is compiled once from the annotation and run many times; what -varies between runs -- for a caller whose leaf reads a nested entity, -the scope to read it in -- is the reading, an argument every parser -hands down to the parsers it is built from and reads nothing of itself. -""" +Parser: TypeAlias = Callable[[object, Loc], Parsed] +"""One value against one annotation, located at `loc`: compiled once, run many times.""" -Leaf: TypeAlias = Callable[[object], "Parser[S] | None"] +Leaf: TypeAlias = Callable[[object], "Parser | None"] """A caller's own shapes: asked first for every annotation, None to decline.""" -Writer: TypeAlias = Callable[[object], JSONValue] -"""A typed value as the JSON a document writes for it: a parser's inverse over the same annotation.""" - -WriterLeaf: TypeAlias = Callable[[object], "Writer | None"] -"""A caller's own shapes, for writing: asked first for every annotation, None to decline.""" +# The qualifiers a TypedDict key may carry. `typing` and `typing_extensions` +# may each define one, and a TypedDict may be written with either. +_REQUIRED: Final[tuple[object, ...]] = (typing.Required, typing_extensions.Required) +_NOT_REQUIRED: Final[tuple[object, ...]] = (typing.NotRequired, typing_extensions.NotRequired) +_READ_ONLY: Final[tuple[object, ...]] = ( + typing_extensions.ReadOnly, + getattr(typing, "ReadOnly", typing_extensions.ReadOnly), +) +_QUALIFIERS: Final[tuple[object, ...]] = (*_REQUIRED, *_NOT_REQUIRED, *_READ_ONLY) -RecordWriter: TypeAlias = Callable[[object], dict[str, JSONValue]] -"""A record dataclass as the JSON object a document writes for it.""" +# A `type` statement makes a `typing.TypeAliasType`, which is not the +# `typing_extensions` one on every version that has both. +_ALIASES: Final[tuple[type, ...]] = ( + typing_extensions.TypeAliasType, + getattr(typing, "TypeAliasType", typing_extensions.TypeAliasType), +) def problem( @@ -116,7 +116,7 @@ def is_integer(value: object) -> TypeIs[int]: def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: """An annotation's type, and the metadata `Annotated` wrapped it in. - `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict + `Required`, `NotRequired` and `ReadOnly` are qualifiers on a TypedDict key, not part of the value's type; peeled with the `Annotated` layers, in whatever order they were written. """ @@ -127,132 +127,173 @@ def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: inner, *extras = get_args(annotation) metadata.extend(extras) annotation = inner - elif origin in (NotRequired, Required, ReadOnly): + elif origin in _QUALIFIERS: (annotation,) = get_args(annotation) else: return annotation, tuple(metadata) -def is_not_required(annotation: object) -> bool: - """Whether a TypedDict key is marked `NotRequired`, under whatever qualifiers.""" +Qualifier: TypeAlias = Literal["Required", "NotRequired", "ReadOnly"] +"""A qualifier on a TypedDict key, by name: `typing` and `typing_extensions` may each spell one.""" + + +def qualifiers(annotation: object) -> frozenset[Qualifier]: + """The qualifiers on a TypedDict key, under whatever `Annotated` layers, in whatever order.""" + found: set[Qualifier] = set() while True: origin = get_origin(annotation) - if origin is NotRequired: - return True if origin is Annotated: annotation = get_args(annotation)[0] - elif origin in (Required, ReadOnly): - (annotation,) = get_args(annotation) + continue + if origin in _REQUIRED: + found.add("Required") + elif origin in _NOT_REQUIRED: + found.add("NotRequired") + elif origin in _READ_ONLY: + found.add("ReadOnly") else: - return False + return frozenset(found) + (annotation,) = get_args(annotation) def is_union(annotation: object) -> bool: - return get_origin(annotation) in (Union, types.UnionType) + return get_origin(annotation) in (typing.Union, types.UnionType) -def is_optional(annotation: object) -> bool: - """Whether a field may be absent: its type admits `UNSET`.""" - inner, _ = strip_annotation(annotation) - return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) +def is_alias(annotation: object) -> bool: + """Whether `annotation` is a type alias that takes no type parameters: `type Level = int`.""" + if not isinstance(annotation, _ALIASES): + return False + return len(cast("typing_extensions.TypeAliasType", annotation).__type_params__) == 0 -def without_unset(inner: object) -> object: - """The type of a present value: the union less `UNSET`.""" - if not is_union(inner): - return inner - arguments = get_args(inner) - present = [argument for argument in arguments if argument is not UNSET] - if len(present) == len(arguments): - return inner - if len(present) == 1: - return present[0] - return Union[tuple(present)] # noqa: UP007 - built from a tuple, which `|` cannot be - - -def own_annotations(klass: type) -> dict[str, object]: - """A class's own annotations, unevaluated. - - From 3.14 a class does not carry an `__annotations__` dict until it is - asked for one, and asking evaluates every annotation at once -- so a - `ClassVar` naming something imported only for the type checker would - fail the whole class. `annotationlib` can hand them back as the text - they were written as, which is what the callers here want anyway: - class variables are skipped by text before anything is evaluated. - Earlier versions leave the dict on the class, strings or values as - the module chose. - """ - if sys.version_info >= (3, 14): - import annotationlib +# --- TypedDicts ---------------------------------------------------------- - return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) - return dict(vars(klass).get("__annotations__", {})) +@dataclass(frozen=True, slots=True) +class TypedDictKeys: + """What a TypedDict says of an object's keys, read as the typing spec defines it. -def is_class_var(annotation: object) -> bool: - """Whether an annotation says `ClassVar`. - - `from __future__ import annotations` leaves them as strings, so this - reads the text when it gets one -- the same thing `dataclasses` does, - and for the same reason: resolving the name needs a module namespace - that is not available while the class is still being built. + `members` holds every key it declares, its bases' included, with the + type of the key's value -- qualifiers and `Annotated` metadata peeled + -- and whether the key is required. `extra_items` is what any other + key may hold: `Never` when the TypedDict is closed, `object` when it + is open, and the `extra_items` type otherwise. `declared` is whether + that was said, by the TypedDict or a base, rather than defaulted: a + TypedDict that says nothing is open. """ - if isinstance(annotation, str): - head = annotation.strip().split("[", 1)[0].strip() - return head.rsplit(".", 1)[-1] == "ClassVar" - return annotation is ClassVar or get_origin(annotation) is ClassVar + members: Mapping[str, tuple[object, bool]] + extra_items: object + declared: bool -@functools.cache -def field_hints(cls: type) -> Mapping[str, object]: - """The dataclass fields of `cls`, resolved, base first. - - Each class's own annotations are resolved in that class's module, - and class variables are skipped *before* resolving, by text -- so a - `ClassVar` whose annotation names something imported only for the - type checker cannot fail registration. `@dataclass` sees the same - set, in the same order. - - Cached per class: a class's annotations are fixed once it exists, - and resolving them costs a third of a read. A name that does not - resolve raises, and a raise is not cached. Read-only, since every - caller shares the one mapping. - """ - hints: dict[str, object] = {} - for ancestor in reversed(cls.__mro__): - raw = { - name: annotation - for name, annotation in own_annotations(ancestor).items() - if not is_class_var(annotation) - } - if len(raw) == 0: - continue - shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) - hints.update(get_type_hints(shell, include_extras=True)) - return types.MappingProxyType(hints) + @property + def required(self) -> frozenset[str]: + """The keys an object of this TypedDict must have.""" + return frozenset(key for key, (_, required) in self.members.items() if required) + @property + def closed(self) -> bool: + """Whether a key it does not declare is not a key of the type: `closed=True`, or `extra_items=Never`.""" + return self.extra_items is Never or self.extra_items is NoReturn -def declared_class_vars(cls: type) -> dict[str, type]: - """Every class variable annotated anywhere in `cls`'s ancestry. + @property + def open(self) -> bool: + """Whether a key it does not declare may hold anything: the default, or `closed=False`.""" + return self.extra_items is object - Mapped to the class that annotated it, so a message can say where the - requirement comes from. Base first, so a redeclaration names the - nearest ancestor. + +@functools.cache +def typeddict_keys(typeddict: type) -> TypedDictKeys: + """What `typeddict` says of an object's keys; `TypeError` if its annotations do not resolve. + + Whether a key is required follows the spec: a `Required` or + `NotRequired` qualifier says, and otherwise `total` of the class that + declared the key does. The qualifiers are read off the annotations as + evaluated, each where its class was defined, because the runtime's + `__required_keys__` is worked out before they are: an annotation + written as a string -- as `from __future__ import annotations` writes + every one -- hides its qualifier. For a key with no qualifier, + `__required_keys__` is right, and carries the `total` that applied. + + Openness follows the spec too: `closed=True` closes the TypedDict, + `extra_items=` types every other key, `closed=False` opens it, and one + that says none of these is as its bases are -- which the runtime does + not record -- or open, when no base says either. """ - found: dict[str, type] = {} - for ancestor in reversed(cls.__mro__): - for name, annotation in own_annotations(ancestor).items(): - if is_class_var(annotation): - found[name] = ancestor - return found + try: + hints = get_type_hints(typeddict, include_extras=True) + except NameError as error: + msg = f"{typeddict.__name__}: {error}; its annotations must resolve in its module" + raise TypeError(msg) from error + required_at_runtime = cast("frozenset[str]", getattr(typeddict, "__required_keys__", ())) + members: dict[str, tuple[object, bool]] = {} + for key, hint in hints.items(): + said = qualifiers(hint) + if "Required" in said: + required = True + elif "NotRequired" in said: + required = False + else: + required = key in required_at_runtime + members[key] = (strip_annotation(hint)[0], required) + extra_items, declared = _openness(typeddict) + return TypedDictKeys(types.MappingProxyType(members), extra_items, declared) + + +def _openness(typeddict: type) -> tuple[object, bool]: + """What a key `typeddict` does not declare may hold, and whether that was said rather than defaulted.""" + extra_items = getattr(typeddict, "__extra_items__", NoExtraItems) + if extra_items is not NoExtraItems: + return _extra_items_type(typeddict, extra_items), True + closed = getattr(typeddict, "__closed__", None) + if closed is not None: + return (Never if closed else object), True + inherited = [_openness(base) for base in _typeddict_bases(typeddict)] + restricted = {extra for extra, _ in inherited if extra is not object} + if len(restricted) > 1: + msg = ( + f"{typeddict.__name__}: its bases disagree on what a key they do not declare may " + "hold; say it on the class, with closed= or extra_items=" + ) + raise TypeError(msg) + if len(restricted) == 1: + return restricted.pop(), True + return object, any(declared for _, declared in inherited) + + +def _extra_items_type(typeddict: type, extra_items: object) -> object: + """The `extra_items` type, evaluated where `typeddict` was defined, `ReadOnly` peeled.""" + if isinstance(extra_items, (str, ForwardRef)): + holder = type( + "_ExtraItems", + (), + {"__annotations__": {"extra": extra_items}, "__module__": typeddict.__module__}, + ) + try: + extra_items = get_type_hints(holder, include_extras=True)["extra"] + except NameError as error: + msg = f"{typeddict.__name__}: {error}; its extra_items must resolve in its module" + raise TypeError(msg) from error + return strip_annotation(extra_items)[0] + + +def _typeddict_bases(typeddict: type) -> tuple[type, ...]: + """The TypedDicts `typeddict` was declared with as bases, type arguments dropped.""" + bases: list[type] = [] + for base in getattr(typeddict, "__orig_bases__", ()): + origin = get_origin(base) or base + if isinstance(origin, type) and is_typeddict(origin): + bases.append(origin) + return tuple(bases) # --- what a message calls a shape, and which shape a value has ------------- -def describe(annotation: object) -> str: +def describe(annotation: object, seen: frozenset[object] = frozenset()) -> str: """The annotation as a message would name it: "an integer", "an object".""" - inner = without_unset(strip_annotation(annotation)[0]) + inner = strip_annotation(annotation)[0] if inner is int: return "an integer" if inner is float: @@ -261,26 +302,33 @@ def describe(annotation: object) -> str: return "a boolean" if inner is str: return "a string" + if inner is None or inner is types.NoneType: + return "null" if inner is JSONValue: return "a JSON value" origin = get_origin(inner) if origin is Literal: return f"one of {tuple(sorted(get_args(inner), key=repr))!r}" if is_union(inner): - return " or ".join(describe(branch) for branch in get_args(inner)) + return " or ".join(describe(branch, seen) for branch in get_args(inner)) if origin is tuple: arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: - return f"an array of {describe(arguments[0])} elements" + return f"an array of {describe(arguments[0], seen)} elements" if len(arguments) == 2: - return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" + return f"a [{describe(arguments[0], seen)}, {describe(arguments[1], seen)}] pair" return f"an array of {len(arguments)} elements" if origin in (Mapping, dict): return "an object" if isinstance(inner, NewType): - return describe(inner.__supertype__) - if is_typeddict(inner) or is_dataclass(inner): + return describe(inner.__supertype__, seen) + if isinstance(inner, type) and is_typeddict(inner): return "an object" + if is_alias(inner): + alias = cast("typing_extensions.TypeAliasType", inner) + if alias in seen: + return f"a {alias.__name__}" + return describe(alias.__value__, seen | {alias}) return "a value" @@ -290,7 +338,14 @@ def shape_of(annotation: object) -> str | None: None means any shape -- a JSON value, a union that mixes them, or a shape the caller's `leaf` reads. """ - inner = without_unset(strip_annotation(annotation)[0]) + inner = strip_annotation(annotation)[0] + seen: set[object] = set() + while (isinstance(inner, NewType) or is_alias(inner)) and inner not in seen: + seen.add(inner) + if isinstance(inner, NewType): + inner = strip_annotation(inner.__supertype__)[0] + else: + inner = strip_annotation(cast("typing_extensions.TypeAliasType", inner).__value__)[0] if inner is int: return "int" if inner is float: @@ -299,6 +354,8 @@ def shape_of(annotation: object) -> str | None: return "bool" if inner is str: return "str" + if inner is None or inner is types.NoneType: + return "null" origin = get_origin(inner) if origin is Literal: # `True` is a bool before it is an int, as `is_integer` says. @@ -309,9 +366,7 @@ def shape_of(annotation: object) -> str | None: return "tuple" if origin in (Mapping, dict): return "mapping" - if isinstance(inner, NewType): - return shape_of(inner.__supertype__) - if is_typeddict(inner) or is_dataclass(inner): + if isinstance(inner, type) and is_typeddict(inner): return "mapping" return None @@ -327,16 +382,18 @@ def has_shape(shape: str | None, value: object) -> bool: return isinstance(value, bool) if shape == "str": return isinstance(value, str) + if shape == "null": + return value is None if shape == "tuple": - return isinstance(value, tuple) + return isinstance(value, (list, tuple)) return isinstance(value, Mapping) # "mapping" # --- the parsers --------------------------------------------------------- -def _scalar(description: str, admits: Callable[[object], bool]) -> Parser[object]: - def parse(value: object, loc: Loc, state: object) -> Parsed: +def _scalar(description: str, admits: Callable[[object], bool]) -> Parser: + def parse(value: object, loc: Loc) -> Parsed: if admits(value): return value, () return value, problem(loc, f"expected {description}, got {value!r}") @@ -344,23 +401,24 @@ def parse(value: object, loc: Loc, state: object) -> Parsed: return parse -_INTEGER: Parser[object] = _scalar("an integer", is_integer) -_NUMBER: Parser[object] = _scalar( +_INTEGER: Final = _scalar("an integer", is_integer) +_NUMBER: Final = _scalar( "a number", lambda value: not isinstance(value, bool) and isinstance(value, (int, float)) ) -_BOOLEAN: Parser[object] = _scalar("a boolean", lambda value: isinstance(value, bool)) -_STRING: Parser[object] = _scalar("a string", lambda value: isinstance(value, str)) -_JSON: Parser[object] = _scalar("a JSON value", is_json) +_BOOLEAN: Final = _scalar("a boolean", lambda value: isinstance(value, bool)) +_STRING: Final = _scalar("a string", lambda value: isinstance(value, str)) +_NULL: Final = _scalar("null", lambda value: value is None) +_JSON: Final = _scalar("a JSON value", is_json) -def one_of(allowed: tuple[object, ...]) -> Parser[object]: +def one_of(allowed: tuple[object, ...]) -> Parser: """A member whose type is a closed set of values. Equal and of the same type: JSON `true` is not the integer 1, though Python says `True == 1`. """ - def parse(value: object, loc: Loc, state: object) -> Parsed: + def parse(value: object, loc: Loc) -> Parsed: if not any(value == entry and type(value) is type(entry) for entry in allowed): return value, problem( loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value" @@ -370,17 +428,17 @@ def parse(value: object, loc: Loc, state: object) -> Parsed: return parse -def sequence_of(element: Parser[S]) -> Parser[S]: +def sequence_of(element: Parser) -> Parser: """A member whose type is an array of one element type, parsed element by element.""" - def parse(value: object, loc: Loc, state: S) -> Parsed: + def parse(value: object, loc: Loc) -> Parsed: if not isinstance(value, (list, tuple)): return value, problem(loc, f"expected a sequence, got {value!r}") entries = cast("list[object] | tuple[object, ...]", value) parsed: list[object] = [] found: list[ValidationProblem] = [] for index, entry in enumerate(entries): - item, problems = element(entry, (*loc, index), state) + item, problems = element(entry, (*loc, index)) parsed.append(item) found.extend(problems) return tuple(parsed), tuple(found) @@ -388,10 +446,10 @@ def parse(value: object, loc: Loc, state: S) -> Parsed: return parse -def fixed_tuple(elements: Sequence[Parser[S]], description: str) -> Parser[S]: +def fixed_tuple(elements: Sequence[Parser], description: str) -> Parser: """A member whose type is an array of a fixed length, parsed position by position.""" - def parse(value: object, loc: Loc, state: S) -> Parsed: + def parse(value: object, loc: Loc) -> Parsed: if not isinstance(value, (list, tuple)): return value, problem(loc, f"expected {description}, got {value!r}") entries = tuple(cast("list[object] | tuple[object, ...]", value)) @@ -400,7 +458,7 @@ def parse(value: object, loc: Loc, state: S) -> Parsed: parsed: list[object] = [] found: list[ValidationProblem] = [] for position, (element, entry) in enumerate(zip(elements, entries, strict=True)): - item, problems = element(entry, (*loc, position), state) + item, problems = element(entry, (*loc, position)) parsed.append(item) found.extend(problems) return tuple(parsed), tuple(found) @@ -408,23 +466,24 @@ def parse(value: object, loc: Loc, state: S) -> Parsed: return parse -def any_of(branches: Sequence[tuple[object, Parser[S]]], description: str) -> Parser[S]: +def any_of(branches: Sequence[tuple[str | None, Parser]], description: str) -> Parser: """A member whose type is a union of shapes, parsed by the branch it fits. - The branch whose top-level shape the value has is the one that - reports -- so an element inside a malformed array is located inside - the array, rather than the whole array being called wrong. A value - fitting no branch's shape is reported once, by what was expected; - one fitting several is parsed by the first that accepts it, and - reported by the first that does not. + Each branch is its top-level shape, from `shape_of`, and its parser. + The branch whose shape the value has is the one that reports -- so an + element inside a malformed array is located inside the array, rather + than the whole array being called wrong. A value fitting no branch's + shape is reported once, by what was expected; one fitting several is + parsed by the first that accepts it with no problem, and otherwise + reported by the first that tried. """ - def parse(value: object, loc: Loc, state: S) -> Parsed: + def parse(value: object, loc: Loc) -> Parsed: first: Parsed | None = None - for annotation, branch in branches: - if not has_shape(shape_of(annotation), value): + for shape, branch in branches: + if not has_shape(shape, value): continue - result = branch(value, loc, state) + result = branch(value, loc) if len(result[1]) == 0: return result if first is None: @@ -436,91 +495,56 @@ def parse(value: object, loc: Loc, state: S) -> Parsed: return parse -Members: TypeAlias = Mapping[str, tuple[bool, "Parser[S]"]] -"""An object's declared keys: whether each is required, and its parser.""" - - -def _keys( - members: Members[S], entries: Mapping[str, object], loc: Loc, state: S -) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: - """The declared keys of one object, each parsed at its own key. - - Closed, like every configuration in this package: a key the type does - not declare is `unknown_key`, a required one missing is `missing_key`, - both located at the key. An optional key left out is parsed as - `UNSET`, so a record never depends on a default for it. - """ - parsed: dict[str, object] = {} - found: list[ValidationProblem] = [] - for key in entries: - if key not in members: - found.extend(problem((*loc, key), f"unexpected key {key!r}", "unknown_key")) - for key, (required, member) in members.items(): - if key not in entries: - if required: - found.extend(problem((*loc, key), f"missing required key {key!r}", "missing_key")) - else: - parsed[key] = UNSET - continue - item, problems = member(entries[key], (*loc, key), state) - parsed[key] = item - found.extend(problems) - return parsed, tuple(found) - - -def object_of(members: Members[S]) -> Parser[S]: - """A member that is itself an object with declared keys, kept as the mapping it came as. - - A key the type does not declare is reported and kept: the member - still says what the document said. - """ - - def parse(value: object, loc: Loc, state: S) -> Parsed: - if not isinstance(value, Mapping): - return value, problem(loc, f"expected an object, got {value!r}") - entries = cast("Mapping[str, object]", value) - parsed, found = _keys(members, entries, loc, state) - return {**entries, **{key: item for key, item in parsed.items() if key in entries}}, found - - return parse - - -def record_of(record: Callable[..., object], members: Members[S]) -> Parser[S]: - """A member that is itself an object with declared keys, built as a dataclass. +def object_of(members: Mapping[str, tuple[Parser, bool]], extra: Parser | None) -> Parser: + """An object with declared keys, each parsed at its own key, and `extra` for every other key. - Built only from an object whose every key read; otherwise the value - comes back as it came, with the reasons. A record has no rules of - its own, and every member it is built from has just been checked, - so building it cannot fail. + `members` maps each declared key to its parser and whether it is + required; a required key missing is `missing_key`. `extra` parses a + key the object does not declare; with no `extra` -- a closed + TypedDict -- such a key is `unknown_key`, and left out of what comes + back. What comes back is a new dict, keys in the order they came. """ - def parse(value: object, loc: Loc, state: S) -> Parsed: + def parse(value: object, loc: Loc) -> Parsed: if not isinstance(value, Mapping): return value, problem(loc, f"expected an object, got {value!r}") entries = cast("Mapping[str, object]", value) - parsed, found = _keys(members, entries, loc, state) - if any(entry.kind != "unknown_key" for entry in found): - return entries, found - return record(**parsed), found + parsed: dict[str, object] = {} + found: list[ValidationProblem] = [] + for key, entry in entries.items(): + if key in members: + continue + if extra is None: + found.extend(problem((*loc, key), f"unexpected key {key!r}", "unknown_key")) + continue + parsed[key], problems = extra(entry, (*loc, key)) + found.extend(problems) + for key, (member, required) in members.items(): + if key in entries: + parsed[key], problems = member(entries[key], (*loc, key)) + found.extend(problems) + elif required: + found.extend(problem((*loc, key), f"missing required key {key!r}", "missing_key")) + return {key: parsed[key] for key in entries if key in parsed}, tuple(found) return parse -def mapping_of(value: Parser[S]) -> Parser[S]: +def mapping_of(value: Parser) -> Parser: """A member whose type is an object with any keys, parsed value by value. The open counterpart of `object_of`: a `Mapping[str, V]` says nothing about which keys there are, only what each value must be. """ - def parse(candidate: object, loc: Loc, state: S) -> Parsed: + def parse(candidate: object, loc: Loc) -> Parsed: if not isinstance(candidate, Mapping): return candidate, problem(loc, f"expected an object, got {candidate!r}") entries = cast("Mapping[str, object]", candidate) parsed: dict[str, object] = {} found: list[ValidationProblem] = [] for key, entry in entries.items(): - item, problems = value(entry, (*loc, key), state) + item, problems = value(entry, (*loc, key)) parsed[key] = item found.extend(problems) return parsed, tuple(found) @@ -530,69 +554,103 @@ def parse(candidate: object, loc: Loc, state: S) -> Parsed: # --- the compiler -------------------------------------------------------- +_Building: TypeAlias = dict[object, Parser] +"""The parsers of one compilation, by the TypedDict or alias each reads. -def members_of(annotations: Mapping[str, object], leaf: Leaf[S]) -> Members[S] | None: - """A member table for an object's keys; None if any key's type has no parser. +A TypedDict or alias being compiled is in it already, as a parser that +reads through to the one being built: a type that holds itself compiles +to a parser that calls itself. +""" - For a caller with a record class of its own to hand to `record_of`; - `parser_for` builds the same table for a record it meets as an - annotation. - """ - members: dict[str, tuple[bool, Parser[S]]] = {} - for key, annotation in annotations.items(): - member = parser_for(annotation, leaf) + +def _union(inner: object, leaf: Leaf, building: _Building) -> Parser | None: + branches: list[tuple[str | None, Parser]] = [] + for branch in get_args(inner): + member = _compile(branch, leaf, building) if member is None: return None - required = not is_not_required(annotation) and not is_optional(annotation) - members[key] = (required, member) - return members - - -def _union(inner: object, leaf: Leaf[S]) -> Parser[S] | None: - compiled = [(branch, parser_for(branch, leaf)) for branch in get_args(inner)] - branches = [(branch, member) for branch, member in compiled if member is not None] - if len(branches) != len(compiled): - return None + branches.append((shape_of(branch), member)) return any_of(branches, describe(inner)) -def _tuple(inner: object, leaf: Leaf[S]) -> Parser[S] | None: +def _tuple(inner: object, leaf: Leaf, building: _Building) -> Parser | None: arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: - element = parser_for(arguments[0], leaf) + element = _compile(arguments[0], leaf, building) return None if element is None else sequence_of(element) - compiled = [parser_for(argument, leaf) for argument in arguments] - elements = [element for element in compiled if element is not None] - if len(elements) != len(compiled): - return None + elements: list[Parser] = [] + for argument in arguments: + element = _compile(argument, leaf, building) + if element is None: + return None + elements.append(element) return fixed_tuple(elements, describe(inner)) -def _mapping(inner: object, leaf: Leaf[S]) -> Parser[S] | None: +def _mapping(inner: object, leaf: Leaf, building: _Building) -> Parser | None: arguments = get_args(inner) if len(arguments) != 2 or arguments[0] is not str: return None - value = parser_for(arguments[1], leaf) + value = _compile(arguments[1], leaf, building) return None if value is None else mapping_of(value) -def no_leaf(annotation: object) -> Parser[object] | None: - """The leaf of a caller with no shapes of its own.""" - return None - - -def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: - """The parser a field annotation implies, or None if it implies none. - - A small compiler over the shapes JSON takes and no others, listed in - the module docstring. `leaf` is asked first, here and at every depth - -- inside a union, an array, an object -- and what it returns is - used as it is. Closed: an annotation outside these implies no - parser, and an entity declaring one is refused at registration. The - field is written as one of these shapes instead, with any finer - rule in the configuration's `problems`. - """ - inner = without_unset(strip_annotation(annotation)[0]) +def _recursive( + key: object, building: _Building, compile_: Callable[[], Parser | None] +) -> Parser | None: + """The parser `compile_` builds for `key`, a TypedDict or alias that may hold itself.""" + if key in building: + return building[key] + built: list[Parser] = [] + building[key] = lambda value, loc: built[0](value, loc) + parser = compile_() + if parser is None: + return None + built.append(parser) + building[key] = parser + return parser + + +def _object(typeddict: type, leaf: Leaf, building: _Building) -> Parser | None: + def compile_() -> Parser | None: + keys = typeddict_keys(typeddict) + members: dict[str, tuple[Parser, bool]] = {} + for key, (annotation, required) in keys.members.items(): + member = _compile(annotation, leaf, building) + if member is None: + return None + members[key] = (member, required) + if keys.closed: + return object_of(members, None) + if keys.open: + return object_of(members, _JSON) + extra = _compile(keys.extra_items, leaf, building) + return None if extra is None else object_of(members, extra) + + return _recursive(typeddict, building, compile_) + + +def _alias( + alias: typing_extensions.TypeAliasType, leaf: Leaf, building: _Building +) -> Parser | None: + return _recursive(alias, building, lambda: _compile(alias.__value__, leaf, building)) + + +def _literal(inner: object) -> Parser | None: + values: tuple[object, ...] = get_args(inner) + if not all(value is None or isinstance(value, (str, int)) for value in values): + return None # an enum member, or bytes: nothing a JSON document holds + # Sorted, because the order `get_args` reports is not the order the + # `Literal` was written in: two `Literal`s over the same values + # compare and hash equal, so the first one built anywhere in the + # process is the one every later one resolves to. The parse is a + # membership test either way; this is so the message listing the + # values does not depend on import order. + return one_of(tuple(sorted(values, key=repr))) + + +def _compile(annotation: object, leaf: Leaf, building: _Building) -> Parser | None: + inner = strip_annotation(annotation)[0] found = leaf(inner) if found is not None: return found @@ -604,282 +662,81 @@ def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: return _BOOLEAN if inner is str: return _STRING + if inner is None or inner is types.NoneType: + return _NULL if inner is JSONValue: return _JSON - if get_origin(inner) is Literal: - # Sorted, because the order `get_args` reports is not the order - # the `Literal` was written in: two `Literal`s over the same - # values compare and hash equal, so the first one built anywhere - # in the process is the one every later one resolves to. The - # parse is a membership test either way; this is so the message - # listing the values does not depend on import order. - return one_of(tuple(sorted(get_args(inner), key=repr))) + origin = get_origin(inner) + if origin is Literal: + return _literal(inner) if is_union(inner): - return _union(inner, leaf) - if get_origin(inner) is tuple: - return _tuple(inner, leaf) - if is_typeddict(inner): - members = members_of(get_type_hints(inner, include_extras=True), leaf) - return None if members is None else object_of(members) - if get_origin(inner) in (Mapping, dict): - return _mapping(inner, leaf) + return _union(inner, leaf, building) + if origin is tuple: + return _tuple(inner, leaf, building) + if origin in (Mapping, dict): + return _mapping(inner, leaf, building) + if isinstance(inner, type) and is_typeddict(inner): + return _object(inner, leaf, building) if isinstance(inner, NewType): # A `NewType` is its supertype to a document; the distinction is # the code's, for a value it has vouched for. - return parser_for(inner.__supertype__, leaf) - # Last, because `is_dataclass` narrows what pyright knows of `inner` - # for every line after it. - if isinstance(inner, type) and is_dataclass(inner): - if "__post_init__" in vars(inner): - # A record is plain data, built whenever its keys read; a - # rule about it belongs in the configuration's `problems`, - # which is asked for every problem rather than the first. - msg = ( - f"{inner.__name__} defines __post_init__; a record is plain data, and a rule " - "about it belongs in the configuration's `problems`" - ) - raise TypeError(msg) - members = members_of(field_hints(inner), leaf) - return None if members is None else record_of(inner, members) + return _compile(inner.__supertype__, leaf, building) + if is_alias(inner): + return _alias(cast("typing_extensions.TypeAliasType", inner), leaf, building) return None -def parser(annotation: object, leaf: Leaf[S]) -> Parser[S]: - """The parser a field annotation implies; `TypeError` if it implies none.""" - found = parser_for(annotation, leaf) - if found is None: - msg = f"{annotation!r} is not a shape JSON takes" - raise TypeError(msg) - return found - - -# --- the writers --------------------------------------------------------- -# -# The inverse of each parser, over the same annotation. A writer is asked -# for a value the entity holds, which its field's type vouches for; the -# checks here are what stand between a hand-built entity holding -# something that is not JSON and a document that is not JSON. - - -def _not_json(value: object) -> TypeError: - return TypeError( - f"{value!r} is not a JSON value; an entity's members are the JSON the document writes" - ) - - -def _as_json(value: object) -> JSONValue: - """A scalar or a JSON value as it is.""" - if is_json(value): - return value - raise _not_json(value) - - -def _copied(value: object) -> JSONValue: - """A JSON value, copied: the document handed out is not a handle on a frozen entity.""" - if is_json(value): - return deepcopy(value) - raise _not_json(value) - - -def each_of(element: Writer) -> Writer: - """An array, each element written by its type.""" - - def write(value: object) -> JSONValue: - if not isinstance(value, (list, tuple)): - raise _not_json(value) - return tuple(element(entry) for entry in cast("list[object] | tuple[object, ...]", value)) - - return write - - -def positions_of(elements: Sequence[Writer]) -> Writer: - """An array of a fixed length, each position written by its type.""" - - def write(value: object) -> JSONValue: - if not isinstance(value, (list, tuple)): - raise _not_json(value) - entries = cast("list[object] | tuple[object, ...]", value) - if len(entries) != len(elements): - raise _not_json(entries) - return tuple(element(entry) for element, entry in zip(elements, entries, strict=True)) - - return write - - -def one_of_writers(branches: Sequence[tuple[object, Writer]]) -> Writer: - """A union, written by the branch whose shape the value has, as the parser chose it.""" - - def write(value: object) -> JSONValue: - for annotation, branch in branches: - if has_shape(shape_of(annotation), value): - return branch(value) - return _as_json(value) - - return write - - -def keys_of(members: Mapping[str, Writer]) -> Writer: - """An object kept as a mapping: declared keys written by their types, the rest copied.""" - - def write(value: object) -> JSONValue: - if not isinstance(value, Mapping): - raise _not_json(value) - entries = cast("Mapping[str, object]", value) - return { - key: members[key](entry) if key in members else _copied(entry) - for key, entry in entries.items() - } - - return write +def parser_for(annotation: object, leaf: Leaf) -> Parser | None: + """The parser an annotation implies, or None if it implies none. - -def fields_of(members: Mapping[str, Writer]) -> RecordWriter: - """A record dataclass as an object: each field written by its type, an absent optional one left out.""" - - def write(value: object) -> dict[str, JSONValue]: - written: dict[str, JSONValue] = {} - for key, member in members.items(): - entry = getattr(value, key) - if entry is not UNSET: - written[key] = member(entry) - return written - - return write - - -def values_of(value: Writer) -> Writer: - """An object of any keys, each value written by its type.""" - - def write(candidate: object) -> JSONValue: - if not isinstance(candidate, Mapping): - raise _not_json(candidate) - entries = cast("Mapping[str, object]", candidate) - return {key: value(entry) for key, entry in entries.items()} - - return write - - -def _writers_of(annotations: Mapping[str, object], leaf: WriterLeaf) -> dict[str, Writer] | None: - members: dict[str, Writer] = {} - for key, annotation in annotations.items(): - member = writer_for(annotation, leaf) - if member is None: - return None - members[key] = member - return members + A small compiler over the shapes JSON takes and no others, listed in + the module docstring. `leaf` is asked first, here and at every depth + -- inside a union, an array, an object -- and what it returns is used + as it is. Closed: an annotation outside these implies no parser. A + TypedDict whose annotations do not resolve, or whose bases disagree + on what its other keys hold, is a `TypeError` naming it. + """ + return _compile(annotation, leaf, {}) -def no_writer_leaf(annotation: object) -> Writer | None: +def no_leaf(annotation: object) -> Parser | None: """The leaf of a caller with no shapes of its own.""" return None -def writer_for(annotation: object, leaf: WriterLeaf) -> Writer | None: - """The writer a field annotation implies, or None if it implies none. - - The inverse of `parser_for` over the same shapes: what the parser - reads from a document, the writer puts back. `leaf` is asked first, - here and at every depth, as the parser's is. - """ - inner = without_unset(strip_annotation(annotation)[0]) - found = leaf(inner) - if found is not None: - return found - if inner is int or inner is float or inner is bool or inner is str: - return _as_json - if get_origin(inner) is Literal: - return _as_json - if inner is JSONValue: - return _copied - if is_union(inner): - compiled = [(branch, writer_for(branch, leaf)) for branch in get_args(inner)] - branches = [(branch, member) for branch, member in compiled if member is not None] - return one_of_writers(branches) if len(branches) == len(compiled) else None - if get_origin(inner) is tuple: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - element = writer_for(arguments[0], leaf) - return None if element is None else each_of(element) - compiled = [writer_for(argument, leaf) for argument in arguments] - elements = [element for element in compiled if element is not None] - return positions_of(elements) if len(elements) == len(compiled) else None - if is_typeddict(inner): - members = _writers_of(get_type_hints(inner, include_extras=True), leaf) - return None if members is None else keys_of(members) - if get_origin(inner) in (Mapping, dict): - arguments = get_args(inner) - if len(arguments) != 2 or arguments[0] is not str: - return None - value = writer_for(arguments[1], leaf) - return None if value is None else values_of(value) - if isinstance(inner, NewType): - return writer_for(inner.__supertype__, leaf) - if isinstance(inner, type) and is_dataclass(inner): - members = _writers_of(field_hints(inner), leaf) - return None if members is None else fields_of(members) - return None - - -def writer(annotation: object, leaf: WriterLeaf) -> Writer: - """The writer a field annotation implies; `TypeError` if it implies none.""" - found = writer_for(annotation, leaf) +def parser(annotation: object, leaf: Leaf) -> Parser: + """The parser an annotation implies; `TypeError` if it implies none.""" + found = parser_for(annotation, leaf) if found is None: msg = f"{annotation!r} is not a shape JSON takes" raise TypeError(msg) return found -def record_writer(record: type, leaf: WriterLeaf) -> RecordWriter: - """The writer of a record dataclass, as the object it writes; `TypeError` for a field no writer reads.""" - members = _writers_of(field_hints(record), leaf) - if members is None: - msg = f"{record.__name__} has a field that is not a shape JSON takes" - raise TypeError(msg) - return fields_of(members) - - __all__ = [ "Leaf", "Loc", - "Members", "Parsed", "Parser", - "RecordWriter", - "Writer", - "WriterLeaf", + "Qualifier", + "TypedDictKeys", "any_of", - "declared_class_vars", "describe", - "each_of", - "field_hints", - "fields_of", "fixed_tuple", "has_shape", - "is_class_var", + "is_alias", "is_integer", - "is_not_required", - "is_optional", "is_union", - "keys_of", "mapping_of", - "members_of", "no_leaf", - "no_writer_leaf", "object_of", "one_of", - "one_of_writers", - "own_annotations", "parser", "parser_for", - "positions_of", "problem", - "record_of", - "record_writer", + "qualifiers", "sequence_of", "shape_of", "strip_annotation", - "values_of", - "without_unset", - "writer", - "writer_for", + "typeddict_keys", ] diff --git a/packages/zarr-metadata/tests/v3/test_typed_json.py b/packages/zarr-metadata/tests/v3/test_typed_json.py index 0a8484d066..994b5e7852 100644 --- a/packages/zarr-metadata/tests/v3/test_typed_json.py +++ b/packages/zarr-metadata/tests/v3/test_typed_json.py @@ -1,42 +1,37 @@ -"""The checker: parsers and writers compiled from annotations for the shapes JSON takes. +"""The checker: parsers compiled from annotations for the shapes JSON takes. -Tested on its own, with no entity in sight. A leaf stands in for -whatever a caller adds -- the entity layer adds the shape of a field -holding another entity -- and `no_leaf` adds nothing. +Tested on its own, with no metadata field in sight. A leaf stands in for +whatever a caller adds -- the definition layer adds the shape of a member +holding another metadata field -- and `no_leaf` adds nothing. """ from __future__ import annotations +import itertools +import sys +import textwrap +import types from collections.abc import Mapping -from dataclasses import dataclass -from typing import ClassVar, Literal, NewType, NotRequired, cast +from typing import Generic, Literal, Never, NewType, NotRequired, TypeVar import pytest -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypeAliasType, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.model import UNSET from zarr_metadata.v3._typed_json import ( Loc, Parsed, Parser, - declared_class_vars, describe, - field_hints, - is_class_var, - is_not_required, - is_optional, no_leaf, - no_writer_leaf, parser, parser_for, - record_writer, shape_of, - writer, - writer_for, + typeddict_keys, ) Level = NewType("Level", int) +Width = TypeAliasType("Width", int) class Options(TypedDict, closed=True): @@ -45,29 +40,34 @@ class Options(TypedDict, closed=True): tag: ReadOnly[NotRequired[str]] -@dataclass(frozen=True) -class Record: +class WithExtras(TypedDict, extra_items=int): level: int - note: str | UNSET = UNSET -@dataclass(frozen=True) -class Checked: +class Open(TypedDict, closed=False): level: int - def __post_init__(self) -> None: - return None +class Small(TypedDict, closed=True): + x: int + + +class Large(TypedDict, closed=True): + x: int + y: int -@dataclass(frozen=True) -class Declared: - level: int - identifier: ClassVar[str] = "declared" - owed: ClassVar[int] + +class Tree(TypedDict, closed=True): + label: str + children: tuple[Tree, ...] def _read(annotation: object, value: object) -> Parsed: - return parser(annotation, no_leaf)(value, (), None) + return parser(annotation, no_leaf)(value, ()) + + +def _found(parsed: Parsed) -> list[tuple[Loc, str]]: + return [(found.loc, found.kind) for found in parsed[1]] @pytest.mark.parametrize( @@ -78,15 +78,17 @@ def _read(annotation: object, value: object) -> Parsed: (float, 2.5, 2.5), (bool, True, True), (str, "x", "x"), + (None, None, None), (Literal["a", "b"], "a", "a"), (Literal[1, 2], 2, 2), (tuple[int, ...], [1, 2], (1, 2)), (tuple[int, str], (1, "a"), (1, "a")), + (tuple[str | None, ...], ["a", None], ("a", None)), (int | str, "a", "a"), (Mapping[str, int], {"k": 1}, {"k": 1}), (JSONValue, {"any": [1, None]}, {"any": [1, None]}), (Level, 5, 5), - (int | UNSET, 4, 4), + (Width, 5, 5), ], ids=[ "int", @@ -94,15 +96,17 @@ def _read(annotation: object, value: object) -> Parsed: "number", "bool", "str", + "null", "literal-str", "literal-int", "sequence", "fixed-tuple", + "sequence-of-optional", "union", "mapping", "json", "newtype", - "optional-present", + "alias", ], ) def test_a_value_of_the_shape_reads_as_itself( @@ -120,6 +124,7 @@ def test_a_value_of_the_shape_reads_as_itself( (int, 2.0, (), "invalid_type"), (float, "2", (), "invalid_type"), (str, 1, (), "invalid_type"), + (None, 0, (), "invalid_type"), (Literal["a"], "b", (), "invalid_value"), (Literal[1], True, (), "invalid_value"), (tuple[int, ...], (1, "x"), (1,), "invalid_type"), @@ -133,6 +138,7 @@ def test_a_value_of_the_shape_reads_as_itself( "float-is-not-int", "str-is-not-number", "int-is-not-str", + "zero-is-not-null", "literal-other", "literal-bool-is-not-int", "element", @@ -150,53 +156,355 @@ def test_error_a_value_of_another_shape_is_located( assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] -def test_a_typed_dict_keeps_what_the_document_said() -> None: - # A closed TypedDict member reports a key it does not declare and - # keeps it: the member still says what the document said. An - # optional key left out stays out. - read = parser(Options, no_leaf) - assert read({"level": 1}, (), None) == ({"level": 1}, ()) - value, problems = read({"level": 1, "extra": 2}, (), None) - assert value == {"level": 1, "extra": 2} - assert [(problem.loc, problem.kind) for problem in problems] == [(("extra",), "unknown_key")] +@pytest.mark.parametrize( + ("annotation", "value", "parsed", "found"), + [ + (Options, {"level": 1}, {"level": 1}, []), + (Options, {"note": "n", "level": 1}, {"note": "n", "level": 1}, []), + (Options, {"level": 1, "extra": 2}, {"level": 1}, [(("extra",), "unknown_key")]), + (WithExtras, {"level": 1, "more": 2}, {"level": 1, "more": 2}, []), + ( + WithExtras, + {"level": 1, "more": "two"}, + {"level": 1, "more": "two"}, + [(("more",), "invalid_type")], + ), + (Open, {"level": 1, "more": [2]}, {"level": 1, "more": [2]}, []), + (Small | Large, {"x": 1, "y": 2}, {"x": 1, "y": 2}, []), + (Small | Large, {"x": 1, "z": 2}, {"x": 1}, [(("z",), "unknown_key")]), + ( + Tree, + {"label": "a", "children": [{"label": "b", "children": []}]}, + {"label": "a", "children": ({"label": "b", "children": ()},)}, + [], + ), + ( + Tree, + {"label": "a", "children": [{"label": 1, "children": []}]}, + {"label": "a", "children": ({"label": 1, "children": ()},)}, + [(("children", 0, "label"), "invalid_type")], + ), + ], + ids=[ + "closed", + "closed-keeps-the-order-keys-came-in", + "closed-leaves-an-unknown-key-out", + "extra-items", + "extra-items-of-the-wrong-type", + "open", + "union-takes-the-branch-that-accepts", + "union-falls-back-to-the-first-that-tried", + "recursive", + "recursive-located", + ], +) +def test_an_object_reads_as_what_its_typeddict_admits( + annotation: object, value: object, parsed: dict[str, object], found: list[tuple[Loc, str]] +) -> None: + # A new dict of the keys the type admits, in the order they came: a + # closed TypedDict reports a key it does not declare and leaves it + # out, `extra_items` types every other key, an open one keeps them. + result = _read(annotation, value) + assert result[0] == parsed + assert list(cast_dict(result[0])) == list(parsed) + assert _found(result) == found + + +def cast_dict(value: object) -> dict[str, object]: + assert isinstance(value, dict) + return value def test_error_a_typed_dict_misses_a_required_key() -> None: - value, problems = parser(Options, no_leaf)({"note": "n"}, (), None) + value, problems = parser(Options, no_leaf)({"note": "n"}, ()) assert value == {"note": "n"} assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "missing_key")] -def test_a_record_is_built_when_every_member_reads() -> None: - # An optional member left out is `UNSET` in the record, so no - # default of the record's own decides what a document said; an - # unknown key is reported and survives. - read = parser(Record, no_leaf) - assert read({"level": 1}, (), None) == (Record(level=1, note=UNSET), ()) - record, problems = read({"level": 1, "typo": 0}, (), None) - assert record == Record(level=1) - assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "unknown_key")] +# --- a TypedDict, read as the typing spec defines it ----------------------- + +_PRELUDE = """\ +import typing +from typing import Annotated, Generic, NotRequired, Required, TypeVar +from typing_extensions import Never, ReadOnly, TypedDict +V = TypeVar("V") +""" + +_modules = itertools.count() + + +def _declared(source: str, *, postponed: bool, monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + """The module `source` defines, written with or without `from __future__ import annotations`. + + Registered in `sys.modules`, because a string annotation is resolved + in the module its class was defined in. + """ + name = f"tests.v3._typeddicts_{next(_modules)}" + module = types.ModuleType(name) + monkeypatch.setitem(sys.modules, name, module) + header = "from __future__ import annotations\n" if postponed else "" + exec(header + _PRELUDE + textwrap.dedent(source), module.__dict__) # noqa: S102 - the declaration under test + return module + +@pytest.mark.parametrize( + ("source", "members", "required", "extra_items", "declared"), + [ + ( + """ + class T(TypedDict, closed=True): + a: int + b: NotRequired[int] + c: ReadOnly[NotRequired[str]] + d: Annotated[NotRequired[int], 'meta'] + """, + {"a", "b", "c", "d"}, + {"a"}, + Never, + True, + ), + ( + """ + class T(TypedDict, total=False, closed=True): + a: int + b: Required[int] + c: Annotated[Required[int], 'meta'] + """, + {"a", "b", "c"}, + {"b", "c"}, + Never, + True, + ), + ( + """ + class Base(TypedDict, total=False): + a: int + b: Required[int] + class T(Base, closed=True): + c: int + d: NotRequired[int] + """, + {"a", "b", "c", "d"}, + {"b", "c"}, + Never, + True, + ), + ( + """ + class Base(TypedDict): + a: ReadOnly[NotRequired[int]] + class T(Base): + a: ReadOnly[int] + """, + {"a"}, + {"a"}, + object, + False, + ), + ( + """ + class T(typing.TypedDict): + a: int + b: typing.NotRequired[int] + """, + {"a", "b"}, + {"a"}, + object, + False, + ), + ("class T(TypedDict):\n a: int\n", {"a"}, {"a"}, object, False), + ("class T(TypedDict, closed=False):\n a: int\n", {"a"}, {"a"}, object, True), + ("class T(TypedDict, extra_items=int):\n a: int\n", {"a"}, {"a"}, int, True), + ("class T(TypedDict, extra_items='int'):\n a: int\n", {"a"}, {"a"}, int, True), + ( + """ + class T(TypedDict, extra_items=ReadOnly[str]): + a: int + """, + {"a"}, + {"a"}, + str, + True, + ), + ( + """ + class Base(TypedDict, closed=True): + a: int + class T(Base): + pass + """, + {"a"}, + {"a"}, + Never, + True, + ), + ( + """ + class Base(TypedDict, extra_items=ReadOnly[int]): + a: int + class T(Base): + b: NotRequired[int] + """, + {"a", "b"}, + {"a"}, + int, + True, + ), + ( + """ + class Base(TypedDict): + a: int + class T(Base, closed=True): + pass + """, + {"a"}, + {"a"}, + Never, + True, + ), + ( + """ + class Base(TypedDict, closed=False): + a: int + class T(Base): + pass + """, + {"a"}, + {"a"}, + object, + True, + ), + ( + """ + class G(TypedDict, Generic[V], closed=True): + a: int + class T(G[int]): + pass + """, + {"a"}, + {"a"}, + Never, + True, + ), + ( + """ + class A(TypedDict, closed=True): + a: int + class B(TypedDict, closed=True): + b: NotRequired[int] + class T(A, B): + pass + """, + {"a", "b"}, + {"a"}, + Never, + True, + ), + ], + ids=[ + "qualifiers-in-any-order", + "total-false-and-required", + "totality-is-per-class", + "read-only-key-required-by-a-subclass", + "stdlib-typeddict", + "open-by-default", + "open-when-said", + "extra-items", + "extra-items-written-as-a-string", + "extra-items-read-only", + "closed-is-inherited", + "extra-items-are-inherited", + "a-subclass-closes-an-open-base", + "open-said-by-a-base", + "through-a-generic-base", + "bases-that-agree", + ], +) +@pytest.mark.parametrize("postponed", [False, True], ids=["evaluated", "postponed"]) +def test_typeddict_keys_follow_the_typing_spec( + source: str, + members: set[str], + required: set[str], + extra_items: object, + declared: bool, + postponed: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Written as strings, as `from __future__ import annotations` writes + # every annotation, a qualifier is invisible to `__required_keys__`; + # read off the evaluated annotations it is not, so both spellings of + # a declaration read alike. Openness is inherited, which the runtime + # does not record. + keys = typeddict_keys(_declared(source, postponed=postponed, monkeypatch=monkeypatch).T) + assert set(keys.members) == members + assert keys.required == required + assert keys.extra_items is extra_items + assert keys.declared is declared + + +def test_error_a_typeddict_whose_bases_disagree_on_other_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _declared( + """ + class A(TypedDict, closed=True): + a: int + class B(TypedDict, extra_items=int): + b: int + class T(A, B): + pass + """, + postponed=False, + monkeypatch=monkeypatch, + ) + with pytest.raises(TypeError, match="T: its bases disagree"): + typeddict_keys(module.T) + + +@pytest.mark.parametrize("name", ["T", "Outer"]) +def test_error_a_typeddict_whose_annotations_do_not_resolve( + name: str, monkeypatch: pytest.MonkeyPatch +) -> None: + # Named by the TypedDict that holds the annotation, at any depth: a + # parser compiled around it cannot be. + module = _declared( + """ + class T(TypedDict): + a: Missing + class Outer(TypedDict): + inner: T + """, + postponed=True, + monkeypatch=monkeypatch, + ) + with pytest.raises(TypeError, match="T: name 'Missing' is not defined"): + parser_for(getattr(module, name), no_leaf) -def test_error_a_record_is_not_built_around_a_member_that_did_not_read() -> None: - value, problems = parser(Record, no_leaf)({"level": "high"}, (), None) - assert value == {"level": "high"} - assert [(problem.loc, problem.kind, problem.message) for problem in problems] == [ - (("level",), "invalid_type", "expected an integer, got 'high'") - ] +T = TypeVar("T") -def test_error_a_record_may_not_define_post_init() -> None: - # A record is plain data, built whenever its keys read; a rule about - # it belongs where the caller asks for rules. - with pytest.raises(TypeError, match="Checked defines __post_init__; a record is plain data"): - parser(Checked, no_leaf) +class Generic_(TypedDict, Generic[T], closed=True): + x: T -def test_error_an_annotation_outside_the_shapes_has_no_parser() -> None: - assert parser_for(set[int], no_leaf) is None + +@pytest.mark.parametrize( + "annotation", + [set[int], list[int], Literal[b"x"], Generic_, Generic_[int]], + ids=["set", "list", "literal-bytes", "generic-typeddict", "generic-alias"], +) +def test_error_an_annotation_outside_the_shapes_has_no_parser(annotation: object) -> None: + assert parser_for(annotation, no_leaf) is None with pytest.raises(TypeError, match="is not a shape JSON takes"): - parser(set[int], no_leaf) + parser(annotation, no_leaf) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="the `type` statement is 3.12 syntax") +def test_a_type_statement_alias_may_hold_itself(monkeypatch: pytest.MonkeyPatch) -> None: + module = _declared( + "type Nested = int | tuple[Nested, ...]\n", postponed=False, monkeypatch=monkeypatch + ) + assert parser(module.Nested, no_leaf)([1, [2, [3]]], ()) == ((1, (2, (3,))), ()) def test_a_leaf_is_asked_first_at_every_depth() -> None: @@ -205,11 +513,11 @@ def test_a_leaf_is_asked_first_at_every_depth() -> None: class Marker: pass - def leaf(annotation: object) -> Parser[None] | None: + def leaf(annotation: object) -> Parser | None: if annotation is not Marker: return None - def parse(value: object, loc: Loc, state: None) -> Parsed: + def parse(value: object, loc: Loc) -> Parsed: if isinstance(value, str) and value.startswith("m:"): return value[2:], () return value, ((loc, "expected a marker"),) # pyright: ignore[reportReturnType] @@ -217,58 +525,7 @@ def parse(value: object, loc: Loc, state: None) -> Parsed: return parse read = parser(tuple[Marker, ...], leaf) - assert read(["m:a", "m:b"], (), None) == (("a", "b"), ()) - - -def test_writers_are_the_parsers_inverse() -> None: - # What a parser reads from a document, a writer puts back: a record - # as the object of its present members, an absent optional member - # left out, a JSON-valued member copied rather than shared. - assert record_writer(Record, no_writer_leaf)(Record(level=1)) == {"level": 1} - assert record_writer(Record, no_writer_leaf)(Record(level=1, note="n")) == { - "level": 1, - "note": "n", - } - assert writer(tuple[int, str], no_writer_leaf)((1, "a")) == (1, "a") - held = {"k": [1, 2]} - written = writer(JSONValue, no_writer_leaf)(held) - assert written == held - assert written is not held - - -def test_error_a_fixed_tuple_of_the_wrong_length_is_not_written() -> None: - write = writer_for(tuple[int, str], no_writer_leaf) - assert write is not None - with pytest.raises(TypeError, match="is not a JSON value"): - write((1,)) - - -def test_field_hints_resolve_per_class_and_skip_class_variables() -> None: - # Under PEP 649 the annotations are strings, resolved where the class - # is; a class variable is not a field, and one a base annotates and - # nothing sets is owed. - assert dict(field_hints(Declared)) == {"level": int} - assert declared_class_vars(Declared) == {"identifier": Declared, "owed": Declared} - hints = field_hints(Declared) - assert field_hints(Declared) is hints - with pytest.raises(TypeError, match="does not support item assignment"): - cast("dict[str, object]", hints)["level"] = str - - -@pytest.mark.parametrize( - ("annotation", "expected"), - [ - ("ClassVar[str]", True), - ("ClassVar", True), - ("typing.ClassVar[str]", True), - ("t.ClassVar[str]", True), - ("str", False), - ("ClassVarLike[str]", False), - ("Final[ClassVar[str]]", False), - ], -) -def test_a_class_var_is_read_from_any_spelling(annotation: str, expected: bool) -> None: - assert is_class_var(annotation) is expected + assert read(["m:a", "m:b"], ()) == (("a", "b"), ()) @pytest.mark.parametrize( @@ -278,8 +535,10 @@ def test_a_class_var_is_read_from_any_spelling(annotation: str, expected: bool) (Literal[1, 2], "int"), (Literal["a", "b"], "str"), (Literal[0, "auto"], None), + (None, "null"), (tuple[int, ...], "tuple"), (Options, "mapping"), + (Width, "int"), (JSONValue, None), ], ) @@ -289,10 +548,4 @@ def test_a_shape_is_what_a_union_dispatches_on(annotation: object, shape: str | def test_a_mixed_literal_is_described() -> None: assert describe(Literal[0, "auto"]) == "one of ('auto', 0)" - - -def test_optional_and_not_required_are_read_off_the_annotation() -> None: - assert is_optional(int | UNSET) - assert not is_optional(int) - assert is_not_required(NotRequired[int]) - assert not is_not_required(int) + assert describe(int | None) == "an integer or null" diff --git a/packages/zarr-metadata/tests/v3/test_typed_json_properties.py b/packages/zarr-metadata/tests/v3/test_typed_json_properties.py new file mode 100644 index 0000000000..ec7584d0bb --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_typed_json_properties.py @@ -0,0 +1,611 @@ +"""The checker against a reference, over TypedDicts and JSON values drawn at random. + +A spec is drawn first: a small description of a type in the checker's +vocabulary -- scalars, `Literal`s, arrays, unions, `Mapping[str, V]`, +`NewType`s and aliases, and TypedDicts with up to two bases, each class +with its own `total`, its keys' qualifiers written in any order, and +what it says of other keys, some holding themselves. It is written out +as source text and built from that twice: with each class's annotations +evaluated, and left as the strings `from __future__ import annotations` +leaves. Beside it stands `conforms`, a predicate over JSON worked out +from the spec alone by the typing spec's rules, not the checker's. + +Values are JSON, their depth capped: drawn from the spec, so they +conform; drawn at large, so most do not; or drawn from the spec and +changed in one place, so they nearly do. On every one the checker has to +agree with the reference, and the two builds with each other. +""" + +from __future__ import annotations + +import copy +import itertools +import sys +import types +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Annotated, Literal, NewType, NotRequired, Required, TypeAlias, Union, cast + +from hypothesis import HealthCheck, event, given, settings +from hypothesis import strategies as st +from typing_extensions import ReadOnly, TypeAliasType, TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata.v3._typed_json import Loc, Parsed, no_leaf, parser, typeddict_keys + +# --- specs ----------------------------------------------------------------- + +Scalar: TypeAlias = Literal["int", "float", "bool", "str", "null", "json"] + + +@dataclass(frozen=True) +class Leaf: + name: Scalar + + +@dataclass(frozen=True) +class OneOf: + values: tuple[str | int | bool | None, ...] + + +@dataclass(frozen=True) +class ArrayOf: + element: Spec + + +@dataclass(frozen=True) +class Fixed: + elements: tuple[Spec, ...] + + +@dataclass(frozen=True) +class AnyOf: + branches: tuple[Spec, ...] + + +@dataclass(frozen=True) +class MappingOf: + value: Spec + + +@dataclass(frozen=True) +class Named: + how: Literal["newtype", "alias"] + inner: Spec + + +@dataclass(frozen=True) +class Itself: + """The TypedDict a member sits in: a class that holds itself.""" + + +@dataclass(frozen=True) +class ExtraItems: + spec: Spec + + +Says: TypeAlias = Literal["unsaid", "closed", "open"] | ExtraItems +"""What a class says of the keys it does not declare: nothing, `closed=True`, `closed=False`, or `extra_items=`.""" + + +@dataclass(frozen=True) +class Member: + key: str + spec: Spec + wrappers: tuple[str, ...] + """`Required`, `NotRequired`, `ReadOnly` and `Annotated`, as drawn, innermost first.""" + + +@dataclass(frozen=True) +class Layer: + """One class in a TypedDict's ancestry, its own keys and keywords.""" + + total: bool + members: tuple[Member, ...] + says: Says + + +@dataclass(frozen=True) +class Object: + layers: tuple[Layer, ...] + """Base first; the last is the TypedDict itself.""" + + +Spec: TypeAlias = Leaf | OneOf | ArrayOf | Fixed | AnyOf | MappingOf | Named | Itself | Object + + +def members_of(spec: Object) -> dict[str, tuple[Spec, bool]]: + """Each key the TypedDict declares, its spec, and whether it is required, by the typing spec. + + A `Required` or `NotRequired` qualifier says; otherwise the `total` of + the class that declared the key does. + """ + found: dict[str, tuple[Spec, bool]] = {} + for layer in spec.layers: + for member in layer.members: + if "Required" in member.wrappers: + required = True + elif "NotRequired" in member.wrappers: + required = False + else: + required = layer.total + found[member.key] = (member.spec, required) + return found + + +def says_of(spec: Object) -> Says: + """What the TypedDict says of other keys: the nearest class that says anything, as the spec has it.""" + said: Says = "unsaid" + for layer in spec.layers: + if layer.says != "unsaid": + said = layer.says + return said + + +def conforms(value: object, spec: Spec, itself: Object | None = None) -> bool: + """Whether JSON `value` is a value of `spec`: the reference, worked out from the spec alone.""" + if isinstance(spec, Leaf): + return { + "int": type(value) is int, + "float": type(value) in (int, float), + "bool": type(value) is bool, + "str": type(value) is str, + "null": value is None, + "json": True, + }[spec.name] + if isinstance(spec, OneOf): + return any(value == entry and type(value) is type(entry) for entry in spec.values) + if isinstance(spec, ArrayOf): + return isinstance(value, (list, tuple)) and all( + conforms(entry, spec.element, itself) for entry in value + ) + if isinstance(spec, Fixed): + return ( + isinstance(value, (list, tuple)) + and len(value) == len(spec.elements) + and all( + conforms(entry, element, itself) + for entry, element in zip(value, spec.elements, strict=True) + ) + ) + if isinstance(spec, AnyOf): + return any(conforms(value, branch, itself) for branch in spec.branches) + if isinstance(spec, MappingOf): + return isinstance(value, dict) and all( + conforms(entry, spec.value, itself) for entry in value.values() + ) + if isinstance(spec, Named): + return conforms(value, spec.inner, itself) + if isinstance(spec, Itself): + assert itself is not None + return conforms(value, itself, itself) + if not isinstance(value, dict): + return False + members = members_of(spec) + for key, (member, required) in members.items(): + if key in value: + if not conforms(value[key], member, spec): + return False + elif required: + return False + others = [key for key in value if key not in members] + said = says_of(spec) + if said == "closed": + return len(others) == 0 + if isinstance(said, ExtraItems): + return all(conforms(value[key], said.spec, spec) for key in others) + return True + + +# --- building a spec, evaluated or postponed -------------------------------- + +_SCRATCH = types.ModuleType("tests.v3._typed_json_properties_scratch") +_SCRATCH.__dict__.update( + { + "Annotated": Annotated, + "JSONValue": JSONValue, + "Literal": Literal, + "Mapping": Mapping, + "NewType": NewType, + "NotRequired": NotRequired, + "ReadOnly": ReadOnly, + "Required": Required, + "TypeAliasType": TypeAliasType, + "Union": Union, + } +) +sys.modules[_SCRATCH.__name__] = _SCRATCH # where a string annotation is resolved +_serial = itertools.count() + +_LEAF_TEXT: dict[str, str] = { + "int": "int", + "float": "float", + "bool": "bool", + "str": "str", + "null": "None", + "json": "JSONValue", +} + + +def _evaluated(text: str) -> object: + return eval(text, _SCRATCH.__dict__) + + +@dataclass(frozen=True) +class Build: + """A spec built in the scratch module: every class's annotations evaluated, or left as strings.""" + + postponed: bool + + def annotation(self, spec: Spec) -> object: + return _evaluated(self.text(spec, None)) + + def text(self, spec: Spec, itself: str | None) -> str: + if isinstance(spec, Leaf): + return _LEAF_TEXT[spec.name] + if isinstance(spec, OneOf): + return f"Literal[{', '.join(repr(value) for value in spec.values)}]" + if isinstance(spec, ArrayOf): + return f"tuple[{self.text(spec.element, itself)}, ...]" + if isinstance(spec, Fixed): + return f"tuple[{', '.join(self.text(element, itself) for element in spec.elements)}]" + if isinstance(spec, AnyOf): + return f"Union[{', '.join(self.text(branch, itself) for branch in spec.branches)}]" + if isinstance(spec, MappingOf): + return f"Mapping[str, {self.text(spec.value, itself)}]" + if isinstance(spec, Named): + name = f"Named{next(_serial)}" + maker = "NewType" if spec.how == "newtype" else "TypeAliasType" + inner = self.text(spec.inner, itself) + _SCRATCH.__dict__[name] = _evaluated(f"{maker}({name!r}, {inner})") + return name + if isinstance(spec, Itself): + assert itself is not None + return repr(itself) # a forward reference: the class is not built yet + return self.typeddict(spec) + + def member(self, member: Member, itself: str) -> str: + text = self.text(member.spec, itself) + for wrapper in member.wrappers: + text = f"Annotated[{text}, 'meta']" if wrapper == "Annotated" else f"{wrapper}[{text}]" + return text + + def typeddict(self, spec: Object) -> str: + names = [f"T{next(_serial)}" for _ in spec.layers] + base: object = TypedDict + for layer, name in zip(spec.layers, names, strict=True): + texts = {member.key: self.member(member, names[-1]) for member in layer.members} + annotations = ( + texts if self.postponed else {key: _evaluated(text) for key, text in texts.items()} + ) + keywords: dict[str, object] = {"total": layer.total} + if layer.says == "closed": + keywords["closed"] = True + elif layer.says == "open": + keywords["closed"] = False + elif isinstance(layer.says, ExtraItems): + extra = self.text(layer.says.spec, None) + keywords["extra_items"] = extra if self.postponed else _evaluated(extra) + namespace = {"__annotations__": annotations, "__module__": _SCRATCH.__name__} + built = types.new_class(name, (base,), keywords, lambda body: body.update(namespace)) # noqa: B023 - run at once + _SCRATCH.__dict__[name] = built + base = built + return names[-1] + + +# --- strategies ------------------------------------------------------------ + +_KEYS = ("a", "b", "c", "d", "x-y") +_LITERALS: tuple[str | int | bool | None, ...] = (None, True, False, 0, 1, -1, "", "a", "b") +_SCALARS = st.one_of( + st.none(), + st.booleans(), + st.integers(-2, 2), + st.floats(allow_nan=False, allow_infinity=False, width=32), + st.text(max_size=2), +) +_leaves = st.one_of( + st.builds(Leaf, st.sampled_from(("int", "float", "bool", "str", "null", "json"))), + st.lists( + st.sampled_from(_LITERALS), min_size=1, max_size=3, unique_by=lambda v: (type(v), v) + ).map(lambda values: OneOf(tuple(values))), +) + + +@st.composite +def _wrappers(draw: st.DrawFn) -> tuple[str, ...]: + chosen = [draw(st.sampled_from(("", "Required", "NotRequired")))] + chosen += ["ReadOnly"] if draw(st.booleans()) else [] + chosen += ["Annotated"] if draw(st.booleans()) else [] + return tuple(draw(st.permutations([wrapper for wrapper in chosen if wrapper != ""]))) + + +@st.composite +def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: + keys = draw(st.lists(st.sampled_from(_KEYS), unique=True, max_size=4)) + count = draw(st.integers(1, 3)) + owners = [draw(st.integers(0, count - 1)) for _ in keys] + layers: list[Layer] = [] + said: Says = "unsaid" + for index in range(count): + # A subclass of a closed TypedDict, or of one with extra items, is + # kept to what the typing spec allows it: to say the same, or + # nothing, and to add no key. + restricted = said == "closed" or isinstance(said, ExtraItems) + choices: list[Says] = ["unsaid", said] if restricted else ["unsaid", "closed", "open"] + if not restricted and draw(st.booleans()): + choices.append(ExtraItems(draw(inner))) + says = draw(st.sampled_from(choices)) + members = [ + Member(key, draw(inner), draw(_wrappers())) + for key, owner in zip(keys, owners, strict=True) + if owner == index and not restricted + ] + if index == count - 1 and not restricted and draw(st.booleans()): + members.append(Member("self", ArrayOf(Itself()), draw(_wrappers()))) + layers.append(Layer(draw(st.booleans()), tuple(members), says)) + said = says if says != "unsaid" else said + return Object(tuple(layers)) + + +def specs(depth: int) -> st.SearchStrategy[Spec]: + if depth == 0: + return _leaves + inner = specs(depth - 1) + return st.one_of( + _leaves, + st.builds(ArrayOf, inner), + st.lists(inner, min_size=1, max_size=3).map(lambda elements: Fixed(tuple(elements))), + st.lists(inner, min_size=2, max_size=3).map(lambda branches: AnyOf(tuple(branches))), + st.builds(MappingOf, inner), + st.builds(Named, st.sampled_from(("newtype", "alias")), inner), + _objects(inner), + ) + + +def json_values(depth: int) -> st.SearchStrategy[object]: + """Any JSON, nested no deeper than `depth`.""" + if depth <= 0: + return _SCALARS + inner = json_values(depth - 1) + keys = st.sampled_from((*_KEYS, "self", "z")) | st.text(max_size=2) + return st.one_of( + _SCALARS, st.lists(inner, max_size=2), st.dictionaries(keys, inner, max_size=2) + ) + + +def conforming(spec: Spec, depth: int, itself: Object | None = None) -> st.SearchStrategy[object]: + """JSON values of `spec`, arrays and objects of any key nested no deeper than `depth`.""" + if isinstance(spec, Leaf): + return { + "int": st.integers(), + "float": st.integers() | st.floats(allow_nan=False, allow_infinity=False), + "bool": st.booleans(), + "str": st.text(max_size=3), + "null": st.none(), + "json": json_values(depth), + }[spec.name] + if isinstance(spec, OneOf): + return st.sampled_from(spec.values) + if isinstance(spec, ArrayOf): + if depth <= 0: + return st.just([]) + return st.lists(conforming(spec.element, depth - 1, itself), max_size=2) + if isinstance(spec, Fixed): + elements = [conforming(element, depth - 1, itself) for element in spec.elements] + return st.tuples(*elements).map(list) + if isinstance(spec, AnyOf): + return st.one_of(*(conforming(branch, depth, itself) for branch in spec.branches)) + if isinstance(spec, MappingOf): + if depth <= 0: + return st.just({}) + return st.dictionaries( + st.text(max_size=2), conforming(spec.value, depth - 1, itself), max_size=2 + ) + if isinstance(spec, Named): + return conforming(spec.inner, depth, itself) + if isinstance(spec, Itself): + assert itself is not None + return conforming(itself, depth, itself) + return _object_values(spec, depth) + + +@st.composite +def _object_values(draw: st.DrawFn, spec: Object, depth: int) -> dict[str, object]: + members = members_of(spec) + value: dict[str, object] = {} + for key, (member, required) in members.items(): + if required or draw(st.booleans()): + value[key] = draw(conforming(member, depth - 1, spec)) + said = says_of(spec) + others = st.text(max_size=3).filter(lambda key: key not in members) + if said in ("unsaid", "open"): + value.update(draw(st.dictionaries(others, json_values(depth - 1), max_size=2))) + elif isinstance(said, ExtraItems): + value.update( + draw(st.dictionaries(others, conforming(said.spec, depth - 1, spec), max_size=2)) + ) + return value + + +def _positions(value: object, path: Loc = ()) -> list[Loc]: + """Every place in `value`: itself, and each position inside an array or object, depth first.""" + found: list[Loc] = [path] + if isinstance(value, dict): + for key, entry in value.items(): + found += _positions(entry, (*path, key)) + elif isinstance(value, (list, tuple)): + for index, entry in enumerate(value): + found += _positions(entry, (*path, index)) + return found + + +def _changed(value: object, path: Loc, change: object) -> object: + """`value` with what sits at `path` replaced by `change`, or removed if `change` is `_GONE`.""" + if len(path) == 0: + return change + step, rest = path[0], path[1:] + if isinstance(value, dict): + entries = dict(value) + if len(rest) == 0 and change is _GONE: + del entries[step] + else: + entries[step] = _changed(entries[step], rest, change) + return entries + assert isinstance(value, (list, tuple)) + assert isinstance(step, int) + entries_ = list(value) + if len(rest) == 0 and change is _GONE: + del entries_[step] + else: + entries_[step] = _changed(entries_[step], rest, change) + return entries_ + + +_GONE = object() + + +@st.composite +def nearly(draw: st.DrawFn, spec: Spec, depth: int) -> object: + """A value of `spec`, changed in one place: a value replaced, a key or entry dropped, or one added.""" + value = draw(conforming(spec, depth)) + path = draw(st.sampled_from(_positions(value))) + how = draw(st.sampled_from(("replace", "drop", "add"))) + if how == "drop" and len(path) != 0: + return _changed(value, path, _GONE) + target = _at(value, path) + if how == "add" and isinstance(target, dict): + key = draw(st.sampled_from((*_KEYS, "self", "z"))) + return _changed(value, path, {**target, key: draw(json_values(1))}) + if how == "add" and isinstance(target, list): + return _changed(value, path, [*target, draw(json_values(1))]) + return _changed(value, path, draw(json_values(1))) + + +def _at(value: object, loc: Loc) -> object: + for step in loc: + if isinstance(value, dict): + value = cast("dict[str, object]", value)[cast("str", step)] + else: + value = cast("list[object]", value)[cast("int", step)] + return value + + +def _is_place(value: object, loc: Loc) -> bool: + """Whether `loc` names a place in `value`.""" + for step in loc: + in_object = isinstance(value, dict) and isinstance(step, str) and step in value + in_array = ( + isinstance(value, (list, tuple)) and isinstance(step, int) and 0 <= step < len(value) + ) + if not (in_object or in_array): + return False + value = _at(value, (step,)) + return True + + +def _same_json(left: object, right: object) -> bool: + """Equal as JSON: an array is an array, list or tuple.""" + if isinstance(left, (list, tuple)) and isinstance(right, (list, tuple)): + return len(left) == len(right) and all(map(_same_json, left, right)) + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all( + _same_json(left[key], right[key]) for key in left + ) + return type(left) is type(right) and left == right + + +# --- the properties --------------------------------------------------------- + +_DEPTH = 3 +"""The operational cap: how deep a spec nests, and how deep a value of it.""" + +_EXAMPLES = settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) + + +def _values(spec: Spec) -> st.SearchStrategy[object]: + return st.one_of(conforming(spec, _DEPTH), nearly(spec, _DEPTH), json_values(_DEPTH)) + + +def _read(annotation: object, value: object) -> Parsed: + return parser(annotation, no_leaf)(value, ()) + + +@_EXAMPLES +@given(st.data()) +def test_the_checker_agrees_with_the_reference(data: st.DataObject) -> None: + # No problem exactly when the value is one of the type; the value that + # comes back is the one read, as JSON; every problem is located at a + # place in the value, or, for a missing key, in the object that lacks + # it; and when every problem is a key a closed TypedDict does not + # declare, what comes back is a value of the type. + spec = data.draw(specs(_DEPTH), label="spec") + annotation = Build(postponed=False).annotation(spec) + value = data.draw(_values(spec), label="value") + event(f"spec {type(spec).__name__}") + before = copy.deepcopy(value) + typed, problems = _read(annotation, value) + expected = conforms(value, spec) + event("conforms" if expected else "does not conform") + assert value == before + assert (len(problems) == 0) == expected, problems + if expected: + assert _same_json(typed, value) + for found in problems: + place = found.loc[:-1] if found.kind == "missing_key" else found.loc + assert _is_place(value, place), found + if len(problems) != 0 and all(found.kind == "unknown_key" for found in problems): + event("only unknown keys") + assert conforms(typed, spec) + + +@_EXAMPLES +@given(st.data()) +def test_a_postponed_typeddict_reads_as_an_evaluated_one(data: st.DataObject) -> None: + # The same classes, their annotations evaluated or left as strings, as + # `from __future__ import annotations` leaves them: read alike, down to + # each problem's message. + spec = data.draw(specs(_DEPTH), label="spec") + evaluated = Build(postponed=False).annotation(spec) + postponed = Build(postponed=True).annotation(spec) + value = data.draw(_values(spec), label="value") + assert _read(evaluated, value) == _read(postponed, value) + + +@_EXAMPLES +@given(st.data()) +def test_typeddict_keys_follow_the_typing_spec(data: st.DataObject) -> None: + # Which keys are required, per class `total` under each key's own + # qualifiers, and what other keys may hold, inherited when a class + # says nothing: as the spec reads the declaration, evaluated or not. + spec = data.draw(_objects(specs(_DEPTH - 1)), label="spec") + members = members_of(spec) + said = says_of(spec) + event(f"{len(spec.layers)} classes") + event(f"says {said if isinstance(said, str) else 'extra_items'}") + for postponed in (False, True): + keys = typeddict_keys(cast("type", Build(postponed=postponed).annotation(spec))) + assert set(keys.members) == set(members) + assert keys.required == {key for key, (_, required) in members.items() if required} + assert keys.closed == (said == "closed") + assert keys.open == (said in ("unsaid", "open")) + assert keys.declared == (said != "unsaid") + + +@_EXAMPLES +@given(st.data()) +def test_a_key_a_closed_typeddict_does_not_declare_is_reported_and_left_out( + data: st.DataObject, +) -> None: + # The value still reads, as the value it was before the key was added; + # the key is the one problem, where it sits. + spec = data.draw( + _objects(specs(_DEPTH - 1)).filter(lambda drawn: says_of(drawn) == "closed"), label="spec" + ) + declared = members_of(spec) + value = data.draw(_object_values(spec, _DEPTH), label="value") + key = data.draw(st.text(max_size=3).filter(lambda drawn: drawn not in declared), label="key") + annotation = Build(postponed=data.draw(st.booleans(), label="postponed")).annotation(spec) + typed, problems = _read(annotation, {**value, key: data.draw(json_values(1), label="held")}) + assert [(found.loc, found.kind) for found in problems] == [((key,), "unknown_key")] + assert _same_json(typed, value) From 83491e05a7e945666f616082c642fa979365a2c5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 10:56:53 +0200 Subject: [PATCH 4/9] feat(zarr-metadata): check, the checker as a public tool The checker was private, so a layer meant to stand alone gave a caller nothing to call until the definition layer. `zarr_metadata.typed_json` is its door: `check(value, SomeTypedDict)` type-checks a JSON value against any TypedDict -- the package's own document and configuration types, or a caller's -- and gives back a value of it, or None, with every problem located; `typeddict_keys` says what a TypedDict declares. The module moves out of `v3`, since nothing in it is version-specific. Every TypedDict the package declares now compiles, and a test says so: the v2 `dtype` alias names itself through a string, which is now resolved in the alias's module. What other runtime checkers do, and what their test suites catch, was surveyed first -- pydantic, typing-inspection, typeguard, cattrs, msgspec, trycast, typedload, beartype, typing_extensions and CPython -- and taken in: - Each annotation is evaluated in the module of the class that wrote it. `get_type_hints` reads what a subclass inherited in the subclass's module, so a base's `x: Foo` read as the subclass's `Foo`, a silent wrong answer, and a string nested in a base's annotation failed to resolve at all (CPython #90531). pydantic and typeguard get the first right; none of them the second. - A union of TypedDicts tagged by a `Literal` key is read by the branch the tag names, so a gzip with a bad level is reported as a gzip; an untagged union is reported by its closest branch; and of clean TypedDict branches, the one declaring the most of the value's keys reads it. - A key both `Required` and `NotRequired`, and runtime keys that are not the annotated keys, are a `TypeError`. - A recursive alias is named in a message rather than expanded. - Tests: every declaration under both `typing.TypedDict` and `typing_extensions.TypedDict`; bases in other modules holding names their subclasses' modules lack or mean otherwise; redeclared and narrowed keys, diamonds, quoted qualifiers, keys that are not names. The property suite now builds each class in a module of its own, mixes postponed and evaluated classes, quotes names by hand, redeclares read-only keys, notes a failing example's source, and asserts it still reaches each of these. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/347.feature.md | 14 + packages/zarr-metadata/changes/347.misc.md | 15 - packages/zarr-metadata/docs/api/index.md | 3 + packages/zarr-metadata/docs/api/typed_json.md | 5 + packages/zarr-metadata/mkdocs.yml | 1 + .../src/zarr_metadata/{v3 => }/_typed_json.py | 300 +++++- .../src/zarr_metadata/typed_json.py | 79 ++ .../zarr-metadata/tests/test_public_api.py | 2 + .../zarr-metadata/tests/test_typed_json.py | 989 ++++++++++++++++++ .../{v3 => }/test_typed_json_properties.py | 307 ++++-- .../zarr-metadata/tests/v3/test_typed_json.py | 551 ---------- 11 files changed, 1605 insertions(+), 661 deletions(-) create mode 100644 packages/zarr-metadata/changes/347.feature.md delete mode 100644 packages/zarr-metadata/changes/347.misc.md create mode 100644 packages/zarr-metadata/docs/api/typed_json.md rename packages/zarr-metadata/src/zarr_metadata/{v3 => }/_typed_json.py (69%) create mode 100644 packages/zarr-metadata/src/zarr_metadata/typed_json.py create mode 100644 packages/zarr-metadata/tests/test_typed_json.py rename packages/zarr-metadata/tests/{v3 => }/test_typed_json_properties.py (63%) delete mode 100644 packages/zarr-metadata/tests/v3/test_typed_json.py diff --git a/packages/zarr-metadata/changes/347.feature.md b/packages/zarr-metadata/changes/347.feature.md new file mode 100644 index 0000000000..bb5f7a14e4 --- /dev/null +++ b/packages/zarr-metadata/changes/347.feature.md @@ -0,0 +1,14 @@ +`zarr_metadata.typed_json.check(value, SomeTypedDict)` type-checks a JSON +value against any of the package's TypedDicts -- `ZarrV3ArrayMetadataJSON`, +`GzipCodecConfiguration`, or one of your own -- and gives back a value of +it, or None, with every problem located. A TypedDict reads as the typing +spec defines it, whatever the runtime's attributes say: a key is required +by its own `Required` or `NotRequired`, or else by the `total` of the class +that declared it, read from the evaluated annotations, so a module written +with `from __future__ import annotations` reads like any other; `closed`, +`extra_items` and openness come from the class, or from its bases when it +says neither; and each annotation is evaluated in the module of the class +that wrote it. A key a closed TypedDict does not declare is reported, as +the new problem kind `unknown_key`, and left out of the value, which still +comes back; a union of TypedDicts tagged by a `Literal` key is read by the +branch the tag names. `typeddict_keys` says what a TypedDict declares. diff --git a/packages/zarr-metadata/changes/347.misc.md b/packages/zarr-metadata/changes/347.misc.md deleted file mode 100644 index 46dd1b3999..0000000000 --- a/packages/zarr-metadata/changes/347.misc.md +++ /dev/null @@ -1,15 +0,0 @@ -`v3/_typed_json` is the type checker the definition layer reads -metadata with, on its own: for each shape JSON takes -- `int`, `float`, -`bool`, `str`, `None`, `JSONValue`, a `Literal`, `tuple[T, ...]` and -`tuple[T1, T2]`, a union, a TypedDict, `Mapping[str, V]`, a `NewType` -and a type alias -- a parser compiled once from the annotation. A -parser is a pure function, `(value, loc) -> (typed value, problems)`, -and a caller's leaf is asked first at every depth for shapes of its -own. A TypedDict is read as the typing spec defines it: `typeddict_keys` -takes each key's `Required` or `NotRequired` from the evaluated -annotations, so a module written with `from __future__ import -annotations` reads like any other, falls back on the declaring class's -`total`, and takes `closed` and `extra_items` from the class or, when it -says neither, from its bases. What comes back is a value of the type: a -key a closed TypedDict does not declare is reported and left out. The -module knows nothing of metadata fields. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 5e230c7aa2..e4e27e6d6e 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -10,6 +10,9 @@ The package is organized to mirror the structure of the Zarr specifications: structural validators, loc-aware parsers, and the `UNSET` sentinel - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models +- [`zarr_metadata.typed_json`](typed_json.md) — `check`, which type-checks + a JSON value against any of the package's `TypedDict`s, read as the + typing spec defines them, with every problem located - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`) - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 diff --git a/packages/zarr-metadata/docs/api/typed_json.md b/packages/zarr-metadata/docs/api/typed_json.md new file mode 100644 index 0000000000..34bd63d385 --- /dev/null +++ b/packages/zarr-metadata/docs/api/typed_json.md @@ -0,0 +1,5 @@ +--- +title: typed_json +--- + +::: zarr_metadata.typed_json diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 411ec7a529..10990e3218 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -16,6 +16,7 @@ nav: - api/index.md - ' zarr_metadata.model': api/model.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': - api/v3/index.md diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py similarity index 69% rename from packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py rename to packages/zarr-metadata/src/zarr_metadata/_typed_json.py index f39f215a96..6d19dd2721 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -18,6 +18,8 @@ its type admits, so a key a closed TypedDict does not declare is reported, as `unknown_key`, and left out. +`check` is the whole of it for a caller holding a JSON value and a +TypedDict, and the public door, `zarr_metadata.typed_json`, exports it. Nothing here knows what a metadata field is. A caller with a shape of its own passes a `leaf`, which is asked first for every annotation at every depth; the parser it returns is used as it is. Parsers are compiled once @@ -28,9 +30,11 @@ from __future__ import annotations import functools +import sys import types import typing from collections.abc import Callable, Mapping, Sequence +from collections.abc import Set as AbstractSet from dataclasses import dataclass from typing import ( TYPE_CHECKING, @@ -42,6 +46,7 @@ NewType, NoReturn, TypeAlias, + TypeVar, cast, get_args, get_origin, @@ -52,12 +57,14 @@ from typing_extensions import NoExtraItems, TypeIs, is_typeddict from zarr_metadata._common import JSONValue -from zarr_metadata.model._validation import ValidationProblem, is_json +from zarr_metadata.model._validation import ValidationProblem, is_json, refine_json if TYPE_CHECKING: from zarr_metadata.model._validation import ProblemKind +T = TypeVar("T") + Loc: TypeAlias = tuple[str | int, ...] """Where in a document a value sits: the keys and indices down to it.""" @@ -167,6 +174,27 @@ def is_alias(annotation: object) -> bool: return len(cast("typing_extensions.TypeAliasType", annotation).__type_params__) == 0 +@functools.cache +def alias_value(alias: typing_extensions.TypeAliasType) -> object: + """What `alias` stands for, evaluated where it was made; `TypeError` if a name in it does not resolve. + + A `type` statement's value is evaluated when asked for. One made with + `TypeAliasType` holds whatever it was given, and an alias that names + itself -- `TypeAliasType("Dtype", str | tuple[tuple[str, "Dtype"], ...])` + -- gives it as a string, which is resolved here in the alias's module. + """ + holder = type( + "_AliasValue", + (), + {"__annotations__": {"value": alias.__value__}, "__module__": alias.__module__}, + ) + try: + return get_type_hints(holder, include_extras=True)["value"] + except NameError as error: + msg = f"{alias.__name__}: {error}; its value must resolve in its module" + raise TypeError(msg) from error + + # --- TypedDicts ---------------------------------------------------------- @@ -221,15 +249,21 @@ def typeddict_keys(typeddict: type) -> TypedDictKeys: that says none of these is as its bases are -- which the runtime does not record -- or open, when no base says either. """ - try: - hints = get_type_hints(typeddict, include_extras=True) - except NameError as error: - msg = f"{typeddict.__name__}: {error}; its annotations must resolve in its module" - raise TypeError(msg) from error + hints = dict(_hints(typeddict)) required_at_runtime = cast("frozenset[str]", getattr(typeddict, "__required_keys__", ())) + optional_at_runtime = cast("frozenset[str]", getattr(typeddict, "__optional_keys__", ())) + if frozenset(hints) != required_at_runtime | optional_at_runtime: + msg = ( + f"{typeddict.__name__}: its annotations declare {sorted(hints)!r}, and the runtime " + f"{sorted(required_at_runtime | optional_at_runtime)!r}" + ) + raise TypeError(msg) members: dict[str, tuple[object, bool]] = {} for key, hint in hints.items(): said = qualifiers(hint) + if "Required" in said and "NotRequired" in said: + msg = f"{typeddict.__name__}.{key}: Required and NotRequired both; the spec allows one" + raise TypeError(msg) if "Required" in said: required = True elif "NotRequired" in said: @@ -241,6 +275,55 @@ def typeddict_keys(typeddict: type) -> TypedDictKeys: return TypedDictKeys(types.MappingProxyType(members), extra_items, declared) +@functools.cache +def _hints(typeddict: type) -> Mapping[str, object]: + """Every key's annotation, evaluated in the module of the class that declared the key. + + As the typing spec has it, and as `get_type_hints` does not quite: + it evaluates all a class inherited in the class's own module, so a + string a base elsewhere wrote inside an annotation -- `tuple["Local", + ...]` -- is read where `Local` may mean something else, or nothing, + since the runtime ties a module only to a string at the top of an + annotation. And it looks names up in the class's module before the + module a string is tied to, so an inherited `x: "Foo"` reads as the + subclass's `Foo` when both modules have one. So a base's keys come + from the base, and a class's own keys -- those no base gave it -- are + evaluated in its module with nothing else in scope. `typing.TypedDict` + on 3.11 does not record a class's bases, so there every key is + evaluated as the class's own, which is right unless a string nested + in a base's annotation names something only the base's module has. + """ + bases = _typeddict_bases(typeddict) + hints: dict[str, object] = {} + for base in bases: + hints.update(_hints(base)) + written = _written(typeddict) + inherited = [_written(base) for base in bases] + own = { + key: annotation + for key, annotation in written.items() + if not any(key in given and given[key] == annotation for given in inherited) + } + holder = type("_Own", (), {"__annotations__": own, "__module__": typeddict.__module__}) + try: + hints.update(get_type_hints(holder, localns={}, include_extras=True)) + except NameError as error: + msg = f"{typeddict.__name__}: {error}; its annotations must resolve in its module" + raise TypeError(msg) from error + return types.MappingProxyType(hints) + + +def _written(typeddict: type) -> dict[str, object]: + """A TypedDict's annotations, its bases' included, as far as they evaluate without failing.""" + if sys.version_info >= (3, 14): + import annotationlib + + return dict( + annotationlib.get_annotations(typeddict, format=annotationlib.Format.FORWARDREF) + ) + return dict(vars(typeddict).get("__annotations__", {})) + + def _openness(typeddict: type) -> tuple[object, bool]: """What a key `typeddict` does not declare may hold, and whether that was said rather than defaulted.""" extra_items = getattr(typeddict, "__extra_items__", NoExtraItems) @@ -326,12 +409,25 @@ def describe(annotation: object, seen: frozenset[object] = frozenset()) -> str: return "an object" if is_alias(inner): alias = cast("typing_extensions.TypeAliasType", inner) - if alias in seen: + if alias in seen or _holds(alias_value(alias), alias, frozenset()): return f"a {alias.__name__}" - return describe(alias.__value__, seen | {alias}) + return describe(alias_value(alias), seen | {alias}) return "a value" +def _holds(annotation: object, alias: object, seen: frozenset[object]) -> bool: + """Whether `alias` occurs in `annotation`, through the aliases in it: whether it holds itself.""" + inner = strip_annotation(annotation)[0] + if inner is alias: + return True + if is_alias(inner): + if inner in seen: + return False + value = alias_value(cast("typing_extensions.TypeAliasType", inner)) + return _holds(value, alias, seen | {inner}) + return any(_holds(argument, alias, seen) for argument in get_args(inner)) + + def shape_of(annotation: object) -> str | None: """The top-level JSON shape an annotation admits, for choosing a union branch. @@ -345,7 +441,7 @@ def shape_of(annotation: object) -> str | None: if isinstance(inner, NewType): inner = strip_annotation(inner.__supertype__)[0] else: - inner = strip_annotation(cast("typing_extensions.TypeAliasType", inner).__value__)[0] + inner = strip_annotation(alias_value(cast("typing_extensions.TypeAliasType", inner)))[0] if inner is int: return "int" if inner is float: @@ -466,35 +562,79 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -def any_of(branches: Sequence[tuple[str | None, Parser]], description: str) -> Parser: - """A member whose type is a union of shapes, parsed by the branch it fits. +Branch: TypeAlias = tuple[str | None, Parser, frozenset[str] | None] +"""A branch of a union: its top-level shape, from `shape_of`; its parser; and its keys, if a TypedDict.""" - Each branch is its top-level shape, from `shape_of`, and its parser. - The branch whose shape the value has is the one that reports -- so an +Tag: TypeAlias = tuple[str, Mapping[tuple[type, object], int]] +"""A key every branch requires as a `Literal`, and which branch each of its values picks.""" + + +def any_of(branches: Sequence[Branch], description: str, tag: Tag | None = None) -> Parser: + """A member whose type is a union of shapes, parsed by the branch it is. + + Only a branch whose top-level shape the value has is tried -- so an element inside a malformed array is located inside the array, rather - than the whole array being called wrong. A value fitting no branch's - shape is reported once, by what was expected; one fitting several is - parsed by the first that accepts it with no problem, and otherwise - reported by the first that tried. + than the whole array being called wrong -- and a value fitting no + branch's shape is reported once, by what was expected. When every + branch is a TypedDict requiring a key as a `Literal` of values of its + own, that key's value picks the branch, and the problems are that + branch's. Otherwise the value is parsed by the first branch it reads + as with no problem -- among TypedDicts, the one declaring the most of + its keys -- and failing that, reported by the branch with the fewest + problems. """ def parse(value: object, loc: Loc) -> Parsed: - first: Parsed | None = None - for shape, branch in branches: + present = _keys_of(value) + if tag is not None and present is not None: + return _by_tag(branches, tag, cast("Mapping[str, object]", value), loc) + clean: list[tuple[int, int, Parsed]] = [] + failed: list[tuple[int, int, Parsed]] = [] + for index, (shape, branch, keys) in enumerate(branches): if not has_shape(shape, value): continue result = branch(value, loc) - if len(result[1]) == 0: + if len(result[1]) != 0: + failed.append((len(result[1]), index, result)) + elif keys is None or present is None: return result - if first is None: - first = result - if first is None: - return value, problem(loc, f"expected {description}, got {value!r}") - return first + else: + clean.append((-len(keys & present), index, result)) + if len(clean) != 0: + return min(clean, key=lambda found: found[:2])[2] + if len(failed) != 0: + return min(failed, key=lambda found: found[:2])[2] + return value, problem(loc, f"expected {description}, got {value!r}") return parse +def _by_tag(branches: Sequence[Branch], tag: Tag, value: Mapping[str, object], loc: Loc) -> Parsed: + """`value` parsed by the branch its tag picks; a tag missing, or one no branch has, reported at it.""" + key, picks = tag + if key not in value: + return value, problem((*loc, key), f"missing required key {key!r}", "missing_key") + said = value[key] + index = picks.get((type(said), said)) if _hashable(said) else None + if index is None: + allowed = tuple(sorted((entry for _, entry in picks), key=repr)) + return value, problem( + (*loc, key), f"expected one of {allowed!r}, got {said!r}", "invalid_value" + ) + return branches[index][1](value, loc) + + +def _keys_of(value: object) -> AbstractSet[str] | None: + """The keys of `value` if it is a JSON object, else None.""" + if isinstance(value, Mapping): + return cast("Mapping[str, object]", value).keys() + return None + + +def _hashable(value: object) -> bool: + return value is None or isinstance(value, (str, int, float)) + + def object_of(members: Mapping[str, tuple[Parser, bool]], extra: Parser | None) -> Parser: """An object with declared keys, each parsed at its own key, and `extra` for every other key. @@ -564,13 +704,48 @@ def parse(candidate: object, loc: Loc) -> Parsed: def _union(inner: object, leaf: Leaf, building: _Building) -> Parser | None: - branches: list[tuple[str | None, Parser]] = [] - for branch in get_args(inner): + arguments = get_args(inner) + branches: list[Branch] = [] + for branch in arguments: member = _compile(branch, leaf, building) if member is None: return None - branches.append((shape_of(branch), member)) - return any_of(branches, describe(inner)) + typeddict = strip_annotation(branch)[0] + keys = ( + frozenset(typeddict_keys(typeddict).members) + if isinstance(typeddict, type) and is_typeddict(typeddict) + else None + ) + branches.append((shape_of(branch), member, keys)) + return any_of(branches, describe(inner), _tag(arguments)) + + +def _tag(arguments: Sequence[object]) -> Tag | None: + """The key that says which of these TypedDicts a value is, if there is one. + + One every branch requires, as a `Literal` whose values no other branch + has: the discriminator a union of configurations spells with `name`. + """ + typeddicts = [strip_annotation(argument)[0] for argument in arguments] + if not all(isinstance(typed, type) and is_typeddict(typed) for typed in typeddicts): + return None + keys = [typeddict_keys(cast("type", typed)) for typed in typeddicts] + shared = set(keys[0].required) + for each in keys[1:]: + shared &= each.required + for key in sorted(shared): + picks: dict[tuple[type, object], int] = {} + for index, each in enumerate(keys): + annotation = strip_annotation(each.members[key][0])[0] + if get_origin(annotation) is not Literal: + break + values: tuple[object, ...] = get_args(annotation) + if any((type(entry), entry) in picks for entry in values): + break + picks.update({(type(entry), entry): index for entry in values}) + else: + return key, types.MappingProxyType(picks) + return None def _tuple(inner: object, leaf: Leaf, building: _Building) -> Parser | None: @@ -633,7 +808,7 @@ def compile_() -> Parser | None: def _alias( alias: typing_extensions.TypeAliasType, leaf: Leaf, building: _Building ) -> Parser | None: - return _recursive(alias, building, lambda: _compile(alias.__value__, leaf, building)) + return _recursive(alias, building, lambda: _compile(alias_value(alias), leaf, building)) def _literal(inner: object) -> Parser | None: @@ -705,22 +880,82 @@ def no_leaf(annotation: object) -> Parser | None: def parser(annotation: object, leaf: Leaf) -> Parser: - """The parser an annotation implies; `TypeError` if it implies none.""" + """The parser an annotation implies; `TypeError` saying what in it no parser reads.""" found = parser_for(annotation, leaf) if found is None: - msg = f"{annotation!r} is not a shape JSON takes" + inner = strip_annotation(annotation)[0] + unread = unread_in(inner, leaf) if isinstance(inner, type) and is_typeddict(inner) else None + msg = unread or f"{annotation!r} is not a shape JSON takes" raise TypeError(msg) return found +def unread_in(typeddict: type, leaf: Leaf) -> str | None: + """What in `typeddict` no parser reads, named down to the TypedDict that holds it; None if all is read.""" + keys = typeddict_keys(typeddict) + others: list[tuple[str, object]] = [] + if not (keys.closed or keys.open): + others.append(("extra_items", keys.extra_items)) + for key, annotation in [*((key, member[0]) for key, member in keys.members.items()), *others]: + try: + member = parser_for(annotation, leaf) + except TypeError as error: + return f"{typeddict.__name__}.{key}: {error}" + if member is not None: + continue + inner = strip_annotation(annotation)[0] + if isinstance(inner, type) and is_typeddict(inner): + deeper = unread_in(inner, leaf) + if deeper is not None: + return f"{typeddict.__name__}.{key}: {deeper}" + return ( + f"{typeddict.__name__}: {key} is not a shape JSON takes; write it as one -- a " + "number, string, boolean, null, array, object, or an alias of one" + ) + return None + + +@functools.cache +def _checker(typeddict: type) -> Parser: + return parser(typeddict, no_leaf) + + +def check( + value: object, shape: type[T], loc: Loc = () +) -> tuple[T | None, tuple[ValidationProblem, ...]]: + """`value` type-checked as `shape`, a TypedDict: a value of it or None, and every problem. + + `value` is refined to JSON first -- arrays as tuples, string keys, + finite floats -- and then checked member by member, each problem + located under `loc`. What comes back holds what `shape` admits and + nothing else: a key a closed TypedDict does not declare is reported, + as `unknown_key`, and left out, and the value still comes back. + Anything else wrong and it does not. `TypeError` for a `shape` that is + not a TypedDict, or holds something no parser reads. + """ + if not is_typeddict(shape): + msg = f"{shape!r} is not a TypedDict" + raise TypeError(msg) + refined, problems = refine_json(value, loc) + if refined is None: + return None, problems + typed, found = _checker(shape)(refined, loc) + readable = all(problem.kind == "unknown_key" for problem in found) + return (cast("T", typed) if readable else None), found + + __all__ = [ + "Branch", "Leaf", "Loc", "Parsed", "Parser", "Qualifier", + "Tag", "TypedDictKeys", + "alias_value", "any_of", + "check", "describe", "fixed_tuple", "has_shape", @@ -739,4 +974,5 @@ def parser(annotation: object, leaf: Leaf) -> Parser: "shape_of", "strip_annotation", "typeddict_keys", + "unread_in", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py new file mode 100644 index 0000000000..a76d863dd3 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -0,0 +1,79 @@ +"""JSON checked against a TypedDict: the public door. + +Every Zarr document and configuration this package describes is a +TypedDict -- `ZarrV3ArrayMetadataJSON`, `GzipCodecConfiguration` -- and +`check` makes any of them executable. It type-checks a JSON value +against the TypedDict and gives back a value of it, or None, and every +problem, each located in the value: + + import json + + from zarr_metadata import ZarrV3ArrayMetadataJSON + from zarr_metadata.typed_json import check + + document, problems = check(json.loads(raw), ZarrV3ArrayMetadataJSON) + for problem in problems: + print(problem.loc, problem.kind, problem.message) + +A TypedDict reads as the typing spec defines it, however its module +writes annotations: + +- A key is required when its `Required` or `NotRequired` qualifier says + so, and otherwise when the `total` of the class that declared it does. + The runtime's `__required_keys__` cannot see a qualifier written as a + string, as `from __future__ import annotations` writes every one; + `typeddict_keys` reads the annotations evaluated, and can. +- A key the TypedDict does not declare is what `closed`, `extra_items` + or, when the class says neither, its bases make it: in a closed + TypedDict it is reported, as `unknown_key`, and left out, and the value + still comes back; with `extra_items=` it is checked as that type; in + an open one it is kept. +- Each annotation is evaluated in the module of the class that wrote + it, as the spec has it, where `get_type_hints` would read what a + subclass inherited in the subclass's module. +- `ReadOnly` and `Annotated` are peeled wherever they are written; a + `NewType` reads as the type it names, a type alias as the type it + stands for, and a TypedDict or alias that holds itself as deep as the + value goes. +- A union of TypedDicts that each require a key as a `Literal` of values + of their own is read by the branch that key names, and its problems + are that branch's. Otherwise a value is read by the first branch it + fits with no problem -- among TypedDicts, the one declaring the most of + its keys -- and failing that, reported by the branch with the fewest + problems. + +What comes back is a value of the TypedDict: arrays as tuples, and each +object a new dict of the keys its type admits. Problems are values, not +exceptions: `ValidationProblem(loc, message, kind)`, with `kind` one of +`missing_key`, `invalid_type`, `invalid_value`, `unknown_key` and +`invalid_json`, so a caller that tolerates a key the TypedDict does not +declare can tell it from a wrong value. + +`check` reads the shapes JSON takes and no others -- `int`, `float` for +any number, `bool`, `str`, `None`, `JSONValue`, a `Literal`, +`tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, +`Mapping[str, V]`, a `NewType` and a type alias -- and a TypedDict +holding anything else is a `TypeError` naming the member, down to the +TypedDict that holds it. So is one the spec itself refuses, such as a +key both `Required` and `NotRequired`, and a generic TypedDict, since no +runtime records what its type arguments bind in its bases' keys. A +string annotation resolves only in its module, so a TypedDict written +inside a function cannot name a class of that function. `typing.TypedDict` +on Python 3.11 does not record a class's bases, so there a subclass +evaluates what it inherits in its own module; `typing_extensions.TypedDict` +records them on every version. +""" + +from zarr_metadata._common import JSONValue +from zarr_metadata._typed_json import Loc, TypedDictKeys, check, typeddict_keys +from zarr_metadata.model._validation import ProblemKind, ValidationProblem + +__all__ = [ + "JSONValue", + "Loc", + "ProblemKind", + "TypedDictKeys", + "ValidationProblem", + "check", + "typeddict_keys", +] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 6613aa394b..026eeeb553 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -286,6 +286,7 @@ def test_all_is_grouped_and_unique() -> None: "HexFloat32", "HexFloat64", "JSONValue", + "Loc", "MetadataValidationError", "NumpyDatetime64", "NumpyTimeUnit", @@ -297,6 +298,7 @@ def test_all_is_grouped_and_unique() -> None: "ShardingIndexLocation", "Struct", "StructField", + "TypedDictKeys", "ValidationProblem", } ) diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py new file mode 100644 index 0000000000..94ca56c855 --- /dev/null +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -0,0 +1,989 @@ +"""The checker: parsers compiled from annotations for the shapes JSON takes. + +Tested on its own, with no metadata field in sight. A leaf stands in for +whatever a caller adds -- the definition layer adds the shape of a member +holding another metadata field -- and `no_leaf` adds nothing. +""" + +from __future__ import annotations + +import importlib +import itertools +import math +import pkgutil +import sys +import textwrap +import types +from collections.abc import Mapping +from typing import TYPE_CHECKING, Generic, Literal, NewType, NotRequired, TypeVar, cast + +import pytest +from typing_extensions import ReadOnly, TypeAliasType, TypedDict, is_typeddict + +import zarr_metadata.v2 +import zarr_metadata.v3 +from zarr_metadata._common import JSONValue +from zarr_metadata._typed_json import ( + Loc, + Parsed, + Parser, + describe, + no_leaf, + parser, + parser_for, + shape_of, + typeddict_keys, +) +from zarr_metadata.model import ZarrV2ArrayMetadata, ZarrV3ArrayMetadata +from zarr_metadata.typed_json import check +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2DataTypeMetadata +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON + +if TYPE_CHECKING: + from _pytest.mark import ParameterSet + +Level = NewType("Level", int) +Width = TypeAliasType("Width", int) + + +class Options(TypedDict, closed=True): + level: int + note: NotRequired[str] + tag: ReadOnly[NotRequired[str]] + + +class WithExtras(TypedDict, extra_items=int): + level: int + + +class Open(TypedDict, closed=False): + level: int + + +class Small(TypedDict, closed=True): + x: int + + +class Large(TypedDict, closed=True): + x: int + y: int + + +class Tree(TypedDict, closed=True): + label: str + children: tuple[Tree, ...] + + +class GzipLevel(TypedDict, closed=True): + level: int + + +class Gzip(TypedDict, closed=True): + name: Literal["gzip"] + configuration: GzipLevel + + +class Blosc(TypedDict, closed=True): + name: Literal["blosc"] + configuration: Options + + +class Pair(TypedDict, closed=True): + a: int + b: int + + +class Triple(TypedDict, closed=True): + a: str + b: str + c: str + + +class Option1(TypedDict, closed=False): + one: NotRequired[int] + + +class Option2(TypedDict, closed=False): + two: NotRequired[tuple[int, ...]] + + +def _read(annotation: object, value: object) -> Parsed: + return parser(annotation, no_leaf)(value, ()) + + +def _found(parsed: Parsed) -> list[tuple[Loc, str]]: + return [(found.loc, found.kind) for found in parsed[1]] + + +@pytest.mark.parametrize( + ("annotation", "value", "parsed"), + [ + (int, 3, 3), + (float, 3, 3), + (float, 2.5, 2.5), + (bool, True, True), + (str, "x", "x"), + (None, None, None), + (Literal["a", "b"], "a", "a"), + (Literal[1, 2], 2, 2), + (tuple[int, ...], [1, 2], (1, 2)), + (tuple[int, str], (1, "a"), (1, "a")), + (tuple[str | None, ...], ["a", None], ("a", None)), + (int | str, "a", "a"), + (Mapping[str, int], {"k": 1}, {"k": 1}), + (JSONValue, {"any": [1, None]}, {"any": [1, None]}), + (Level, 5, 5), + (Width, 5, 5), + ], + ids=[ + "int", + "number-from-int", + "number", + "bool", + "str", + "null", + "literal-str", + "literal-int", + "sequence", + "fixed-tuple", + "sequence-of-optional", + "union", + "mapping", + "json", + "newtype", + "alias", + ], +) +def test_a_value_of_the_shape_reads_as_itself( + annotation: object, value: object, parsed: object +) -> None: + # The shapes JSON takes and no others, each compiled once from its + # annotation; an array comes back as a tuple, the rest as it came. + assert _read(annotation, value) == (parsed, ()) + + +@pytest.mark.parametrize( + ("annotation", "value", "loc", "kind"), + [ + (int, True, (), "invalid_type"), + (int, 2.0, (), "invalid_type"), + (float, "2", (), "invalid_type"), + (str, 1, (), "invalid_type"), + (None, 0, (), "invalid_type"), + (Literal["a"], "b", (), "invalid_value"), + (Literal[1], True, (), "invalid_value"), + (tuple[int, ...], (1, "x"), (1,), "invalid_type"), + (tuple[int, ...], 5, (), "invalid_type"), + (int | str, 2.5, (), "invalid_type"), + (Mapping[str, int], {"k": "v"}, ("k",), "invalid_type"), + (JSONValue, {"k": object()}, (), "invalid_type"), + ( + Blosc | Gzip, + {"name": "gzip", "configuration": {"level": "x"}}, + ("configuration", "level"), + "invalid_type", + ), + (Blosc | Gzip, {"name": "zstd", "configuration": {}}, ("name",), "invalid_value"), + (Blosc | Gzip, {"configuration": {}}, ("name",), "missing_key"), + (Triple | Pair, {"a": 1, "b": "x"}, ("b",), "invalid_type"), + ], + ids=[ + "bool-is-not-int", + "float-is-not-int", + "str-is-not-number", + "int-is-not-str", + "zero-is-not-null", + "literal-other", + "literal-bool-is-not-int", + "element", + "not-a-sequence", + "no-branch", + "mapping-value", + "not-json", + "a-tag-picks-the-branch-that-reports", + "a-tag-no-branch-has", + "a-tag-missing", + "untagged-the-closest-branch-reports", + ], +) +def test_error_a_value_of_another_shape_is_located( + annotation: object, value: object, loc: Loc, kind: str +) -> None: + parsed, problems = _read(annotation, value) + assert parsed == value + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] + + +@pytest.mark.parametrize( + ("annotation", "value", "parsed", "found"), + [ + (Options, {"level": 1}, {"level": 1}, []), + (Options, {"note": "n", "level": 1}, {"note": "n", "level": 1}, []), + (Options, {"level": 1, "extra": 2}, {"level": 1}, [(("extra",), "unknown_key")]), + (WithExtras, {"level": 1, "more": 2}, {"level": 1, "more": 2}, []), + ( + WithExtras, + {"level": 1, "more": "two"}, + {"level": 1, "more": "two"}, + [(("more",), "invalid_type")], + ), + (Open, {"level": 1, "more": [2]}, {"level": 1, "more": [2]}, []), + (Small | Large, {"x": 1, "y": 2}, {"x": 1, "y": 2}, []), + (Small | Large, {"x": 1, "z": 2}, {"x": 1}, [(("z",), "unknown_key")]), + (Option1 | Option2, {"two": [1]}, {"two": (1,)}, []), + ( + Tree, + {"label": "a", "children": [{"label": "b", "children": []}]}, + {"label": "a", "children": ({"label": "b", "children": ()},)}, + [], + ), + ( + Tree, + {"label": "a", "children": [{"label": 1, "children": []}]}, + {"label": "a", "children": ({"label": 1, "children": ()},)}, + [(("children", 0, "label"), "invalid_type")], + ), + ], + ids=[ + "closed", + "closed-keeps-the-order-keys-came-in", + "closed-leaves-an-unknown-key-out", + "extra-items", + "extra-items-of-the-wrong-type", + "open", + "union-takes-the-branch-that-accepts", + "union-falls-back-to-the-first-that-tried", + "union-takes-the-branch-declaring-the-most-keys", + "recursive", + "recursive-located", + ], +) +def test_an_object_reads_as_what_its_typeddict_admits( + annotation: object, value: object, parsed: dict[str, object], found: list[tuple[Loc, str]] +) -> None: + # A new dict of the keys the type admits, in the order they came: a + # closed TypedDict reports a key it does not declare and leaves it + # out, `extra_items` types every other key, an open one keeps them. + result = _read(annotation, value) + assert result[0] == parsed + assert list(cast_dict(result[0])) == list(parsed) + assert _found(result) == found + + +def cast_dict(value: object) -> dict[str, object]: + assert isinstance(value, dict) + return value + + +def test_error_a_typed_dict_misses_a_required_key() -> None: + value, problems = parser(Options, no_leaf)({"note": "n"}, ()) + assert value == {"note": "n"} + assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "missing_key")] + + +# --- a TypedDict, read as the typing spec defines it ----------------------- + +_PRELUDE = """\ +from typing import Annotated, Generic, NotRequired, Required, TypeVar +from typing_extensions import Never, ReadOnly +from {typeddicts} import TypedDict +V = TypeVar("V") +""" + +_modules = itertools.count() + +_TYPING_TAKES_PEP_728 = sys.version_info >= (3, 15) +"""Whether `typing.TypedDict` takes `closed=` and `extra_items=`, as `typing_extensions` does.""" + + +def _declared( + source: str, + *, + postponed: bool, + monkeypatch: pytest.MonkeyPatch, + typeddicts: str = "typing_extensions", +) -> types.ModuleType: + """The module `source` defines, written with or without `from __future__ import annotations`. + + Registered in `sys.modules`, because a string annotation is resolved + in the module its class was defined in. `typeddicts` is the module the + `TypedDict` it declares with comes from. + """ + name = f"tests._typeddicts_{next(_modules)}" + module = types.ModuleType(name) + monkeypatch.setitem(sys.modules, name, module) + header = "from __future__ import annotations\n" if postponed else "" + text = header + _PRELUDE.format(typeddicts=typeddicts) + textwrap.dedent(source) + exec(text, module.__dict__) # noqa: S102 - the declaration under test + return module + + +_DECLARATIONS = [ + pytest.param( + """ + class T(TypedDict, closed=True): + a: int + b: NotRequired[int] + c: ReadOnly[NotRequired[str]] + d: Annotated[NotRequired[int], 'meta'] + """, + {"a", "b", "c", "d"}, + {"a"}, + "Never", + True, + id="qualifiers-in-any-order", + ), + pytest.param( + """ + class T(TypedDict, total=False, closed=True): + a: int + b: Required[int] + c: Annotated[Required[int], 'meta'] + """, + {"a", "b", "c"}, + {"b", "c"}, + "Never", + True, + id="total-false-and-required", + ), + pytest.param( + """ + class Base(TypedDict, total=False): + a: int + b: Required[int] + class T(Base, closed=True): + c: int + d: NotRequired[int] + """, + {"a", "b", "c", "d"}, + {"b", "c"}, + "Never", + True, + id="totality-is-per-class", + ), + pytest.param( + """ + class Base(TypedDict): + a: ReadOnly[NotRequired[int]] + class T(Base): + a: ReadOnly[int] + """, + {"a"}, + {"a"}, + "object", + False, + id="read-only-key-required-by-a-subclass", + ), + pytest.param( + """ + class Base(TypedDict): + a: ReadOnly[int | str] + b: ReadOnly[NotRequired[int]] + class T(Base): + a: ReadOnly[int] + b: ReadOnly[Required[int]] + """, + {"a", "b"}, + {"a", "b"}, + "object", + False, + id="read-only-keys-narrowed", + ), + pytest.param( + """ + class Base(TypedDict, total=False): + a: int + class T(Base): + a: int + """, + {"a"}, + {"a"}, + "object", + False, + id="redeclared-by-a-total-subclass", + ), + pytest.param( + """ + class T(TypedDict, total=False): + a: Annotated['Annotated[Required[str], "x"]', "y"] + b: str + """, + {"a", "b"}, + {"a"}, + "object", + False, + id="a-qualifier-quoted-inside-annotated", + ), + pytest.param( + """ + T = TypedDict("T", {"class": int, "that's": NotRequired[str], "": ReadOnly[bool]}) + """, + {"class", "that's", ""}, + {"class", ""}, + "object", + False, + id="functional-syntax-with-keys-that-are-not-names", + ), + pytest.param( + "class T(TypedDict):\n a: int\n", + {"a"}, + {"a"}, + "object", + False, + id="open-by-default", + ), + pytest.param( + "class T(TypedDict, closed=False):\n a: int\n", + {"a"}, + {"a"}, + "object", + True, + id="open-when-said", + ), + pytest.param( + "class T(TypedDict, extra_items=int):\n a: int\n", + {"a"}, + {"a"}, + "int", + True, + id="extra-items", + ), + pytest.param( + "class T(TypedDict, extra_items='int'):\n a: int\n", + {"a"}, + {"a"}, + "int", + True, + id="extra-items-written-as-a-string", + ), + pytest.param( + "class T(TypedDict, extra_items='tuple[T, ...]'):\n a: int\n", + {"a"}, + {"a"}, + "tuple[T, ...]", + True, + id="extra-items-naming-its-own-class", + ), + pytest.param( + "class T(TypedDict, extra_items=ReadOnly[str]):\n a: int\n", + {"a"}, + {"a"}, + "str", + True, + id="extra-items-read-only", + ), + pytest.param( + """ + class Base(TypedDict, closed=True): + a: int + class T(Base): + pass + """, + {"a"}, + {"a"}, + "Never", + True, + id="closed-is-inherited", + ), + pytest.param( + """ + class Base(TypedDict, extra_items=ReadOnly[int]): + a: int + class T(Base): + b: NotRequired[int] + """, + {"a", "b"}, + {"a"}, + "int", + True, + id="extra-items-are-inherited", + ), + pytest.param( + """ + class Base(TypedDict, extra_items=int): + a: int + class Left(Base): + pass + class Right(Base): + pass + class T(Left, Right): + pass + """, + {"a"}, + {"a"}, + "int", + True, + id="a-diamond-over-extra-items", + ), + pytest.param( + """ + class Base(TypedDict): + a: int + class T(Base, closed=True): + pass + """, + {"a"}, + {"a"}, + "Never", + True, + id="a-subclass-closes-an-open-base", + ), + pytest.param( + """ + class Base(TypedDict, closed=False): + a: int + class T(Base): + pass + """, + {"a"}, + {"a"}, + "object", + True, + id="open-said-by-a-base", + ), + pytest.param( + """ + class G(TypedDict, Generic[V], closed=True): + a: int + class T(G[int]): + pass + """, + {"a"}, + {"a"}, + "Never", + True, + id="through-a-generic-base", + ), + pytest.param( + """ + class A(TypedDict, closed=True): + a: int + class B(TypedDict, closed=True): + b: NotRequired[int] + class T(A, B): + pass + """, + {"a", "b"}, + {"a"}, + "Never", + True, + id="bases-that-agree", + ), + pytest.param( + """ + class A(TypedDict, closed=False): + a: int + class B(TypedDict, closed=True): + b: int + class T(A, B): + pass + """, + {"a", "b"}, + {"a", "b"}, + "Never", + True, + id="a-closed-base-beside-an-open-one", + ), +] + + +def _under_both_typeddicts(declarations: list[ParameterSet]) -> list[ParameterSet]: + """Each declaration with `typing_extensions.TypedDict`, and with `typing.TypedDict` where it can say it.""" + both: list[ParameterSet] = [] + for declaration in declarations: + source = cast("str", declaration.values[0]) + both.append(pytest.param(*declaration.values, "typing_extensions", id=f"{declaration.id}")) + if _TYPING_TAKES_PEP_728 or ("closed=" not in source and "extra_items=" not in source): + both.append(pytest.param(*declaration.values, "typing", id=f"{declaration.id}-typing")) + return both + + +@pytest.mark.parametrize( + ("source", "members", "required", "extra_items", "declared", "typeddicts"), + _under_both_typeddicts(_DECLARATIONS), +) +@pytest.mark.parametrize("postponed", [False, True], ids=["evaluated", "postponed"]) +def test_typeddict_keys_follow_the_typing_spec( + source: str, + members: set[str], + required: set[str], + extra_items: str, + declared: bool, + typeddicts: str, + postponed: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Written as strings, as `from __future__ import annotations` writes + # every annotation, a qualifier is invisible to `__required_keys__`, + # as it is to `typing.TypedDict` on 3.11 under `ReadOnly` or + # `Annotated`; read off the evaluated annotations it is not, so every + # spelling of a declaration reads alike. Openness is inherited, which + # the runtime does not record. + module = _declared(source, postponed=postponed, monkeypatch=monkeypatch, typeddicts=typeddicts) + keys = typeddict_keys(module.T) + assert set(keys.members) == members + assert keys.required == required + assert keys.extra_items == eval(extra_items, vars(module)) + assert keys.declared is declared + + +_WRITTEN_WHERE = """ + class _Hidden(TypedDict): + x: int + class Base(TypedDict, total=False): + nested: tuple['_Hidden', ...] + top: '_Hidden' + kept: ReadOnly[int | str] +""" + + +@pytest.mark.parametrize( + "child_postponed", [False, True], ids=["child-evaluated", "child-postponed"] +) +@pytest.mark.parametrize("base_postponed", [False, True], ids=["base-evaluated", "base-postponed"]) +def test_a_key_reads_where_its_class_was_written( + base_postponed: bool, child_postponed: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + # The subclass's module never imports `_Hidden`, and the runtime ties a + # string to its module only at the top of an annotation; each key still + # reads in the module of the class that declared it. + base = _declared(_WRITTEN_WHERE, postponed=base_postponed, monkeypatch=monkeypatch) + child = _declared( + f"from {base.__name__} import Base\nclass T(Base):\n own: int\n kept: ReadOnly[int]\n", + postponed=child_postponed, + monkeypatch=monkeypatch, + ) + assert typeddict_keys(child.T).required == {"kept", "own"} + value = {"nested": [{"x": 1}], "top": {"x": 2}, "kept": 3, "own": 4} + assert parser(child.T, no_leaf)(value, ()) == ({**value, "nested": ({"x": 1},)}, ()) + + +@pytest.mark.parametrize( + "postponed", + [ + pytest.param( + False, + id="evaluated", + marks=pytest.mark.skipif( + sys.version_info < (3, 14), reason="an unquoted later name needs PEP 649" + ), + ), + pytest.param(True, id="postponed"), + ], +) +def test_a_typeddict_may_name_a_class_written_after_it( + postponed: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _declared( + """ + class T(TypedDict): + later: NotRequired[Later] + class Later(TypedDict): + x: int + """, + postponed=postponed, + monkeypatch=monkeypatch, + ) + assert parser(module.T, no_leaf)({"later": {"x": 1}}, ()) == ({"later": {"x": 1}}, ()) + + +@pytest.mark.parametrize( + ("base_source", "postponed"), + [ + ("Foo = int\nclass Base(TypedDict):\n x: Foo\n items: tuple[Foo, ...]\n", True), + ("Foo = int\nclass Base(TypedDict):\n x: 'Foo'\n items: tuple['Foo', ...]\n", False), + ], + ids=["postponed", "quoted-by-hand"], +) +def test_a_name_reads_as_the_module_that_wrote_it_means_it( + base_source: str, postponed: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + # The subclass's module means something else by `Foo`; the keys still + # read as their base wrote them, at the top of an annotation or inside. + base = _declared(base_source, postponed=postponed, monkeypatch=monkeypatch) + child = _declared( + f"from {base.__name__} import Base\nFoo = str\nclass T(Base):\n pass\n", + postponed=True, + monkeypatch=monkeypatch, + ) + read = parser(child.T, no_leaf) + assert read({"x": 1, "items": [2]}, ())[1] == () + assert [(found.loc, found.kind) for found in read({"x": "s", "items": ["t"]}, ())[1]] == [ + (("x",), "invalid_type"), + (("items", 0), "invalid_type"), + ] + + +def test_error_a_key_both_required_and_not_required(monkeypatch: pytest.MonkeyPatch) -> None: + # The spec allows one; the runtime and other checkers disagree on which wins. + module = _declared( + "class T(TypedDict):\n a: Required[NotRequired[int]]\n", + postponed=True, + monkeypatch=monkeypatch, + ) + with pytest.raises(TypeError, match="T.a: Required and NotRequired both"): + typeddict_keys(module.T) + + +def test_error_a_typeddict_whose_runtime_keys_are_not_its_annotations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Were they to disagree, a key would stop being required unnoticed. + module = _declared( + "class T(TypedDict):\n a: int\n", postponed=False, monkeypatch=monkeypatch + ) + monkeypatch.setattr(module.T, "__required_keys__", frozenset({"a", "b"})) + with pytest.raises(TypeError, match="T: its annotations declare"): + typeddict_keys(module.T) + + +def test_error_a_typeddict_whose_bases_disagree_on_other_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _declared( + """ + class A(TypedDict, closed=True): + a: int + class B(TypedDict, extra_items=int): + b: int + class T(A, B): + pass + """, + postponed=False, + monkeypatch=monkeypatch, + ) + with pytest.raises(TypeError, match="T: its bases disagree"): + typeddict_keys(module.T) + + +@pytest.mark.parametrize("name", ["T", "Outer"]) +def test_error_a_typeddict_whose_annotations_do_not_resolve( + name: str, monkeypatch: pytest.MonkeyPatch +) -> None: + # Named by the TypedDict that holds the annotation, at any depth: a + # parser compiled around it cannot be. + module = _declared( + """ + class T(TypedDict): + a: Missing + class Outer(TypedDict): + inner: T + """, + postponed=True, + monkeypatch=monkeypatch, + ) + with pytest.raises(TypeError, match="T: name 'Missing' is not defined"): + parser_for(getattr(module, name), no_leaf) + + +def test_error_a_key_that_does_not_resolve_where_it_was_written_names_its_class( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Asked through a subclass elsewhere, the class that wrote it is named. + base = _declared( + "class Base(TypedDict):\n items: tuple['Missing', ...]\n", + postponed=False, + monkeypatch=monkeypatch, + ) + child = _declared( + f"from {base.__name__} import Base\nclass T(Base):\n own: int\n", + postponed=False, + monkeypatch=monkeypatch, + ) + with pytest.raises(TypeError, match="Base: name 'Missing' is not defined"): + typeddict_keys(child.T) + + +def test_error_a_typeddict_written_in_a_function_resolves_only_in_its_module() -> None: + # This module postpones annotations, and `Local` is no name of it. + class Local(TypedDict, closed=True): + x: int + + class Holder(TypedDict, closed=True): + local: Local + + with pytest.raises(TypeError, match="Holder: name 'Local' is not defined"): + typeddict_keys(Holder) + + +T = TypeVar("T") + + +class Generic_(TypedDict, Generic[T], closed=True): + x: T + + +@pytest.mark.parametrize( + "annotation", + [set[int], list[int], Literal[b"x"], Generic_, Generic_[int]], + ids=["set", "list", "literal-bytes", "generic-typeddict", "generic-alias"], +) +def test_error_an_annotation_outside_the_shapes_has_no_parser(annotation: object) -> None: + assert parser_for(annotation, no_leaf) is None + with pytest.raises(TypeError, match="is not a shape JSON takes"): + parser(annotation, no_leaf) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="the `type` statement is 3.12 syntax") +def test_a_type_statement_alias_may_hold_itself(monkeypatch: pytest.MonkeyPatch) -> None: + module = _declared( + "type Nested = int | tuple[Nested, ...]\n", postponed=False, monkeypatch=monkeypatch + ) + assert parser(module.Nested, no_leaf)([1, [2, [3]]], ()) == ((1, (2, (3,))), ()) + + +def test_a_leaf_is_asked_first_at_every_depth() -> None: + # The one hook: a caller's own shapes, consulted before the built-in + # ones, inside an array or an object as at the top. + class Marker: + pass + + def leaf(annotation: object) -> Parser | None: + if annotation is not Marker: + return None + + def parse(value: object, loc: Loc) -> Parsed: + if isinstance(value, str) and value.startswith("m:"): + return value[2:], () + return value, ((loc, "expected a marker"),) # pyright: ignore[reportReturnType] + + return parse + + read = parser(tuple[Marker, ...], leaf) + assert read(["m:a", "m:b"], ()) == (("a", "b"), ()) + + +@pytest.mark.parametrize( + ("annotation", "shape"), + [ + (Literal[True], "bool"), + (Literal[1, 2], "int"), + (Literal["a", "b"], "str"), + (Literal[0, "auto"], None), + (None, "null"), + (tuple[int, ...], "tuple"), + (Options, "mapping"), + (Width, "int"), + (JSONValue, None), + ], +) +def test_a_shape_is_what_a_union_dispatches_on(annotation: object, shape: str | None) -> None: + assert shape_of(annotation) == shape + + +def test_a_shape_is_described_as_a_message_would_name_it() -> None: + assert describe(Literal[0, "auto"]) == "one of ('auto', 0)" + assert describe(int | None) == "an integer or null" + assert describe(Width) == "an integer" + assert describe(ZarrV2DataTypeMetadata) == "a ZarrV2DataTypeMetadata" + + +# --- check: the public door ------------------------------------------------ + +_V3_DOCUMENT = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +_V2_DOCUMENT = { + **ZarrV2ArrayMetadata.create_default(shape=(4,)).to_json(), + "dtype": [["a", " None: + # Any TypedDict the package declares, and any other: a value of it, + # arrays as tuples, holding what it admits and nothing else. + typed, problems = check(value, shape) + assert [(problem.loc, problem.kind) for problem in problems] == found + assert typed is not None + assert parser(shape, no_leaf)(typed, ())[1] == () + assert set(typed) == {key for key in value if (key,) not in {loc for loc, _ in found}} + + +def test_error_check_is_asked_of_a_typeddict() -> None: + with pytest.raises(TypeError, match="is not a TypedDict"): + check({}, dict) + + +def test_error_check_names_what_no_parser_reads() -> None: + # Down to the TypedDict that holds it, at any depth. + with pytest.raises(TypeError, match=r"Holds.unreadable: Unreadable: members is not a shape"): + check({}, Holds) + + +@pytest.mark.parametrize( + ("value", "loc", "kind"), + [ + ({"level": math.nan}, ("level",), "invalid_value"), + ({"level": 1, "note": math.inf}, ("note",), "invalid_value"), + ({"level": 1, 2: 3}, (), "invalid_type"), + ({"level": 1, "note": {1}}, ("note",), "invalid_type"), + ({"level": 1, "note": [{3: 1}]}, ("note", 0), "invalid_type"), + ], + ids=[ + "nan", + "infinity", + "a-key-that-is-not-a-string", + "a-set", + "nested-key-that-is-not-a-string", + ], +) +def test_error_check_locates_a_value_that_is_not_json( + value: dict[object, object], loc: Loc, kind: str +) -> None: + # Refused before any TypedDict is asked: what `check` compares is JSON. + typed, problems = check(value, Options) + assert typed is None + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] + + +def test_error_check_refuses_a_value_of_another_type() -> None: + typed, problems = check({"level": "high"}, Options) + assert typed is None + assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "invalid_type")] + + +def _declared_typeddicts() -> list[type]: + """Every TypedDict a public module of the package's JSON types exports.""" + found: dict[type, None] = {} + for package in (zarr_metadata.v2, zarr_metadata.v3): + modules = [package.__name__] + [ + info.name + for info in pkgutil.walk_packages(package.__path__, prefix=f"{package.__name__}.") + if not any(part.startswith("_") for part in info.name.split(".")) + ] + for name in modules: + module = importlib.import_module(name) + for exported in getattr(module, "__all__", ()): + value = getattr(module, exported) + if isinstance(value, type) and is_typeddict(value): + found[value] = None + return list(found) + + +@pytest.mark.parametrize( + "typeddict", _declared_typeddicts(), ids=lambda typeddict: typeddict.__name__ +) +def test_every_typeddict_the_package_declares_compiles(typeddict: type) -> None: + # A declaration no parser reads would be a document type `check` could + # not be asked about. + assert parser_for(typeddict, no_leaf) is not None diff --git a/packages/zarr-metadata/tests/v3/test_typed_json_properties.py b/packages/zarr-metadata/tests/test_typed_json_properties.py similarity index 63% rename from packages/zarr-metadata/tests/v3/test_typed_json_properties.py rename to packages/zarr-metadata/tests/test_typed_json_properties.py index ec7584d0bb..00a137f560 100644 --- a/packages/zarr-metadata/tests/v3/test_typed_json_properties.py +++ b/packages/zarr-metadata/tests/test_typed_json_properties.py @@ -22,16 +22,18 @@ import itertools import sys import types +import typing from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Annotated, Literal, NewType, NotRequired, Required, TypeAlias, Union, cast -from hypothesis import HealthCheck, event, given, settings +import pytest +from hypothesis import HealthCheck, event, find, given, note, settings from hypothesis import strategies as st from typing_extensions import ReadOnly, TypeAliasType, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._typed_json import Loc, Parsed, no_leaf, parser, typeddict_keys +from zarr_metadata._typed_json import Loc, Parsed, no_leaf, parser, typeddict_keys # --- specs ----------------------------------------------------------------- @@ -98,17 +100,23 @@ class Member: @dataclass(frozen=True) class Layer: - """One class in a TypedDict's ancestry, its own keys and keywords.""" + """One class in a TypedDict's ancestry: its own keys and keywords, and how its module writes it.""" total: bool members: tuple[Member, ...] says: Says + postponed: bool = False + """In a mixed build, whether the class's annotations are left as strings.""" + quoted: bool = False + """Evaluated, whether the class writes the name of another class inside an annotation as a string.""" @dataclass(frozen=True) class Object: layers: tuple[Layer, ...] - """Base first; the last is the TypedDict itself.""" + """Base first; the last is the TypedDict itself. Each is written in a module of its own.""" + stdlib: bool = False + """Whether the classes are `typing.TypedDict`s rather than `typing_extensions` ones.""" Spec: TypeAlias = Leaf | OneOf | ArrayOf | Fixed | AnyOf | MappingOf | Named | Itself | Object @@ -123,16 +131,18 @@ def members_of(spec: Object) -> dict[str, tuple[Spec, bool]]: found: dict[str, tuple[Spec, bool]] = {} for layer in spec.layers: for member in layer.members: - if "Required" in member.wrappers: - required = True - elif "NotRequired" in member.wrappers: - required = False - else: - required = layer.total - found[member.key] = (member.spec, required) + found[member.key] = (member.spec, _required(member, layer.total)) return found +def _required(member: Member, total: bool) -> bool: + if "Required" in member.wrappers: + return True + if "NotRequired" in member.wrappers: + return False + return total + + def says_of(spec: Object) -> Says: """What the TypedDict says of other keys: the nearest class that says anything, as the spec has it.""" said: Says = "unsaid" @@ -199,22 +209,18 @@ def conforms(value: object, spec: Spec, itself: Object | None = None) -> bool: # --- building a spec, evaluated or postponed -------------------------------- -_SCRATCH = types.ModuleType("tests.v3._typed_json_properties_scratch") -_SCRATCH.__dict__.update( - { - "Annotated": Annotated, - "JSONValue": JSONValue, - "Literal": Literal, - "Mapping": Mapping, - "NewType": NewType, - "NotRequired": NotRequired, - "ReadOnly": ReadOnly, - "Required": Required, - "TypeAliasType": TypeAliasType, - "Union": Union, - } -) -sys.modules[_SCRATCH.__name__] = _SCRATCH # where a string annotation is resolved +_PRELUDE: dict[str, object] = { + "Annotated": Annotated, + "JSONValue": JSONValue, + "Literal": Literal, + "Mapping": Mapping, + "NewType": NewType, + "NotRequired": NotRequired, + "ReadOnly": ReadOnly, + "Required": Required, + "TypeAliasType": TypeAliasType, + "Union": Union, +} _serial = itertools.count() _LEAF_TEXT: dict[str, str] = { @@ -226,57 +232,96 @@ def conforms(value: object, spec: Spec, itself: Object | None = None) -> bool: "json": "JSONValue", } +Mode: TypeAlias = Literal["evaluated", "postponed", "mixed"] +"""How a build writes its classes' annotations: all evaluated, all strings, or as each class says.""" + +_ONE_MODULE_FOR_STDLIB = sys.version_info < (3, 12) +"""`typing.TypedDict` records a class's bases from 3.12; before, a subclass reads its keys in one module.""" -def _evaluated(text: str) -> object: - return eval(text, _SCRATCH.__dict__) + +def _module() -> types.ModuleType: + """A fresh module, registered where a string annotation is resolved, knowing only the prelude.""" + module = types.ModuleType(f"tests._typed_json_properties_{next(_serial)}") + module.__dict__.update(_PRELUDE) + sys.modules[module.__name__] = module + return module + + +def _evaluated(text: str, home: types.ModuleType) -> object: + return eval(text, home.__dict__) @dataclass(frozen=True) class Build: - """A spec built in the scratch module: every class's annotations evaluated, or left as strings.""" + """A spec built as source, each class in a module of its own, the names it uses imported there.""" - postponed: bool + mode: Mode + source: list[str] = field(default_factory=list) + """What was built, as it would be written: for a failing example's notes.""" def annotation(self, spec: Spec) -> object: - return _evaluated(self.text(spec, None)) + home = _module() + return _evaluated(self.text(spec, None, home, quoted=False), home) - def text(self, spec: Spec, itself: str | None) -> str: + def _named(self, name: str, made: object, home: types.ModuleType, *, quoted: bool) -> str: + home.__dict__[name] = made + return repr(name) if quoted else name + + def text(self, spec: Spec, itself: str | None, home: types.ModuleType, *, quoted: bool) -> str: + """`spec` written in `home`, a name inside another annotation quoted when `quoted`.""" if isinstance(spec, Leaf): return _LEAF_TEXT[spec.name] if isinstance(spec, OneOf): return f"Literal[{', '.join(repr(value) for value in spec.values)}]" if isinstance(spec, ArrayOf): - return f"tuple[{self.text(spec.element, itself)}, ...]" + return f"tuple[{self.text(spec.element, itself, home, quoted=quoted)}, ...]" if isinstance(spec, Fixed): - return f"tuple[{', '.join(self.text(element, itself) for element in spec.elements)}]" + elements = ", ".join( + self.text(element, itself, home, quoted=quoted) for element in spec.elements + ) + return f"tuple[{elements}]" if isinstance(spec, AnyOf): - return f"Union[{', '.join(self.text(branch, itself) for branch in spec.branches)}]" + branches = ", ".join( + self.text(branch, itself, home, quoted=quoted) for branch in spec.branches + ) + return f"Union[{branches}]" if isinstance(spec, MappingOf): - return f"Mapping[str, {self.text(spec.value, itself)}]" + return f"Mapping[str, {self.text(spec.value, itself, home, quoted=quoted)}]" if isinstance(spec, Named): name = f"Named{next(_serial)}" maker = "NewType" if spec.how == "newtype" else "TypeAliasType" - inner = self.text(spec.inner, itself) - _SCRATCH.__dict__[name] = _evaluated(f"{maker}({name!r}, {inner})") - return name + inner = self.text(spec.inner, itself, home, quoted=quoted and spec.how == "alias") + written = f"{maker}({name!r}, {inner})" + self.source.append(f"# {home.__name__}\n{name} = {written}") + return self._named(name, _evaluated(written, home), home, quoted=quoted) if isinstance(spec, Itself): assert itself is not None return repr(itself) # a forward reference: the class is not built yet - return self.typeddict(spec) + name, built = self.typeddict(spec) + return self._named(name, built, home, quoted=quoted) - def member(self, member: Member, itself: str) -> str: - text = self.text(member.spec, itself) + def member(self, member: Member, itself: str, home: types.ModuleType, *, quoted: bool) -> str: + text = self.text(member.spec, itself, home, quoted=quoted) for wrapper in member.wrappers: text = f"Annotated[{text}, 'meta']" if wrapper == "Annotated" else f"{wrapper}[{text}]" return text - def typeddict(self, spec: Object) -> str: + def typeddict(self, spec: Object) -> tuple[str, object]: + """The TypedDict `spec` is, a class per layer, each in its module: its name, and it.""" names = [f"T{next(_serial)}" for _ in spec.layers] - base: object = TypedDict + shared = _module() if spec.stdlib and _ONE_MODULE_FOR_STDLIB else None + base: object = typing.TypedDict if spec.stdlib else TypedDict + base_name = "typing.TypedDict" if spec.stdlib else "TypedDict" for layer, name in zip(spec.layers, names, strict=True): - texts = {member.key: self.member(member, names[-1]) for member in layer.members} + home = shared or _module() + postponed = self.mode == "postponed" or (self.mode == "mixed" and layer.postponed) + quoted = not postponed and layer.quoted + texts = { + member.key: self.member(member, names[-1], home, quoted=quoted) + for member in layer.members + } annotations = ( - texts if self.postponed else {key: _evaluated(text) for key, text in texts.items()} + texts if postponed else {key: _evaluated(text, home) for key, text in texts.items()} ) keywords: dict[str, object] = {"total": layer.total} if layer.says == "closed": @@ -284,13 +329,24 @@ def typeddict(self, spec: Object) -> str: elif layer.says == "open": keywords["closed"] = False elif isinstance(layer.says, ExtraItems): - extra = self.text(layer.says.spec, None) - keywords["extra_items"] = extra if self.postponed else _evaluated(extra) - namespace = {"__annotations__": annotations, "__module__": _SCRATCH.__name__} + extra = self.text(layer.says.spec, None, home, quoted=False) + keywords["extra_items"] = extra if postponed else _evaluated(extra, home) + namespace = {"__annotations__": annotations, "__module__": home.__name__} built = types.new_class(name, (base,), keywords, lambda body: body.update(namespace)) # noqa: B023 - run at once - _SCRATCH.__dict__[name] = built - base = built - return names[-1] + home.__dict__[name] = built + written = "".join( + f"\n {key}: {text!r}" if postponed else f"\n {key}: {text}" + for key, text in texts.items() + ) + header = "from __future__ import annotations\n" if postponed else "" + self.source.append( + f"# {home.__name__}\n{header}class {name}({base_name}, {keywords!r}):{written or ' ...'}" + ) + base, base_name = built, name + return names[-1], base + + def text_of(self) -> str: + return "\n\n".join(self.source) # --- strategies ------------------------------------------------------------ @@ -320,6 +376,10 @@ def _wrappers(draw: st.DrawFn) -> tuple[str, ...]: return tuple(draw(st.permutations([wrapper for wrapper in chosen if wrapper != ""]))) +_TYPING_TAKES_PEP_728 = sys.version_info >= (3, 15) +"""Whether `typing.TypedDict` takes `closed=` and `extra_items=`, as `typing_extensions` does.""" + + @st.composite def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: keys = draw(st.lists(st.sampled_from(_KEYS), unique=True, max_size=4)) @@ -327,6 +387,7 @@ def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: owners = [draw(st.integers(0, count - 1)) for _ in keys] layers: list[Layer] = [] said: Says = "unsaid" + declared: dict[str, tuple[Member, bool]] = {} for index in range(count): # A subclass of a closed TypedDict, or of one with extra items, is # kept to what the typing spec allows it: to say the same, or @@ -336,16 +397,56 @@ def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: if not restricted and draw(st.booleans()): choices.append(ExtraItems(draw(inner))) says = draw(st.sampled_from(choices)) + total = draw(st.booleans()) members = [ Member(key, draw(inner), draw(_wrappers())) for key, owner in zip(keys, owners, strict=True) if owner == index and not restricted ] + if not restricted: + members += draw(_redeclared(declared, total)) if index == count - 1 and not restricted and draw(st.booleans()): members.append(Member("self", ArrayOf(Itself()), draw(_wrappers()))) - layers.append(Layer(draw(st.booleans()), tuple(members), says)) + postponed, quoted = draw(st.booleans()), draw(st.booleans()) + layers.append(Layer(total, tuple(members), says, postponed, quoted)) + declared.update({member.key: (member, _required(member, total)) for member in members}) said = says if says != "unsaid" else said - return Object(tuple(layers)) + speaks = any(layer.says != "unsaid" for layer in layers) + # `typing.TypedDict` from 3.13 refuses a read-only key redeclaring one it + # thinks mutable, and cannot see `ReadOnly` in a postponed base: it would + # not build what the spec allows, so it declares no key twice here. + redeclares = sum(len(layer.members) for layer in layers) > len( + {member.key for layer in layers for member in layer.members} + ) + stdlib = (_TYPING_TAKES_PEP_728 or not speaks) and not redeclares and draw(st.booleans()) + return Object(tuple(layers), stdlib) + + +@st.composite +def _redeclared( + draw: st.DrawFn, declared: dict[str, tuple[Member, bool]], total: bool +) -> list[Member]: + """None, or one read-only key a base declared, narrowed as the typing spec lets a subclass. + + Its type may narrow -- a union to some of its branches -- and a key + that was not required may become required, never the other way. + """ + candidates = sorted( + key for key, (member, _) in declared.items() if "ReadOnly" in member.wrappers + ) + if len(candidates) == 0 or not draw(st.booleans()): + return [] + key = draw(st.sampled_from(candidates)) + earlier, was_required = declared[key] + spec = earlier.spec + if isinstance(spec, AnyOf): + kept = sorted(draw(st.sets(st.integers(0, len(spec.branches) - 1), min_size=1))) + branches = tuple(spec.branches[index] for index in kept) + spec = branches[0] if len(branches) == 1 else AnyOf(branches) + wrappers = draw(_wrappers()) + if was_required and not _required(Member(key, spec, wrappers), total): + wrappers = ("Required", *(wrapper for wrapper in wrappers if wrapper != "NotRequired")) + return [Member(key, spec, wrappers)] def specs(depth: int) -> st.SearchStrategy[Spec]: @@ -517,6 +618,8 @@ def _same_json(left: object, right: object) -> bool: # --- the properties --------------------------------------------------------- +_MODES: tuple[Mode, ...] = ("evaluated", "postponed", "mixed") + _DEPTH = 3 """The operational cap: how deep a spec nests, and how deep a value of it.""" @@ -540,7 +643,9 @@ def test_the_checker_agrees_with_the_reference(data: st.DataObject) -> None: # it; and when every problem is a key a closed TypedDict does not # declare, what comes back is a value of the type. spec = data.draw(specs(_DEPTH), label="spec") - annotation = Build(postponed=False).annotation(spec) + build = Build(data.draw(st.sampled_from(_MODES), label="mode")) + annotation = build.annotation(spec) + note(build.text_of()) value = data.draw(_values(spec), label="value") event(f"spec {type(spec).__name__}") before = copy.deepcopy(value) @@ -566,10 +671,13 @@ def test_a_postponed_typeddict_reads_as_an_evaluated_one(data: st.DataObject) -> # `from __future__ import annotations` leaves them: read alike, down to # each problem's message. spec = data.draw(specs(_DEPTH), label="spec") - evaluated = Build(postponed=False).annotation(spec) - postponed = Build(postponed=True).annotation(spec) + evaluated = Build("evaluated") + written = Build(data.draw(st.sampled_from(("postponed", "mixed")), label="mode")) + read_evaluated, read_written = evaluated.annotation(spec), written.annotation(spec) + note(evaluated.text_of()) + note(written.text_of()) value = data.draw(_values(spec), label="value") - assert _read(evaluated, value) == _read(postponed, value) + assert _read(read_evaluated, value) == _read(read_written, value) @_EXAMPLES @@ -583,8 +691,10 @@ def test_typeddict_keys_follow_the_typing_spec(data: st.DataObject) -> None: said = says_of(spec) event(f"{len(spec.layers)} classes") event(f"says {said if isinstance(said, str) else 'extra_items'}") - for postponed in (False, True): - keys = typeddict_keys(cast("type", Build(postponed=postponed).annotation(spec))) + for mode in _MODES: + build = Build(mode) + keys = typeddict_keys(cast("type", build.annotation(spec))) + note(build.text_of()) assert set(keys.members) == set(members) assert keys.required == {key for key, (_, required) in members.items() if required} assert keys.closed == (said == "closed") @@ -605,7 +715,78 @@ def test_a_key_a_closed_typeddict_does_not_declare_is_reported_and_left_out( declared = members_of(spec) value = data.draw(_object_values(spec, _DEPTH), label="value") key = data.draw(st.text(max_size=3).filter(lambda drawn: drawn not in declared), label="key") - annotation = Build(postponed=data.draw(st.booleans(), label="postponed")).annotation(spec) + build = Build(data.draw(st.sampled_from(_MODES), label="mode")) + annotation = build.annotation(spec) + note(build.text_of()) typed, problems = _read(annotation, {**value, key: data.draw(json_values(1), label="held")}) assert [(found.loc, found.kind) for found in problems] == [((key,), "unknown_key")] assert _same_json(typed, value) + + +# --- reach: what the strategy has to keep drawing ----------------------------- + + +def _nests_a_name(spec: Spec, *, nested: bool = False) -> bool: + """Whether `spec` names another class or alias inside an annotation, where a quoted name loses its module.""" + if isinstance(spec, (Named, Object)): + return nested + if isinstance(spec, ArrayOf): + return _nests_a_name(spec.element, nested=True) + if isinstance(spec, MappingOf): + return _nests_a_name(spec.value, nested=True) + if isinstance(spec, Fixed): + return any(_nests_a_name(element, nested=True) for element in spec.elements) + if isinstance(spec, AnyOf): + return any(_nests_a_name(branch, nested=True) for branch in spec.branches) + return False + + +def _hides_a_qualifier(spec: Object) -> bool: + """Whether a postponed class writes a qualifier its `total` contradicts, which the runtime then misses.""" + return any( + layer.postponed + and any( + ("NotRequired" in member.wrappers and layer.total) + or ("Required" in member.wrappers and not layer.total) + for member in layer.members + ) + for layer in spec.layers + ) + + +_REACHES = { + "a-key-redeclared": lambda spec: any( + member.key in {earlier.key for layer in spec.layers[:index] for earlier in layer.members} + for index, later in enumerate(spec.layers) + for member in later.members + ), + "openness-inherited": lambda spec: ( + spec.layers[-1].says == "unsaid" and any(layer.says != "unsaid" for layer in spec.layers) + ), + "typing-typeddict": lambda spec: spec.stdlib and len(spec.layers) > 1, + "classes-evaluated-and-postponed": lambda spec: ( + len({layer.postponed for layer in spec.layers}) == 2 + ), + "a-qualifier-the-runtime-cannot-see": _hides_a_qualifier, + "a-quoted-name-a-subclass-inherits": lambda spec: ( + len(spec.layers) > 1 + and any( + layer.quoted and any(_nests_a_name(member.spec) for member in layer.members) + for layer in spec.layers[:-1] + ) + ), + "a-class-that-holds-itself": lambda spec: any( + member.key == "self" for layer in spec.layers for member in layer.members + ), +} + + +@pytest.mark.parametrize("reach", sorted(_REACHES)) +def test_the_strategy_reaches(reach: str) -> None: + # Each case the properties are meant to cover, drawn at least once in a + # modest search: an edit to the strategy cannot quietly lose one. + find( + _objects(specs(_DEPTH - 1)), + _REACHES[reach], + settings=settings(max_examples=2000, database=None, deadline=None), + ) diff --git a/packages/zarr-metadata/tests/v3/test_typed_json.py b/packages/zarr-metadata/tests/v3/test_typed_json.py deleted file mode 100644 index 994b5e7852..0000000000 --- a/packages/zarr-metadata/tests/v3/test_typed_json.py +++ /dev/null @@ -1,551 +0,0 @@ -"""The checker: parsers compiled from annotations for the shapes JSON takes. - -Tested on its own, with no metadata field in sight. A leaf stands in for -whatever a caller adds -- the definition layer adds the shape of a member -holding another metadata field -- and `no_leaf` adds nothing. -""" - -from __future__ import annotations - -import itertools -import sys -import textwrap -import types -from collections.abc import Mapping -from typing import Generic, Literal, Never, NewType, NotRequired, TypeVar - -import pytest -from typing_extensions import ReadOnly, TypeAliasType, TypedDict - -from zarr_metadata._common import JSONValue -from zarr_metadata.v3._typed_json import ( - Loc, - Parsed, - Parser, - describe, - no_leaf, - parser, - parser_for, - shape_of, - typeddict_keys, -) - -Level = NewType("Level", int) -Width = TypeAliasType("Width", int) - - -class Options(TypedDict, closed=True): - level: int - note: NotRequired[str] - tag: ReadOnly[NotRequired[str]] - - -class WithExtras(TypedDict, extra_items=int): - level: int - - -class Open(TypedDict, closed=False): - level: int - - -class Small(TypedDict, closed=True): - x: int - - -class Large(TypedDict, closed=True): - x: int - y: int - - -class Tree(TypedDict, closed=True): - label: str - children: tuple[Tree, ...] - - -def _read(annotation: object, value: object) -> Parsed: - return parser(annotation, no_leaf)(value, ()) - - -def _found(parsed: Parsed) -> list[tuple[Loc, str]]: - return [(found.loc, found.kind) for found in parsed[1]] - - -@pytest.mark.parametrize( - ("annotation", "value", "parsed"), - [ - (int, 3, 3), - (float, 3, 3), - (float, 2.5, 2.5), - (bool, True, True), - (str, "x", "x"), - (None, None, None), - (Literal["a", "b"], "a", "a"), - (Literal[1, 2], 2, 2), - (tuple[int, ...], [1, 2], (1, 2)), - (tuple[int, str], (1, "a"), (1, "a")), - (tuple[str | None, ...], ["a", None], ("a", None)), - (int | str, "a", "a"), - (Mapping[str, int], {"k": 1}, {"k": 1}), - (JSONValue, {"any": [1, None]}, {"any": [1, None]}), - (Level, 5, 5), - (Width, 5, 5), - ], - ids=[ - "int", - "number-from-int", - "number", - "bool", - "str", - "null", - "literal-str", - "literal-int", - "sequence", - "fixed-tuple", - "sequence-of-optional", - "union", - "mapping", - "json", - "newtype", - "alias", - ], -) -def test_a_value_of_the_shape_reads_as_itself( - annotation: object, value: object, parsed: object -) -> None: - # The shapes JSON takes and no others, each compiled once from its - # annotation; an array comes back as a tuple, the rest as it came. - assert _read(annotation, value) == (parsed, ()) - - -@pytest.mark.parametrize( - ("annotation", "value", "loc", "kind"), - [ - (int, True, (), "invalid_type"), - (int, 2.0, (), "invalid_type"), - (float, "2", (), "invalid_type"), - (str, 1, (), "invalid_type"), - (None, 0, (), "invalid_type"), - (Literal["a"], "b", (), "invalid_value"), - (Literal[1], True, (), "invalid_value"), - (tuple[int, ...], (1, "x"), (1,), "invalid_type"), - (tuple[int, ...], 5, (), "invalid_type"), - (int | str, 2.5, (), "invalid_type"), - (Mapping[str, int], {"k": "v"}, ("k",), "invalid_type"), - (JSONValue, {"k": object()}, (), "invalid_type"), - ], - ids=[ - "bool-is-not-int", - "float-is-not-int", - "str-is-not-number", - "int-is-not-str", - "zero-is-not-null", - "literal-other", - "literal-bool-is-not-int", - "element", - "not-a-sequence", - "no-branch", - "mapping-value", - "not-json", - ], -) -def test_error_a_value_of_another_shape_is_located( - annotation: object, value: object, loc: Loc, kind: str -) -> None: - parsed, problems = _read(annotation, value) - assert parsed == value - assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] - - -@pytest.mark.parametrize( - ("annotation", "value", "parsed", "found"), - [ - (Options, {"level": 1}, {"level": 1}, []), - (Options, {"note": "n", "level": 1}, {"note": "n", "level": 1}, []), - (Options, {"level": 1, "extra": 2}, {"level": 1}, [(("extra",), "unknown_key")]), - (WithExtras, {"level": 1, "more": 2}, {"level": 1, "more": 2}, []), - ( - WithExtras, - {"level": 1, "more": "two"}, - {"level": 1, "more": "two"}, - [(("more",), "invalid_type")], - ), - (Open, {"level": 1, "more": [2]}, {"level": 1, "more": [2]}, []), - (Small | Large, {"x": 1, "y": 2}, {"x": 1, "y": 2}, []), - (Small | Large, {"x": 1, "z": 2}, {"x": 1}, [(("z",), "unknown_key")]), - ( - Tree, - {"label": "a", "children": [{"label": "b", "children": []}]}, - {"label": "a", "children": ({"label": "b", "children": ()},)}, - [], - ), - ( - Tree, - {"label": "a", "children": [{"label": 1, "children": []}]}, - {"label": "a", "children": ({"label": 1, "children": ()},)}, - [(("children", 0, "label"), "invalid_type")], - ), - ], - ids=[ - "closed", - "closed-keeps-the-order-keys-came-in", - "closed-leaves-an-unknown-key-out", - "extra-items", - "extra-items-of-the-wrong-type", - "open", - "union-takes-the-branch-that-accepts", - "union-falls-back-to-the-first-that-tried", - "recursive", - "recursive-located", - ], -) -def test_an_object_reads_as_what_its_typeddict_admits( - annotation: object, value: object, parsed: dict[str, object], found: list[tuple[Loc, str]] -) -> None: - # A new dict of the keys the type admits, in the order they came: a - # closed TypedDict reports a key it does not declare and leaves it - # out, `extra_items` types every other key, an open one keeps them. - result = _read(annotation, value) - assert result[0] == parsed - assert list(cast_dict(result[0])) == list(parsed) - assert _found(result) == found - - -def cast_dict(value: object) -> dict[str, object]: - assert isinstance(value, dict) - return value - - -def test_error_a_typed_dict_misses_a_required_key() -> None: - value, problems = parser(Options, no_leaf)({"note": "n"}, ()) - assert value == {"note": "n"} - assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "missing_key")] - - -# --- a TypedDict, read as the typing spec defines it ----------------------- - -_PRELUDE = """\ -import typing -from typing import Annotated, Generic, NotRequired, Required, TypeVar -from typing_extensions import Never, ReadOnly, TypedDict -V = TypeVar("V") -""" - -_modules = itertools.count() - - -def _declared(source: str, *, postponed: bool, monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: - """The module `source` defines, written with or without `from __future__ import annotations`. - - Registered in `sys.modules`, because a string annotation is resolved - in the module its class was defined in. - """ - name = f"tests.v3._typeddicts_{next(_modules)}" - module = types.ModuleType(name) - monkeypatch.setitem(sys.modules, name, module) - header = "from __future__ import annotations\n" if postponed else "" - exec(header + _PRELUDE + textwrap.dedent(source), module.__dict__) # noqa: S102 - the declaration under test - return module - - -@pytest.mark.parametrize( - ("source", "members", "required", "extra_items", "declared"), - [ - ( - """ - class T(TypedDict, closed=True): - a: int - b: NotRequired[int] - c: ReadOnly[NotRequired[str]] - d: Annotated[NotRequired[int], 'meta'] - """, - {"a", "b", "c", "d"}, - {"a"}, - Never, - True, - ), - ( - """ - class T(TypedDict, total=False, closed=True): - a: int - b: Required[int] - c: Annotated[Required[int], 'meta'] - """, - {"a", "b", "c"}, - {"b", "c"}, - Never, - True, - ), - ( - """ - class Base(TypedDict, total=False): - a: int - b: Required[int] - class T(Base, closed=True): - c: int - d: NotRequired[int] - """, - {"a", "b", "c", "d"}, - {"b", "c"}, - Never, - True, - ), - ( - """ - class Base(TypedDict): - a: ReadOnly[NotRequired[int]] - class T(Base): - a: ReadOnly[int] - """, - {"a"}, - {"a"}, - object, - False, - ), - ( - """ - class T(typing.TypedDict): - a: int - b: typing.NotRequired[int] - """, - {"a", "b"}, - {"a"}, - object, - False, - ), - ("class T(TypedDict):\n a: int\n", {"a"}, {"a"}, object, False), - ("class T(TypedDict, closed=False):\n a: int\n", {"a"}, {"a"}, object, True), - ("class T(TypedDict, extra_items=int):\n a: int\n", {"a"}, {"a"}, int, True), - ("class T(TypedDict, extra_items='int'):\n a: int\n", {"a"}, {"a"}, int, True), - ( - """ - class T(TypedDict, extra_items=ReadOnly[str]): - a: int - """, - {"a"}, - {"a"}, - str, - True, - ), - ( - """ - class Base(TypedDict, closed=True): - a: int - class T(Base): - pass - """, - {"a"}, - {"a"}, - Never, - True, - ), - ( - """ - class Base(TypedDict, extra_items=ReadOnly[int]): - a: int - class T(Base): - b: NotRequired[int] - """, - {"a", "b"}, - {"a"}, - int, - True, - ), - ( - """ - class Base(TypedDict): - a: int - class T(Base, closed=True): - pass - """, - {"a"}, - {"a"}, - Never, - True, - ), - ( - """ - class Base(TypedDict, closed=False): - a: int - class T(Base): - pass - """, - {"a"}, - {"a"}, - object, - True, - ), - ( - """ - class G(TypedDict, Generic[V], closed=True): - a: int - class T(G[int]): - pass - """, - {"a"}, - {"a"}, - Never, - True, - ), - ( - """ - class A(TypedDict, closed=True): - a: int - class B(TypedDict, closed=True): - b: NotRequired[int] - class T(A, B): - pass - """, - {"a", "b"}, - {"a"}, - Never, - True, - ), - ], - ids=[ - "qualifiers-in-any-order", - "total-false-and-required", - "totality-is-per-class", - "read-only-key-required-by-a-subclass", - "stdlib-typeddict", - "open-by-default", - "open-when-said", - "extra-items", - "extra-items-written-as-a-string", - "extra-items-read-only", - "closed-is-inherited", - "extra-items-are-inherited", - "a-subclass-closes-an-open-base", - "open-said-by-a-base", - "through-a-generic-base", - "bases-that-agree", - ], -) -@pytest.mark.parametrize("postponed", [False, True], ids=["evaluated", "postponed"]) -def test_typeddict_keys_follow_the_typing_spec( - source: str, - members: set[str], - required: set[str], - extra_items: object, - declared: bool, - postponed: bool, - monkeypatch: pytest.MonkeyPatch, -) -> None: - # Written as strings, as `from __future__ import annotations` writes - # every annotation, a qualifier is invisible to `__required_keys__`; - # read off the evaluated annotations it is not, so both spellings of - # a declaration read alike. Openness is inherited, which the runtime - # does not record. - keys = typeddict_keys(_declared(source, postponed=postponed, monkeypatch=monkeypatch).T) - assert set(keys.members) == members - assert keys.required == required - assert keys.extra_items is extra_items - assert keys.declared is declared - - -def test_error_a_typeddict_whose_bases_disagree_on_other_keys( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = _declared( - """ - class A(TypedDict, closed=True): - a: int - class B(TypedDict, extra_items=int): - b: int - class T(A, B): - pass - """, - postponed=False, - monkeypatch=monkeypatch, - ) - with pytest.raises(TypeError, match="T: its bases disagree"): - typeddict_keys(module.T) - - -@pytest.mark.parametrize("name", ["T", "Outer"]) -def test_error_a_typeddict_whose_annotations_do_not_resolve( - name: str, monkeypatch: pytest.MonkeyPatch -) -> None: - # Named by the TypedDict that holds the annotation, at any depth: a - # parser compiled around it cannot be. - module = _declared( - """ - class T(TypedDict): - a: Missing - class Outer(TypedDict): - inner: T - """, - postponed=True, - monkeypatch=monkeypatch, - ) - with pytest.raises(TypeError, match="T: name 'Missing' is not defined"): - parser_for(getattr(module, name), no_leaf) - - -T = TypeVar("T") - - -class Generic_(TypedDict, Generic[T], closed=True): - x: T - - -@pytest.mark.parametrize( - "annotation", - [set[int], list[int], Literal[b"x"], Generic_, Generic_[int]], - ids=["set", "list", "literal-bytes", "generic-typeddict", "generic-alias"], -) -def test_error_an_annotation_outside_the_shapes_has_no_parser(annotation: object) -> None: - assert parser_for(annotation, no_leaf) is None - with pytest.raises(TypeError, match="is not a shape JSON takes"): - parser(annotation, no_leaf) - - -@pytest.mark.skipif(sys.version_info < (3, 12), reason="the `type` statement is 3.12 syntax") -def test_a_type_statement_alias_may_hold_itself(monkeypatch: pytest.MonkeyPatch) -> None: - module = _declared( - "type Nested = int | tuple[Nested, ...]\n", postponed=False, monkeypatch=monkeypatch - ) - assert parser(module.Nested, no_leaf)([1, [2, [3]]], ()) == ((1, (2, (3,))), ()) - - -def test_a_leaf_is_asked_first_at_every_depth() -> None: - # The one hook: a caller's own shapes, consulted before the built-in - # ones, inside an array or an object as at the top. - class Marker: - pass - - def leaf(annotation: object) -> Parser | None: - if annotation is not Marker: - return None - - def parse(value: object, loc: Loc) -> Parsed: - if isinstance(value, str) and value.startswith("m:"): - return value[2:], () - return value, ((loc, "expected a marker"),) # pyright: ignore[reportReturnType] - - return parse - - read = parser(tuple[Marker, ...], leaf) - assert read(["m:a", "m:b"], ()) == (("a", "b"), ()) - - -@pytest.mark.parametrize( - ("annotation", "shape"), - [ - (Literal[True], "bool"), - (Literal[1, 2], "int"), - (Literal["a", "b"], "str"), - (Literal[0, "auto"], None), - (None, "null"), - (tuple[int, ...], "tuple"), - (Options, "mapping"), - (Width, "int"), - (JSONValue, None), - ], -) -def test_a_shape_is_what_a_union_dispatches_on(annotation: object, shape: str | None) -> None: - assert shape_of(annotation) == shape - - -def test_a_mixed_literal_is_described() -> None: - assert describe(Literal[0, "auto"]) == "one of ('auto', 0)" - assert describe(int | None) == "an integer or null" From 7386ad9449f5bf65e7e6d6520743a5eb8687ee48 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 11:43:08 +0200 Subject: [PATCH 5/9] build(zarr-metadata): hypothesis for the checker's property tests The first tests that import it are the checker's, so it joins the test group here. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/pyproject.toml | 2 +- packages/zarr-metadata/uv.lock | 99 +++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index daa6cbd7e2..31d548bf2a 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://zarr-metadata.readthedocs.io/" [dependency-groups] -test = ["pytest", "pydantic>=2.13", "jsonschema"] +test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so every site renders with the same toolchain, and the diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index 9a03d18a61..07ad9a0ebe 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -249,6 +249,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/63/e876e789525063c840ccfa8857febdabd6523bcef9ce7eb979b9305ea895/griffelib-2.3.0-py3-none-any.whl", hash = "sha256:1b8f9cd525681c26b1d6d574faa1371651e8459ca51d209684f50b8096ae06e0", size = 169423, upload-time = "2026-09-04T15:08:12.956Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/31/b54cd138ee36a32d0047d40fe7bdcb63cefd57ce098a26024161332d15fd/hypothesis-6.168.1.tar.gz", hash = "sha256:fd8acb5c67f260dfbdcc1b2dc95ff48b2b3f90708ae595c76f591bbf3f6f8eeb", size = 510823, upload-time = "2026-09-23T04:36:04.09Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/95/3091b9ad24baa9666d49feea433a8c4060ad47245f0d529d736af3672ddd/hypothesis-6.168.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:5d9cc2d2a4779c92788389564fc6760df7c2909185f6431378133dea6933194f", size = 791345, upload-time = "2026-09-23T04:35:39.379Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b3/b954ffa69f91d111f801030c28d6721f422222af245a846571fc87874242/hypothesis-6.168.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d0e1ec15d32af11da6784d2506de0c901e4c1abf97a3cdcfc144225457783f4e", size = 787104, upload-time = "2026-09-23T04:35:21.799Z" }, + { url = "https://files.pythonhosted.org/packages/e5/dc/732df08f570d845a480d3729bfb46beccc441bfabe2804bb3a458649be73/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cc38519a57c4b919b2c59714a67501d20b46b0c96a0078cbdc9f86e728efce3", size = 1123848, upload-time = "2026-09-23T04:33:48.857Z" }, + { url = "https://files.pythonhosted.org/packages/88/06/e1e937df42243267ba25ede5e9b29921186f290f6c0515729481cc41726e/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7e229e937e761830c7fa6437fb12cb801616a1737af0898abf5edba57d2ad691", size = 1147698, upload-time = "2026-09-23T04:35:33.439Z" }, + { url = "https://files.pythonhosted.org/packages/be/d1/3b41c008dc2a49e43bc4893dcd27ea9d9fef64fef50382ca5f7107680b1f/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:569ba21e5543840895d17633cc6716a9578ac83b1193a2978b3f24cc16b93085", size = 1149280, upload-time = "2026-09-23T04:35:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/71/ea/d54781058fe6a487b81ec6c636f3806fff6a417b48cf3fcd2b003d80f3ce/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a232665716607c3c50c9a3aeeac5929be1d4e58703f5e7cb51db7099445a8c6", size = 1191693, upload-time = "2026-09-23T04:35:57.896Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f2/43bdd07450978611d25989ffdcbe1579acd46c1500a484f8838359fe7880/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9022fe330d8839259da9b496573c4b83ea248bf32029eba23ff589f3c25754bf", size = 1169725, upload-time = "2026-09-23T04:34:32.59Z" }, + { url = "https://files.pythonhosted.org/packages/d1/72/fd4b657cba66940f208bc0eccaceaa777553009d26e86068ff3ab5e55b0b/hypothesis-6.168.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:69be660b831d55ab35619b30abd311a6cb8cd0068a1192210d194a776b211094", size = 1129181, upload-time = "2026-09-23T04:35:27.495Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3f/516a29d7a2bd31e28972de635350eab086c94faa07afcdd3ad0f06e70db8/hypothesis-6.168.1-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e2cccca80ee4ffe8f66edf5b2adb2a16b8cd3c5110b3a774d70f2fc75a42aa", size = 1160162, upload-time = "2026-09-23T04:35:29.333Z" }, + { url = "https://files.pythonhosted.org/packages/54/09/e2c7e32f281b31d4c19cc2fa935bfdde5b31ad4848a485e5ebb8de2389da/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:11cbd2a2539194f0960de4ed9f980882e3a3f55c476afcc6f4cc342eec5a07d2", size = 1299714, upload-time = "2026-09-23T04:35:12.529Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bc/85850bdb1fc35d0fa9e299ca3b2e354f403959666c7ef4d0bfda9c308eee/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:aa1620111616c660a1db03347b1790b5914ef30f86c6e8ddd6c765785291f0b4", size = 1425335, upload-time = "2026-09-23T04:35:23.557Z" }, + { url = "https://files.pythonhosted.org/packages/40/c0/66bdb24f32d32aa5636655b6037af07962118b89fae16ee74bc9a1bb982e/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:a07c8665c95bb856f99ecd60b6f50d94dcf8157415387c32dcd5bb5b5061d032", size = 1376916, upload-time = "2026-09-23T04:34:59.742Z" }, + { url = "https://files.pythonhosted.org/packages/31/f0/7ad0b17bda81c473cbc539009148f3e0950880c4618028691bcf62eeb8d2/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:8a50b6335d5fa2312dd9f5d50faeb953e61475b301d2c073cd7a4c5efd7e6171", size = 1280975, upload-time = "2026-09-23T04:34:21.966Z" }, + { url = "https://files.pythonhosted.org/packages/e2/01/8d04d1eecf03a6097e26400e8393eff7a81fd52c877a664ee4e97adf7880/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:4ccb0276db8ad197a63dff25c4868e11123346347df6900625d2b1f794ce7c19", size = 1300238, upload-time = "2026-09-23T04:34:52.825Z" }, + { url = "https://files.pythonhosted.org/packages/f5/74/eda0c70b3845789c713fe2ac72d3d4586da83f38a34f1f7aefbda0b5b01e/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85ca7788574e06d0591d51e19ecbb1d09bc82d50c6eae5de0fe8317590b1e138", size = 1336088, upload-time = "2026-09-23T04:33:42.002Z" }, + { url = "https://files.pythonhosted.org/packages/80/c9/43d8528e6d43eebb7688d36e530d1fc7763809b8f923b62cc8b4d95f99ff/hypothesis-6.168.1-cp310-abi3-win32.whl", hash = "sha256:77378e04aa47a22a615af80f42970dc44782f02b8a82ac105b619a2cc7f5eee2", size = 678004, upload-time = "2026-09-23T04:34:29.564Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8b/f7d4de5b57972f7f68a6c7fb5698f60425589eae6fcb175f285f2525e8c0/hypothesis-6.168.1-cp310-abi3-win_amd64.whl", hash = "sha256:f94d3dbc31112f8fbcb3af19a662ee5e90e380e6ae676357903caa14a1604763", size = 684722, upload-time = "2026-09-23T04:34:41.279Z" }, + { url = "https://files.pythonhosted.org/packages/a5/fd/528eb221a82da0aa477e34c7af6ed2b4a5109b326638d4418b9e49435e96/hypothesis-6.168.1-cp310-abi3-win_arm64.whl", hash = "sha256:9bd7e8c6e2c3f6a5befee6b03ebd2170da2e50658075c6decad184777c72a726", size = 682711, upload-time = "2026-09-23T04:33:46.085Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1b/620484ba32fbb655ccbd02f3c510e7a961e840a466c3672ef0b877ab8716/hypothesis-6.168.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9fd235c4ccfefe18fd97c181bdf247187cb3d4fdc8800a4d623322234ce4cfd3", size = 792049, upload-time = "2026-09-23T04:34:28.21Z" }, + { url = "https://files.pythonhosted.org/packages/03/df/2ad250c2c4ee4d5f43d6d9bc5d852e7b81c771e4c8946212931b8cafe849/hypothesis-6.168.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0722212cf825d001a0d3b4db9303cc31af467f6814b69acab91d91c3f629c95a", size = 787932, upload-time = "2026-09-23T04:35:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/24/69/4f8e8bf1d5ad81352a05aa0835baef82e691e3e43933bafe6d6f2c171763/hypothesis-6.168.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94a1053575f133ff6978876fd4b4c2a153c1111f1b5f14d6cfb355c306fc4cb5", size = 1124000, upload-time = "2026-09-23T04:34:14.106Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e7/b72a34e0c6b434d3ad2b2e0040fa88d06a15a7880c59bddacf1e27d4902a/hypothesis-6.168.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2c74d496304574cfed537e4f657620be7fee3e287381c0aa4b481211e6ec92", size = 1170172, upload-time = "2026-09-23T04:34:30.93Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6c/afe1b73b58479ba0d75e5408259978d580e96fee812eb4ea2bb83a76f7ff/hypothesis-6.168.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cf4cc6819ba9689effba38dce98a3025554669f76be07e47cc1002e93e8c4918", size = 1300163, upload-time = "2026-09-23T04:35:53.549Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3d/aad54ffe7942c06da79c719fc98daa1dffdd8ac2d384ca64cc3611b1a833/hypothesis-6.168.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7888e0876e1d93fb92a6a4869834b0d0de18f07f6600edbf3d1cad16cf089357", size = 1336274, upload-time = "2026-09-23T04:35:10.655Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cb/8d2a3474e7921cfb5e59f933ac6fe1ad83b95cc98a871af8f0a68dcf946e/hypothesis-6.168.1-cp311-cp311-win_amd64.whl", hash = "sha256:4aa312b6447743a72283698292ea0e1c4d684279b38dc7f270729c4efda6c27a", size = 684479, upload-time = "2026-09-23T04:33:59.307Z" }, + { url = "https://files.pythonhosted.org/packages/7e/6f/9d5ae55d81e10f0fc46868c19ab9d468e9959e82ff8cc8c26990b16114f3/hypothesis-6.168.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:dfe14a88b1ed03ce47333d800cc37949da73feb0cd9767e7ef076d2202768928", size = 793124, upload-time = "2026-09-23T04:33:52.589Z" }, + { url = "https://files.pythonhosted.org/packages/14/11/54c19d259d7f92176aa2199c90f7951e4e396a26898a596dc875ccb9c389/hypothesis-6.168.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8acd000c3bf233cdcd28c1ff03f1bf56cb03821b1c742d00d57978b20fc6e6f8", size = 784674, upload-time = "2026-09-23T04:34:38.137Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ae/f80a9810f88bd654ab0d0ed0652c274e4c0384e22d4f9442b6753b9200a6/hypothesis-6.168.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29280d4743546ddcf74644f9acdc5726972cba9182699a86b7e77f3f8f6e5dde", size = 1122860, upload-time = "2026-09-23T04:33:51.375Z" }, + { url = "https://files.pythonhosted.org/packages/f4/5d/aa5b7ffd57ec475a9252c55985c07a366823c3f85267d44648e94c1ec321/hypothesis-6.168.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd9a488d46b8c7da3103058c7569376a3a08994aee9a7d6f1d160d3d3b49b82d", size = 1168922, upload-time = "2026-09-23T04:34:44.567Z" }, + { url = "https://files.pythonhosted.org/packages/65/10/4862448192d4c9cc61a3b8a620730800f56b1e83b0bfda63dbc728526eb3/hypothesis-6.168.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451ee6c42955887a9e1850c53813e31182eab1663e4d3dacea4e535dedc45b5d", size = 1298568, upload-time = "2026-09-23T04:35:31.191Z" }, + { url = "https://files.pythonhosted.org/packages/15/ad/55e73fda9afbfcf3336b2eee564833bc9a4fee1f7a32fe5f9d183742da39/hypothesis-6.168.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bc2c036f4a9534af64977e38b2976295136f83417fdb95bed816a52cc3023497", size = 1335091, upload-time = "2026-09-23T04:34:23.42Z" }, + { url = "https://files.pythonhosted.org/packages/ef/99/b6340fa21a89b0088ed023fe984109e1c47a07dfd741fd6d8e17a9e2795f/hypothesis-6.168.1-cp312-cp312-win_amd64.whl", hash = "sha256:d603d7c96d030a63701b6ad9bff0bc4870e4c226335b3db509f1eed1f7fc061b", size = 682011, upload-time = "2026-09-23T04:34:34.565Z" }, + { url = "https://files.pythonhosted.org/packages/f5/23/093b9dc768e89d2cec74d29115f4004fe1be64218bac65f8d817d727d30b/hypothesis-6.168.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aebf7e9d7ba9f920e154abc65851d4affab497daeaea285474bed93840a983ea", size = 793062, upload-time = "2026-09-23T04:33:58.177Z" }, + { url = "https://files.pythonhosted.org/packages/df/d7/f191686b44b3f892ae1404e238a55260152a645a2deafe8b2949c7e90fe9/hypothesis-6.168.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b26616604cac458dd83e75900fefd9190f1074e67d5d9dc4d9e321d30f295eca", size = 784518, upload-time = "2026-09-23T04:34:02.012Z" }, + { url = "https://files.pythonhosted.org/packages/10/25/b7a5dca1911c9012a564d0b7aaeb44e393ade9424a695dea179ff32d79fe/hypothesis-6.168.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb6d89cff2edcd45e5af957c7b6a71ae7887419561f7b9c7055e1c50dea41017", size = 1122844, upload-time = "2026-09-23T04:35:06.897Z" }, + { url = "https://files.pythonhosted.org/packages/ea/60/ddacd0d244e6719ea9a9d322d4d4b6b054fd27ada94ec7aa7dce7ef11297/hypothesis-6.168.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a4ce6310ec21000446ae75cdc6667b81d4ee0f2d226a0dfaf8baca42dd151d0", size = 1168779, upload-time = "2026-09-23T04:33:56.885Z" }, + { url = "https://files.pythonhosted.org/packages/dc/07/343bab6676a41f38cb7c5912ab3ca059258bf40e41c181d1b77cc7251801/hypothesis-6.168.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a22c6e730c327d22a4d61b53f66639cdf6f0fe4a5a1b3595c36533e5b8127169", size = 1298434, upload-time = "2026-09-23T04:34:48.576Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bd/4b72fafa5f46a455024a9242b62505b1a53fad95ace162e831c4ffb233b9/hypothesis-6.168.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dcd66493fe180d38641b02dd0456cc1e429655f8940dd1396ab874808f16dd34", size = 1334975, upload-time = "2026-09-23T04:35:01.531Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ae/48643915cda400ca2dccb9bcdc363690b13bcd5c179172bc6450c3d76ce8/hypothesis-6.168.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8a96f2e9fa2a57f95865aaf092943775e7ea96f72535371b2be20fe5464298c", size = 681978, upload-time = "2026-09-23T04:36:00.056Z" }, + { url = "https://files.pythonhosted.org/packages/0c/0e/e6999cddbaef975ecc1b0a7e9028a3d1a1014b542fa003889d0cb3870fa9/hypothesis-6.168.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:35b6582ecc22ed682ea39bbf972f8b8b1798043a3bc624ae1dce02d42fcf3ad4", size = 793097, upload-time = "2026-09-23T04:35:03.374Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f3/2ff6bf4bf43ab12641b163bd8fea4b6cddc9a0f07499c4779519099034c9/hypothesis-6.168.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:48e467c73b7fb46bc072323c5d67e8e683cf32218fbba47c4e6ee0fe0d72687f", size = 784662, upload-time = "2026-09-23T04:35:08.976Z" }, + { url = "https://files.pythonhosted.org/packages/10/3a/b3653de877981cdab6861476c3b68d02b7c682c2875557cff7b92dbde068/hypothesis-6.168.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c82fcc3dece367bd43f3eca5c2746cf60b73f61bba071ef9421d4cfd6188a85", size = 1123106, upload-time = "2026-09-23T04:34:26.476Z" }, + { url = "https://files.pythonhosted.org/packages/49/25/24c725cb94c8953a98d8bdec83971f76c8f4b103070df1fdacd8b7f37253/hypothesis-6.168.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36031c8f8fd4bec7185cf70ed080ba54d45c106cf4f575bc8c2482c01f5f9b21", size = 1168942, upload-time = "2026-09-23T04:35:41.237Z" }, + { url = "https://files.pythonhosted.org/packages/1d/e4/cbe45b9e78e4dd9c18ed2328ff95f659f59235b04ae3c73cf7e9c498adc9/hypothesis-6.168.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ddfe21791579fdc90423973c733578efa23b010cbfb685dcd5201cf7b0a928b6", size = 1298937, upload-time = "2026-09-23T04:33:50.181Z" }, + { url = "https://files.pythonhosted.org/packages/93/e9/cacaa3f58ee6eb98ce29a2182483d57686011bba35c8fb8a9bc8bf981c7f/hypothesis-6.168.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ecf86b1c9c8700f6d45324becbbe25dd581c69dc03e1357a64f4bf9a1afe823f", size = 1335198, upload-time = "2026-09-23T04:36:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/31/d8/ca51626959f59a557656cf8024d9afda4d213257c170412fa3193d11e11f/hypothesis-6.168.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:d5ab336a7dd233858bc3066a69479e295da788cef345d7e4efd7b77dad724db0", size = 624105, upload-time = "2026-09-23T04:34:08.466Z" }, + { url = "https://files.pythonhosted.org/packages/83/61/6ce96e7d58fc7a592ad7c84d8c7cf9ea36db51468b30ac8306bbf4c3e6ba/hypothesis-6.168.1-cp314-cp314-win_amd64.whl", hash = "sha256:63244ba76464202d3d24126d2e4c2c1abf36db60fffe16ebaa93292572c2a42a", size = 681859, upload-time = "2026-09-23T04:34:50.801Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b1/6c7345e2b37ea8a1df2520a0ecfbdc0243a897895ff950309ca7f687429a/hypothesis-6.168.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:6cec76c8cf76738ad4c0c0c5d6e4632a32d74748639ea6ce26fa672490d9bf95", size = 791685, upload-time = "2026-09-23T04:33:54.243Z" }, + { url = "https://files.pythonhosted.org/packages/72/8c/4f6cfafde4b31add263d6a562f615939cd49649e87a3695ac8693506c811/hypothesis-6.168.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9da10ba2dbbcaba93c6a11eb394fd1a0cfee6e2ba21345c3d6f2fc7401d80670", size = 783237, upload-time = "2026-09-23T04:34:25.049Z" }, + { url = "https://files.pythonhosted.org/packages/a0/25/646c301a9d1fe3825a2ff8362594c4aee0ceb2ce192f49469f7f2824a742/hypothesis-6.168.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3218c6d18a041c0eba0cccd88b20d74be24687e42a73338abd0a31ec5294c265", size = 1121414, upload-time = "2026-09-23T04:34:09.876Z" }, + { url = "https://files.pythonhosted.org/packages/27/4f/f84ec8434ef765e6a5c4f9942bbb7652be444b23b9ff868da43e96e61e86/hypothesis-6.168.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760cad0526a84122bbe2a0157217e16d76d649d333422142853c7f1ba43ca44d", size = 1167536, upload-time = "2026-09-23T04:33:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/38/74/c53334d793b0c54054ffc781cdf79ccae6ee51daf347bd9cfa7e0015cece/hypothesis-6.168.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:50a60b57b485dff68b56f99e4506b7f1b52994f801ee1aaa4cb9125a90bf7834", size = 1297122, upload-time = "2026-09-23T04:34:17.235Z" }, + { url = "https://files.pythonhosted.org/packages/43/e7/b115a1faea6d40bddce6342da703b7536d803d2384fa50d277dc55170c64/hypothesis-6.168.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68017d433ca5c558c34e3308a6653fc63077749af8bf86c10850cb6bb3cb559f", size = 1334055, upload-time = "2026-09-23T04:35:37.298Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/f272620d304dd67ff7815509745e2fb8d2bf58e4e1da4b7842a83f1de2da/hypothesis-6.168.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ff92099a0185ba14bf4bc2111299b456074596199148ae84590eee4d480fac2e", size = 681804, upload-time = "2026-09-23T04:35:35.374Z" }, + { url = "https://files.pythonhosted.org/packages/54/e9/45595951f8bc9bd03f032325e7b176ca990a0c86bf2e09d481e8ee5a5cab/hypothesis-6.168.1-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:e9510e33467d586a7700d56b9a6ee65b44a06832c1cab3d270369d7c53381276", size = 791046, upload-time = "2026-09-23T04:35:55.724Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3e/718431d0ab72c4c07454f71040b4de298a63e765f20088557dc1bef0a48b/hypothesis-6.168.1-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:99970e3a75f11dde0414d3d6724bd8b7a2bd1deeb9a509ec9236c91d41a5cff0", size = 782982, upload-time = "2026-09-23T04:34:06.017Z" }, + { url = "https://files.pythonhosted.org/packages/d7/05/c60db6df1456a70a3c589e42acaa446c22f56ed342c1b8dd04b8e49df8fd/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5ea78773de5bcc25266e37eb086472e90a85fc81b8d4a9ca8afc15858ee15fb", size = 1120961, upload-time = "2026-09-23T04:33:43.494Z" }, + { url = "https://files.pythonhosted.org/packages/8b/fe/d943250395750ecc3b4ec95b1ab3c8119f6cbad08c1ba9169ffee7d89049/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:13f502ce2423e46dfcd2af38950a86ecf97084d94e721f7f13dd5a70afb36e0a", size = 1143877, upload-time = "2026-09-23T04:34:56.065Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9d/8c1f8e11f20fd81ce40caed7ad6d6a9776e24775070e87f3aa9ba3683272/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b4bcaa8b0e753fc8921011d5d10c8a7c3caf814c6a968bc140a08917973fdc", size = 1146503, upload-time = "2026-09-23T04:34:04.791Z" }, + { url = "https://files.pythonhosted.org/packages/71/d8/ac2e9fa653d2f05a13724837007e857d376d81fdf728543ab85f6c7f7350/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7ed811908d38ca8e05c20ad25ecb61edc1e20a519667d4b668be49dd09e6bf3e", size = 1189216, upload-time = "2026-09-23T04:34:36.271Z" }, + { url = "https://files.pythonhosted.org/packages/2b/7b/640952448781c7df5098880e2e2d6e7702b3a1c0badd4c0ea5fca6d6ab53/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9239c868418088d58ae18d041f9f424bd7a75dd143f209c35fde38770b65f74f", size = 1166919, upload-time = "2026-09-23T04:35:16.498Z" }, + { url = "https://files.pythonhosted.org/packages/b6/de/0d46b330a28f7614850fb1a0564cf88af7c0b29cbd791e9d04ade7f66f13/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:5cac279798ecc92d3301ec4421c0458e22627f8a79fdde8a6c98ab1d4b8811f6", size = 1126652, upload-time = "2026-09-23T04:35:49.432Z" }, + { url = "https://files.pythonhosted.org/packages/92/36/6adeca1724ed4966095406ffe7308fcff338cf9f5b7443d5ea34a01ab6e1/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bc660dc14d633641e22e407417d35bf2925331273ed71bc6ab5536dfca5c72d", size = 1155687, upload-time = "2026-09-23T04:35:18.234Z" }, + { url = "https://files.pythonhosted.org/packages/bd/00/e9b7c7e4a0a75d1621be95a126999352e01a16d68b0a59b396695c7b4991/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:157a29e4d6bcb77ae732cb83d6546beef47265ecac88408fdd5d5b2bd325685d", size = 1296471, upload-time = "2026-09-23T04:34:12.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/52cb320e9e07cc67f718506617b5900344370394e39f4affada317aab313/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:b125aac89594ee2a76687b724f057b2423ea83dfee6b8ff59f54289d6eef19a1", size = 1421838, upload-time = "2026-09-23T04:34:20.451Z" }, + { url = "https://files.pythonhosted.org/packages/9b/93/969c33a87cd2865beffebe491294b8115e7e7421c6840f79a67486535bfc/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:4b960c9d31b85dbc6a25888897c02bbea5f498844932841612925184fc78d9eb", size = 1373994, upload-time = "2026-09-23T04:34:00.608Z" }, + { url = "https://files.pythonhosted.org/packages/34/c9/5fe4ade4bd23baa2e4cd27668bb8ac10e840ce19fbcfc74116ea529cbaa7/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:d4d76f35874471fe9b8e1bb696fb532363ef3d9588161a06d30520f638f15a39", size = 1278216, upload-time = "2026-09-23T04:34:07.253Z" }, + { url = "https://files.pythonhosted.org/packages/a1/17/30b9a1ad975e90700dea74ce8b08ab07b9e58c4d83918422bc8841942c1e/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:d5147c5f7497d109eed0267c498adaf5cad6e4beeaca6fad6057f2007bae70a7", size = 1297647, upload-time = "2026-09-23T04:35:51.491Z" }, + { url = "https://files.pythonhosted.org/packages/e0/2f/233a9d5adafad6e187ab8f4d9275592cf120b56e63ea1cfb02213c01021e/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:a909f89758e184c4f86418598de297a2ced5b922979c0f98c5d73ff6bafe38e5", size = 1333790, upload-time = "2026-09-23T04:34:42.828Z" }, + { url = "https://files.pythonhosted.org/packages/10/d7/4c9d4d597f70316d781cd99b600adae090a6f5d675487196e10510a23afe/hypothesis-6.168.1-cp315-abi3.abi3t-win32.whl", hash = "sha256:66b9fb6f02840f8ffa2a4e6359aff6ecc1b2c6630c8a2066771044fa22956c7a", size = 675206, upload-time = "2026-09-23T04:35:14.592Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a8/f9fc0c0dbb9e80238fc7145cb4ce52e0897ea417a23217022911855bccda/hypothesis-6.168.1-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:9246eaa80044c33a182be661cbed7e6a7314c61359ef501ea9e9efd90d50ca3b", size = 681503, upload-time = "2026-09-23T04:34:11.223Z" }, + { url = "https://files.pythonhosted.org/packages/11/b4/962ae350c10860be37c9abf98c0ef1ef23e4baca3ee7ad77208941e370e5/hypothesis-6.168.1-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:52e0c804cbf697d4760400c84444a91cbd5dfe7ae0ef1bb3228f74fb5547eb9e", size = 679199, upload-time = "2026-09-23T04:34:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c5/2bfdcc601b08eb9611bc524d18ca0a8cdbbdd7bd07bc68c38e0196a00668/hypothesis-6.168.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:91e3700d9c35e184dd253cff2f151f890557242bede75a2535901e9062d42975", size = 792943, upload-time = "2026-09-23T04:35:25.308Z" }, + { url = "https://files.pythonhosted.org/packages/4b/52/36ae5128d9102d73dc578cd7b241c33f5483bd56e05799a2d06043a1ce7d/hypothesis-6.168.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c19bc5da324a5899527ea749522e9ceb47f1eb54772869673abe59ae0b703aef", size = 788802, upload-time = "2026-09-23T04:34:39.655Z" }, + { url = "https://files.pythonhosted.org/packages/23/34/698fc3481757503dc60f07547c29098dce97ac035a1ee134d66d88814281/hypothesis-6.168.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a4c9e46b2f349658d3013a357e1fbf114bc08abced7ee8e91cd6e0bc718b195", size = 1124766, upload-time = "2026-09-23T04:34:57.869Z" }, + { url = "https://files.pythonhosted.org/packages/02/c9/1f871666106d90048027fc6b309ec505f0fb44e4519add73ddaad98bcd9a/hypothesis-6.168.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a0beee16fcbc0516951f2ef3ef1a479ad11d9726da49840139303b36ebc1d0", size = 1171766, upload-time = "2026-09-23T04:34:54.476Z" }, + { url = "https://files.pythonhosted.org/packages/17/44/eff662526259ac4447dde4abe4fc285d0e8e5ff5345d35af836c6f0f12cc/hypothesis-6.168.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:4a8beb513c066fcb187b885fdfa93da4006f223d50d605b687bfb7e60aac33e9", size = 685449, upload-time = "2026-09-23T04:33:44.837Z" }, +] + [[package]] name = "idna" version = "3.20" @@ -977,6 +1065,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -1051,6 +1148,7 @@ docs = [ { name = "ruff" }, ] test = [ + { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic" }, { name = "pytest" }, @@ -1069,6 +1167,7 @@ docs = [ { name = "ruff", specifier = "==0.16.8" }, ] test = [ + { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic", specifier = ">=2.13" }, { name = "pytest" }, From 5341ead663ab1c7dc46b8127b6973dfe80238dda Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 11:43:09 +0200 Subject: [PATCH 6/9] docs(zarr-metadata): problem() says what a caller does with it Nothing raises `MetadataValidationError` from a rule; a caller that collects problems extends with the tuple `problem()` returns. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/src/zarr_metadata/_typed_json.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 6d19dd2721..2b7de8fd70 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -100,10 +100,9 @@ def problem( ) -> tuple[ValidationProblem, ...]: """One problem, as the one-element tuple every parser returns. - A tuple so that a parser can return it directly and a rule can - `found.extend(problem(...))` and raise `MetadataValidationError(found)` - once. The default `kind` names a type mismatch; a value rule passes - `"invalid_value"`. + A tuple so that a parser can return it directly and a caller that + collects problems can `found.extend(problem(...))`. The default `kind` + names a type mismatch; a value rule passes `"invalid_value"`. """ return (ValidationProblem(loc, message, kind),) From bf489502d72457b8d3034c6a1b29c311fbbdeb78 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 23 Sep 2026 12:26:19 +0200 Subject: [PATCH 7/9] fix(zarr-metadata): check refuses null `check(None, SomeTypedDict)` came back `(None, ())`: no value and no problem. `refine_json` gives JSON's `null` as None with no problem, and `check` read a None refinement as "not JSON, and its problems say why". It now asks the problems whether the value is JSON, so `null` is checked like any other value, and refused as not an object. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/src/zarr_metadata/_typed_json.py | 2 +- packages/zarr-metadata/tests/test_typed_json.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 2b7de8fd70..bf5b5448f8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -936,7 +936,7 @@ def check( msg = f"{shape!r} is not a TypedDict" raise TypeError(msg) refined, problems = refine_json(value, loc) - if refined is None: + if len(problems) != 0: return None, problems typed, found = _checker(shape)(refined, loc) readable = all(problem.kind == "unknown_key" for problem in found) diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index 94ca56c855..f225ba9eae 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -956,6 +956,14 @@ def test_error_check_locates_a_value_that_is_not_json( assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] +def test_error_check_refuses_null() -> None: + # JSON's null is JSON, and not an object: refined, it is None with no + # problem, which is not a verdict. + typed, problems = check(None, Options) + assert typed is None + assert [(problem.loc, problem.kind) for problem in problems] == [((), "invalid_type")] + + def test_error_check_refuses_a_value_of_another_type() -> None: typed, problems = check({"level": "high"}, Options) assert typed is None From d29adafed6e7bfff95f3b77c93ddc38f38ec29d1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 18:09:27 +0200 Subject: [PATCH 8/9] refactor(zarr-metadata): JSON and its problems below the model The checker imported `ValidationProblem`, `ProblemKind`, `refine_json` and `is_json` from the model, so the layer that stands alone depended on the one it feeds. They move, with the rest of what reads JSON and reports on it, into `zarr_metadata._json`: the problem types, `MetadataValidationError`, `refine_json` and `refine_user_data`, `validate_json`, `is_json`, `parse_json` and `arrays_to_tuples`. `zarr_metadata.model` exports every name it did; the checker, and the model's own modules, import from where the names now live. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/src/zarr_metadata/_json.py | 267 ++++++++++++++++++ .../src/zarr_metadata/_typed_json.py | 4 +- .../src/zarr_metadata/model/__init__.py | 14 +- .../src/zarr_metadata/model/_array.py | 6 +- .../src/zarr_metadata/model/_group.py | 12 +- .../src/zarr_metadata/model/_validation.py | 252 +---------------- .../src/zarr_metadata/typed_json.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 8 +- .../zarr-metadata/tests/model/test_group.py | 8 +- .../tests/model/test_refine_json.py | 2 +- 10 files changed, 310 insertions(+), 265 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/_json.py diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py new file mode 100644 index 0000000000..b4dcad614f --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -0,0 +1,267 @@ +"""JSON values, and the problems found with them: where reading starts, below every layer. + +`refine_json` is the one walk that normalizes and judges a value, and +`refine_user_data` the same walk over user data, where a non-finite number +is the float it is. +`ValidationProblem` and `MetadataValidationError` are what every +layer reports with: the checker, the definitions and the model alike. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Literal, TypeGuard, cast, get_args + +from zarr_metadata._common import JSONValue + +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +- `unknown_key`: a key an object's type does not declare, where the type + says it is closed, as a closed TypedDict does. Whether a Zarr + configuration is closed is rarely said (zarr-developers/zarr-specs#270 + has been open since 2023), and many readers refuse such a key. It gets + a kind of its own so that a caller who tolerates it can tell it from a + wrong value, and so that it never masks the other findings about the + same object. +""" + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single problem found in a value: where it is, what is wrong, and what kind of wrong. + + `loc` is the path from the root of what was judged to the offending + value, e.g. `("codecs", 0, "name")` in a document, and an empty `loc` + refers to that root. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. + """ + + loc: tuple[str | int, ...] + message: str + kind: ProblemKind + + def __post_init__(self) -> None: + # The runtime half of the annotations: a rule written without a type + # checker, as an extension's may be, fails where it builds a problem + # rather than reporting one at a location that is not one. + loc = cast("object", self.loc) + if not isinstance(loc, tuple) or not all( + isinstance(part, str) or (isinstance(part, int) and not isinstance(part, bool)) + for part in cast("tuple[object, ...]", loc) + ): + msg = f"a ValidationProblem's loc is a tuple of keys and indices, got {loc!r}" + raise TypeError(msg) + message = cast("object", self.message) + if not isinstance(message, str): + msg = f"a ValidationProblem's message is a string, got {message!r}" + raise TypeError(msg) + kind = cast("object", self.kind) + if not isinstance(kind, str) or kind not in get_args(ProblemKind): + msg = f"a ValidationProblem's kind is one of {get_args(ProblemKind)!r}, got {kind!r}" + raise TypeError(msg) + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails validation, by the entry points that raise rather than report. + + Carries every problem found (not just the first) in `.problems`, as an + immutable tuple: a raised error is a finished report, and a caller + inspecting it must not be able to edit the record. + """ + + problems: tuple[ValidationProblem, ...] + + def __init__(self, problems: Sequence[ValidationProblem]) -> None: + self.problems = tuple(problems) + for entry in cast("tuple[object, ...]", self.problems): + # The runtime half of the annotation: a caller that is not + # type-checked, and hands over anything else, fails here + # rather than far away, where a `loc` is read off it. + if not isinstance(entry, ValidationProblem): + msg = ( + "MetadataValidationError takes ValidationProblem values, " + f"got {type(entry).__name__}" + ) + raise TypeError(msg) + super().__init__("\n".join(str(problem) for problem in self.problems)) + + def __reduce__( + self, + ) -> tuple[ + type[MetadataValidationError], tuple[tuple[ValidationProblem, ...]], dict[str, object] + ]: + # An exception pickles and copies as its class called with its + # `args`, which here are the message; it is built from its problems, + # and the rest of its state -- its notes among it -- follows. + return (type(self), (self.problems,), self.__dict__) + + +def prefixed( + loc_head: str | int, problems: Sequence[ValidationProblem] +) -> tuple[ValidationProblem, ...]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return tuple(ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems) + + +def validate_json(value: object) -> tuple[ValidationProblem, ...]: + """Return every reason `value` is not JSON-serializable (recursively): `refine_json`'s problems.""" + return refine_json(value)[1] + + +def refine_json( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """`value` as JSON with arrays as tuples, or None with every reason it is not JSON. + + One walk normalizes and judges: a mapping becomes a `dict` with + string keys, a sequence a tuple, and a float must be finite. A value + that is not JSON is None, with the problems located at the leaves + that are not. JSON's `null` is None too, with no problem, so whether + `value` is JSON is whether there are problems. + """ + return _refine(value, loc, finite=True) + + +def refine_user_data( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """User data refined as `refine_json` refines JSON, a non-finite number being the float it is. + + A node's attributes are user data. The spec asks only that each be a + JSON value and interprets none, and zarr-python writes them with the + defaults of Python's `json` module, so an attribute can hold `NaN`, + `Infinity` or `-Infinity` -- xarray's `_FillValue`, a CF + `missing_value` -- which RFC 8259 lacks. Wherever the spec interprets + a value it spells those numbers as strings, so a validator walks what + it interprets with `refine_json`, and only user data with this. + """ + return _refine(value, loc, finite=False) + + +_Refined = tuple[JSONValue | None, tuple[ValidationProblem, ...]] + + +def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refined: + """`refine_json`, a non-finite number being JSON unless `finite`.""" + if isinstance(value, float): + if not finite or math.isfinite(value): + return value, () + return None, ( + ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value"), + ) + if isinstance(value, (str, int, bool)) or value is None: + return value, () + if isinstance(value, Mapping): + # Walked here rather than through `_refine_members`, so that each + # level of nesting costs one frame, as deep as the interpreter goes. + members: dict[str, JSONValue] = {} + found_in_members: list[ValidationProblem] = [] + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + found_in_members.append( + ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") + ) + continue + member, found = _refine(item, (*loc, key), finite=finite) + found_in_members.extend(found) + if len(found) == 0: + members[key] = member + return (members if len(found_in_members) == 0 else None), tuple(found_in_members) + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + entries: list[JSONValue] = [] + found_in_entries: list[ValidationProblem] = [] + for index, item in enumerate(cast("Sequence[object]", value)): + entry, found = _refine(item, (*loc, index), finite=finite) + found_in_entries.extend(found) + if len(found) == 0: + entries.append(entry) + return (tuple(entries) if len(found_in_entries) == 0 else None), tuple(found_in_entries) + return None, ( + ValidationProblem(loc, f"not a JSON-serializable value: {value!r}", "invalid_type"), + ) + + +def is_canonical_json(value: object, *, finite: bool = True) -> TypeGuard[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`. + + A non-finite number counts only when `finite` is false, as a document's + guard passes it: where one may be is the document's validator's to say. + """ + if isinstance(value, float): + return not finite or math.isfinite(value) + if isinstance(value, (str, int, bool)) or value is None: + return True + if isinstance(value, (list, tuple)): + sequence = cast("list[object] | tuple[object, ...]", value) + return all(is_canonical_json(item, finite=finite) for item in sequence) + if isinstance(value, dict): + mapping = cast("dict[object, object]", value) + return all( + isinstance(key, str) and is_canonical_json(item, finite=finite) + for key, item in mapping.items() + ) + return False + + +def is_json(value: object) -> TypeGuard[JSONValue]: + """Whether `value` is a canonical JSON structure (recursively).""" + return is_canonical_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" + refined, problems = refine_json(value) + if len(problems) != 0: + raise MetadataValidationError(problems) + return refined + + +def arrays_to_tuples(obj: object) -> object: + """Recursively materialize mappings and convert array-like values to tuples.""" + if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): + sequence = cast("Sequence[object]", obj) + converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) + if isinstance(obj, tuple) and all( + converted is original + for converted, original in zip(converted_sequence, sequence, strict=True) + ): + return sequence + return converted_sequence + if isinstance(obj, Mapping): + mapping = cast("Mapping[object, object]", obj) + converted: dict[object, object] = { + key: arrays_to_tuples(value) for key, value in mapping.items() + } + if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): + return mapping + return converted + return obj + + +__all__ = [ + "MetadataValidationError", + "ProblemKind", + "ValidationProblem", + "arrays_to_tuples", + "is_canonical_json", + "is_json", + "parse_json", + "prefixed", + "refine_json", + "refine_user_data", + "validate_json", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index bf5b5448f8..b8dbc1cd80 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -57,10 +57,10 @@ from typing_extensions import NoExtraItems, TypeIs, is_typeddict from zarr_metadata._common import JSONValue -from zarr_metadata.model._validation import ValidationProblem, is_json, refine_json +from zarr_metadata._json import ValidationProblem, is_json, refine_json if TYPE_CHECKING: - from zarr_metadata.model._validation import ProblemKind + from zarr_metadata._json import ProblemKind T = TypeVar("T") diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index edf3561d1d..2f58f9b8f2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -11,6 +11,14 @@ missing store keys and undecodable bytes. """ +from zarr_metadata._json import ( + MetadataValidationError, + ProblemKind, + ValidationProblem, + is_json, + parse_json, + validate_json, +) from zarr_metadata.model._array import ( ZarrV2ArrayMetadata, ZarrV2ArrayMetadataPartial, @@ -37,26 +45,20 @@ GROUP_METADATA_REQUIRED_KEYS_V2, GROUP_METADATA_REQUIRED_KEYS_V3, GROUP_METADATA_STANDARD_KEYS_V3, - MetadataValidationError, - ProblemKind, - ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, is_group_metadata_v2, is_group_metadata_v3, - is_json, is_metadata_field_v3, parse_array_metadata_v2, parse_array_metadata_v3, parse_group_metadata_v2, parse_group_metadata_v3, - parse_json, parse_metadata_field_v3, validate_array_metadata_v2, validate_array_metadata_v3, validate_group_metadata_v2, validate_group_metadata_v3, - validate_json, validate_metadata_field_v3, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index dde0ffa748..df68471532 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,12 +10,14 @@ from typing_extensions import TypedDict, Unpack +from zarr_metadata._json import ( + MetadataValidationError, + ValidationProblem, +) from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, - MetadataValidationError, StoreKey, - ValidationProblem, dump_store_json, load_store_json, parse_array_metadata_v2, diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 35857f719f..f36bd03f47 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -10,6 +10,13 @@ from typing_extensions import TypedDict, Unpack +from zarr_metadata._json import ( + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + refine_json, + refine_user_data, +) from zarr_metadata.model._array import ( ZarrV3ArrayMetadata, must_understand_subset, @@ -17,16 +24,11 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, - MetadataValidationError, StoreKey, - ValidationProblem, - arrays_to_tuples, dump_store_json, load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, - refine_json, - refine_user_data, validate_consolidated_metadata_v3, ) from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 0b03b82c16..eca0d0ae3c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -17,232 +17,24 @@ from __future__ import annotations import json -import math from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Final, Literal, TypeGuard, TypeVar, cast, get_args - -from zarr_metadata._common import JSONValue +from typing import Final, TypeGuard, TypeVar, cast + +from zarr_metadata._json import ( + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + refine_user_data, + validate_json, +) +from zarr_metadata._json import is_canonical_json as _is_canonical_json +from zarr_metadata._json import prefixed as _prefix from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] -"""Machine-readable classification of a `ValidationProblem`. - -- `missing_key`: a required key (document key or store key) is absent. -- `invalid_type`: a value has the wrong structural type (e.g. a string where - a mapping is required, a non-JSON-serializable object). -- `invalid_value`: a value has an acceptable type but an invalid content - (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). -- `invalid_json`: bytes that do not decode as JSON. -- `unknown_key`: a key an object's type does not declare, where the type - says it is closed, as a closed TypedDict does. Whether a Zarr - configuration is closed is rarely said (zarr-developers/zarr-specs#270 - has been open since 2023), and many readers refuse such a key. It gets - a kind of its own so that a caller who tolerates it can tell it from a - wrong value, and so that it never masks the other findings about the - same object. -""" - - -@dataclass(frozen=True, slots=True) -class ValidationProblem: - """A single structural problem found while validating a metadata document. - - `loc` is the path from the root of what was judged to the offending - value, e.g. `("codecs", 0, "name")`, and an empty `loc` refers to that - root: for the validators here, the document. - `kind` classifies the failure mode for programmatic dispatch; `message` - is the human-readable description. - """ - - loc: tuple[str | int, ...] - message: str - kind: ProblemKind - - def __post_init__(self) -> None: - # The runtime half of the annotations: a rule written without a type - # checker, as an extension's may be, fails where it builds a problem - # rather than reporting one at a location that is not one. - loc = cast("object", self.loc) - if not isinstance(loc, tuple) or not all( - isinstance(part, str) or (isinstance(part, int) and not isinstance(part, bool)) - for part in cast("tuple[object, ...]", loc) - ): - msg = f"a ValidationProblem's loc is a tuple of keys and indices, got {loc!r}" - raise TypeError(msg) - message = cast("object", self.message) - if not isinstance(message, str): - msg = f"a ValidationProblem's message is a string, got {message!r}" - raise TypeError(msg) - kind = cast("object", self.kind) - if not isinstance(kind, str) or kind not in get_args(ProblemKind): - msg = f"a ValidationProblem's kind is one of {get_args(ProblemKind)!r}, got {kind!r}" - raise TypeError(msg) - - def __str__(self) -> str: - location = ".".join(str(part) for part in self.loc) if self.loc else "" - return f"{location}: {self.message}" - - -class MetadataValidationError(ValueError): - """Raised when a value fails structural metadata validation. - - Carries every problem found (not just the first) in `.problems`, as an - immutable tuple: a raised error is a finished report, and a caller - inspecting it must not be able to edit the record. - """ - - problems: tuple[ValidationProblem, ...] - - def __init__(self, problems: Sequence[ValidationProblem]) -> None: - self.problems = tuple(problems) - for entry in cast("tuple[object, ...]", self.problems): - # The runtime half of the annotation: a caller that is not - # type-checked, and hands over anything else, fails here - # rather than far away, where a `loc` is read off it. - if not isinstance(entry, ValidationProblem): - msg = ( - "MetadataValidationError takes ValidationProblem values, " - f"got {type(entry).__name__}" - ) - raise TypeError(msg) - super().__init__("\n".join(str(problem) for problem in self.problems)) - - def __reduce__( - self, - ) -> tuple[ - type[MetadataValidationError], tuple[tuple[ValidationProblem, ...]], dict[str, object] - ]: - # An exception pickles and copies as its class called with its - # `args`, which here are the message; it is built from its problems, - # and the rest of its state -- its notes among it -- follows. - return (type(self), (self.problems,), self.__dict__) - - -def _prefix( - loc_head: str | int, problems: Sequence[ValidationProblem] -) -> tuple[ValidationProblem, ...]: - """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" - return tuple(ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems) - - -def validate_json(value: object) -> tuple[ValidationProblem, ...]: - """Return every reason `value` is not JSON-serializable (recursively): `refine_json`'s problems.""" - return refine_json(value)[1] - - -def refine_json( - value: object, loc: tuple[str | int, ...] = () -) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: - """`value` as JSON with arrays as tuples, or None with every reason it is not JSON. - - One walk normalizes and judges: a mapping becomes a `dict` with - string keys, a sequence a tuple, and a float must be finite. A value - that is not JSON is None, with the problems located at the leaves - that are not. JSON's `null` is None too, with no problem, so whether - `value` is JSON is whether there are problems. - """ - return _refine(value, loc, finite=True) - - -def refine_user_data( - value: object, loc: tuple[str | int, ...] = () -) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: - """User data refined as `refine_json` refines JSON, a non-finite number being the float it is. - - A node's attributes are user data. The spec asks only that each be a - JSON value and interprets none, and zarr-python writes them with the - defaults of Python's `json` module, so an attribute can hold `NaN`, - `Infinity` or `-Infinity` -- xarray's `_FillValue`, a CF - `missing_value` -- which RFC 8259 lacks. Wherever the spec interprets - a value it spells those numbers as strings, so a validator walks what - it interprets with `refine_json`, and only user data with this. - """ - return _refine(value, loc, finite=False) - - -_Refined = tuple[JSONValue | None, tuple[ValidationProblem, ...]] - - -def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refined: - """`refine_json`, a non-finite number being JSON unless `finite`.""" - if isinstance(value, float): - if not finite or math.isfinite(value): - return value, () - return None, ( - ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value"), - ) - if isinstance(value, (str, int, bool)) or value is None: - return value, () - if isinstance(value, Mapping): - # Walked here rather than through `_refine_members`, so that each - # level of nesting costs one frame, as deep as the interpreter goes. - members: dict[str, JSONValue] = {} - found_in_members: list[ValidationProblem] = [] - for key, item in cast("Mapping[object, object]", value).items(): - if not isinstance(key, str): - found_in_members.append( - ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") - ) - continue - member, found = _refine(item, (*loc, key), finite=finite) - found_in_members.extend(found) - if len(found) == 0: - members[key] = member - return (members if len(found_in_members) == 0 else None), tuple(found_in_members) - if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): - entries: list[JSONValue] = [] - found_in_entries: list[ValidationProblem] = [] - for index, item in enumerate(cast("Sequence[object]", value)): - entry, found = _refine(item, (*loc, index), finite=finite) - found_in_entries.extend(found) - if len(found) == 0: - entries.append(entry) - return (tuple(entries) if len(found_in_entries) == 0 else None), tuple(found_in_entries) - return None, ( - ValidationProblem(loc, f"not a JSON-serializable value: {value!r}", "invalid_type"), - ) - - -def _is_canonical_json(value: object, *, finite: bool = True) -> TypeGuard[JSONValue]: - """Whether `value` already uses the concrete containers in `JSONValue`. - - A non-finite number counts only when `finite` is false, as a document's - guard passes it: where one may be is the document's validator's to say. - """ - if isinstance(value, float): - return not finite or math.isfinite(value) - if isinstance(value, (str, int, bool)) or value is None: - return True - if isinstance(value, (list, tuple)): - sequence = cast("list[object] | tuple[object, ...]", value) - return all(_is_canonical_json(item, finite=finite) for item in sequence) - if isinstance(value, dict): - mapping = cast("dict[object, object]", value) - return all( - isinstance(key, str) and _is_canonical_json(item, finite=finite) - for key, item in mapping.items() - ) - return False - - -def is_json(value: object) -> TypeGuard[JSONValue]: - """Whether `value` is a canonical JSON structure (recursively).""" - return _is_canonical_json(value) - - -def parse_json(value: object) -> JSONValue: - """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" - refined, problems = refine_json(value) - if len(problems) != 0: - raise MetadataValidationError(problems) - return refined - - # The standard top-level keys of a v3 array metadata document. Anything outside # this set is an extension field. Built from the TypedDict's required/optional # key sets (which resolve inherited keys, unlike `__annotations__`). @@ -980,25 +772,3 @@ def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: validator is what keeps one out of anywhere else. """ return json.dumps(value, indent=indent, allow_nan=True).encode("utf-8") - - -def arrays_to_tuples(obj: object) -> object: - """Recursively materialize mappings and convert array-like values to tuples.""" - if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): - sequence = cast("Sequence[object]", obj) - converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) - if isinstance(obj, tuple) and all( - converted is original - for converted, original in zip(converted_sequence, sequence, strict=True) - ): - return sequence - return converted_sequence - if isinstance(obj, Mapping): - mapping = cast("Mapping[object, object]", obj) - converted: dict[object, object] = { - key: arrays_to_tuples(value) for key, value in mapping.items() - } - if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): - return mapping - return converted - return obj diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index a76d863dd3..6fb5c14db4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -65,8 +65,8 @@ """ from zarr_metadata._common import JSONValue +from zarr_metadata._json import ProblemKind, ValidationProblem from zarr_metadata._typed_json import Loc, TypedDictKeys, check, typeddict_keys -from zarr_metadata.model._validation import ProblemKind, ValidationProblem __all__ = [ "JSONValue", diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index b92b86e14e..5fab5d9b9b 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -13,6 +13,7 @@ from typing_extensions import Unpack from tests.model._cases import Expect, ExpectFail, mutate_nested_containers +from zarr_metadata._json import arrays_to_tuples, prefixed from zarr_metadata.model import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V3, @@ -41,7 +42,6 @@ validate_json, validate_metadata_field_v3, ) -from zarr_metadata.model._validation import _prefix, arrays_to_tuples if TYPE_CHECKING: from zarr_metadata._common import JSONValue @@ -1349,10 +1349,10 @@ def test_error_the_error_refuses_what_is_not_a_problem() -> None: def test_prefix_prepends_loc_head() -> None: - """_prefix prepends a loc head to each problem's loc.""" + """`prefixed` prepends a loc head to each problem's loc.""" problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] - prefixed = _prefix(0, problems) - assert prefixed == ( + located = prefixed(0, problems) + assert located == ( ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type"), ) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 391f1240c5..845e303fd4 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -9,6 +9,11 @@ import pytest from tests.model._cases import mutate_nested_containers +from zarr_metadata._json import ( + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, +) from zarr_metadata.model import UNSET from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._group import ( @@ -20,9 +25,6 @@ ZarrV3GroupMetadataPartial, ) from zarr_metadata.model._validation import ( - MetadataValidationError, - ValidationProblem, - arrays_to_tuples, is_group_metadata_v2, is_group_metadata_v3, parse_group_metadata_v2, diff --git a/packages/zarr-metadata/tests/model/test_refine_json.py b/packages/zarr-metadata/tests/model/test_refine_json.py index 2440cf95c2..5ce759d40a 100644 --- a/packages/zarr-metadata/tests/model/test_refine_json.py +++ b/packages/zarr-metadata/tests/model/test_refine_json.py @@ -8,7 +8,7 @@ import pytest -from zarr_metadata.model._validation import refine_json, refine_user_data +from zarr_metadata._json import refine_json, refine_user_data if TYPE_CHECKING: from collections.abc import Callable From d564657641318fd664db47d5db36450df555392a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 26 Sep 2026 23:03:48 +0200 Subject: [PATCH 9/9] docs(zarr-metadata): the checker's fragment takes this PR's number Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/{347.feature.md => 4432.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{347.feature.md => 4432.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/347.feature.md b/packages/zarr-metadata/changes/4432.feature.md similarity index 100% rename from packages/zarr-metadata/changes/347.feature.md rename to packages/zarr-metadata/changes/4432.feature.md