diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 266224164..40d2d3b3f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -29,8 +29,9 @@ env: WEAVIATE_135: 1.35.23 WEAVIATE_136: 1.36.23 WEAVIATE_137: 1.37.14 - WEAVIATE_138: 1.38.9 - WEAVIATE_139: 1.39.0 + WEAVIATE_138: 1.38.20 + WEAVIATE_139: 1.39.10 + WEAVIATE_140: 1.40.0 jobs: lint-and-format: @@ -150,7 +151,7 @@ jobs: - name: start weaviate run: | source ./ci/compose.sh - export WEAVIATE_VERSION=$WEAVIATE_139 + export WEAVIATE_VERSION=$WEAVIATE_140 docker compose -f ci/docker-compose-async.yml up -d wait "http://localhost:8090" - name: Run the e2e suite inside Pyodide under Node @@ -223,11 +224,11 @@ jobs: fail-fast: false matrix: versions: [ - { py: "3.10", weaviate: $WEAVIATE_139, grpc: "1.59.0"}, - { py: "3.11", weaviate: $WEAVIATE_139, grpc: "1.66.0"}, - { py: "3.12", weaviate: $WEAVIATE_139, grpc: "1.70.0"}, - { py: "3.13", weaviate: $WEAVIATE_139, grpc: "1.72.1"}, - { py: "3.14", weaviate: $WEAVIATE_139, grpc: "1.76.0"} + { py: "3.10", weaviate: $WEAVIATE_140, grpc: "1.59.0"}, + { py: "3.11", weaviate: $WEAVIATE_140, grpc: "1.66.0"}, + { py: "3.12", weaviate: $WEAVIATE_140, grpc: "1.70.0"}, + { py: "3.13", weaviate: $WEAVIATE_140, grpc: "1.72.1"}, + { py: "3.14", weaviate: $WEAVIATE_140, grpc: "1.76.0"} ] optional_dependencies: [false] steps: @@ -279,11 +280,11 @@ jobs: fail-fast: false matrix: versions: [ - { py: "3.10", weaviate: $WEAVIATE_139}, - { py: "3.11", weaviate: $WEAVIATE_139}, - { py: "3.12", weaviate: $WEAVIATE_139}, - { py: "3.13", weaviate: $WEAVIATE_139}, - { py: "3.14", weaviate: $WEAVIATE_139} + { py: "3.10", weaviate: $WEAVIATE_140}, + { py: "3.11", weaviate: $WEAVIATE_140}, + { py: "3.12", weaviate: $WEAVIATE_140}, + { py: "3.13", weaviate: $WEAVIATE_140}, + { py: "3.14", weaviate: $WEAVIATE_140} ] optional_dependencies: [false] steps: @@ -385,7 +386,8 @@ jobs: $WEAVIATE_136, $WEAVIATE_137, $WEAVIATE_138, - $WEAVIATE_139 + $WEAVIATE_139, + $WEAVIATE_140 ] steps: - name: Checkout diff --git a/docs/changelog.rst b/docs/changelog.rst index 007e75ab7..c795db55e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,9 @@ This minor version includes: - Backwards compatibility: - Every ``_``-prefixed spelling is still importable from the module that defines it, and is the same object as its public counterpart, so ``isinstance`` checks and existing imports of it are unaffected - A private spelling that was only ever reachable through a module that happened to import it — the executor modules, and a few of the ``config_*`` modules — is no longer reachable there. Import configuration types from ``weaviate.classes.config``, or from ``weaviate.collections.classes.config`` + - Minor bug fixes and improvements: + - Fix ``collection.config.get()`` dropping the multi-vector and MUVERA settings of an HFresh vector index + - Add the ``LOADING``, ``LAZY_LOADING``, ``SHUTDOWN``, ``UNAVAILABLE`` and ``RECOVERING`` shard statuses to ``ShardTypes``, and type ``ShardStatus.per_node_status`` with it Version 4.23.1 -------------- diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index d0ecad251..70f0af070 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -881,7 +881,7 @@ def test_collection_update_shards(collection_factory: CollectionFactory) -> None collection.tenants.create([Tenant(name="tenant1"), Tenant(name="tenant2")]) for shard in collection.config.get_shards(): if shard.per_node_status: - assert all(per_node == "READY" for per_node in shard.per_node_status) + assert all(per_node == "READY" for per_node in shard.per_node_status.values()) assert shard.status == "READY" # all possibilites of calling the function diff --git a/integration/test_collection_hfresh.py b/integration/test_collection_hfresh.py index 9c93fb0a5..a692c0a5f 100644 --- a/integration/test_collection_hfresh.py +++ b/integration/test_collection_hfresh.py @@ -7,6 +7,7 @@ VectorDistances, VectorIndexType, Vectorizers, + _MuveraConfig, _VectorIndexConfigHFresh, ) @@ -122,3 +123,32 @@ def test_collection_hfresh_export_and_reimport(collection_factory: CollectionFac assert config == new assert config.to_dict() == new.to_dict() client.collections.delete(name) + + +def test_collection_hfresh_multi_vector_muvera(collection_factory: CollectionFactory) -> None: + collection_dummy = collection_factory("dummy") + if collection_dummy._connection._weaviate_version.is_lower_than(1, 40, 0): + pytest.skip("Multi-vector HFresh is not supported in Weaviate versions lower than 1.40.0") + + collection = collection_factory( + vector_config=[ + Configure.MultiVectors.self_provided( + name="colbert", + encoding=Configure.VectorIndex.MultiVector.Encoding.muvera( + ksim=4, dprojections=16, repetitions=10 + ), + vector_index_config=Configure.VectorIndex.hfresh(), + ), + ], + ) + + config = collection.config.get() + + assert config.vector_config is not None + index_config = config.vector_config["colbert"].vector_index_config + assert isinstance(index_config, _VectorIndexConfigHFresh) + assert index_config.multi_vector is not None + assert isinstance(index_config.multi_vector.encoding, _MuveraConfig) + assert index_config.multi_vector.encoding.ksim == 4 + assert index_config.multi_vector.encoding.dprojections == 16 + assert index_config.multi_vector.encoding.repetitions == 10 diff --git a/integration/test_rbac.py b/integration/test_rbac.py index 4f89cf6d2..5f0fe5479 100644 --- a/integration/test_rbac.py +++ b/integration/test_rbac.py @@ -372,31 +372,6 @@ ), 32, ), - ( - Permissions.alias(alias="", collection="*", read=True, delete=True), - Role( - name="AlliasRole2", - alias_permissions=[ - AliasPermissionOutput( - alias="*", - collection="*", - actions={Actions.Alias.READ, Actions.Alias.DELETE}, - ) - ], - cluster_permissions=[], - users_permissions=[], - collections_permissions=[], - roles_permissions=[], - data_permissions=[], - backups_permissions=[], - mcp_permissions=[], - nodes_permissions=[], - tenants_permissions=[], - replicate_permissions=[], - groups_permissions=[], - ), - 32, # Minimum version for alias permissions - ), ( Permissions.alias(alias="*", collection="*", read=True, delete=True), Role( diff --git a/mock_tests/test_collection.py b/mock_tests/test_collection.py index 4910ae2fd..49133f4a4 100644 --- a/mock_tests/test_collection.py +++ b/mock_tests/test_collection.py @@ -27,6 +27,7 @@ ReplicationConfig, ReplicationDeletionStrategy, ShardingConfig, + ShardStatus, StopwordsConfig, StopwordsPreset, VectorDistances, @@ -563,6 +564,36 @@ def test_delete_vector_index(weaviate_mock: HTTPServer) -> None: client.collections.use("test").config.delete_vector_index(42) # type: ignore[arg-type] +def test_get_shards_parses_all_statuses(weaviate_mock: HTTPServer) -> None: + weaviate_mock.expect_request("/v1/schema/Test/shards", method="GET").respond_with_json( + response_json=[ + { + "name": "shard1", + "status": "UNAVAILABLE", + "vectorQueueSize": 0, + "per_node_status": {"node1": "READY", "node2": "UNAVAILABLE"}, + }, + {"name": "shard2", "status": "RECOVERING", "vectorQueueSize": 3}, + ], + status=200, + ) + + with weaviate.connect_to_local( + port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True + ) as client: + shards = client.collections.use("test").config.get_shards() + + assert shards == [ + ShardStatus( + name="shard1", + status="UNAVAILABLE", + vector_queue_size=0, + per_node_status={"node1": "READY", "node2": "UNAVAILABLE"}, + ), + ShardStatus(name="shard2", status="RECOVERING", vector_queue_size=3, per_node_status=None), + ] + + def test_create_from_dict_skips_dropped_vectors(weaviate_mock: HTTPServer) -> None: """Entries with vectorIndexType "none" cannot be re-created and are stripped before the POST.""" bodies: List[Dict[str, Any]] = [] diff --git a/requirements-devel.txt b/requirements-devel.txt index c7c99e56a..b141d0892 100644 --- a/requirements-devel.txt +++ b/requirements-devel.txt @@ -1,6 +1,6 @@ httpx==0.26.0 validators==0.34.0 -authlib==1.6.12 +authlib==1.7.2 grpcio==1.75.1 grpcio-tools==1.75.1 grpcio-health-checking==1.75.1 diff --git a/test/collection/test_config_methods.py b/test/collection/test_config_methods.py index 748fdf7e1..38ad9616b 100644 --- a/test/collection/test_config_methods.py +++ b/test/collection/test_config_methods.py @@ -1,8 +1,14 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional import pytest -from weaviate.collections.classes.config import VectorIndexType, _VectorIndexConfigNone +from weaviate.collections.classes.config import ( + VectorIndexType, + _MultiVectorConfig, + _MuveraConfig, + _VectorIndexConfigHFresh, + _VectorIndexConfigNone, +) from weaviate.exceptions import SchemaValidationError from weaviate.collections.classes.config_methods import ( _collection_config_from_json, @@ -83,6 +89,75 @@ def test_collection_config_from_json_with_dropped_vector_index() -> None: assert as_dict["vectorConfig"]["kept"]["vectorIndexType"] == VectorIndexType.HNSW.value +HFRESH_CONFIG = { + "distance": "cosine", + "maxPostingSizeKB": 48, + "replicas": 4, + "searchProbe": 256, + "rq": {"enabled": True, "bits": 1, "rescoreLimit": 350, "trainingLimit": 0}, +} + + +@pytest.mark.parametrize( + "multivector,expected", + [ + ( + { + "enabled": True, + "aggregation": "", + "muvera": {"enabled": True, "ksim": 4, "dprojections": 16, "repetitions": 10}, + }, + _MultiVectorConfig( + aggregation="", + encoding=_MuveraConfig(enabled=True, ksim=4, dprojections=16, repetitions=10), + ), + ), + ( + { + "enabled": True, + "aggregation": "", + "muvera": {"enabled": False, "ksim": 4, "dprojections": 16, "repetitions": 10}, + }, + _MultiVectorConfig(aggregation="", encoding=None), + ), + ( + { + "enabled": False, + "aggregation": "", + "muvera": {"enabled": False, "ksim": 4, "dprojections": 16, "repetitions": 10}, + }, + None, + ), + # servers before 1.40 omit the multivector block for HFresh + (None, None), + ], +) +def test_collection_config_from_json_hfresh_multi_vector( + multivector: Optional[Dict[str, Any]], expected: Optional[_MultiVectorConfig] +) -> None: + index_config = dict(HFRESH_CONFIG) + if multivector is not None: + index_config["multivector"] = multivector + schema = _schema_with_vector_config( + { + "mv": { + "vectorizer": {"none": {}}, + "vectorIndexType": "hfresh", + "vectorIndexConfig": index_config, + } + } + ) + + config = _collection_config_from_json(schema) + + assert config.vector_config is not None + vic = config.vector_config["mv"].vector_index_config + assert isinstance(vic, _VectorIndexConfigHFresh) + assert vic.multi_vector == expected + assert vic.max_posting_size_kb == 48 + assert vic.search_probe == 256 + + def test_collection_config_from_json_missing_vector_index_config_raises() -> None: """A non-dropped vector missing its vectorIndexConfig must fail fast, not parse as None.""" schema = _schema_with_vector_config( diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index 390578756..07f92cb79 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -2407,7 +2407,16 @@ class _CollectionConfigSimple(_ConfigBase): CollectionConfigSimple = _CollectionConfigSimple -ShardTypes = Literal["READONLY", "READY", "INDEXING"] +ShardTypes = Literal[ + "READONLY", + "READY", + "INDEXING", + "LOADING", + "LAZY_LOADING", + "SHUTDOWN", + "UNAVAILABLE", + "RECOVERING", +] @dataclass @@ -2415,7 +2424,7 @@ class _ShardStatus: name: str status: ShardTypes vector_queue_size: int - per_node_status: Optional[Dict[str, str]] + per_node_status: Optional[Dict[str, ShardTypes]] ShardStatus = _ShardStatus diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index c87350c3f..846c06fd5 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -227,7 +227,7 @@ def __get_hfresh_config(config: Dict[str, Any]) -> _VectorIndexConfigHFresh: replicas=config["replicas"], search_probe=config["searchProbe"], quantizer=quantizer, - multi_vector=None, + multi_vector=__get_multivector(config), ) diff --git a/weaviate/embedded.py b/weaviate/embedded.py index 545b3bbc4..6deef4684 100644 --- a/weaviate/embedded.py +++ b/weaviate/embedded.py @@ -31,7 +31,7 @@ DEFAULT_PORT = 8079 DEFAULT_GRPC_PORT = 50060 -WEAVIATE_VERSION = "1.39.7" +WEAVIATE_VERSION = "1.40.0" @dataclass