From f2247426aba6f2cd2fbe993a21269786f656caa7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 21 May 2026 06:32:23 +0200 Subject: [PATCH 1/9] Revert "CI: Lock REST Fixture to 1.10.1 (#3240)" This reverts commit 03bbbe384a809226acd1fcde239786cf31080f50. --- dev/docker-compose-integration.yml | 2 +- tests/integration/test_catalog.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 387e54ac2c..03f5684ce4 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -44,7 +44,7 @@ services: retries: 5 start_period: 90s rest: - image: apache/iceberg-rest-fixture:1.10.1 + image: apache/iceberg-rest-fixture container_name: pyiceberg-rest networks: iceberg_net: diff --git a/tests/integration/test_catalog.py b/tests/integration/test_catalog.py index 751dbe0479..590c901914 100644 --- a/tests/integration/test_catalog.py +++ b/tests/integration/test_catalog.py @@ -618,7 +618,6 @@ def test_register_table_existing(test_catalog: Catalog, table_schema_nested: Sch @pytest.mark.integration -@pytest.mark.skip(reason="Requires Iceberg REST Fixtures 1.11.x") def test_rest_custom_namespace_separator(rest_catalog: RestCatalog, table_schema_simple: Schema) -> None: """ Tests that the REST catalog correctly picks up the namespace-separator from the config endpoint. From d7c7b1958d1c9064cb947f0e2e743aff645ba977 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 21 May 2026 06:41:26 +0200 Subject: [PATCH 2/9] Bump to Iceberg 1.11.0 --- dev/spark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile index 0e1f29d152..e6cba7b589 100644 --- a/dev/spark/Dockerfile +++ b/dev/spark/Dockerfile @@ -19,7 +19,7 @@ FROM apache/spark:${BASE_IMAGE_SPARK_VERSION} # Dependency versions - keep these compatible # Changing these will invalidate the JAR download cache layer -ARG ICEBERG_VERSION=1.10.1 +ARG ICEBERG_VERSION=1.11.0 ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13 ARG HADOOP_VERSION=3.4.1 ARG AWS_SDK_VERSION=2.24.6 From ab93c5d6c922faa7fdef70e7b6f0dc6010fae74a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 26 May 2026 12:58:43 +0200 Subject: [PATCH 3/9] Update test --- tests/integration/test_catalog.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_catalog.py b/tests/integration/test_catalog.py index 590c901914..cc1fd80e15 100644 --- a/tests/integration/test_catalog.py +++ b/tests/integration/test_catalog.py @@ -689,19 +689,19 @@ def test_namespace_with_slash(test_catalog: Catalog) -> None: namespace = ("new/db",) - if test_catalog.namespace_exists(namespace): + if namespace in test_catalog.list_namespaces(): test_catalog.drop_namespace(namespace) - assert not test_catalog.namespace_exists(namespace) + assert namespace not in test_catalog.list_namespaces() test_catalog.create_namespace(namespace) - assert test_catalog.namespace_exists(namespace) + assert namespace in test_catalog.list_namespaces() properties = test_catalog.load_namespace_properties(namespace) assert properties is not None test_catalog.drop_namespace(namespace) - assert not test_catalog.namespace_exists(namespace) + assert namespace not in test_catalog.list_namespaces() @pytest.mark.integration From 7294ea906a4418f3fd8891f139e33cf4d2bd3a5a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Jun 2026 04:41:20 +0200 Subject: [PATCH 4/9] CI: Pass args through docker-compose --- .gitignore | 2 ++ dev/docker-compose-integration.yml | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ef8c522482..792ec66e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ .cache tmp/ site +.env +dev/.env # intellij files .idea diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 03f5684ce4..32cdccbb1a 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -18,7 +18,10 @@ services: spark-iceberg: image: pyiceberg-spark:latest - build: spark/ + build: + context: spark/ + args: + MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo.maven.apache.org/maven2} container_name: pyiceberg-spark networks: iceberg_net: @@ -93,7 +96,10 @@ services: " hive: image: pyiceberg-hive:latest - build: hive/ + build: + context: hive/ + args: + MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo1.maven.org/maven2} container_name: pyiceberg-hive hostname: hive networks: From 27819c2a6ec82116d96394ba0a6992a0910ea389 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Jun 2026 05:34:26 +0200 Subject: [PATCH 5/9] Disallow slashes in namespaces Similar to Java --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index d7cdba012b..eb0b712d1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3320,7 +3320,7 @@ def does_support_slash_in_identifier(catalog: Catalog) -> bool: from pyiceberg.catalog.sql import SqlCatalog if isinstance(catalog, RestCatalog): - return property_as_bool(catalog.properties, "supports_slash_in_identifier", True) + return property_as_bool(catalog.properties, "supports_slash_in_identifier", False) from pyiceberg.catalog.hive import HiveCatalog if isinstance(catalog, (HiveCatalog, NoopCatalog, SqlCatalog)): From 3495a2c70b90bd3fcb44f3c2fb9ae163cc38cd2f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Jun 2026 05:38:46 +0200 Subject: [PATCH 6/9] Revert "CI: Pass args through docker-compose" This reverts commit 7294ea906a4418f3fd8891f139e33cf4d2bd3a5a. --- .gitignore | 2 -- dev/docker-compose-integration.yml | 10 ++-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 792ec66e6f..ef8c522482 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,6 @@ .cache tmp/ site -.env -dev/.env # intellij files .idea diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 32cdccbb1a..03f5684ce4 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -18,10 +18,7 @@ services: spark-iceberg: image: pyiceberg-spark:latest - build: - context: spark/ - args: - MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo.maven.apache.org/maven2} + build: spark/ container_name: pyiceberg-spark networks: iceberg_net: @@ -96,10 +93,7 @@ services: " hive: image: pyiceberg-hive:latest - build: - context: hive/ - args: - MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo1.maven.org/maven2} + build: hive/ container_name: pyiceberg-hive hostname: hive networks: From f210bca80fbc1f7dc3ea985af922390b9accdca0 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Jun 2026 05:45:47 +0200 Subject: [PATCH 7/9] Reapply "CI: Pass args through docker-compose" This reverts commit 3495a2c70b90bd3fcb44f3c2fb9ae163cc38cd2f. --- .gitignore | 2 ++ dev/docker-compose-integration.yml | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ef8c522482..792ec66e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ .cache tmp/ site +.env +dev/.env # intellij files .idea diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 03f5684ce4..32cdccbb1a 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -18,7 +18,10 @@ services: spark-iceberg: image: pyiceberg-spark:latest - build: spark/ + build: + context: spark/ + args: + MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo.maven.apache.org/maven2} container_name: pyiceberg-spark networks: iceberg_net: @@ -93,7 +96,10 @@ services: " hive: image: pyiceberg-hive:latest - build: hive/ + build: + context: hive/ + args: + MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo1.maven.org/maven2} container_name: pyiceberg-hive hostname: hive networks: From 141bc6519c128f3238d7b3fd175f1b66a36adf6a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Jun 2026 05:57:22 +0200 Subject: [PATCH 8/9] Add missing column --- pyiceberg/table/inspect.py | 2 ++ tests/integration/test_inspect_table.py | 1 + 2 files changed, 3 insertions(+) diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index 41d5bf76be..5cae743313 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -404,6 +404,7 @@ def _get_all_manifests_schema(self) -> pa.Schema: all_manifests_schema = self._get_manifests_schema() all_manifests_schema = all_manifests_schema.append(pa.field("reference_snapshot_id", pa.int64(), nullable=False)) + all_manifests_schema = all_manifests_schema.append(pa.field("key_metadata", pa.binary(), nullable=True)) return all_manifests_schema def _generate_manifests_table(self, snapshot: Snapshot | None, is_all_manifests_table: bool = False) -> pa.Table: @@ -468,6 +469,7 @@ def _partition_summaries_to_rows( } if is_all_manifests_table: manifest_row["reference_snapshot_id"] = snapshot.snapshot_id + manifest_row["key_metadata"] = manifest.key_metadata manifests.append(manifest_row) return pa.Table.from_pylist( diff --git a/tests/integration/test_inspect_table.py b/tests/integration/test_inspect_table.py index 03d4437d18..4d8dfbe9bb 100644 --- a/tests/integration/test_inspect_table.py +++ b/tests/integration/test_inspect_table.py @@ -1012,6 +1012,7 @@ def test_inspect_all_manifests(spark: SparkSession, session_catalog: Catalog, fo "deleted_delete_files_count", "partition_summaries", "reference_snapshot_id", + "key_metadata", ] int_cols = [ From 029b5694d5cbf76b428ca7faf1dfeb6d3a7991f6 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Jun 2026 05:57:33 +0200 Subject: [PATCH 9/9] Revert "Reapply "CI: Pass args through docker-compose"" This reverts commit f210bca80fbc1f7dc3ea985af922390b9accdca0. --- .gitignore | 2 -- dev/docker-compose-integration.yml | 10 ++-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 792ec66e6f..ef8c522482 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,6 @@ .cache tmp/ site -.env -dev/.env # intellij files .idea diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 32cdccbb1a..03f5684ce4 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -18,10 +18,7 @@ services: spark-iceberg: image: pyiceberg-spark:latest - build: - context: spark/ - args: - MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo.maven.apache.org/maven2} + build: spark/ container_name: pyiceberg-spark networks: iceberg_net: @@ -96,10 +93,7 @@ services: " hive: image: pyiceberg-hive:latest - build: - context: hive/ - args: - MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo1.maven.org/maven2} + build: hive/ container_name: pyiceberg-hive hostname: hive networks: