From 27e3c938e07b1a9ee860da2c59a3eecb7e993a76 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 15 Oct 2026 10:59:30 +0000 Subject: [PATCH 1/9] pyarrow: add 16.1.0 for apache-beam's pyarrow<17.0.0 pin apache-beam 2.54.0-2.61.0 (the window apache-flink needs for its cimport'd Cython bindings) all pin pyarrow<17.0.0; our registry only carried 25.0.1/26.0.0. 16.1.0 shares the existing bundled-Arrow-C++ build (env-var driven, same as 25.0.1/26.0.0); the build matrix drops cp313/cp314/cp314t for this version since it predates cp313's release by ~5 months and upstream itself never shipped past cp312 for it. --- .github/workflows/build-pyarrow.yml | 9 +++++++++ docs/packages/pyarrow.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 43c0b6219050..e273c9aa20c0 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -77,6 +77,15 @@ jobs: - cp313-cp313 - cp314-cp314 - cp314-cp314t + exclude: + # 16.1.0 predates cp313 (released Oct 2024, ~5 months after this + # release) and never shipped a wheel past cp312 upstream either. + - version: "16.1.0" + python: cp313-cp313 + - version: "16.1.0" + python: cp314-cp314 + - version: "16.1.0" + python: cp314-cp314t env: PYARROW_VERSION: ${{ matrix.version }} diff --git a/docs/packages/pyarrow.yaml b/docs/packages/pyarrow.yaml index 95b97b7635b5..948bfb0b19eb 100644 --- a/docs/packages/pyarrow.yaml +++ b/docs/packages/pyarrow.yaml @@ -2,6 +2,7 @@ package-name: pyarrow source-code: https://github.com/apache/arrow license: Apache-2.0 versions: +- version: 16.1.0 - version: 25.0.1 tag: pyarrow-v25.0.1-20260902125649 files: From 333f257040e4803047c93dc2af37b23211f71d4a Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 18:00:11 +0000 Subject: [PATCH 2/9] pyarrow: fix bundled gflags configure under CMake >= 4 ARROW_DEPENDENCY_SOURCE=BUNDLED builds gflags via ExternalProject_Add using its vendored cmake_minimum_required(VERSION 2.8.12), below CMake 4's removed-compatibility floor of 3.5. The riscv64 image's pipx-installed cmake is >= 4, so the gflags_ep configure step hard-errors before any other bundled dependency gets a chance to build. Export CMAKE_POLICY_VERSION_MINIMUM=3.5 for the whole build: CMake reads it from the environment with the same effect as the -D cache flag, so it reaches every nested cmake invocation ExternalProject_Add makes without needing to patch Arrow's own CMakeLists.txt. --- .github/workflows/build-pyarrow.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index e273c9aa20c0..fc39c607b23f 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -144,6 +144,14 @@ jobs: DIST=/tmp/arrow-dist BUILD=/tmp/arrow-build + # BUNDLED thirdparty deps are built via ExternalProject_Add with their own + # vendored cmake_minimum_required (e.g. gflags' is 2.8.12), below CMake 4's + # floor of 3.5. The image's cmake (pipx-installed, >= 4) hard-errors on that + # instead of warning. We don't control the ExternalProject configure line, but + # CMake also reads CMAKE_POLICY_VERSION_MINIMUM from the environment, which + # reaches every nested cmake invocation this build makes (gotcha 257). + export CMAKE_POLICY_VERSION_MINIMUM=3.5 + # ninja and OpenSSL headers aren't in the base image dnf install -y --setopt=install_weak_deps=False ninja-build openssl-devel From 77b935d3096bb5b337377ce12c915398a50d39dc Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Wed, 28 Oct 2026 11:01:21 +0000 Subject: [PATCH 3/9] pyarrow: force-include for 16.1.0's bundled Abseil Arrow 16.1.0 pins Abseil 20211102.0 for its BUNDLED absl_ep. That release's absl/strings/internal/str_format/extension.h forward-declares `enum class FormatConversionChar : uint8_t` and `enum class FormatConversionCharSet : uint64_t` without including , relying on libstdc++ headers to pull it in transitively. Since GCC 13, libstdc++ no longer does, and manylinux_2_39_riscv64 ships GCC 14, so both declarations fail and str_format's extension.cc and float_conversion.cc cascade into "FormatConversionCharSet was not declared" / "no member named conversion_char". Abseil added the include upstream in later LTS releases; 25.0.1/26.0.0 pin 20250127.0 and never hit it. Pass -DCMAKE_CXX_FLAGS="-include cstdint": Arrow copies CMAKE_CXX_FLAGS into EP_CXX_FLAGS, which every bundled ExternalProject receives through EP_COMMON_CMAKE_ARGS, so absl_ep (and gRPC/ORC built on top of it) get the header without patching Arrow or Abseil. Same remedy as build-aimrocks.yml uses for the same libstdc++ change. --- .github/workflows/build-pyarrow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index fc39c607b23f..c9ad33175df8 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -159,6 +159,8 @@ jobs: pushd "$BUILD" # The image ships OpenSSL shared-only, so force it shared (the rest stay static). # UNITY_BUILD off: bundled c-ares isn't unity-clean and upstream never compiles it. + # `-include cstdint`: older bundled Abseil (16.1.0 pins 20211102.0) relies on + # arriving transitively, which GCC 13+'s libstdc++ no longer does; EP_CXX_FLAGS hands it on. cmake \ -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_DEPENDENCY_USE_SHARED=OFF \ @@ -195,6 +197,7 @@ jobs: -DARROW_RPATH_ORIGIN=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_CXX_FLAGS="-include cstdint" \ -DCMAKE_INSTALL_PREFIX="$DIST" \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_UNITY_BUILD=OFF \ From aac9d3e6fc9a02a7c58d1b63b3e0da1f12d7a724 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 08:38:24 +0000 Subject: [PATCH 4/9] pyarrow: cap setuptools below 82 for 16.1.0's pkg_resources import pyarrow 16.1.0's setup.py does `import pkg_resources` at module scope (and uses pkg_resources.resource_filename in build_ext). setuptools 82.0.0 removed pkg_resources, and the unpinned preinstall resolved to 84.2.1, so `build --no-isolation` died in get_requires_for_build_wheel with ModuleNotFoundError. Cap at <82 (81.0.0 still ships it), as asyncpg does (gotcha 29). --- .github/workflows/build-pyarrow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index c9ad33175df8..2e06226ad1ba 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -244,7 +244,8 @@ jobs: # numpy/libcst: riscv64 wheels only in our registry "$pybin" -m pip install -U --extra-index-url "$PIP_EXTRA_INDEX_URL" \ --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build + # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29) + "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \ From b5394c4ef7e8d4f684b376b3628758ebf67ea695 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 23:41:45 +0000 Subject: [PATCH 5/9] pyarrow: cap setuptools_scm below 10 for 16.1.0's absolute write_to 16.1.0's setup.py sets use_scm_version's write_to to os.path.join(setup_dir, 'pyarrow/_generated_version.py') with an abspath'd setup_dir. setuptools_scm 10.1 added _sanitize_relative_path to its build_py hook and rejects that with "Version file path must be relative"; 9.2.2 accepts an absolute write_to under root by design. --- .github/workflows/build-pyarrow.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 2e06226ad1ba..0fbc2d7d25c9 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -244,8 +244,9 @@ jobs: # numpy/libcst: riscv64 wheels only in our registry "$pybin" -m pip install -U --extra-index-url "$PIP_EXTRA_INDEX_URL" \ --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" - # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29) - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build "setuptools<82" + # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29), + # and passes an absolute write_to, rejected by setuptools_scm >= 10.1 (gotcha 663) + "$pybin" -m pip install -U "cython>=3.1" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \ From 61b744ddc9fceae0c01e1ef124ed7ae7ce32d566 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 30 Oct 2026 16:33:38 +0000 Subject: [PATCH 6/9] pyarrow: cap cython below 3.1 for 16.1.0's nogil-strict table.pxi Cython 3.1 (May 2025) tightened with-nogil GIL-safety checking; pyarrow/table.pxi's `options.unify_schemas = promote_options != "none"` inside a nogil block is a Python string compare that only Cython's older, more lenient check let through. Upstream's own build-system.requires floor for 16.1.0 is cython>=0.29.31 -- release-era Cython was 3.0.x. Scoped to 16.1.0 alone: the matrix already excludes cp313/cp314/cp314t for it, and those are the only legs that need 3.1's free-threading support. --- .github/workflows/build-pyarrow.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 0fbc2d7d25c9..5e502ec2bbfd 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -246,7 +246,20 @@ jobs: --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29), # and passes an absolute write_to, rejected by setuptools_scm >= 10.1 (gotcha 663) - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" + CYTHON_SPEC="cython>=3.1" + if [ "$ARROW_VERSION" = "16.1.0" ]; then + # Cython 3.1 (May 2025, ~a year after this release) tightened `with nogil:` + # GIL-safety checking; pyarrow/table.pxi's `options.unify_schemas = + # promote_options != "none"` inside a nogil block is a Python string + # compare that only Cython's older, more lenient check let through + # (gotcha 23's shape). Upstream's own build-system.requires floor for + # 16.1.0 is just cython>=0.29.31 -- release-era Cython was 3.0.x, so cap + # below the breaking version rather than patch pyarrow's source. Safe to + # scope to this version alone: the matrix excludes cp313/cp314/cp314t for + # it above, and those are the only legs that need 3.1's free-threading support. + CYTHON_SPEC="cython>=3,<3.1" + fi + "$pybin" -m pip install -U "$CYTHON_SPEC" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \ From 87ed5acb24e90036721f812f47c62de5b438b4b1 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 31 Oct 2026 07:38:21 +0000 Subject: [PATCH 7/9] pyarrow: cap pytest below 9.1 for 16.1.0's legacy conftest.py hook 16.1.0's bundled conftest.py defines pytest_ignore_collect(path, config), the pre-9.1 hookspec signature; pytest 9.1 dropped the legacy `path` LEGACY_PATH parameter, so pluggy's hookimpl validation rejects it at collection time with "Argument(s) {'path'} are declared in the hookimpl but can not be found in the hookspec". Confirmed by diffing pytest's hookspec.py across 7.4.4/8.x/9.0.0/9.1.0: the signature holds the legacy path parameter through 9.0.0 and drops it at 9.1.0. Same release-era-drift shape as the earlier cython cap, this time in the test phase's floating pytest install rather than the build tool; scoped to 16.1.0 alone. Needed PYARROW_VERSION threaded into the test step's docker run, which previously only had it available in the outer job env, not inside the container. --- .github/workflows/build-pyarrow.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 5e502ec2bbfd..3da883be4289 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -285,7 +285,7 @@ jobs: docker run -i --rm \ -v "$PWD/arrow:/arrow" \ -v "$PWD/output:/output" \ - -e PIP_EXTRA_INDEX_URL \ + -e PIP_EXTRA_INDEX_URL -e PYARROW_VERSION \ -e ARROW_DATASET -e ARROW_ACERO -e ARROW_PARQUET \ -e PARQUET_REQUIRE_ENCRYPTION -e ARROW_ORC -e ARROW_SUBSTRAIT \ -e ARROW_FLIGHT -e ARROW_S3 -e ARROW_GCS -e ARROW_AZURE \ @@ -346,9 +346,18 @@ jobs: # hypothesis is only-binary too: upstream ships no cp314t riscv64 wheel (it builds # riscv64 abi3-only, and abi3t starts at 3.15), so without this pip falls back to its # sdist and the Rust build dies on a triple rustup does not have. Ours comes from the registry. + PYTEST_SPEC="pytest" + if [ "$PYARROW_VERSION" = "16.1.0" ]; then + # 16.1.0's bundled conftest.py defines pytest_ignore_collect(path, config), + # the pre-9.1 hookspec signature; pytest 9.1 (Nov 2026) dropped the legacy + # `path` LEGACY_PATH parameter, so pluggy's hookimpl validation now rejects + # it at collection time -- release-era pytest still had it (gotcha 23's shape, + # this time in the test deps rather than the build tool). + PYTEST_SPEC="pytest<9.1" + fi "$pybin" -m pip install -U --only-binary=numpy,cffi,hypothesis \ --extra-index-url "$PIP_EXTRA_INDEX_URL" \ - pytest hypothesis cffi pytz packaging "numpy>=2.0.0" + "$PYTEST_SPEC" hypothesis cffi pytz packaging "numpy>=2.0.0" # pandas may lack a free-threaded riscv64 wheel; skip its tests if so if "$pybin" -m pip install -U --only-binary=pandas \ --extra-index-url "$PIP_EXTRA_INDEX_URL" pandas; then From 267c83d528b3a91b70554359c78a2e463d228e9f Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 1 Nov 2026 00:36:36 +0000 Subject: [PATCH 8/9] pyarrow: cap pytest below 9 for 16.1.0, not just below 9.1 The <9.1 cap wasn't low enough. pytest escalates a RemovedInN warning to a hard error starting at major version N: confirmed with a two-line repro plugin exercising 16.1.0's legacy pytest_ignore_collect(path, config) hook under both 8.4.0 and 9.0.3 -- 8.4.0 only warns (PytestRemovedIn9Warning) and collection passes, 9.0.3 raises that exact warning as an error during plugin registration. pyarrow's own pytest config (python/setup.cfg) only errors on a SparseDataFrame FutureWarning, so the escalation is pytest's own default, not something pyarrow's filterwarnings adds. Confirmed again from the actual CI log: pytest 9.0.3 (resolved under the old <9.1 cap) fails collection with exactly this PytestRemovedIn9Warning-as-error. --- .github/workflows/build-pyarrow.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 3da883be4289..f630f78f8f94 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -349,11 +349,15 @@ jobs: PYTEST_SPEC="pytest" if [ "$PYARROW_VERSION" = "16.1.0" ]; then # 16.1.0's bundled conftest.py defines pytest_ignore_collect(path, config), - # the pre-9.1 hookspec signature; pytest 9.1 (Nov 2026) dropped the legacy - # `path` LEGACY_PATH parameter, so pluggy's hookimpl validation now rejects - # it at collection time -- release-era pytest still had it (gotcha 23's shape, - # this time in the test deps rather than the build tool). - PYTEST_SPEC="pytest<9.1" + # the legacy (path: py.path.local) hookimpl signature. pytest's own policy + # escalates a RemovedInN warning to a hard error starting at major version N: + # 8.4.0 only warns (PytestRemovedIn9Warning) and collection still passes, but + # 9.0.3 raises that same warning as an error during plugin registration -- + # confirmed by running a two-line repro plugin under both. <9.1 (tried first) + # was not low enough; the escalation already happens at 9.0. Release-era + # pytest was comfortably below both (gotcha 23's shape, in the test deps + # rather than the build tool). + PYTEST_SPEC="pytest<9" fi "$pybin" -m pip install -U --only-binary=numpy,cffi,hypothesis \ --extra-index-url "$PIP_EXTRA_INDEX_URL" \ From a35f40ce2de87d953bf1863da9eabaaff70d7526 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 1 Nov 2026 12:40:22 +0000 Subject: [PATCH 9/9] pyarrow: cap pandas<3 and deselect 4 jemalloc-assumption tests for 16.1.0 Two independent, confirmed-from-the-CI-log root causes for the 46 test failures once collection started passing: - pandas 3.0 made string-backed dtypes the default for columns that used to come back as plain object dtype, and changed pa.array(Categorical)'s inferred value type to large_string. 16.1.0's pandas-compat code and its own test suite's expected values both predate that, so the unconstrained `pip install pandas` resolving to today's 3.1.0 broke ~42 tests purely on dtype-equality assertions (test_backwards_compatible_index_naming: dtype='object' vs 'str'; test_dictionary_from_pandas: string vs large_string). Capped to pandas<3, same release-era-drift shape as the earlier cython/pytest caps, this time in a floating test dependency. - test_memory.py hardcodes `should_have_jemalloc = sys.platform == "linux"` (removed in later pyarrow releases -- it's gone by 25.0.1) and never checks whether this build actually enabled jemalloc. We build with ARROW_JEMALLOC=OFF ("jemalloc is page-size fragile"), so 4 tests fail on that wrong assumption regardless of pandas: ArrowNotImplementedError, "Unsupported backend 'jemalloc'", "'jemalloc' in ['mimalloc', 'system']", and an allocation byte-count tied to the allocator. Excluded by test name via `-k` (not --deselect: --pyargs resolves test files under the interpreter's own site-packages path, which varies per matrix.python). Both scoped to 16.1.0 alone. --- .github/workflows/build-pyarrow.yml | 37 +++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index f630f78f8f94..35775441dd39 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -362,9 +362,38 @@ jobs: "$pybin" -m pip install -U --only-binary=numpy,cffi,hypothesis \ --extra-index-url "$PIP_EXTRA_INDEX_URL" \ "$PYTEST_SPEC" hypothesis cffi pytz packaging "numpy>=2.0.0" + + PANDAS_SPEC="pandas" + PYTEST_K="" + if [ "$PYARROW_VERSION" = "16.1.0" ]; then + # pandas 3.0 made string-backed dtypes the default for columns that used + # to come back as plain object dtype (and changed pa.array(Categorical)'s + # inferred value type to large_string); 16.1.0's pandas-compat code and its + # own test suite's expected values both predate that and assume object/ + # string, so an unconstrained `pip install pandas` (today's 3.1.0) breaks + # ~42 of the suite's tests on dtype-equality assertions alone -- confirmed + # from the CI log (test_backwards_compatible_index_naming et al. failing + # with dtype='object' vs dtype='str', test_dictionary_from_pandas with + # string vs large_string). Same release-era-drift shape as the cython/ + # pytest caps above, this time in a floating *test* dependency. + PANDAS_SPEC="pandas<3" + # test_memory.py hardcodes `should_have_jemalloc = sys.platform == "linux"` + # (removed in later pyarrow releases, e.g. it's gone by 25.0.1) -- it never + # checks whether *this* build actually enabled jemalloc. We build with + # ARROW_JEMALLOC=OFF (see the env block above: "jemalloc is page-size + # fragile"), so these 4 tests fail here on every Linux build that disables + # it, regardless of pandas: confirmed from the CI log, each failing on the + # exact jemalloc/mimalloc assertion (ArrowNotImplementedError, "Unsupported + # backend 'jemalloc'", "'jemalloc' in ['mimalloc', 'system']", an + # allocation byte-count tied to the allocator). Excluded by test function + # name via -k (not --deselect: --pyargs resolves test files under the + # interpreter's own site-packages path, which varies per matrix.python, so + # there is no single static path --deselect could name). + PYTEST_K="not (test_env_var or test_specific_memory_pools or test_supported_memory_backends or test_table_uses_memory_pool)" + fi # pandas may lack a free-threaded riscv64 wheel; skip its tests if so if "$pybin" -m pip install -U --only-binary=pandas \ - --extra-index-url "$PIP_EXTRA_INDEX_URL" pandas; then + --extra-index-url "$PIP_EXTRA_INDEX_URL" "$PANDAS_SPEC"; then export PYARROW_TEST_PANDAS=ON else echo "WARNING: pandas has no riscv64 wheel for ${{ matrix.python }}; disabling pandas tests." @@ -372,7 +401,11 @@ jobs: fi "$pybin" -c 'import pyarrow; pyarrow.create_library_symlinks()' - "$pybin" -m pytest -r s --pyargs pyarrow + if [ -n "$PYTEST_K" ]; then + "$pybin" -m pytest -r s --pyargs pyarrow -k "$PYTEST_K" + else + "$pybin" -m pytest -r s --pyargs pyarrow + fi PYARROW_TEST_EOF publish: