pyarrow: Add version 16.1.0

.github/workflows/build-pyarrow.yml

@@ -77,6 +77,15 @@ jobs:
           - cp313-cp313
           - cp314-cp314
           - cp314-cp314t
+        exclude:
+          # 16.1.0 predates cp313 (released Oct 2024, ~5 months after this
+          # release) and never shipped a wheel past cp312 upstream either.
+          - version: "16.1.0"
+            python: cp313-cp313
+          - version: "16.1.0"
+            python: cp314-cp314
+          - version: "16.1.0"
+            python: cp314-cp314t
 
     env:
       PYARROW_VERSION: ${{ matrix.version }}
@@ -135,13 +144,23 @@ jobs:
           DIST=/tmp/arrow-dist
           BUILD=/tmp/arrow-build
 
+          # BUNDLED thirdparty deps are built via ExternalProject_Add with their own
+          # vendored cmake_minimum_required (e.g. gflags' is 2.8.12), below CMake 4's
+          # floor of 3.5. The image's cmake (pipx-installed, >= 4) hard-errors on that
+          # instead of warning. We don't control the ExternalProject configure line, but
+          # CMake also reads CMAKE_POLICY_VERSION_MINIMUM from the environment, which
+          # reaches every nested cmake invocation this build makes (gotcha 257).
+          export CMAKE_POLICY_VERSION_MINIMUM=3.5
+
           # ninja and OpenSSL headers aren't in the base image
           dnf install -y --setopt=install_weak_deps=False ninja-build openssl-devel
 
           mkdir -p "$BUILD"
           pushd "$BUILD"
           # The image ships OpenSSL shared-only, so force it shared (the rest stay static).
           # UNITY_BUILD off: bundled c-ares isn't unity-clean and upstream never compiles it.
+          # `-include cstdint`: older bundled Abseil (16.1.0 pins 20211102.0) relies on <cstdint>
+          # arriving transitively, which GCC 13+'s libstdc++ no longer does; EP_CXX_FLAGS hands it on.
           cmake \
               -DARROW_DEPENDENCY_SOURCE=BUNDLED \
               -DARROW_DEPENDENCY_USE_SHARED=OFF \
@@ -178,6 +197,7 @@ jobs:
               -DARROW_RPATH_ORIGIN=ON \
               -DARROW_USE_CCACHE=ON \
               -DCMAKE_BUILD_TYPE=release \
+              -DCMAKE_CXX_FLAGS="-include cstdint" \
               -DCMAKE_INSTALL_PREFIX="$DIST" \
               -DCMAKE_INSTALL_LIBDIR=lib \
               -DCMAKE_UNITY_BUILD=OFF \
@@ -224,7 +244,22 @@ jobs:
           # numpy/libcst: riscv64 wheels only in our registry
           "$pybin" -m pip install -U --extra-index-url "$PIP_EXTRA_INDEX_URL" \
               --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6"
-          "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build
+          # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29),
+          # and passes an absolute write_to, rejected by setuptools_scm >= 10.1 (gotcha 663)
+          CYTHON_SPEC="cython>=3.1"
+          if [ "$ARROW_VERSION" = "16.1.0" ]; then
+              # Cython 3.1 (May 2025, ~a year after this release) tightened `with nogil:`
+              # GIL-safety checking; pyarrow/table.pxi's `options.unify_schemas =
+              # promote_options != "none"` inside a nogil block is a Python string
+              # compare that only Cython's older, more lenient check let through
+              # (gotcha 23's shape). Upstream's own build-system.requires floor for
+              # 16.1.0 is just cython>=0.29.31 -- release-era Cython was 3.0.x, so cap
+              # below the breaking version rather than patch pyarrow's source. Safe to
+              # scope to this version alone: the matrix excludes cp313/cp314/cp314t for
+              # it above, and those are the only legs that need 3.1's free-threading support.
+              CYTHON_SPEC="cython>=3,<3.1"
+          fi
+          "$pybin" -m pip install -U "$CYTHON_SPEC" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82"
           mkdir -p /output
           pushd "$ARROW_SRC/python"
           "$pybin" -m build --wheel --no-isolation \
@@ -250,7 +285,7 @@ jobs:
           docker run -i --rm \
             -v "$PWD/arrow:/arrow" \
             -v "$PWD/output:/output" \
-            -e PIP_EXTRA_INDEX_URL \
+            -e PIP_EXTRA_INDEX_URL -e PYARROW_VERSION \
             -e ARROW_DATASET -e ARROW_ACERO -e ARROW_PARQUET \
             -e PARQUET_REQUIRE_ENCRYPTION -e ARROW_ORC -e ARROW_SUBSTRAIT \
             -e ARROW_FLIGHT -e ARROW_S3 -e ARROW_GCS -e ARROW_AZURE \
@@ -311,20 +346,66 @@ jobs:
           # hypothesis is only-binary too: upstream ships no cp314t riscv64 wheel (it builds
           # riscv64 abi3-only, and abi3t starts at 3.15), so without this pip falls back to its
           # sdist and the Rust build dies on a triple rustup does not have. Ours comes from the registry.
+          PYTEST_SPEC="pytest"
+          if [ "$PYARROW_VERSION" = "16.1.0" ]; then
+              # 16.1.0's bundled conftest.py defines pytest_ignore_collect(path, config),
+              # the legacy (path: py.path.local) hookimpl signature. pytest's own policy
+              # escalates a RemovedInN warning to a hard error starting at major version N:
+              # 8.4.0 only warns (PytestRemovedIn9Warning) and collection still passes, but
+              # 9.0.3 raises that same warning as an error during plugin registration --
+              # confirmed by running a two-line repro plugin under both. <9.1 (tried first)
+              # was not low enough; the escalation already happens at 9.0. Release-era
+              # pytest was comfortably below both (gotcha 23's shape, in the test deps
+              # rather than the build tool).
+              PYTEST_SPEC="pytest<9"
+          fi
           "$pybin" -m pip install -U --only-binary=numpy,cffi,hypothesis \
               --extra-index-url "$PIP_EXTRA_INDEX_URL" \
-              pytest hypothesis cffi pytz packaging "numpy>=2.0.0"
+              "$PYTEST_SPEC" hypothesis cffi pytz packaging "numpy>=2.0.0"
+
+          PANDAS_SPEC="pandas"
+          PYTEST_K=""
+          if [ "$PYARROW_VERSION" = "16.1.0" ]; then
+              # pandas 3.0 made string-backed dtypes the default for columns that used
+              # to come back as plain object dtype (and changed pa.array(Categorical)'s
+              # inferred value type to large_string); 16.1.0's pandas-compat code and its
+              # own test suite's expected values both predate that and assume object/
+              # string, so an unconstrained `pip install pandas` (today's 3.1.0) breaks
+              # ~42 of the suite's tests on dtype-equality assertions alone -- confirmed
+              # from the CI log (test_backwards_compatible_index_naming et al. failing
+              # with dtype='object' vs dtype='str', test_dictionary_from_pandas with
+              # string vs large_string). Same release-era-drift shape as the cython/
+              # pytest caps above, this time in a floating *test* dependency.
+              PANDAS_SPEC="pandas<3"
+              # test_memory.py hardcodes `should_have_jemalloc = sys.platform == "linux"`
+              # (removed in later pyarrow releases, e.g. it's gone by 25.0.1) -- it never
+              # checks whether *this* build actually enabled jemalloc. We build with
+              # ARROW_JEMALLOC=OFF (see the env block above: "jemalloc is page-size
+              # fragile"), so these 4 tests fail here on every Linux build that disables
+              # it, regardless of pandas: confirmed from the CI log, each failing on the
+              # exact jemalloc/mimalloc assertion (ArrowNotImplementedError, "Unsupported
+              # backend 'jemalloc'", "'jemalloc' in ['mimalloc', 'system']", an
+              # allocation byte-count tied to the allocator). Excluded by test function
+              # name via -k (not --deselect: --pyargs resolves test files under the
+              # interpreter's own site-packages path, which varies per matrix.python, so
+              # there is no single static path --deselect could name).
+              PYTEST_K="not (test_env_var or test_specific_memory_pools or test_supported_memory_backends or test_table_uses_memory_pool)"
+          fi
           # pandas may lack a free-threaded riscv64 wheel; skip its tests if so
           if "$pybin" -m pip install -U --only-binary=pandas \
-              --extra-index-url "$PIP_EXTRA_INDEX_URL" pandas; then
+              --extra-index-url "$PIP_EXTRA_INDEX_URL" "$PANDAS_SPEC"; then
               export PYARROW_TEST_PANDAS=ON
           else
               echo "WARNING: pandas has no riscv64 wheel for ${{ matrix.python }}; disabling pandas tests."
               export PYARROW_TEST_PANDAS=OFF
           fi
 
           "$pybin" -c 'import pyarrow; pyarrow.create_library_symlinks()'
-          "$pybin" -m pytest -r s --pyargs pyarrow
+          if [ -n "$PYTEST_K" ]; then
+              "$pybin" -m pytest -r s --pyargs pyarrow -k "$PYTEST_K"
+          else
+              "$pybin" -m pytest -r s --pyargs pyarrow
+          fi
           PYARROW_TEST_EOF
 
   publish:

docs/packages/pyarrow.yaml

@@ -2,6 +2,7 @@ package-name: pyarrow
 source-code: https://github.com/apache/arrow
 license: Apache-2.0
 versions:
+- version: 16.1.0
 - version: 25.0.1
   tag: pyarrow-v25.0.1-20260902125649
   files: