From 313a54b29a908c96c9bee0070b53f5718a7ac529 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 29 Jan 2026 16:19:06 +0000 Subject: [PATCH 1/2] Require PyArrow >=16 --- .github/workflows/tests.yaml | 4 ++-- continuous_integration/environment-3.11.yaml | 2 +- continuous_integration/recipes/dask/meta.yaml | 4 ++-- distributed/protocol/arrow.py | 4 ++-- distributed/shuffle/tests/test_shuffle.py | 1 - pyproject.toml | 2 -- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cd3e97260a0..6bc200c3dcd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -75,12 +75,12 @@ jobs: - os: ubuntu-latest environment: mindeps label: pandas - extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1] + extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0] partition: "ci1" - os: ubuntu-latest environment: mindeps label: pandas - extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1] + extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0] partition: "not ci1" - os: ubuntu-latest diff --git a/continuous_integration/environment-3.11.yaml b/continuous_integration/environment-3.11.yaml index 641db193ae9..976ea2fb5c2 100644 --- a/continuous_integration/environment-3.11.yaml +++ b/continuous_integration/environment-3.11.yaml @@ -28,7 +28,7 @@ dependencies: - pre-commit - prometheus_client - psutil - - pyarrow=14 + - pyarrow=16.0 - pytest - pytest-cov - pytest-faulthandler diff --git a/continuous_integration/recipes/dask/meta.yaml b/continuous_integration/recipes/dask/meta.yaml index b48c799914d..109ec9db498 100644 --- a/continuous_integration/recipes/dask/meta.yaml +++ b/continuous_integration/recipes/dask/meta.yaml @@ -27,11 +27,11 @@ requirements: - {{ pin_compatible('distributed', exact=True) }} - cytoolz >=0.11.2 - lz4 >=4.3.2 - - numpy >=1.21 + - numpy >=1.24 - pandas >=2 - bokeh >=3.1.0 - jinja2 >=2.10.3 - - pyarrow >=7.0 + - pyarrow >=16 run_constrained: - openssl !=1.1.1e diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py index 8a9814a443e..95097fb56b0 100644 --- a/distributed/protocol/arrow.py +++ b/distributed/protocol/arrow.py @@ -4,9 +4,9 @@ from distributed.protocol.serialize import dask_deserialize, dask_serialize -if pyarrow.__version__ < "0.10": +if int(pyarrow.__version__.split(".")[0]) < 16: raise ImportError( - "Need pyarrow >= 0.10 . " + "Need pyarrow >= 16.0. " "See https://arrow.apache.org/docs/python/install.html" ) diff --git a/distributed/shuffle/tests/test_shuffle.py b/distributed/shuffle/tests/test_shuffle.py index 9b2c521b980..b89913b77c2 100644 --- a/distributed/shuffle/tests/test_shuffle.py +++ b/distributed/shuffle/tests/test_shuffle.py @@ -1084,7 +1084,6 @@ async def test_heartbeat(c, s, a, b): @pytest.mark.skipif("not pa", reason="Requires PyArrow") -@pytest.mark.filterwarnings("ignore:DatetimeTZBlock") # pandas >=2.2 vs. pyarrow <15 @pytest.mark.parametrize("drop_column", [True, False]) def test_processing_chain(tmp_path, drop_column): """ diff --git a/pyproject.toml b/pyproject.toml index 0b2a9176292..bb22eaab6b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,8 +178,6 @@ filterwarnings = [ '''ignore:datetime\.datetime\.utc(fromtimestamp|now)\(\) is deprecated and scheduled for removal in a future version.*:DeprecationWarning:tornado''', # https://github.com/dateutil/dateutil/issues/1284 '''ignore:datetime\.datetime\.utc(fromtimestamp|now)\(\) is deprecated and scheduled for removal in a future version.*:DeprecationWarning:dateutil''', - # https://github.com/dask/dask/pull/10622 - '''ignore:Minimal version of pyarrow will soon be increased to 14.0.1''', '''ignore:the matrix subclass is not the recommended way''', '''ignore:The current Dask DataFrame implementation is deprecated.*:DeprecationWarning''', ] From eac62176f03586bf3b43701dbbbd214ecf5e22d6 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 29 Jan 2026 16:36:22 +0000 Subject: [PATCH 2/2] lint --- distributed/protocol/arrow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py index 95097fb56b0..dfa98fdcda1 100644 --- a/distributed/protocol/arrow.py +++ b/distributed/protocol/arrow.py @@ -6,8 +6,7 @@ if int(pyarrow.__version__.split(".")[0]) < 16: raise ImportError( - "Need pyarrow >= 16.0. " - "See https://arrow.apache.org/docs/python/install.html" + "Need pyarrow >=16.0. See https://arrow.apache.org/docs/python/install.html" )