diff --git a/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json b/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json index 37dd25bf9029..5d04b2c0a8c7 100644 --- a/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json +++ b/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 3 + "modification": 5 } diff --git a/sdks/python/apache_beam/testing/benchmarks/cloudml/constraints.txt b/sdks/python/apache_beam/testing/benchmarks/cloudml/constraints.txt new file mode 100644 index 000000000000..b2f76d200850 --- /dev/null +++ b/sdks/python/apache_beam/testing/benchmarks/cloudml/constraints.txt @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Constraints file to pin versions and avoid pip ResolutionTooDeep. +# This file is used with: pip install -c constraints.txt -r requirements.txt + +# Core dependencies +tfx_bsl==1.15.1 +tensorflow-transform==1.15.0 + +# TensorFlow ecosystem +tensorflow==2.15.1 +tensorflow-metadata==1.15.0 +tf-keras==2.15.1 + +# NumPy and data handling +numpy==1.26.4 +pyarrow==10.0.1 + +# Google Cloud (pin to avoid transitive resolution) +google-cloud-aiplatform==1.60.0 +google-api-core==2.19.1 + +# Note: google-auth is NOT constrained - let pip resolve it to satisfy +# apache-beam's google-genai requirement (>=2.48.1) + +# Note: tensorflow-serving-api is NOT constrained - let pip resolve it within +# the range specified in requirements.txt (>=2.15,<2.16) diff --git a/sdks/python/apache_beam/testing/benchmarks/cloudml/requirements.txt b/sdks/python/apache_beam/testing/benchmarks/cloudml/requirements.txt index ab94ec5e9acf..5f754e27148e 100644 --- a/sdks/python/apache_beam/testing/benchmarks/cloudml/requirements.txt +++ b/sdks/python/apache_beam/testing/benchmarks/cloudml/requirements.txt @@ -15,12 +15,15 @@ # limitations under the License. # -dill==0.4.1 -tfx_bsl==1.16.1 -tensorflow-transform==1.16.0 -tensorflow>=2.16,<2.17 +# Core TFT dependencies with version bounds. +# Note: To avoid pip ResolutionTooDeep errors, always install using the constraints file: +# pip install -c constraints.txt -r requirements.txt +dill>=0.3,<0.5 +tfx_bsl>=1.15,<1.17 +tensorflow-transform>=1.15,<1.17 +tensorflow>=2.15,<2.16 numpy>=1.22.0,<2.0 -tensorflow-metadata>=1.16.1,<1.17.0 +tensorflow-metadata>=1.15,<1.16 pyarrow>=10,<11 -tensorflow-serving-api>=2.16.1,<2.20 -tf-keras>=2.16.0,<2.17 +tensorflow-serving-api>=2.15,<2.16 +tf-keras>=2.15,<2.16 diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 7c84700e29fa..480e2a62a2ef 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -573,13 +573,9 @@ task installTFTRequirements { exec { workingDir "$rootProject.projectDir/sdks/python/apache_beam/testing/benchmarks/cloudml/" executable 'sh' - // installGcpTest already installed apache-beam[gcp]. tensorflow-transform also - // lists that dependency, so a plain pip install -r can re-resolve the GCP extra - // and hit ResolutionTooDeep. Install TFT with --no-deps instead. - args '-c', ". ${envdir}/bin/activate && " + - "grep -v '^tensorflow-transform' requirements.txt > /tmp/cloudml_tft_base_requirements.txt && " + - "pip install -r /tmp/cloudml_tft_base_requirements.txt && " + - "pip install --no-deps tensorflow-transform==1.16.0" + // Use constraints file to pin versions while allowing pip to + // resolve compatible versions within the specified ranges in requirements.txt + args '-c', ". ${envdir}/bin/activate && pip install -c constraints.txt -r requirements.txt" } } }