diff --git a/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json b/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json index 37dd25bf9029..5d04b2c0a8c7 100644 --- a/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json +++ b/.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 3 + "modification": 5 } diff --git a/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_driver_requirements.txt b/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_driver_requirements.txt new file mode 100644 index 000000000000..86e78aa309d6 --- /dev/null +++ b/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_driver_requirements.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Driver requirements for CloudML TFT benchmarks. +# tensorflow-transform and tfx_bsl are installed separately with --no-deps in +# installTFTRequirements to avoid re-resolving apache-beam[gcp] during setup. +dill==0.4.1 +tensorflow>=2.16,<2.17 +numpy>=1.22.0,<2.0 +tensorflow-metadata>=1.16.1,<1.17.0 +pyarrow>=10,<11 +tensorflow-serving-api>=2.16.1,<2.20 +tf-keras>=2.16.0,<2.17 diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 27cf2869600c..ba4e6ddd6400 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -571,13 +571,12 @@ task installTFTRequirements { exec { workingDir "$rootProject.projectDir/sdks/python/apache_beam/testing/benchmarks/cloudml/" executable 'sh' - // installGcpTest already installed apache-beam[gcp]. tensorflow-transform also - // lists that dependency, so a plain pip install -r can re-resolve the GCP extra - // and hit ResolutionTooDeep. Install TFT with --no-deps instead. + // installGcpTest already installed apache-beam[gcp]. tensorflow-transform and + // tfx_bsl also list that dependency, so a plain pip install -r can re-resolve + // the GCP extra and hit ResolutionTooDeep. Install them with --no-deps instead. args '-c', ". ${envdir}/bin/activate && " + - "grep -v '^tensorflow-transform' requirements.txt > /tmp/cloudml_tft_base_requirements.txt && " + - "pip install -r /tmp/cloudml_tft_base_requirements.txt && " + - "pip install --no-deps tensorflow-transform==1.16.0" + "pip install -r cloudml_driver_requirements.txt && " + + "pip install --no-deps tfx_bsl==1.16.1 tensorflow-transform==1.16.0" } } }