remove gsutil usage

.github/workflows/beam_PreCommit_GHA.yml

@@ -80,6 +80,22 @@ jobs:
         comment_phrase: ${{ matrix.job_phrase }}
         github_token: ${{ secrets.GITHUB_TOKEN }}
         github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    - name: Check for gsutil references
+      run: |
+        echo "Checking codebase for gsutil..."
+        # Search for 'gsutil', excluding this workflow file itself to avoid false positives.
+        if git grep -n "gsutil" -- ':!.github/workflows/beam_PreCommit_GHA.yml'; then
+          echo "ERROR: Found references to gsutil in the codebase. Please use 'gcloud storage' instead."
+          exit 1
+        elif [ "$(date +%Y%m)" -ge 202704 ]; then
+          echo "ERROR: Current date is April 2027 or later."
+          echo "Please verify gsutil deprecation date is still March 2027 (Reference: https://docs.cloud.google.com/storage/docs/gsutil)."
+          echo "If so, then delete this workflow step."
+          exit 1
+        else
+          echo "SUCCESS: No references to gsutil found."
+        fi
+      shell: bash
     - name: Setup environment
       uses: ./.github/actions/setup-environment-action
       with:

.github/workflows/build_wheels.yml

@@ -201,7 +201,7 @@ jobs:
     if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
     steps:
       - name: Remove existing files on GCS bucket
-        run: gsutil rm -r ${GCP_PATH} || true
+        run: gcloud storage rm -r ${GCP_PATH} || true
 
   upload_source_to_gcs:
     name: Upload python source distribution to GCS bucket
@@ -217,7 +217,7 @@ jobs:
           name: source_zip
           path: source/
       - name: Copy sources to GCS bucket
-        run: gsutil cp -r -a public-read source/* ${GCP_PATH}
+        run: gcloud storage cp -r --predefined-acl=publicRead source/* ${GCP_PATH}
 
   build_wheels:
     name: Build python ${{matrix.py_version}} wheels on ${{matrix.os_python.arch}} for ${{ matrix.os_python.os }}
@@ -330,7 +330,7 @@ jobs:
         merge-multiple: true
         path: wheelhouse/
     - name: Copy wheels to GCS bucket
-      run: gsutil cp -r -a public-read wheelhouse/* ${GCP_PATH}
+      run: gcloud storage cp -r --predefined-acl=publicRead wheelhouse/* ${GCP_PATH}
     - name: Create github action information file on GCS bucket
       run: |
         cat > github_action_info <<EOF
@@ -348,9 +348,9 @@ jobs:
         GITHUB_BASE_REF=$GITHUB_BASE_REF
         EOF
         echo $(cat github_action_info)
-        gsutil cp -a public-read github_action_info ${GCP_PATH}
+        gcloud storage cp --predefined-acl=publicRead github_action_info ${GCP_PATH}
     - name: Upload GitHub event file to GCS bucket
-      run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${GCP_PATH}
+      run: gcloud storage cp --predefined-acl=publicRead ${GITHUB_EVENT_PATH} ${GCP_PATH}
 
   list_files_on_gcs:
     name: List files on Google Cloud Storage Bucket
@@ -361,7 +361,7 @@ jobs:
     if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
     steps:
     - name: List file on Google Cloud Storage Bucket
-      run: gsutil ls "${GCP_PATH}*"
+      run: gcloud storage ls "${GCP_PATH}*"
 
   branch_repo_nightly:
     permissions:

.github/workflows/run_rc_validation_go_wordcount.yml

@@ -74,7 +74,7 @@ jobs:
       # --- Prism Steps ---
       - name: Download Input File from GCS (Prism)
         working-directory: ./${{ steps.setup_go.outputs.work_dir }}
-        run: gsutil cp ${{ env.GCS_INPUT_PATH }} ./kinglear.txt
+        run: gcloud storage cp ${{ env.GCS_INPUT_PATH }} ./kinglear.txt
 
       - name: Run Go WordCount with PrismRunner
         working-directory: ./${{ steps.setup_go.outputs.work_dir }}
@@ -118,17 +118,17 @@ jobs:
             --environment_config=apache/beam_go_sdk:${{ env.CONTAINER_TAG }}
 
       - name: Check Dataflow Output in GCS
-        working-directory: ./${{ steps.setup_go.outputs.work_dir }} # Added working directory for consistency, though not strictly needed for gsutil
+        working-directory: ./${{ steps.setup_go.outputs.work_dir }} # Added working directory for consistency, though not strictly needed for gcloud storage
         env:
           # Re-define the output path pattern for checking
           GCS_OUTPUT_PATH_PATTERN: ${{ env.GCS_OUTPUT_PREFIX }}/${{ env.RC_TAG }}/dataflow/output*
         run: |
           echo "Checking for Dataflow output files in GCS at: $GCS_OUTPUT_PATH_PATTERN"
-          # Use gsutil stat. The -q flag suppresses errors for non-existent files,
-          # allowing us to check the exit code. Exit code 0 means found, 1 means not found.
-          if gsutil -q stat $GCS_OUTPUT_PATH_PATTERN; then
+          # Use gcloud storage ls. Redirect stdout/stderr to suppress output.
+          # Exit code 0 means found, non-zero means not found.
+          if gcloud storage ls $GCS_OUTPUT_PATH_PATTERN > /dev/null 2>&1; then
             echo "Output files found in GCS."
-            FILE_COUNT=$(gsutil ls $GCS_OUTPUT_PATH_PATTERN | wc -l)
+            FILE_COUNT=$(gcloud storage ls $GCS_OUTPUT_PATH_PATTERN | wc -l)
             if [ "$FILE_COUNT" -gt 0 ]; then echo "Found $FILE_COUNT output file(s)."; else echo "Error: Output path exists but contains no files."; exit 1; fi
           else
             echo "Error: Output files not found in GCS at $GCS_OUTPUT_PATH_PATTERN"

.github/workflows/run_rc_validation_python_mobile_gaming.yml

@@ -174,7 +174,7 @@ jobs:
       - name: Create GCS Bucket (if needed - reusing input bucket)
         run: |
           echo "Ensuring GCS Bucket exists: ${{ env.GCS_BUCKET }} in project ${{ env.GCP_PROJECT_ID }}"
-          gsutil mb -p ${{ env.GCP_PROJECT_ID }} ${{ env.GCS_BUCKET }} || echo "Bucket ${{ env.GCS_BUCKET }} likely already exists."
+          gcloud storage buckets create ${{ env.GCS_BUCKET }} --project=${{ env.GCP_PROJECT_ID }} || echo "Bucket ${{ env.GCS_BUCKET }} likely already exists."
         shell: bash
 
       - name: Create PubSub Topic
@@ -533,8 +533,8 @@ jobs:
         if: always()
         run: |
           echo "Deleting objects in GCS Bucket: ${{ env.GCS_BUCKET }}/temp/"
-          gsutil -m rm -r "${{ env.GCS_BUCKET }}/temp/leaderboard/**" || echo "Failed to delete objects in GCS leaderboard temp folder."
-          gsutil -m rm -r "${{ env.GCS_BUCKET }}/temp/gamestats/**" || echo "Failed to delete objects in GCS gamestats temp folder."
+          gcloud storage rm -r "${{ env.GCS_BUCKET }}/temp/leaderboard/**" || echo "Failed to delete objects in GCS leaderboard temp folder."
+          gcloud storage rm -r "${{ env.GCS_BUCKET }}/temp/gamestats/**" || echo "Failed to delete objects in GCS gamestats temp folder."
           echo "Removing local log and jobid files..."
           rm -f leaderboard_dataflow_submit.log gamestats_dataflow_submit.log injector_run.log
           rm -f leaderboard_dataflow_jobid.txt # Remove Leaderboard jobid file here

.github/workflows/run_rc_validation_python_yaml.yml

@@ -265,9 +265,9 @@ jobs:
           sleep 60
           # Check if any files matching the pattern exist within the unique output folder.
           echo "Checking for files matching pattern: ${OUTPUT_PATTERN}"
-          if gsutil ls "${OUTPUT_PATTERN}" > /dev/null 2>&1; then
+          if gcloud storage ls "${OUTPUT_PATTERN}" > /dev/null 2>&1; then
             echo "SUCCESS: Found output files matching pattern in GCS."
-            gsutil ls "${OUTPUT_PATTERN}" # List found files
+            gcloud storage ls "${OUTPUT_PATTERN}" # List found files
           else
             echo "ERROR: No output files found matching pattern '${OUTPUT_PATTERN}' in GCS bucket."
             exit 1
@@ -280,7 +280,7 @@ jobs:
         run: |
           echo "Deleting unique run folder in GCS: ${GCS_UNIQUE_FOLDER_PREFIX}"
           # Delete the entire unique folder for this run, including temp, staging, and output
-          gsutil -m rm -r "${GCS_UNIQUE_FOLDER_PREFIX}" || echo "Failed to delete unique run folder ${GCS_UNIQUE_FOLDER_PREFIX} in GCS. Manual cleanup might be required."
+          gcloud storage rm -r "${GCS_UNIQUE_FOLDER_PREFIX}" || echo "Failed to delete unique run folder ${GCS_UNIQUE_FOLDER_PREFIX} in GCS. Manual cleanup might be required."
 
           echo "Removing local log, yaml, and jobid files..."
           rm -f yaml_dataflow_submit.log ${{ env.YAML_PIPELINE_FILE }} yaml_dataflow_jobid.txt

.test-infra/dataproc/flink_cluster.sh

@@ -91,7 +91,7 @@ YARN_APPLICATION_MASTER=""
 
 function upload_init_actions() {
   echo "Uploading initialization actions to GCS bucket: $GCS_BUCKET"
-  gsutil cp -r $INIT_ACTIONS_FOLDER_NAME/* $GCS_BUCKET/$INIT_ACTIONS_FOLDER_NAME
+  gcloud storage cp -r $INIT_ACTIONS_FOLDER_NAME/* $GCS_BUCKET/$INIT_ACTIONS_FOLDER_NAME
 }
 
 function get_leader() {

.test-infra/metrics/build.gradle

@@ -54,6 +54,7 @@ composeUp {
   dependsOn "createEmptyConfig"
 }
 dockerCompose {
+  projectName = 'beammetrics'
   environment.put 'DOCKER_CONFIG', project.rootProject.buildDir
 }
 

.test-infra/metrics/influxdb/Dockerfile

@@ -16,14 +16,13 @@
 # limitations under the License.
 ################################################################################
 
-FROM python:3.10-slim
-
-RUN pip install --no-cache-dir gsutil
+FROM alpine:latest
 
 WORKDIR /
 
-RUN gsutil cp gs://apache-beam-testing-metrics/influxdb-backup.tar.gz . && \
-tar xzf influxdb-backup.tar.gz
+RUN apk add --no-cache curl tar && \
+    curl -sS https://storage.googleapis.com/apache-beam-testing-metrics/influxdb-backup.tar.gz -o influxdb-backup.tar.gz && \
+    tar xzf influxdb-backup.tar.gz
 
 
 FROM influxdb:1.8.0

.test-infra/metrics/kubernetes/beam-influxdb-autobackup.yaml

@@ -47,9 +47,12 @@ spec:
                 - mountPath: /backup
                   name: shared-data
             - name: copy-to-gsc-bucket
-              image: gcr.io/apache-beam-testing/gsutil
+              image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable-slim
+              env:
+                - name: CLOUDSDK_CORE_PROJECT
+                  value: apache-beam-testing
               command: ['sh', '-c', 'tar czf influxdb-backup.tar.gz /backup
-                        && gsutil cp influxdb-backup.tar.gz
+                        && gcloud storage cp influxdb-backup.tar.gz
                         gs://apache-beam-testing-metrics/']
               volumeMounts:
                 - mountPath: /backup

examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java

@@ -214,7 +214,7 @@ public interface Options extends PipelineOptions {
     day's worth (roughly) of data.
 
     Note: You may want to use a small sample dataset to test it locally/quickly : gs://apache-beam-samples/game/small/gaming_data.csv
-    You can also download it via the command line gsutil cp gs://apache-beam-samples/game/small/gaming_data.csv ./destination_folder/gaming_data.csv */
+    You can also download it via the command line gcloud storage cp gs://apache-beam-samples/game/small/gaming_data.csv ./destination_folder/gaming_data.csv */
     @Default.String("gs://apache-beam-samples/game/gaming_data*.csv")
     String getInput();
 

examples/multi-language/README.md

@@ -124,7 +124,7 @@ mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.multilanguage.Sk
 the digit. The second item is the predicted label of the digit.
 
 ```
-gsutil cat gs://$GCP_BUCKET/multi-language-beam/output*
+gcloud storage cat gs://$GCP_BUCKET/multi-language-beam/output*
 ```
 
 #### Instructions for running the Java pipeline at HEAD (Beam 2.41.0 and 2.42.0).
@@ -171,7 +171,7 @@ export GCP_REGION=<GCP region>
 export EXPANSION_SERVICE_PORT=<PORT>
 
 # This removes any existing output.
-gsutil rm gs://$GCP_BUCKET/multi-language-beam/output*
+gcloud storage rm gs://$GCP_BUCKET/multi-language-beam/output*
 
 ./gradlew :examples:multi-language:sklearnMinstClassification --args=" \
 --runner=DataflowRunner \
@@ -188,7 +188,7 @@ gsutil rm gs://$GCP_BUCKET/multi-language-beam/output*
 of the digit. The second item is the predicted label of the digit.
 
 ```
-gsutil cat gs://$GCP_BUCKET/multi-language-beam/output*
+gcloud storage cat gs://$GCP_BUCKET/multi-language-beam/output*
 ```
 
 ### Python Dataframe Wordcount

examples/notebooks/beam-ml/automatic_model_refresh.ipynb

@@ -298,7 +298,7 @@
         "model = tf.keras.applications.resnet.ResNet101()\n",
         "model.save('resnet101_weights_tf_dim_ordering_tf_kernels.keras')\n",
         "# After saving the model locally, upload the model to GCS bucket and provide that gcs bucket `URI` as `model_uri` to the `TFModelHandler`\n",
-        "!gsutil cp resnet101_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/dataflow/resnet101_weights_tf_dim_ordering_tf_kernels.keras"
+        "!gcloud storage cp resnet101_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/dataflow/resnet101_weights_tf_dim_ordering_tf_kernels.keras"
       ]
     },
     {
@@ -603,7 +603,7 @@
       "source": [
         "model = tf.keras.applications.resnet.ResNet152()\n",
         "model.save('resnet152_weights_tf_dim_ordering_tf_kernels.keras')\n",
-        "!gsutil cp resnet152_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/resnet152_weights_tf_dim_ordering_tf_kernels.keras"
+        "!gcloud storage cp resnet152_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/resnet152_weights_tf_dim_ordering_tf_kernels.keras"
       ]
     },
     {

examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb

@@ -324,7 +324,7 @@
       "source": [
         "# Creates a data directory with our dataset SMSSpamCollection\n",
         "!mkdir -p data\n",
-        "!gsutil cp gs://apachebeamdt/SMSSpamCollection data/"
+        "!gcloud storage cp gs://apachebeamdt/SMSSpamCollection data/"
       ]
     },
     {
@@ -995,7 +995,7 @@
       "source": [
         "!pip install --quiet apache-beam\n",
         "!mkdir -p data\n",
-        "!gsutil cp gs://apachebeamdt/SMSSpamCollection data/"
+        "!gcloud storage cp gs://apachebeamdt/SMSSpamCollection data/"
       ]
     },
     {

examples/notebooks/get-started/learn_beam_transforms_by_doing.ipynb

@@ -324,7 +324,7 @@
       "outputs": [],
       "source": [
         "!mkdir -p data\n",
-        "!gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/"
+        "!gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/"
       ]
     },
     {

examples/notebooks/get-started/learn_beam_windowing_by_doing.ipynb

@@ -173,7 +173,7 @@
       "source": [
         "# Copy the dataset file into the local file system from Google Cloud Storage.\n",
         "!mkdir -p data\n",
-        "!gsutil cp gs://batch-processing-example/air-quality-india.csv data/"
+        "!gcloud storage cp gs://batch-processing-example/air-quality-india.csv data/"
       ]
     },
     {

examples/notebooks/get-started/try-apache-beam-go.ipynb

@@ -85,7 +85,7 @@
         "\n", 
         "# Copy the input file into the local filesystem.\n", 
         "run('mkdir -p data')\n", 
-        "run('gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/')"
+        "run('gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/')"
       ], 
       "cell_type": "code", 
       "execution_count": 1, 
@@ -98,7 +98,7 @@
             "\n", 
             ">> mkdir -p data\n", 
             "\n", 
-            ">> gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", 
+            ">> gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", 
             "Copying gs://dataflow-samples/shakespeare/kinglear.txt...\n", 
             "/ [1 files][153.6 KiB/153.6 KiB]                                                \n", 
             "Operation completed over 1 objects/153.6 KiB.                                    \n", 

examples/notebooks/get-started/try-apache-beam-java.ipynb

@@ -104,7 +104,7 @@
         "\n", 
         "# Copy the input file into the local filesystem.\n", 
         "run('mkdir -p data')\n", 
-        "run('gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/')"
+        "run('gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/')"
       ], 
       "execution_count": 1, 
       "outputs": [
@@ -113,7 +113,7 @@
           "text": [
             ">> mkdir -p data\n", 
             "\n", 
-            ">> gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", 
+            ">> gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", 
             "Copying gs://dataflow-samples/shakespeare/kinglear.txt...\n", 
             "/ [1 files][153.6 KiB/153.6 KiB]                                                \n", 
             "Operation completed over 1 objects/153.6 KiB.                                    \n", 

examples/notebooks/get-started/try-apache-beam-py.ipynb

@@ -107,7 +107,7 @@
         "\n", 
         "# Copy the input file into the local file system.\n", 
         "run('mkdir -p data')\n", 
-        "run('gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/')"
+        "run('gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/')"
       ], 
       "execution_count": 1, 
       "outputs": [
@@ -118,7 +118,7 @@
             "\n", 
             ">> mkdir -p data\n", 
             "\n", 
-            ">> gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", 
+            ">> gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", 
             "Copying gs://dataflow-samples/shakespeare/kinglear.txt...\n", 
             "/ [1 files][153.6 KiB/153.6 KiB]                                                \n", 
             "Operation completed over 1 objects/153.6 KiB.                                    \n",