From 53d3b2f34ac7e764fe9e523d68478729224bb7a7 Mon Sep 17 00:00:00 2001 From: Chandra Date: Sun, 25 Jan 2026 13:24:41 +0000 Subject: [PATCH 1/4] feat: run benchmarks in CI --- cloudbuild/micro_benchmarks.yaml | 110 +++++++++++++++++++ cloudbuild/run_micro_benchmarks.sh | 32 ++++++ tests/perf/microbenchmarks/reads/config.yaml | 8 +- 3 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 cloudbuild/micro_benchmarks.yaml create mode 100644 cloudbuild/run_micro_benchmarks.sh diff --git a/cloudbuild/micro_benchmarks.yaml b/cloudbuild/micro_benchmarks.yaml new file mode 100644 index 000000000..2f1fdd707 --- /dev/null +++ b/cloudbuild/micro_benchmarks.yaml @@ -0,0 +1,110 @@ +substitutions: + _REGION: "us-central1" + _ZONE: "us-central1-a" + _SHORT_BUILD_ID: ${BUILD_ID:0:8} + _VM_NAME: "py-sdk-micro-benchmarks-ci-${_SHORT_BUILD_ID}" + _ULIMIT: "10000" # 10k, for gRPC bidi streams + + + +steps: + # Step 0: Generate a persistent SSH key for this build run. + # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + # Generate the SSH key + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + # Save the public key content to a file for the cleanup step + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + waitFor: ["-"] + + # Step 1 Create a GCE VM to run the tests. + # The VM is created in the same zone as the buckets to test rapid storage features. + # It's given the 'cloud-platform' scope to allow it to access GCS and other services. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "create" + - "${_VM_NAME}" + - "--project=${PROJECT_ID}" + - "--zone=${_ZONE}" + - "--machine-type=c4-standard-192" + - "--network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default" + - "--metadata=enable-osconfig=TRUE,enable-oslogin=TRUE" + - "--maintenance-policy=MIGRATE" + - "--provisioning-model=STANDARD" + - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" + - "--scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write" + - "--tags=allow-ssh,http-server,https-server,lb-health-check" + - "--create-disk=auto-delete=yes,boot=yes,device-name=${_VM_NAME},disk-resource-policy=projects/${PROJECT_ID}/regions/${_REGION}/resourcePolicies/default-schedule-1,image=projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20251120,mode=rw,provisioned-iops=3600,provisioned-throughput=290,size=100,type=hyperdisk-balanced" + - "--no-shielded-secure-boot" + - "--shielded-vtpm" + - "--shielded-integrity-monitoring" + - "--labels=goog-ops-agent-policy=v2-x86-template-1-4-0,goog-ec-src=vm_add-gcloud" + - "--reservation-affinity=any" + - "--network-performance-configs=total-egress-bandwidth-tier=TIER_1" + waitFor: ["-"] + + # Step 2: Run the integration tests inside the newly created VM and cleanup. + # This step uses 'gcloud compute ssh' to execute a remote script. + # The VM is deleted after tests are run, regardless of success. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-tests-and-delete-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + # Wait for the VM to be fully initialized and SSH to be ready. + for i in {1..10}; do + if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then + break + fi + echo "Waiting for VM to become available... (attempt $i/10)" + sleep 15 + done + # copy the script to the VM + gcloud compute scp cloudbuild/run_micro_benchmarks.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + # Execute the script on the VM via SSH. + # Capture the exit code to ensure cleanup happens before the build fails. + set +e + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" + EXIT_CODE=$? + set -e + + echo "--- Deleting GCE VM ---" + gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet + + # Exit with the original exit code from the test script. + exit $$EXIT_CODE + waitFor: + - "create-vm" + - "generate-ssh-key" + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + echo "--- Removing SSH key from OS Login profile to prevent accumulation ---" + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + waitFor: + - "run-tests-and-delete-vm" + +timeout: "3600s" # 60 minutes + +options: + logging: CLOUD_LOGGING_ONLY + pool: + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" diff --git a/cloudbuild/run_micro_benchmarks.sh b/cloudbuild/run_micro_benchmarks.sh new file mode 100644 index 000000000..e7d1178dc --- /dev/null +++ b/cloudbuild/run_micro_benchmarks.sh @@ -0,0 +1,32 @@ + +set -euxo pipefail +echo '--- Installing git and cloning repository on VM ---' +sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv + +# Clone the repository and checkout the specific commit from the build trigger. +git clone https://github.com/googleapis/python-storage.git +cd python-storage +git fetch origin "refs/pull/${_PR_NUMBER}/head" +git checkout ${COMMIT_SHA} + + +echo '--- Installing Python and dependencies on VM ---' +python3 -m venv env +source env/bin/activate + +echo 'Install testing libraries explicitly, as they are not in setup.py' +pip install --upgrade pip +pip install pytest pytest-timeout pytest-subtests pytest-asyncio +pip install google-cloud-testutils google-cloud-kms +pip install -e ".[grpc,testing]" + +echo '--- Setting up environment variables on VM ---' +export ZONAL_BUCKET=${_ZONAL_BUCKET} +export RUN_ZONAL_SYSTEM_TESTS=True +CURRENT_ULIMIT=$(ulimit -n) +echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT +pytest -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' \ + --log-date-format='%H:%M:%S' \ + --benchmark-json=output.json \ + -vv -s tests/perf/microbenchmarks/reads/test_reads.py::test_downloads_single_proc_single_coro diff --git a/tests/perf/microbenchmarks/reads/config.yaml b/tests/perf/microbenchmarks/reads/config.yaml index 25bfd92c8..a7a1c545e 100644 --- a/tests/perf/microbenchmarks/reads/config.yaml +++ b/tests/perf/microbenchmarks/reads/config.yaml @@ -1,11 +1,11 @@ common: bucket_types: - - "regional" + # - "regional" - "zonal" file_sizes_mib: - - 1024 # 1GiB - chunk_sizes_mib: [100] - rounds: 10 + - 100 # 1GiB + chunk_sizes_mib: [10] + rounds: 3 workload: From 2f13bb89f510328491b0e99dbc4d4174c4fe86c1 Mon Sep 17 00:00:00 2001 From: Chandra Date: Sun, 25 Jan 2026 13:59:15 +0000 Subject: [PATCH 2/4] user correct script name and keep it in a variable --- cloudbuild/backup.yaml | 19 +++++++++++++++++++ cloudbuild/micro_benchmarks.yaml | 5 +++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 cloudbuild/backup.yaml diff --git a/cloudbuild/backup.yaml b/cloudbuild/backup.yaml new file mode 100644 index 000000000..9a839de4f --- /dev/null +++ b/cloudbuild/backup.yaml @@ -0,0 +1,19 @@ +gcloud compute instances create +py-sdk-micro-benchmarks-ci-test-2 +--project=gcs-aiml-clients-testing-101 +--zone=us-central1-a +--machine-type=c4-standard-192 +--network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default +--metadata=enable-osconfig=TRUE,enable-oslogin=TRUE +--maintenance-policy=MIGRATE +--provisioning-model=STANDARD +--service-account=gcsfs-perf-ci-runner@gcs-aiml-clients-testing-101.iam.gserviceaccount.com +--scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write +--tags=allow-ssh,http-server,https-server,lb-health-check +--create-disk=auto-delete=yes,boot=yes,device-name=py-sdk-micro-benchmarks-ci-test,disk-resource-policy=projects/gcs-aiml-clients-testing-101/regions/us-central1/resourcePolicies/default-schedule-1,image=projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20251120,mode=rw,provisioned-iops=3600,provisioned-throughput=290,size=100,type=hyperdisk-balanced +--no-shielded-secure-boot +--shielded-vtpm +--shielded-integrity-monitoring +--labels=goog-ops-agent-policy=v2-x86-template-1-4-0,goog-ec-src=vm_add-gcloud +--reservation-affinity=any +--network-performance-configs=total-egress-bandwidth-tier=TIER_1 \ No newline at end of file diff --git a/cloudbuild/micro_benchmarks.yaml b/cloudbuild/micro_benchmarks.yaml index 2f1fdd707..b30de522d 100644 --- a/cloudbuild/micro_benchmarks.yaml +++ b/cloudbuild/micro_benchmarks.yaml @@ -4,6 +4,7 @@ substitutions: _SHORT_BUILD_ID: ${BUILD_ID:0:8} _VM_NAME: "py-sdk-micro-benchmarks-ci-${_SHORT_BUILD_ID}" _ULIMIT: "10000" # 10k, for gRPC bidi streams + _BENCHMARK_SCRIPT: "run_micro_benchmarks.sh" @@ -72,12 +73,12 @@ steps: sleep 15 done # copy the script to the VM - gcloud compute scp cloudbuild/run_micro_benchmarks.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + gcloud compute scp cloudbuild/${_BENCHMARK_SCRIPT} ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. set +e - gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} ${_BENCHMARK_SCRIPT} EXIT_CODE=$? set -e From 8594296b9e81900a70cdffd0294c9a97445596b6 Mon Sep 17 00:00:00 2001 From: Chandra Date: Sun, 25 Jan 2026 14:08:26 +0000 Subject: [PATCH 3/4] add trailing `"` --- cloudbuild/micro_benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/micro_benchmarks.yaml b/cloudbuild/micro_benchmarks.yaml index b30de522d..35b036125 100644 --- a/cloudbuild/micro_benchmarks.yaml +++ b/cloudbuild/micro_benchmarks.yaml @@ -78,7 +78,7 @@ steps: # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. set +e - gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} ${_BENCHMARK_SCRIPT} + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} ${_BENCHMARK_SCRIPT}" EXIT_CODE=$? set -e From bc6bf69b969a1426bb3977bacf36b9ae9b2da77b Mon Sep 17 00:00:00 2001 From: Chandra Date: Sun, 25 Jan 2026 14:20:17 +0000 Subject: [PATCH 4/4] add bash --- cloudbuild/micro_benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/micro_benchmarks.yaml b/cloudbuild/micro_benchmarks.yaml index 35b036125..34dde3885 100644 --- a/cloudbuild/micro_benchmarks.yaml +++ b/cloudbuild/micro_benchmarks.yaml @@ -78,7 +78,7 @@ steps: # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. set +e - gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} ${_BENCHMARK_SCRIPT}" + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n {_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash ${_BENCHMARK_SCRIPT}" EXIT_CODE=$? set -e