diff --git a/beam/util/build-beam-artifacts.sh b/beam/util/build-beam-artifacts.sh index ed83f6ed9..b92107a21 100755 --- a/beam/util/build-beam-artifacts.sh +++ b/beam/util/build-beam-artifacts.sh @@ -11,9 +11,18 @@ readonly BEAM_JOB_SERVICE_DESTINATION="$1" readonly BEAM_CONTAINER_IMAGE_DESTINATION="$2" readonly BEAM_SOURCE_VERSION="${3:-master}" +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + function build_job_service() { ./gradlew :beam-runners-flink_2.11-job-server:shadowJar - gsutil cp \ + ${GSUTIL} cp \ ./runners/flink/job-server/build/libs/beam-runners-flink_2.11-job-server-*-SNAPSHOT.jar \ ${BEAM_JOB_SERVICE_DESTINATION}/beam-runners-flink_2.11-job-server-${BEAM_SOURCE_VERSION}-SNAPSHOT.jar } diff --git a/cloudbuild/run-presubmit-on-k8s.sh b/cloudbuild/run-presubmit-on-k8s.sh index 5e2001c91..138f22cff 100644 --- a/cloudbuild/run-presubmit-on-k8s.sh +++ b/cloudbuild/run-presubmit-on-k8s.sh @@ -8,6 +8,16 @@ readonly DATAPROC_IMAGE_VERSION=$3 readonly POD_NAME=presubmit-${DATAPROC_IMAGE_VERSION//./-}-${BUILD_ID//_/-} +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + + gcloud container clusters get-credentials "${CLOUDSDK_CONTAINER_CLUSTER}" LOGS_SINCE_TIME=$(date --iso-8601=seconds) @@ -52,7 +62,7 @@ if [[ ${exit_code} != 0 ]]; then LOG_GCS_PATH="gs://${BUCKET}/${BUILD_ID}/logs/${POD_NAME}.log" echo "Attempting to upload logs to ${LOG_GCS_PATH}" - if kubectl logs "${POD_NAME}" | gsutil cp - "${LOG_GCS_PATH}"; then + if kubectl logs "${POD_NAME}" | ${GSUTIL} cp - "${LOG_GCS_PATH}"; then echo "Logs for failed pod ${POD_NAME} uploaded to: ${LOG_GCS_PATH}" else echo "Log upload to ${LOG_GCS_PATH} failed." diff --git a/datasketches/datasketches.sh b/datasketches/datasketches.sh index b0aee99c3..94c44f6b9 100755 --- a/datasketches/datasketches.sh +++ b/datasketches/datasketches.sh @@ -35,6 +35,15 @@ readonly DS_LIBPATH="/usr/lib/datasketches" readonly SPARK_VERSION=$(spark-submit --version 2>&1 | sed -n 's/.*version[[:blank:]]\+\([0-9]\+\.[0-9]\).*/\1/p' | head -n1) readonly SPARK_JAVA_EXAMPLE_JAR="gs://spark-lib/datasketches/spark-java-thetasketches-1.0-SNAPSHOT.jar" +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + function download_libraries() { mkdir -p ${DS_LIBPATH} @@ -57,7 +66,7 @@ function download_libraries() function download_example_jar() { if [[ "${SPARK_VERSION}" < "3.5" ]]; then - gsutil cp "${SPARK_JAVA_EXAMPLE_JAR}" "${DS_LIBPATH}" + ${GSUTIL} cp "${SPARK_JAVA_EXAMPLE_JAR}" "${DS_LIBPATH}" if [ $? -eq 0 ]; then echo "Downloaded "${SPARK_JAVA_EXAMPLE_JAR}" successfully" else diff --git a/gobblin/gobblin.sh b/gobblin/gobblin.sh index 16e50287b..02dedddf1 100755 --- a/gobblin/gobblin.sh +++ b/gobblin/gobblin.sh @@ -25,6 +25,15 @@ readonly HADOOP_LIB="/usr/lib/hadoop/lib" readonly JAR_NAME_CANONICALIZER="s/([-a-zA-Z0-9]+?)[-]([0-9][0-9.]+?)([-.].*?)?.jar/\1/" +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + function maybe_symlink() { local jar=$1 if [[ ! -f "${HADOOP_LIB}/${jar}" ]]; then @@ -91,7 +100,7 @@ EOF function install_package() { # Download binary. local temp=$(mktemp -d) - gsutil cp "${PACKAGE_URL}" "${temp}/package.tar.gz" + ${GSUTIL} cp "${PACKAGE_URL}" "${temp}/package.tar.gz" tar -xf "${temp}/package.tar.gz" -C "${temp}" # Setup package. diff --git a/gpu/manual-test-runner.sh b/gpu/manual-test-runner.sh index 37982bfe4..42f032a19 100644 --- a/gpu/manual-test-runner.sh +++ b/gpu/manual-test-runner.sh @@ -38,6 +38,17 @@ export BUCKET="$(jq -r .BUCKET env.json)" gcs_log_dir="gs://${BUCKET}/${BUILD_ID}/logs" +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +GSUTIL_OPTS="" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" + GSUTIL_OPTS="-m" +fi + function exit_handler() { RED='\\e[0;31m' GREEN='\\e[0;32m' @@ -48,7 +59,7 @@ function exit_handler() { # TODO: remove any test related resources in the project echo 'Uploading local logs to GCS bucket.' - gsutil -m rsync -r "${log_dir}/" "${gcs_log_dir}/" + ${GSUTIL} ${GSUTIL_OPTS} rsync -r "${log_dir}/" "${gcs_log_dir}/" if [[ -f "${tmp_dir}/tests_success" ]]; then echo -e "${GREEN}Workflow succeeded${NC}, check logs at ${log_dir}/ or ${gcs_log_dir}/" diff --git a/hbase/test_hbase.py b/hbase/test_hbase.py index 2d8d07f19..c8c54b180 100644 --- a/hbase/test_hbase.py +++ b/hbase/test_hbase.py @@ -15,11 +15,11 @@ def setUp(self): super().setUp() self.GCS_BUCKET = "test-hbase-{}-{}".format(self.datetime_str(), self.random_str()) - self.assert_command('gsutil mb -c regional -l {} gs://{}'.format( + self.assert_command('gcloud storage buckets create --default-storage-class=regional --location {} gs://{}'.format( self.REGION, self.GCS_BUCKET)) def tearDown(self): - self.assert_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET)) + self.assert_command('gcloud storage rm --recursive --continue-on-error gs://{}'.format(self.GCS_BUCKET)) super().tearDown() def verify_instance(self, name): diff --git a/hive-lineage/hive-lineage.sh b/hive-lineage/hive-lineage.sh index 84da22515..f42761028 100644 --- a/hive-lineage/hive-lineage.sh +++ b/hive-lineage/hive-lineage.sh @@ -46,9 +46,18 @@ function set_hive_lineage_conf() { done } +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + function install_jars() { echo "Installing openlineage-hive hook" - gsutil cp -P "$INSTALLATION_SOURCE/hive-openlineage-hook-$HIVE_OL_HOOK_VERSION.jar" "$HIVE_LIB_DIR/hive-openlineage-hook.jar" + ${GSUTIL} cp -P "$INSTALLATION_SOURCE/hive-openlineage-hook-$HIVE_OL_HOOK_VERSION.jar" "$HIVE_LIB_DIR/hive-openlineage-hook.jar" } function restart_hive_server2_master() { diff --git a/knox/verify_knox.sh b/knox/verify_knox.sh index 5241a1252..b5674f3a6 100644 --- a/knox/verify_knox.sh +++ b/knox/verify_knox.sh @@ -40,10 +40,19 @@ function test_installation() { } +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + # to test update, we will upload a new topology to gs bucket, and check whether it appears # we assume that knox initialization action is the very first one, /etc/google-dataproc/startup-scripts/dataproc-initialization-script-0 function test_update_new_topology() { - gsutil cp /etc/knox/conf/topologies/example-hive-pii.xml "${KNOX_GW_CONFIG_GCS}/topologies/update_topology.xml" + ${GSUTIL} cp /etc/knox/conf/topologies/example-hive-pii.xml "${KNOX_GW_CONFIG_GCS}/topologies/update_topology.xml" sudo /bin/bash /etc/google-dataproc/startup-scripts/dataproc-initialization-script-0 update test_installation update_topology [[ $? == 1 ]] && return 1 diff --git a/push-to-gcs.sh b/push-to-gcs.sh index 9e4b2f5ca..2e3cbfd84 100755 --- a/push-to-gcs.sh +++ b/push-to-gcs.sh @@ -22,6 +22,15 @@ readonly HEAD="$1" readonly MODULE="$2" readonly GCS_FOLDER=gs://dataproc-initialization-actions/${MODULE}/ +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" +fi + [[ -n "${HEAD}" && -n "${MODULE}" ]] # Verify the repo has no uncommitted changes. @@ -51,6 +60,6 @@ for file in "${MODULE}/"*.sh; do fi done -gsutil -m rsync -R -x "__pycache__/.*" "${MODULE}/" "${GCS_FOLDER}" +${GSUTIL} rsync -r -x "__pycache__/.*" "${MODULE}/" "${GCS_FOLDER}" echo "Pushed ${MODULE}/ to ${GCS_FOLDER}." diff --git a/rapids/manual-test-runner.sh b/rapids/manual-test-runner.sh index 371917a40..fc125f118 100644 --- a/rapids/manual-test-runner.sh +++ b/rapids/manual-test-runner.sh @@ -37,6 +37,17 @@ export BUCKET="$(jq -r .BUCKET env.json)" gcs_log_dir="gs://${BUCKET}/${BUILD_ID}/logs" +function version_le() { [[ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]; } +function version_lt() { [[ "$1" = "$2" ]] && return 1 || version_le "$1" "$2"; } + +GCLOUD_SDK_VERSION="$(gcloud --version | awk -F'SDK ' '/Google Cloud SDK/ {print $2}')" +GSUTIL="gcloud storage" +GSUTIL_OPTS="" +if version_lt "${GCLOUD_SDK_VERSION}" "402.0.0"; then + GSUTIL="gsutil" + GSUTIL_OPTS="-m" +fi + function exit_handler() { RED='\\e[0;31m' GREEN='\\e[0;32m' @@ -47,7 +58,7 @@ function exit_handler() { # TODO: remove any test related resources in the project echo 'Uploading local logs to GCS bucket.' - gsutil -m rsync -r "${log_dir}/" "${gcs_log_dir}/" + ${GSUTIL} ${GSUTIL_OPTS} rsync -r "${log_dir}/" "${gcs_log_dir}/" if [[ -f "${tmp_dir}/tests_success" ]]; then echo -e "${GREEN}Workflow succeeded, check logs at ${log_dir}/ or ${gcs_log_dir}/${NC}" @@ -74,4 +85,3 @@ export INTERNAL_IP_SSH="true" screen -US "${session_name}" -c rapids/bazel.screenrc -