diff --git a/build.sh b/build.sh index 926a27c..406127e 100755 --- a/build.sh +++ b/build.sh @@ -7,7 +7,7 @@ # SINGLE_VERSION - Specifies the image version - (must match with subdirectory in repo) # VERSIONS - Must be set to a list with possible versions (subdirectories) -set -eE +set -E [ -n "${DEBUG:-}" ] && set -x # shellcheck shell=bash @@ -22,53 +22,6 @@ error() { echo "ERROR: $*" ; false ; } trap 'echo "errexit on line $LINENO, $0" >&2' ERR -# _parse_output_inner -# ------------------- -# Helper function for 'parse_output'. -# We need to avoid case statements in $() for older Bash versions (per issue -# postgresql-container#35, mac ships with 3.2). -# Example of problematic statement: echo $(case i in i) echo i;; esac) -_parse_output_inner () -{ - set -o pipefail - { - case $stream in - stdout|1|"") - eval "$command" | tee >(cat - >&"$stdout_fd") - ;; - stderr|2) - set +x # avoid stderr pollution - eval "$command" {free_fd}>&1 1>&"$stdout_fd" 2>&"$free_fd" | tee >(cat - >&"$stderr_fd") - ;; - esac - # Inherit correct exit status. - (exit "${PIPESTATUS[0]}") - } | eval "$filter" -} - - -# parse_output COMMAND FILTER_COMMAND OUTVAR [STREAM={stderr|stdout}] -# ------------------------------------------------------------------- -# Parse standard (error) output of COMMAND with FILTER_COMMAND and store the -# output into variable named OUTVAR. STREAM might be 'stdout' or 'stderr', -# defaults to 'stdout'. The filtered output stays (live) printed to terminal. -# This method doesn't create any explicit temporary files. -# Defines: -# ${$OUTVAR}: Set to FILTER_COMMAND output. -parse_output () -{ - local command=$1 filter=$2 var=$3 stream=$4 - echo "-> building using $command" - local raw_output='' rc=0 - { - # shellcheck disable=SC2034 - raw_output=$(_parse_output_inner) - } {stdout_fd}>&1 {stderr_fd}>&2 - rc=$? - eval "$var=\$raw_output" - (exit $rc) -} - # "best-effort" cleanup of image function clean_image { for id_file in .image-id .image-id-from; do @@ -196,44 +149,54 @@ function docker_build_with_version { if [[ "$SKIP_SQUASH" -eq 0 ]] && [[ "$is_podman" -eq 1 ]]; then BUILD_OPTIONS+=" --squash" fi - - command="docker build ${BUILD_OPTIONS} -f $dockerfile ${DOCKER_BUILD_CONTEXT}" - echo "-> building using $command" - set +x -o pipefail - tmp_file=$(mktemp "/tmp/${dir}-${OS}.XXXXXX") - $command 2>&1 | tee "$tmp_file" - ret_code=$? - set -x +o pipefail - echo "Return code from docker build is '$ret_code'." - last_row=$(< "$tmp_file" tail -n 1) - if [[ $ret_code != "0" ]]; then - if [[ "${OS}" == "rhel8" ]] || [[ "${OS}" == "rhel9" ]] || [[ "${OS}" == "rhel10" ]]; then - # Do not fail in case of sending log to pastebin or logdetective fails. - set +e - analyze_logs_by_logdetective "${tmp_file}" - set -e - fi - else - # Structure of log build is as follows: - # COMMIT - # --> e191d12b5928 - # e191d12b5928360dd6024fe80d31e08f994d42577f76b9b143e014749afc8ab4 - # shellcheck disable=SC2016 - if [[ "$last_row" =~ (^-->)?(Using cache )?[a-fA-F0-9]+$ ]]; then - IMAGE_ID="$last_row" + i=1 + build_failed=1 + while [ $i -le 2 ]; do + command="docker build ${BUILD_OPTIONS} -f $dockerfile ${DOCKER_BUILD_CONTEXT}" + echo "-> building using $command" + set +x -o pipefail + tmp_file=$(mktemp "/tmp/${dir}-${OS}.XXXXXX") + $command 2>&1 | tee "$tmp_file" + ret_code=$? + set -x +o pipefail + echo "Return code from docker build is '$ret_code'." + last_row=$(< "$tmp_file" tail -n 1) + if [[ $ret_code != "0" ]]; then + # In case of failure, we want to analyze the logs and send them to pastebin or logdetective. + # The failure can be ethel issue like network failure or registry failure. + # Red Hat Enterprise Linux 9 for x86_64 - AppStre 0.0 B/s | 0 B 00:00 + # Errors during downloading metadata for repository 'rhel-9-for-x86_64-appstream-rpms': + # - Curl error (56): Failure when receiving data from the peer for https:// [Received HTTP code 503 from proxy after CONNECT] + # Error: Failed to download metadata for repo 'rhel-9-for-x86_64-appstream-rpms': + if [[ "${OS}" == "rhel8" ]] || [[ "${OS}" == "rhel9" ]] || [[ "${OS}" == "rhel10" ]]; then + # Do not fail in case of sending log to pastebin or logdetective fails. + analyze_logs_by_logdetective "${tmp_file}" + fi + ((i++)) + sleep 5 + echo "Retrying to build image for version $dir, attempt $i of 2." + else + # Structure of log build is as follows: + # COMMIT + # --> e191d12b5928 + # e191d12b5928360dd6024fe80d31e08f994d42577f76b9b143e014749afc8ab4 + # shellcheck disable=SC2016 + if [[ "$last_row" =~ (^-->)?(Using cache )?[a-fA-F0-9]+$ ]]; then + IMAGE_ID="$last_row" + fi + echo "$IMAGE_ID" > .image-id + tag_image + build_failed=0 + break fi - fi - rm -f "$tmp_file" - - # shellcheck disable=SC2016 + rm -f "$tmp_file" + done + if [[ $build_failed -ne 0 ]]; then + echo "-> Build failed for version $dir and OS $OS after 2 attempts, giving up." + exit 1 + fi -# parse_output 'docker build '"$BUILD_OPTIONS"' -f "$dockerfile" "${DOCKER_BUILD_CONTEXT}"' \ -# "tail -n 1 | awk '/Successfully built|(^--> )?(Using cache )?[a-fA-F0-9]+$/{print \$NF}'" \ -# IMAGE_ID -# analyze_logs_by_logdetective "$?" "${tmp_file}" - echo "$IMAGE_ID" > .image-id - tag_image } function tag_image { @@ -276,7 +239,9 @@ fi echo "Built versions are: $dirs" for dir in ${dirs}; do + # shellcheck disable=SC2164 pushd "${dir}" > /dev/null docker_build_with_version Dockerfile."$OS" + # shellcheck disable=SC2164 popd > /dev/null done