diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index d392615213..b15abad9d3 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -27,13 +27,16 @@ run-name: > || github.event.pull_request.title }} jobs: - validate: + + build_info: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' @@ -44,7 +47,7 @@ jobs: - name: Install required packages run: | sudo apt update -y - sudo apt-get install -y file + sudo apt-get install -y file jq - name: Install Python dependencies run: | @@ -59,23 +62,557 @@ jobs: python3 -u gha-script/validate_builds.py ${PR_NUMBER:-false} 2>&1 | tee build_log my_pid_status=${PIPESTATUS[0]} - build_size=$(stat -c %s build_log) - if [ "$my_pid_status" -ne 0 ]; then - echo "Script failed for PR #${PR_NUMBER}" - if [ "$build_size" -lt 1800000 ]; then - cat build_log - else - echo "Build log too large, showing last 100 lines" - tail -100 build_log - fi - exit 1 + echo "Script failed for PR #${PR_NUMBER}" + echo "::group::Validation Logs" + tail -200 build_log + echo "::endgroup::" + exit 1 + else + echo "Script completed successfully for PR #${PR_NUMBER}" + fi + + - name: Fetch base branch + run: git fetch origin ${{ github.base_ref }} --depth=1 + + - name: Locate and parse build_info.json + run: | + + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1) + + if [ -z "$BUILD_INFO_FILE" ]; then + echo "No build_info.json modified, trying to detect from changed files..." + + PACKAGE_DIR=$(echo "$CHANGED_FILES" | head -n 1 | cut -d'/' -f1-2) + + BUILD_INFO_FILE="$PACKAGE_DIR/build_info.json" + + if [ ! -f "$BUILD_INFO_FILE" ]; then + echo "Could not locate build_info.json!" + exit 1 + fi + + echo "Using fallback build_info: $BUILD_INFO_FILE" + fi + + PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) + VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) + + echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV + echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "CHANGED_FILES<> $GITHUB_ENV + echo "$CHANGED_FILES" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + + - name: Read build_info.json + run: | + chmod +x ./gha-script/read_buildinfo.sh + bash ./gha-script/read_buildinfo.sh + + - name: Create scanner-env.sh + run: | + mkdir -p package-cache + + PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) + WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) + + # Robust docker_build extraction + if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then + DOCKER_BUILD="true" else - echo "Script completed successfully for PR #${PR_NUMBER}" - if [ "$build_size" -lt 1800000 ]; then - cat build_log - else - echo "Build log too large, showing last 100 lines" - tail -100 build_log - fi + DOCKER_BUILD="false" + fi + + cat < package-cache/scanner-env.sh + export PACKAGE_NAME=$PACKAGE_NAME + export VERSION=$VERSION + export PACKAGE_DIR=$PACKAGE_DIR + export WHEEL_BUILD=$WHEEL_BUILD + export BUILD_DOCKER=$DOCKER_BUILD + export CHANGED_FILES="$CHANGED_FILES" + EOF + + mv variable.sh package-cache/ + + - name: Archive package cache + run: tar -czf package-cache.tar.gz package-cache/ + + - name: Upload package cache + uses: actions/upload-artifact@v6 + with: + name: package-cache + path: package-cache.tar.gz + + build: + needs: build_info + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + + steps: + - uses: actions/checkout@v6 + + - name: Download package-cache + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Build Package + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + + echo "------------------- variable.sh -----------------------------" + cat package-cache/variable.sh + echo "------------------- scanner-env.sh -----------------------------" + cat package-cache/scanner-env.sh + + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + +# ===================== WHEEL JOBS ===================== + + + wheel_build_py39: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.9" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py310: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.10" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py311: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.11" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + + wheel_build_py312: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.12" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + wheel_build_py313: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.13" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py314: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.14" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + + build_docker: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Download package-cache + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Build Docker Image + run: | + echo "===== Sourcing environment =====" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) + + if [[ "$BUILD_DOCKER" != "true" ]]; then + echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" + exit 0 + fi + + if [[ -z "$DOCKERFILE_CHANGED" ]]; then + echo "Skipping Docker build as no Dockerfile changes detected in PR" + exit 0 + fi + + echo "Dockerfile change detected, proceeding with build..." + + echo "===== Starting Docker build =====" + chmod +x ./gha-script/build_docker.sh + bash ./gha-script/build_docker.sh + + echo "===== Docker images after build =====" + docker images + + echo "===== Saving Docker image =====" + docker save -o package-cache/image.tar "$IMAGE_NAME" + + ls -lh package-cache/image.tar + + + + + + + + + + + + diff --git a/gha-script/build_package.sh b/gha-script/build_package.sh index 8c1cd2ee24..40749d0b98 100755 --- a/gha-script/build_package.sh +++ b/gha-script/build_package.sh @@ -1,4 +1,4 @@ - #!/bin/bash -e +#!/bin/bash -e sudo apt update -y && sudo apt-get install file -y #pip3 install --upgrade requests @@ -37,15 +37,7 @@ else fi fi -# python3 script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" > build_log & -# SCRIPT_PID=$! -# while ps -p $SCRIPT_PID > /dev/null -# do -# echo "$SCRIPT_PID is running" -# sleep 100 -# done -# wait $SCRIPT_PID python3 gha-script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" 2>&1 | tee build_log my_pid_status=${PIPESTATUS[0]} @@ -55,21 +47,9 @@ if [ $my_pid_status != 0 ]; then echo "Script execution failed for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" " echo "*************************************************************************************" - if [ $build_size -lt 1800000 ]; - then - cat build_log - else - tail -100 build_log - fi exit 1 else echo "Script execution completed successfully for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" " - echo "*************************************************************************************" - if [ $build_size -lt 1800000 ]; - then - cat build_log - else - tail -100 build_log - fi + echo "*************************************************************************************" fi exit 0 diff --git a/gha-script/build_wheels.sh b/gha-script/build_wheels.sh index 37d49b67b6..b7593de1bb 100644 --- a/gha-script/build_wheels.sh +++ b/gha-script/build_wheels.sh @@ -42,15 +42,6 @@ WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh # path to post_process_wheel script (suffix addition, license addition, metadata addition) POST_PROCESS_SCRIPT_PATH=gha-script/post_process_wheel.py -#python3 gha-script/build_wheels.py "$WHEEL_SCRIPT" "$PYTHON_VERSION" "$docker_image" "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" > build_log & - -# SCRIPT_PID=$! -# while ps -p $SCRIPT_PID > /dev/null -# do -# echo "$SCRIPT_PID is running" -# sleep 100 -# done -# wait $SCRIPT_PID python3 gha-script/build_wheels.py "$WHEEL_SCRIPT" "$PYTHON_VERSION" "$docker_image" "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$POST_PROCESS_SCRIPT_PATH" 2>&1 | tee wheel_build_log wheel_status=${PIPESTATUS[0]} @@ -59,21 +50,9 @@ if [ $wheel_status != 0 ]; then echo "Wheel build failed for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" " echo "*************************************************************************************" - if [ $log_size -lt 1800000 ]; - then - cat wheel_build_log - else - tail -100 wheel_build_log - fi exit 1 else echo "Script execution completed successfully for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" " echo "*************************************************************************************" - if [ $log_size -lt 1800000 ]; - then - cat wheel_build_log - else - tail -100 wheel_build_log - fi fi exit 0