[improve][broker] Upgrade bookkeeper to 4.17.4

.github/PULL_REQUEST_TEMPLATE.md

@@ -70,29 +70,3 @@ This change added tests and can be verified as follows:
 - [ ] The admin CLI options
 - [ ] The metrics
 - [ ] Anything that affects deployment
-
-### Documentation
-
-<!-- DO NOT REMOVE THIS SECTION. CHECK THE PROPER BOX ONLY. -->
-
-- [ ] `doc` <!-- Your PR contains doc changes. -->
-- [ ] `doc-required` <!-- Your PR changes impact docs and you will update later -->
-- [ ] `doc-not-needed` <!-- Your PR changes do not impact docs -->
-- [ ] `doc-complete` <!-- Docs have been already added -->
-
-### Matching PR in forked repository
-
-PR in forked repository: <!-- ENTER URL HERE -->
-
-<!--
-After opening this PR, the build in apache/pulsar will fail and instructions will
-be provided for opening a PR in the PR author's forked repository.
-
-apache/pulsar pull requests should be first tested in your own fork since the 
-apache/pulsar CI based on GitHub Actions has constrained resources and quota.
-GitHub Actions provides separate quota for pull requests that are executed in 
-a forked repository.
-
-The tests will be run in the forked repository until all PR review comments have
-been handled, the tests pass and the PR is approved by a reviewer.
--->

.github/actions/clean-disk/action.yml

@@ -23,44 +23,98 @@ inputs:
   mode:
     description: "Use 'full' to clean as much as possible"
     required: false
+  clean-threshold-gb:
+    description: "Skip cleaning if available disk space exceeds this threshold (in GB)"
+    required: false
+    default: '20'
+  full-clean-threshold-gb:
+    description: "In 'full' mode, skip additional cleaning steps if available disk space exceeds this threshold (in GB)"
+    required: false
+    default: '40'
 runs:
   using: composite
   steps:
-    - run: |
+    - shell: bash
+      env:
+        CLEAN_MODE: ${{ inputs.mode }}
+        CLEAN_THRESHOLD_GB: ${{ inputs.clean-threshold-gb }}
+        FULL_CLEAN_THRESHOLD_GB: ${{ inputs.full-clean-threshold-gb }}
+      run: |
         if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-          directories=(/usr/local/lib/android /opt/ghc)
-          if [[ "${{ inputs.mode }}" == "full" ]]; then
-            # remove these directories only when mode is 'full'
-            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
+
+          available_gb() {
+            df --output=avail -BG / | tail -1 | tr -d ' G'
+          }
+
+          show_disk() {
+            if mountpoint -q /mnt; then df -BM / /mnt; else df -BM /; fi
+          }
+
+          remove_dir() {
+            local directory=$1
+            if [ -d "$directory" ]; then
+              echo "::group::Removing $directory"
+              local emptydir=/tmp/empty$$/
+              mkdir -p "$emptydir"
+              # fast way to delete a lot of files on linux
+              time sudo eatmydata rsync -a --delete "$emptydir" "${directory}/"
+              time sudo eatmydata rm -rf "${directory}"
+              rmdir "$emptydir" 2>/dev/null || true
+              time show_disk
+              echo "::endgroup::"
+            fi
+          }
+
+          # Remove directories one at a time, stopping when threshold is met
+          clean_dirs() {
+            local threshold=$1
+            shift
+            for directory in "$@"; do
+              if (( $(available_gb) >= threshold )); then
+                return
+              fi
+              remove_dir "$directory"
+            done
+          }
+
+          if [[ "$CLEAN_MODE" == "full" ]]; then
+            threshold=$FULL_CLEAN_THRESHOLD_GB
+          else
+            threshold=$CLEAN_THRESHOLD_GB
           fi
-          emptydir=/tmp/empty$$/
-          mkdir $emptydir
-          echo "::group::Available diskspace"
-          time df -BM / /mnt
+
+          echo "::group::Available diskspace (threshold: ${threshold}GB)"
+          time show_disk
           echo "::endgroup::"
-          for directory in "${directories[@]}"; do
-            echo "::group::Removing $directory"
-            # fast way to delete a lot of files on linux
-            time sudo eatmydata rsync -a --delete $emptydir ${directory}/
-            time sudo eatmydata rm -rf ${directory}
-            time df -BM / /mnt
-            echo "::endgroup::"
-          done
-          if [[ "${{ inputs.mode }}" == "full" ]]; then
-            echo "::group::Moving /var/lib/docker to /mnt/docker"
-            sudo systemctl stop docker
-            echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
-            sudo mv /var/lib/docker /mnt/docker
-            sudo systemctl start docker
-            time df -BM / /mnt
+
+          if (( $(available_gb) >= threshold )); then
+            echo "Available disk space meets threshold, skipping cleanup."
+            exit 0
+          fi
+
+          clean_dirs "$threshold" /usr/local/lib/android /opt/ghc
+
+          if [[ "$CLEAN_MODE" == "full" ]]; then
+            clean_dirs "$threshold" /usr/share/dotnet /opt/hostedtoolcache/CodeQL
+            if (( $(available_gb) < threshold )) && mountpoint -q /mnt; then
+              echo "::group::Moving /var/lib/docker to /mnt/docker"
+              sudo systemctl stop docker
+              echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+              sudo mv /var/lib/docker /mnt/docker
+              sudo systemctl start docker
+              time show_disk
+              echo "::endgroup::"
+            fi
+          fi
+
+          if (( $(available_gb) < threshold )); then
+            echo "::group::Cleaning apt state"
+            time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
+            time show_disk
             echo "::endgroup::"
           fi
-          echo "::group::Cleaning apt state"
-          time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
-          time df -BM / /mnt
+
+          echo "::group::Available diskspace"
+          time show_disk
           echo "::endgroup::"
         fi
-        echo "::group::Available diskspace"
-        time df -BM / /mnt
-        echo "::endgroup::"
-      shell: bash

.github/actions/ssh-access/action.yml

@@ -138,15 +138,16 @@ runs:
             if command -v upterm &>/dev/null; then
                 shopt -s nullglob
                 echo "SSH connection information"
-                upterm session current --admin-socket ~/.upterm/*.sock || {
+                export UPTERM_ADMIN_SOCKET=$(find $HOME/.upterm $XDG_RUNTIME_DIR/upterm /run/user/$(id -u)/upterm -name "*.sock" | head -n 1)
+                upterm session current || {
                     echo "upterm isn't running. Not waiting any longer."
                     exit 0
                 }
                 timeout=${{ inputs.timeout }}
                 echo "Waiting $timeout seconds..."
                 sleep $timeout
                 echo "Keep waiting as long as there's a connected session"
-                while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
+                while upterm session current|grep Connected &>/dev/null; do
                     sleep 30
                 done
                 echo "No session is connected. Not waiting any longer."

.github/actions/tune-runner-vm/action.yml

@@ -22,7 +22,8 @@ description: tunes the GitHub Runner VM operation system
 runs:
   using: composite
   steps:
-    - run: |
+    - shell: bash
+      run: |
         if [[ "$OSTYPE" == "linux-gnu"* ]]; then
             echo "::group::Configure and tune OS"
             # Ensure that reverse lookups for current hostname are handled properly
@@ -33,49 +34,40 @@ runs:
             # consumption is high.
             # Set vm.swappiness=1 to avoid swapping and allow high RAM usage
             echo 1 | sudo tee /proc/sys/vm/swappiness
-            (
-              shopt -s nullglob
-              # Set swappiness to 1 for all cgroups and sub-groups
-              for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
-                echo 1 | sudo tee $swappiness_file > /dev/null
-              done
-            ) || true
 
             # use "madvise" Linux Transparent HugePages (THP) setting
             # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
             # "madvise" is generally a better option than the default "always" setting
-            # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
+            # recommendation from https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b
             echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
             echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
-            echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+            echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
             echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
 
             # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
             # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
             # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
             sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+            if mountpoint -q /mnt; then
+              sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+            fi
             # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
             # https://www.spinics.net/lists/linux-ide/msg52562.html
-            for i in /sys/block/sd*/queue/discard_max_bytes; do
-              echo 0 | sudo tee $i
+            for i in /sys/block/*/queue/discard_max_bytes; do
+              if [ -f "$i" ]; then
+                echo 0 | sudo tee "$i"
+              fi
             done
-            # disable any background jobs that run SSD discard/trim
-            sudo systemctl disable fstrim.timer || true
-            sudo systemctl stop fstrim.timer || true
-            sudo systemctl disable fstrim.service || true
-            sudo systemctl stop fstrim.service || true
+            # disable unnecessary timers
+            sudo systemctl stop fstrim.timer fstrim.service \
+              podman-auto-update.timer sysstat-collect.timer sysstat-summary.timer \
+              phpsessionclean.timer man-db.timer motd-news.timer \
+              dpkg-db-backup.timer e2scrub_all.timer \
+              update-notifier-download.timer update-notifier-motd.timer || true
 
-            # stop php-fpm
-            sudo systemctl stop php8.0-fpm.service || true
-            sudo systemctl stop php7.4-fpm.service || true
-            # stop mono-xsp4
-            sudo systemctl disable mono-xsp4.service || true
-            sudo systemctl stop mono-xsp4.service || true
-            sudo killall mono || true
-
-            # stop Azure Linux agent to save RAM
-            sudo systemctl stop walinuxagent.service || true
+            # stop unnecessary services
+            sudo systemctl stop php8.3-fpm.service ModemManager.service \
+              multipathd.service udisks2.service walinuxagent.service || true
 
             echo '::endgroup::'
 
@@ -87,10 +79,4 @@ runs:
             echo "::group::Available diskspace"
             df -BM
             echo "::endgroup::"
-            # show cggroup
-            echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
-            CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
-            sudo cgget -a $CURRENT_CGGROUP || true
-            echo '::endgroup::'
-        fi
-      shell: bash
+        fi

.github/workflows/ci-go-functions.yaml

@@ -60,22 +60,14 @@ jobs:
             echo docs_only=false >> $GITHUB_OUTPUT
           fi
 
-      - name: Check if the PR has been approved for testing
-        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
-        env:
-          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          build/pulsar_ci_tool.sh check_ready_to_test
-
   check-style:
     needs: preconditions
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     name: Go ${{ matrix.go-version }} Functions style check
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        go-version: ['1.24']
+        go-version: ['1.25']
 
     steps:
       - name: Check out code into the Go module directory
@@ -93,7 +85,7 @@ jobs:
       - name: InstallTool
         run: |
           cd pulsar-function-go
-          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v2.0.2
+          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh| sh -s v2.12.2
           ./bin/golangci-lint --version
 
       - name: Build

.github/workflows/pulsar-ci-flaky.yaml

@@ -153,14 +153,6 @@ jobs:
           github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report'
           }}" >> $GITHUB_OUTPUT
 
-      - name: Check if the PR has been approved for testing
-        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
-        env:
-          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          build/pulsar_ci_tool.sh check_ready_to_test
-
   build-and-test:
     needs: preconditions
     name: Flaky tests suite

.github/workflows/pulsar-ci.yaml

@@ -201,14 +201,6 @@ jobs:
           github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report'
           }}" >> $GITHUB_OUTPUT
 
-      - name: Check if the PR has been approved for testing
-        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
-        env:
-          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          build/pulsar_ci_tool.sh check_ready_to_test
-
   build-and-license-check:
     needs: preconditions
     name: Build and License check
@@ -596,7 +588,7 @@ jobs:
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392
+        uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
         with:
           platforms: arm64