From 8012f4184a3cb4bcbb114885247c9b4afbe0d8a4 Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Wed, 10 Jun 2026 02:16:22 +0000
Subject: [PATCH 01/14] HRX CI: run on self-hosted ROCm GPU runners for
 gfx1151/gfx1201

Replace the ubuntu-latest CPU job with a matrix on
linux-gfx1151-gpu-rocm and linux-gfx120X-gpu-rocm, building for the
runner's gfx target and running the sample MUL_MAT correctness/perf
configs on the HRX0 backend. Fetch per-arch rocwmma dev artifacts.

BENCH_REF temporarily points at the p066-multi-arch-ci branch of the
bench tooling; restore to main when this merges.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/build-hrx.yml | 68 ++++++++++++++++++++++++++-------
 1 file changed, 55 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 071f7a677ae3..13eb1a5762e2 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -16,18 +16,56 @@ concurrency:
 
 jobs:
   hrx-build:
-    name: ubuntu-latest
-    runs-on: ubuntu-latest
+    name: ${{ matrix.name }} (${{ matrix.gpu_target }})
+    strategy:
+      # Per-arch failures are independent; keep both signals.
+      fail-fast: false
+      matrix:
+        include:
+          - name: gfx1151_strix-halo
+            gpu_target: gfx1151
+            runs_on: linux-gfx1151-gpu-rocm
+            extra_rocm_artifacts: rocwmma_dev_gfx1151
+          - name: gfx1201_9070
+            gpu_target: gfx1201
+            runs_on: linux-gfx120X-gpu-rocm
+            extra_rocm_artifacts: rocwmma_dev_gfx120X-all
+    runs-on: ${{ matrix.runs_on }}
+    timeout-minutes: 60
+    defaults:
+      run:
+        # The container default is sh; the bench scripts require bash.
+        shell: bash
+    # The self-hosted ROCm runners require jobs to run in a container; plain
+    # run steps fail in the runner-controller's direct-execution hook.
+    # Image and options mirror TheRock CI
+    # (build_tools/github_actions/fetch_test_configurations.py).
+    container:
+      image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26
+      options: >-
+        --ipc host
+        --user 0:0
+        --ulimit memlock=-1:-1
+        --security-opt seccomp=unconfined
+        --group-add video
+        --device /dev/kfd
+        --device /dev/dri
+        --group-add 993
+        --group-add 992
+        --group-add 110
+        --env-file /etc/podinfo/gha-gpu-isolation-settings
     env:
       HRX_WORK_DIR: ${{ github.workspace }}
       # Public location for bench tooling (rsuderman/llamacpp_ci and fork
       # AaronStGeorge/llamacpp_ci)
       #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced.
       BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci
-      BENCH_REF: 'main'
+      #TODO: restore to main once p066-multi-arch-ci merges in the bench repo.
+      BENCH_REF: 'p066-multi-arch-ci'
       BENCH_DIR: ${{ github.workspace }}/bench
       LLAMA_SRC_DIR: ${{ github.workspace }}/llama-src
-      HRX_ARTIFACT_SET: 'core-with-upstream-hip'
+      HRX_EXTRA_ROCM_ARTIFACTS: ${{ matrix.extra_rocm_artifacts }}
+      GGML_HRX_AMDGPU_TARGETS: ${{ matrix.gpu_target }}
       CCACHE_COMPILERCHECK: content
 
     steps:
@@ -49,7 +87,7 @@ jobs:
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.21
         with:
-          key: ubuntu-latest-${{ env.HRX_ARTIFACT_SET }}
+          key: ${{ matrix.gpu_target }}
           evict-old-files: 1d
           save: ${{ github.event_name == 'pull_request' }}
 
@@ -63,33 +101,37 @@ jobs:
         run: "${BENCH_DIR}/scripts/hrx/build-hrx.sh"
 
       - name: Validate HRX
+        # hrx-info has been observed hanging on some runners; fail fast.
+        timeout-minutes: 5
         run: "${BENCH_DIR}/scripts/hrx/validate-hrx.sh"
 
       - name: Build llama.cpp with HRX
         run: "${BENCH_DIR}/scripts/hrx/build-llama-hrx.sh"
 
-      - name: Run sample MUL_MAT correctness config on CPU
+      - name: Run sample MUL_MAT correctness config on HRX
+        timeout-minutes: 15
         run: |
           . "${BENCH_DIR}/scripts/hrx/env.sh"
           "${BENCH_DIR}/tools/run-op-test.py" \
             --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \
             --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \
             --op MUL_MAT \
-            --backend CPU \
-            --output benchmark-results/sample-mul-mat-f16-cpu-test.jsonl
+            --backend HRX0 \
+            --output benchmark-results/sample-mul-mat-f16-hrx-test.jsonl
 
-      - name: Run sample MUL_MAT benchmark config on CPU
+      - name: Run sample MUL_MAT benchmark config on HRX
+        timeout-minutes: 15
         run: |
           . "${BENCH_DIR}/scripts/hrx/env.sh"
           "${BENCH_DIR}/tools/run-op-perf.py" \
             --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \
             --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \
             --op MUL_MAT \
-            --backend CPU \
-            --output benchmark-results/sample-mul-mat-f16-cpu-perf.jsonl
+            --backend HRX0 \
+            --output benchmark-results/sample-mul-mat-f16-hrx-perf.jsonl
 
-      - name: Upload CPU benchmark results
+      - name: Upload benchmark results
         uses: actions/upload-artifact@v5
         with:
-          name: cpu-benchmark-results
+          name: benchmark-results-${{ matrix.name }}
           path: benchmark-results/

From d8af41d314b1a805fa24ffd38f8465a5afbeb0af Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Thu, 11 Jun 2026 14:14:29 +0000
Subject: [PATCH 02/14] Options should already be set up in the container

---
 .github/workflows/build-hrx.yml | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 13eb1a5762e2..aade97c88cb9 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -34,26 +34,9 @@ jobs:
     timeout-minutes: 60
     defaults:
       run:
-        # The container default is sh; the bench scripts require bash.
         shell: bash
-    # The self-hosted ROCm runners require jobs to run in a container; plain
-    # run steps fail in the runner-controller's direct-execution hook.
-    # Image and options mirror TheRock CI
-    # (build_tools/github_actions/fetch_test_configurations.py).
     container:
       image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26
-      options: >-
-        --ipc host
-        --user 0:0
-        --ulimit memlock=-1:-1
-        --security-opt seccomp=unconfined
-        --group-add video
-        --device /dev/kfd
-        --device /dev/dri
-        --group-add 993
-        --group-add 992
-        --group-add 110
-        --env-file /etc/podinfo/gha-gpu-isolation-settings
     env:
       HRX_WORK_DIR: ${{ github.workspace }}
       # Public location for bench tooling (rsuderman/llamacpp_ci and fork

From 59a9acd20db94ac01ffdb47af1122c6a1d3f0dbe Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Thu, 11 Jun 2026 15:06:24 +0000
Subject: [PATCH 03/14] debug

---
 .github/workflows/build-hrx.yml | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index aade97c88cb9..12ca56199b7f 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -34,9 +34,16 @@ jobs:
     timeout-minutes: 60
     defaults:
       run:
-        shell: bash
+        # --noprofile skips loading ~/.bash_profile
+        # --norc      skips loading ~/.bashrc
+        # -exo pipefail:
+        #   -e exit immediately if any command fails
+        #   -x print each command before executing
+        #   -o pipefail ensure pipeline fails if any command in it fails
+        # {0} github actions placeholder for the command to run
+        shell: bash --noprofile --norc -exo pipefail {0}
     container:
-      image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26
+      image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:fba5f55a122dbb15925e98c51fe65bffe88c36e11ebb25b73daf2bea04202dc3
     env:
       HRX_WORK_DIR: ${{ github.workspace }}
       # Public location for bench tooling (rsuderman/llamacpp_ci and fork
@@ -52,6 +59,12 @@ jobs:
       CCACHE_COMPILERCHECK: content
 
     steps:
+      - name: Debug
+        run: |
+          id
+          ls -ln /__w/_temp | head
+          ls -ln /__w | head
+
       - name: Checkout llama.cpp (under test)
         uses: actions/checkout@v6
         with:

From ab92848afa845d6a1a9c2c38dbd698accae4b81c Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Thu, 11 Jun 2026 15:58:47 +0000
Subject: [PATCH 04/14] HRX CI: run container as root, write build outputs off
 the bind mount

Run the job container with --user 0:0 so checkout can write the
runner's _work tree on runners whose agent owns it as root (the plain
shark65 runner); the image's default tester user (uid 1001) only works
on the prolense fleet where _work is 1001-owned. Add --device kfd/dri
for GPU access on runners that don't inject devices.

Point HRX_WORK_DIR and LLAMA_BUILD_DIR at a container-local /work path so
ROCm downloads and build trees are reclaimed on container removal instead
of accumulating in the persistent runner's bind-mounted workspace.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/build-hrx.yml | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 12ca56199b7f..f71c8fd853ff 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -44,8 +44,24 @@ jobs:
         shell: bash --noprofile --norc -exo pipefail {0}
     container:
       image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:fba5f55a122dbb15925e98c51fe65bffe88c36e11ebb25b73daf2bea04202dc3
+      # --user 0:0: Actions needs to access some files set up as part of the
+      #             Actions runner. In June 2026 some runners are set up with a
+      #             "tester" user with UID/GID 1001 which matches the user in
+      #             the container, and some are set up as root. Running as root
+      #             is the common denominator, accessing a file owned by root
+      #             with user "tester" gives an EACCES.
+      # --device kfd/dri: GPU access; the plain runner adds no devices for us.
+      options: >-
+        --user 0:0
+        --device /dev/kfd
+        --device /dev/dri
     env:
-      HRX_WORK_DIR: ${{ github.workspace }}
+      # Write to a non bind-mounted directory. In June 2026 a debug tests showed
+      # some runners hat assets dated from 6 months ago in the bind-mounted
+      # github workspace, it seems like nothing cleans it up currently. Writing
+      # to an non-bind mounted directory ensures that rocm-artifacts/builds are
+      # cleaned up when docker conteriner is removed.
+      HRX_WORK_DIR: /work
       # Public location for bench tooling (rsuderman/llamacpp_ci and fork
       # AaronStGeorge/llamacpp_ci)
       #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced.
@@ -54,6 +70,8 @@ jobs:
       BENCH_REF: 'p066-multi-arch-ci'
       BENCH_DIR: ${{ github.workspace }}/bench
       LLAMA_SRC_DIR: ${{ github.workspace }}/llama-src
+      # Build tree off the bind mount too (source stays under the workspace).
+      LLAMA_BUILD_DIR: /work/llama-build
       HRX_EXTRA_ROCM_ARTIFACTS: ${{ matrix.extra_rocm_artifacts }}
       GGML_HRX_AMDGPU_TARGETS: ${{ matrix.gpu_target }}
       CCACHE_COMPILERCHECK: content

From 28e5c78a6f4312c2b915e018febc0208b3bb2ad7 Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Thu, 11 Jun 2026 17:17:41 +0000
Subject: [PATCH 05/14] cleanup comments

---
 .github/workflows/build-hrx.yml | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index f71c8fd853ff..00cd55c21a8d 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -44,23 +44,25 @@ jobs:
         shell: bash --noprofile --norc -exo pipefail {0}
     container:
       image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:fba5f55a122dbb15925e98c51fe65bffe88c36e11ebb25b73daf2bea04202dc3
-      # --user 0:0: Actions needs to access some files set up as part of the
-      #             Actions runner. In June 2026 some runners are set up with a
-      #             "tester" user with UID/GID 1001 which matches the user in
-      #             the container, and some are set up as root. Running as root
-      #             is the common denominator, accessing a file owned by root
-      #             with user "tester" gives an EACCES.
+      # --user 0:0: actions running inside the container need to write to some
+      #             files set up by the runner agent before the container runs.
+      #             In June 2026 some runner agents set those files up with
+      #             permissions for a "tester" user with UID/GID 1001 which
+      #             matches the user in no_rocm_image_ubuntu24_04, and some are
+      #             set up as root; accessing a file owned by root with user
+      #             "tester" gives an EACCES. Running as root is the common
+      #             denominator.
       # --device kfd/dri: GPU access; the plain runner adds no devices for us.
       options: >-
         --user 0:0
         --device /dev/kfd
         --device /dev/dri
     env:
-      # Write to a non bind-mounted directory. In June 2026 a debug tests showed
-      # some runners hat assets dated from 6 months ago in the bind-mounted
-      # github workspace, it seems like nothing cleans it up currently. Writing
-      # to an non-bind mounted directory ensures that rocm-artifacts/builds are
-      # cleaned up when docker conteriner is removed.
+      # Write out of tree build/install to a non-bind-mounted directory. In June
+      # 2026 a debug test showed some runners had assets dated from 6 months ago
+      # in the bind-mounted github workspace, it seems like nothing cleans it up
+      # currently. Writing to an non-bind-mounted directory ensures that
+      # rocm-artifacts/builds are cleaned up when docker container is removed.
       HRX_WORK_DIR: /work
       # Public location for bench tooling (rsuderman/llamacpp_ci and fork
       # AaronStGeorge/llamacpp_ci)
@@ -68,6 +70,8 @@ jobs:
       BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci
       #TODO: restore to main once p066-multi-arch-ci merges in the bench repo.
       BENCH_REF: 'p066-multi-arch-ci'
+      # actions/checkout@v6 does auto-cleanup, an in-tree build would be
+      # auto-cleaned as well.
       BENCH_DIR: ${{ github.workspace }}/bench
       LLAMA_SRC_DIR: ${{ github.workspace }}/llama-src
       # Build tree off the bind mount too (source stays under the workspace).

From 5393cbb8d0901e0a97403c581e0ec8f1052a43d4 Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Thu, 11 Jun 2026 17:25:24 +0000
Subject: [PATCH 06/14] HRX CI: source runtime-env.sh in op-test/perf steps

The bench tooling moved runtime env setup (LD_LIBRARY_PATH incl. the HRX
install lib, and HSA_FORCE_FINE_GRAIN_PCIE=1 for gfx12) out of
fetch-rocm-assets.sh into runtime-env.sh. Build/validate scripts source
it internally; the op-test and op-perf steps invoke the python tools
directly, so source runtime-env.sh there too. Without it those steps
would now run with neither the HRX libs on LD_LIBRARY_PATH nor the gfx12
fine-grained-memory flag, which is what produced the MUL_MAT NaNs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/build-hrx.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 00cd55c21a8d..1a2e4010b32b 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -68,8 +68,7 @@ jobs:
       # AaronStGeorge/llamacpp_ci)
       #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced.
       BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci
-      #TODO: restore to main once p066-multi-arch-ci merges in the bench repo.
-      BENCH_REF: 'p066-multi-arch-ci'
+      BENCH_REF: 'main'
       # actions/checkout@v6 does auto-cleanup, an in-tree build would be
       # auto-cleaned as well.
       BENCH_DIR: ${{ github.workspace }}/bench
@@ -130,6 +129,7 @@ jobs:
         timeout-minutes: 15
         run: |
           . "${BENCH_DIR}/scripts/hrx/env.sh"
+          . "${BENCH_DIR}/scripts/hrx/runtime-env.sh"
           "${BENCH_DIR}/tools/run-op-test.py" \
             --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \
             --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \
@@ -141,6 +141,7 @@ jobs:
         timeout-minutes: 15
         run: |
           . "${BENCH_DIR}/scripts/hrx/env.sh"
+          . "${BENCH_DIR}/scripts/hrx/runtime-env.sh"
           "${BENCH_DIR}/tools/run-op-perf.py" \
             --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \
             --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \

From d6bd3f885dbcc34c148100009c4dbdd9596d9c0a Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Fri, 12 Jun 2026 15:07:57 +0000
Subject: [PATCH 07/14] ggml-hrx: port to renamed HRX export-info field
 (constant_byte_length)

Upstream HRX renamed hrx_executable_export_info_t.constant_count ->
constant_byte_length (now the byte length of the constants buffer, not a uint32
count). Update the catalog-kernel ABI check accordingly:
  constant_count * sizeof(uint32_t) == constants_size
becomes
  constant_byte_length == constants_size
and fix the corresponding diagnostic log. Needed to build ggml-hrx against the
bumped HRX (ROCm/hrx upstream main); verified building against e8275fbb5.
---
 ggml/src/ggml-hrx/ggml-hrx.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp
index 120a422b4d10..1f14a69a281a 100644
--- a/ggml/src/ggml-hrx/ggml-hrx.cpp
+++ b/ggml/src/ggml-hrx/ggml-hrx.cpp
@@ -2521,7 +2521,7 @@ static bool ggml_backend_hrx_load_catalog_provider(
                         executable, export_ordinal, &export_info)) &&
                     export_info.binding_count == entry->binding_count &&
                     export_info.parameter_count == entry->parameter_count &&
-                    export_info.constant_count * sizeof(uint32_t) == entry->constants_size;
+                    export_info.constant_byte_length == entry->constants_size;
     if (!ok) {
         GGML_LOG_WARN(
             "%s: HRX catalog kernel %s has unsupported ABI "
@@ -2530,7 +2530,7 @@ static bool ggml_backend_hrx_load_catalog_provider(
             entry->name,
             export_info.binding_count,
             entry->binding_count,
-            export_info.constant_count,
+            export_info.constant_byte_length,
             entry->constants_size,
             export_info.parameter_count,
             entry->parameter_count,

From 94782ce98e13465a1d364a29aa85e9637dae75bd Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Fri, 12 Jun 2026 15:14:07 +0000
Subject: [PATCH 08/14] cleanup

---
 .github/workflows/build-hrx.yml | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 1a2e4010b32b..6d6cfed199e6 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -31,7 +31,7 @@ jobs:
             runs_on: linux-gfx120X-gpu-rocm
             extra_rocm_artifacts: rocwmma_dev_gfx120X-all
     runs-on: ${{ matrix.runs_on }}
-    timeout-minutes: 60
+    timeout-minutes: 15
     defaults:
       run:
         # --noprofile skips loading ~/.bash_profile
@@ -45,14 +45,14 @@ jobs:
     container:
       image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:fba5f55a122dbb15925e98c51fe65bffe88c36e11ebb25b73daf2bea04202dc3
       # --user 0:0: actions running inside the container need to write to some
-      #             files set up by the runner agent before the container runs.
-      #             In June 2026 some runner agents set those files up with
-      #             permissions for a "tester" user with UID/GID 1001 which
+      #             files set up outside the container by the runner agent.  In
+      #             June 2026 some runner agents set those files up with
+      #             permissions for a "tester" user with UID/GID 1001, which
       #             matches the user in no_rocm_image_ubuntu24_04, and some are
       #             set up as root; accessing a file owned by root with user
       #             "tester" gives an EACCES. Running as root is the common
       #             denominator.
-      # --device kfd/dri: GPU access; the plain runner adds no devices for us.
+      # --device kfd/dri: GPU access.
       options: >-
         --user 0:0
         --device /dev/kfd
@@ -80,12 +80,6 @@ jobs:
       CCACHE_COMPILERCHECK: content
 
     steps:
-      - name: Debug
-        run: |
-          id
-          ls -ln /__w/_temp | head
-          ls -ln /__w | head
-
       - name: Checkout llama.cpp (under test)
         uses: actions/checkout@v6
         with:

From a364547005d810c121b202b968aa66de324a8d4c Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Fri, 12 Jun 2026 15:23:23 +0000
Subject: [PATCH 09/14] ci: on hrx-integration, run only build-hrx.yml (push +
 pull_request)

The fork's hrx-integration branch should run only the HRX CI, not the upstream
llama.cpp matrix.
- build-hrx.yml: add a push trigger for hrx-integration (was pull_request-only).
- Every other workflow whose push/pull_request/pull_request_target could match
  hrx-integration now carries branches-ignore: ['hrx-integration'] (build.yml,
  build-vulkan/cann/apple/android/riscv/self-hosted, server*, check-vendor,
  hip-quality-check, python-*, labeler, copilot-setup-steps, etc.).
- gguf-publish is tag-only (can't fire on a branch push) and the rest were
  already scoped to master.

Verified: only build-hrx.yml fires on hrx-integration for push and PR events.
Caveat: these touch upstream workflow files, so upstream merges into
hrx-integration may conflict on them (build-hrx.yml itself does not).
---
 .github/workflows/build-android.yml             | 1 +
 .github/workflows/build-apple.yml               | 1 +
 .github/workflows/build-cann.yml                | 1 +
 .github/workflows/build-hrx.yml                 | 3 +++
 .github/workflows/build-riscv.yml               | 1 +
 .github/workflows/build-self-hosted.yml         | 1 +
 .github/workflows/build-vulkan.yml              | 1 +
 .github/workflows/build.yml                     | 1 +
 .github/workflows/check-vendor.yml              | 1 +
 .github/workflows/copilot-setup-steps.yml       | 2 ++
 .github/workflows/hip-quality-check.yml         | 1 +
 .github/workflows/labeler.yml                   | 3 ++-
 .github/workflows/pre-tokenizer-hashes.yml      | 2 ++
 .github/workflows/python-check-requirements.yml | 2 ++
 .github/workflows/python-lint.yml               | 1 +
 .github/workflows/python-type-check.yml         | 2 ++
 .github/workflows/server-webui.yml              | 1 +
 .github/workflows/server.yml                    | 1 +
 .github/workflows/update-ops-docs.yml           | 2 ++
 19 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml
index 5fc24d8d3492..ec40c35f22e8 100644
--- a/.github/workflows/build-android.yml
+++ b/.github/workflows/build-android.yml
@@ -16,6 +16,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-android.yml',
diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml
index b99e614666e1..ca3b87bfd56b 100644
--- a/.github/workflows/build-apple.yml
+++ b/.github/workflows/build-apple.yml
@@ -19,6 +19,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-apple.yml',
diff --git a/.github/workflows/build-cann.yml b/.github/workflows/build-cann.yml
index d39b87637339..f8d408a5f0f3 100644
--- a/.github/workflows/build-cann.yml
+++ b/.github/workflows/build-cann.yml
@@ -16,6 +16,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-cann.yml',
diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 6d6cfed199e6..7d4f4250fb4c 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -1,6 +1,9 @@
 name: HRX CI
 
 on:
+  push:
+    branches:
+      - hrx-integration
   pull_request:
     types: [opened, synchronize, reopened]
     branches:
diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml
index 9733dbaa7a21..11e6eab022ac 100644
--- a/.github/workflows/build-riscv.yml
+++ b/.github/workflows/build-riscv.yml
@@ -16,6 +16,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-riscv.yml',
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index eeea820ba169..c699f2d5f345 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -24,6 +24,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-self-hosted.yml',
diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml
index de38bb2db6d3..85c7978d2ee9 100644
--- a/.github/workflows/build-vulkan.yml
+++ b/.github/workflows/build-vulkan.yml
@@ -18,6 +18,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-vulkan.yml',
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f4ae3675602e..968f0ade7e2d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,6 +25,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build.yml',
diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml
index 1671ed7b8bd2..548efed185ff 100644
--- a/.github/workflows/check-vendor.yml
+++ b/.github/workflows/check-vendor.yml
@@ -11,6 +11,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       'vendor/**',
diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml
index 6f648bac45b7..8904fd959466 100644
--- a/.github/workflows/copilot-setup-steps.yml
+++ b/.github/workflows/copilot-setup-steps.yml
@@ -5,9 +5,11 @@ name: "Copilot Setup Steps"
 on:
   workflow_dispatch:
   push:
+    branches-ignore: ['hrx-integration']
     paths:
       - .github/workflows/copilot-setup-steps.yml
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     paths:
       - .github/workflows/copilot-setup-steps.yml
 
diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.yml
index d00d30ed65ce..6c404cb8faa1 100644
--- a/.github/workflows/hip-quality-check.yml
+++ b/.github/workflows/hip-quality-check.yml
@@ -13,6 +13,7 @@ on:
     ]
 
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/hip-quality-check.yml',
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index eab20c68811e..81bbd9c6b165 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -1,6 +1,7 @@
 name: "Pull Request Labeler"
 on:
-- pull_request_target
+  pull_request_target:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
 
 jobs:
   labeler:
diff --git a/.github/workflows/pre-tokenizer-hashes.yml b/.github/workflows/pre-tokenizer-hashes.yml
index 7126b62b690b..475a0fa4419e 100644
--- a/.github/workflows/pre-tokenizer-hashes.yml
+++ b/.github/workflows/pre-tokenizer-hashes.yml
@@ -2,10 +2,12 @@ name: Check Pre-Tokenizer Hashes
 
 on:
     push:
+        branches-ignore: ['hrx-integration']
         paths:
             - 'convert_hf_to_gguf.py'
             - 'convert_hf_to_gguf_update.py'
     pull_request:
+        branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
         paths:
             - 'convert_hf_to_gguf.py'
             - 'convert_hf_to_gguf_update.py'
diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml
index 1219b8745927..b260497f2cf1 100644
--- a/.github/workflows/python-check-requirements.yml
+++ b/.github/workflows/python-check-requirements.yml
@@ -2,12 +2,14 @@ name: Python check requirements.txt
 
 on:
   push:
+    branches-ignore: ['hrx-integration']
     paths:
       - '.github/workflows/python-check-requirements.yml'
       - 'scripts/check-requirements.sh'
       - 'convert*.py'
       - '**/requirements*.txt'
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     paths:
       - '.github/workflows/python-check-requirements.yml'
       - 'scripts/check-requirements.sh'
diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml
index 1e5d64c1aee6..0f28f6b035bd 100644
--- a/.github/workflows/python-lint.yml
+++ b/.github/workflows/python-lint.yml
@@ -9,6 +9,7 @@ on:
       '**/*.py'
     ]
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/python-lint.yml',
diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml
index dc7aebe24ca2..88bff2fe8998 100644
--- a/.github/workflows/python-type-check.yml
+++ b/.github/workflows/python-type-check.yml
@@ -2,6 +2,7 @@ name: Python Type-Check
 
 on:
   push:
+    branches-ignore: ['hrx-integration']
     paths:
       - '.github/workflows/python-type-check.yml'
       - 'ty.toml'
@@ -9,6 +10,7 @@ on:
       - '**/requirements*.txt'
       # - 'pyrightconfig.json'
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     paths:
       - '.github/workflows/python-type-check.yml'
       - 'ty.toml'
diff --git a/.github/workflows/server-webui.yml b/.github/workflows/server-webui.yml
index 492107ffd851..c6d520a744d6 100644
--- a/.github/workflows/server-webui.yml
+++ b/.github/workflows/server-webui.yml
@@ -17,6 +17,7 @@ on:
       'tools/server/public/**'
     ]
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/server-webui.yml',
diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml
index 750c29f08e5c..2a5f37aa8570 100644
--- a/.github/workflows/server.yml
+++ b/.github/workflows/server.yml
@@ -28,6 +28,7 @@ on:
       'tools/server/**.*'
     ]
   pull_request:
+    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/server.yml',
diff --git a/.github/workflows/update-ops-docs.yml b/.github/workflows/update-ops-docs.yml
index 2ab06eb9811d..bac72a33f296 100644
--- a/.github/workflows/update-ops-docs.yml
+++ b/.github/workflows/update-ops-docs.yml
@@ -2,11 +2,13 @@ name: Update Operations Documentation
 
 on:
     push:
+        branches-ignore: ['hrx-integration']
         paths:
             - 'docs/ops.md'
             - 'docs/ops/**'
             - 'scripts/create_ops_docs.py'
     pull_request:
+        branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
         paths:
             - 'docs/ops.md'
             - 'docs/ops/**'

From 7a40d707aa540b4a262f5a312c4efb9ac2482752 Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Fri, 12 Jun 2026 15:42:26 +0000
Subject: [PATCH 10/14] ignore

---
 .github/workflows/build-android.yml             | 2 +-
 .github/workflows/build-apple.yml               | 2 +-
 .github/workflows/build-cann.yml                | 2 +-
 .github/workflows/build-riscv.yml               | 2 +-
 .github/workflows/build-self-hosted.yml         | 2 +-
 .github/workflows/build-vulkan.yml              | 2 +-
 .github/workflows/build.yml                     | 2 +-
 .github/workflows/check-vendor.yml              | 2 +-
 .github/workflows/copilot-setup-steps.yml       | 2 +-
 .github/workflows/hip-quality-check.yml         | 2 +-
 .github/workflows/labeler.yml                   | 2 +-
 .github/workflows/pre-tokenizer-hashes.yml      | 2 +-
 .github/workflows/python-check-requirements.yml | 2 +-
 .github/workflows/python-lint.yml               | 2 +-
 .github/workflows/python-type-check.yml         | 2 +-
 .github/workflows/server-webui.yml              | 2 +-
 .github/workflows/server.yml                    | 2 +-
 .github/workflows/update-ops-docs.yml           | 2 +-
 18 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml
index ec40c35f22e8..7a41f4733660 100644
--- a/.github/workflows/build-android.yml
+++ b/.github/workflows/build-android.yml
@@ -16,7 +16,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-android.yml',
diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml
index ca3b87bfd56b..f443178958cb 100644
--- a/.github/workflows/build-apple.yml
+++ b/.github/workflows/build-apple.yml
@@ -19,7 +19,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-apple.yml',
diff --git a/.github/workflows/build-cann.yml b/.github/workflows/build-cann.yml
index f8d408a5f0f3..8cf3b4c356d1 100644
--- a/.github/workflows/build-cann.yml
+++ b/.github/workflows/build-cann.yml
@@ -16,7 +16,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-cann.yml',
diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml
index 11e6eab022ac..5fc7c8dbc407 100644
--- a/.github/workflows/build-riscv.yml
+++ b/.github/workflows/build-riscv.yml
@@ -16,7 +16,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-riscv.yml',
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index c699f2d5f345..2dc44a7c947c 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -24,7 +24,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-self-hosted.yml',
diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml
index 85c7978d2ee9..8615eec563b4 100644
--- a/.github/workflows/build-vulkan.yml
+++ b/.github/workflows/build-vulkan.yml
@@ -18,7 +18,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-vulkan.yml',
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 968f0ade7e2d..05a25904fef4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,7 +25,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build.yml',
diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml
index 548efed185ff..97093c7a39de 100644
--- a/.github/workflows/check-vendor.yml
+++ b/.github/workflows/check-vendor.yml
@@ -11,7 +11,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       'vendor/**',
diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml
index 8904fd959466..43e8aca91274 100644
--- a/.github/workflows/copilot-setup-steps.yml
+++ b/.github/workflows/copilot-setup-steps.yml
@@ -9,7 +9,7 @@ on:
     paths:
       - .github/workflows/copilot-setup-steps.yml
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     paths:
       - .github/workflows/copilot-setup-steps.yml
 
diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.yml
index 6c404cb8faa1..cd92020c28d1 100644
--- a/.github/workflows/hip-quality-check.yml
+++ b/.github/workflows/hip-quality-check.yml
@@ -13,7 +13,7 @@ on:
     ]
 
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/hip-quality-check.yml',
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 81bbd9c6b165..ef53773bf284 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -1,7 +1,7 @@
 name: "Pull Request Labeler"
 on:
   pull_request_target:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
 
 jobs:
   labeler:
diff --git a/.github/workflows/pre-tokenizer-hashes.yml b/.github/workflows/pre-tokenizer-hashes.yml
index 475a0fa4419e..111d632d816c 100644
--- a/.github/workflows/pre-tokenizer-hashes.yml
+++ b/.github/workflows/pre-tokenizer-hashes.yml
@@ -7,7 +7,7 @@ on:
             - 'convert_hf_to_gguf.py'
             - 'convert_hf_to_gguf_update.py'
     pull_request:
-        branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+        branches-ignore: ['hrx-integration']
         paths:
             - 'convert_hf_to_gguf.py'
             - 'convert_hf_to_gguf_update.py'
diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml
index b260497f2cf1..926929e9250d 100644
--- a/.github/workflows/python-check-requirements.yml
+++ b/.github/workflows/python-check-requirements.yml
@@ -9,7 +9,7 @@ on:
       - 'convert*.py'
       - '**/requirements*.txt'
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     paths:
       - '.github/workflows/python-check-requirements.yml'
       - 'scripts/check-requirements.sh'
diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml
index 0f28f6b035bd..4a813dbcf80e 100644
--- a/.github/workflows/python-lint.yml
+++ b/.github/workflows/python-lint.yml
@@ -9,7 +9,7 @@ on:
       '**/*.py'
     ]
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/python-lint.yml',
diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml
index 88bff2fe8998..bf7580960057 100644
--- a/.github/workflows/python-type-check.yml
+++ b/.github/workflows/python-type-check.yml
@@ -10,7 +10,7 @@ on:
       - '**/requirements*.txt'
       # - 'pyrightconfig.json'
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     paths:
       - '.github/workflows/python-type-check.yml'
       - 'ty.toml'
diff --git a/.github/workflows/server-webui.yml b/.github/workflows/server-webui.yml
index c6d520a744d6..132b7a50b822 100644
--- a/.github/workflows/server-webui.yml
+++ b/.github/workflows/server-webui.yml
@@ -17,7 +17,7 @@ on:
       'tools/server/public/**'
     ]
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/server-webui.yml',
diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml
index 2a5f37aa8570..4d642e541481 100644
--- a/.github/workflows/server.yml
+++ b/.github/workflows/server.yml
@@ -28,7 +28,7 @@ on:
       'tools/server/**.*'
     ]
   pull_request:
-    branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+    branches-ignore: ['hrx-integration']
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/server.yml',
diff --git a/.github/workflows/update-ops-docs.yml b/.github/workflows/update-ops-docs.yml
index bac72a33f296..934ea783db4a 100644
--- a/.github/workflows/update-ops-docs.yml
+++ b/.github/workflows/update-ops-docs.yml
@@ -8,7 +8,7 @@ on:
             - 'docs/ops/**'
             - 'scripts/create_ops_docs.py'
     pull_request:
-        branches-ignore: ['hrx-integration']  # fork: hrx-integration runs only build-hrx.yml
+        branches-ignore: ['hrx-integration']
         paths:
             - 'docs/ops.md'
             - 'docs/ops/**'

From 2116f41595992c58cc8294cd0c19ae3026fcceef Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Fri, 12 Jun 2026 16:44:16 +0000
Subject: [PATCH 11/14] ci(hrx): runner-identity debug block + GPU health-check
 step

- Debug (runner identity): first step, if: always(), logs RUNNER_NAME, hostname,
  uname, /dev/kfd|dri, lspci, RUNNER_*/GITHUB_* env -- so the specific
  self-hosted runner behind a failure is identifiable from the log alone.
- GPU health check: after 'Fetch ROCm assets' (prefix binaries present), runs
  the bench runner-health.sh (rocminfo/amd-smi/rocm-smi diagnostics + fail-fast
  on /dev/kfd missing, dead rocminfo, or expected gfx arch not enumerated).

Caveat: neither detects the gfx1151 coherence NaN flake (GPU reports healthy);
the debug block enables post-hoc identification. runner-health.sh must be on the
bench at BENCH_REF (AaronStGeorge/llamacpp_ci) for the health step to resolve.
---
 .github/workflows/build-hrx.yml | 34 +++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 7d4f4250fb4c..6ff59e7d8608 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -83,6 +83,31 @@ jobs:
       CCACHE_COMPILERCHECK: content
 
     steps:
+      # Logged first (if: always(), before any checkout) so the specific
+      # self-hosted runner behind a failure is greppable from the job log via
+      # RUNNER_NAME, without querying the GitHub API. NOTE: a gfx1201
+      # "Initialize containers" failure happens before any step runs, so this
+      # block cannot capture that mode -- only the API runner_name identifies it.
+      - name: Debug (runner identity)
+        if: always()
+        run: |
+          set +e
+          echo "===== RUNNER IDENTITY ====="
+          echo "RUNNER_NAME=${RUNNER_NAME}"
+          echo "RUNNER_OS=${RUNNER_OS}  RUNNER_ENVIRONMENT=${RUNNER_ENVIRONMENT}"
+          echo "GITHUB_RUN_ID=${GITHUB_RUN_ID} attempt=${GITHUB_RUN_ATTEMPT} job=${GITHUB_JOB}"
+          echo "matrix.name=${{ matrix.name }} gpu_target=${{ matrix.gpu_target }} runs_on=${{ matrix.runs_on }}"
+          echo "hostname=$(hostname)"
+          uname -a || true
+          cat /etc/os-release || true
+          id || true
+          ls -l /dev/kfd || echo "MISSING /dev/kfd"
+          ls -l /dev/dri || echo "MISSING /dev/dri"
+          lspci | grep -iE 'vga|display|amd/ati' || echo "lspci unavailable / no GPU match"
+          env | grep -E '^(RUNNER_|GITHUB_)' | sort || true
+          echo "===== END RUNNER IDENTITY ====="
+          true
+
       - name: Checkout llama.cpp (under test)
         uses: actions/checkout@v6
         with:
@@ -111,6 +136,15 @@ jobs:
       - name: Fetch ROCm assets
         run: "${BENCH_DIR}/scripts/hrx/fetch-rocm-assets.sh"
 
+      # Dump GPU/board identity (serial/VBIOS/fw via rocminfo/amd-smi/rocm-smi)
+      # and fail fast on gross breakage (no /dev/kfd, dead rocminfo, expected
+      # gfx arch not enumerated) before the costly HRX/llama build. Runs here
+      # because the diag binaries only exist in the composed prefix after the
+      # fetch step. Does NOT detect the gfx1151 coherence NaN flake.
+      - name: GPU health check
+        timeout-minutes: 3
+        run: "${BENCH_DIR}/scripts/hrx/runner-health.sh"
+
       - name: Build HRX
         run: "${BENCH_DIR}/scripts/hrx/build-hrx.sh"
 

From a31d88925540eeaa05e641d9b32b2ce3413aa5f2 Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Fri, 12 Jun 2026 16:49:48 +0000
Subject: [PATCH 12/14] ci(hrx): add MUL_MAT failure-rate loop (per-runner
 diagnostic)

Run the failing mul_mat_f16 config N times (MULMAT_LOOP_N, default 50) in one
job and print the fail/pass count + RUNNER_NAME, to answer: on a bad machine
does it fail every iteration or only some, and what's the rate per machine?
Advisory (continue-on-error) and placed BEFORE the gating correctness step so it
still runs on a failing machine. Correlate the count with RUNNER_NAME from the
Debug step.
---
 .github/workflows/build-hrx.yml | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 6ff59e7d8608..c0369507e492 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -71,7 +71,7 @@ jobs:
       # AaronStGeorge/llamacpp_ci)
       #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced.
       BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci
-      BENCH_REF: 'main'
+      BENCH_REF: 'runner-health'
       # actions/checkout@v6 does auto-cleanup, an in-tree build would be
       # auto-cleaned as well.
       BENCH_DIR: ${{ github.workspace }}/bench
@@ -156,6 +156,37 @@ jobs:
       - name: Build llama.cpp with HRX
         run: "${BENCH_DIR}/scripts/hrx/build-llama-hrx.sh"
 
+      # Diagnostic: run the failing MUL_MAT config N times in THIS job to measure
+      # the per-RUNNER failure rate -- does a bad machine fail every iteration or
+      # only some? Advisory (continue-on-error, runs BEFORE the gating step so it
+      # still executes on a failing machine). Correlate the printed count with
+      # RUNNER_NAME (see the Debug step). Tune N via the MULMAT_LOOP_N var.
+      - name: MUL_MAT failure-rate loop (diagnostic)
+        continue-on-error: true
+        timeout-minutes: 20
+        run: |
+          . "${BENCH_DIR}/scripts/hrx/env.sh"
+          . "${BENCH_DIR}/scripts/hrx/runtime-env.sh"
+          set +e   # count pass/fail from the output, don't let a NaN exit abort
+          TB="${LLAMA_BUILD_DIR}/bin/test-backend-ops"
+          TF="${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt"
+          N="${MULMAT_LOOP_N:-50}"
+          pass=0; fail=0; other=0
+          for i in $(seq 1 "$N"); do
+            out="$("$TB" test -o MUL_MAT -b HRX0 --test-file "$TF" 2>&1 | sed 's/\x1b\[[0-9;]*m//g')"
+            if echo "$out" | grep -q 'NaN at index'; then
+              fail=$((fail+1))
+              [ "$fail" -le 3 ] && echo "iter $i FAIL: $(echo "$out" | grep -m1 'NaN at index')"
+            elif echo "$out" | grep -q '1/1 tests passed'; then
+              pass=$((pass+1))
+            else
+              other=$((other+1))
+              [ "$other" -le 2 ] && echo "iter $i OTHER: $(echo "$out" | tail -2 | tr '\n' ' ')"
+            fi
+          done
+          echo "MULMAT_RATE runner=${RUNNER_NAME} fail=${fail} pass=${pass} other=${other} total=${N}"
+          echo "::notice::MUL_MAT rate on ${RUNNER_NAME}: ${fail}/${N} FAIL (pass=${pass}, other=${other})"
+
       - name: Run sample MUL_MAT correctness config on HRX
         timeout-minutes: 15
         run: |

From a0e938b4d0d57a856f729bc2215754f0eb06d3fa Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Mon, 15 Jun 2026 04:21:37 +0000
Subject: [PATCH 13/14] ci(hrx): move runner-identity dump into bench script

Replace the inline "Debug (runner identity)" block with a call to
runner-identity.sh (in the bench repo, alongside runner-health.sh). The step
now runs right after the bench checkout (earliest the script is available)
instead of first; matrix values are passed via MATRIX_* env since a script
can't read ${{ matrix.* }}. if: always() retained.

Requires runner-identity.sh on the bench BENCH_REF branch (committed on
AaronStGeorge/llamacpp_ci runner-health).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/build-hrx.yml | 38 +++++++++++----------------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index c0369507e492..80f197cde7da 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -83,31 +83,6 @@ jobs:
       CCACHE_COMPILERCHECK: content
 
     steps:
-      # Logged first (if: always(), before any checkout) so the specific
-      # self-hosted runner behind a failure is greppable from the job log via
-      # RUNNER_NAME, without querying the GitHub API. NOTE: a gfx1201
-      # "Initialize containers" failure happens before any step runs, so this
-      # block cannot capture that mode -- only the API runner_name identifies it.
-      - name: Debug (runner identity)
-        if: always()
-        run: |
-          set +e
-          echo "===== RUNNER IDENTITY ====="
-          echo "RUNNER_NAME=${RUNNER_NAME}"
-          echo "RUNNER_OS=${RUNNER_OS}  RUNNER_ENVIRONMENT=${RUNNER_ENVIRONMENT}"
-          echo "GITHUB_RUN_ID=${GITHUB_RUN_ID} attempt=${GITHUB_RUN_ATTEMPT} job=${GITHUB_JOB}"
-          echo "matrix.name=${{ matrix.name }} gpu_target=${{ matrix.gpu_target }} runs_on=${{ matrix.runs_on }}"
-          echo "hostname=$(hostname)"
-          uname -a || true
-          cat /etc/os-release || true
-          id || true
-          ls -l /dev/kfd || echo "MISSING /dev/kfd"
-          ls -l /dev/dri || echo "MISSING /dev/dri"
-          lspci | grep -iE 'vga|display|amd/ati' || echo "lspci unavailable / no GPU match"
-          env | grep -E '^(RUNNER_|GITHUB_)' | sort || true
-          echo "===== END RUNNER IDENTITY ====="
-          true
-
       - name: Checkout llama.cpp (under test)
         uses: actions/checkout@v6
         with:
@@ -120,6 +95,19 @@ jobs:
           ref: ${{ env.BENCH_REF }}
           path: bench
 
+      # Runner identity dump -- logs RUNNER_NAME so the specific self-hosted
+      # runner behind a failure is greppable from the job log. Runs right after
+      # the bench checkout (earliest a bench script is available); if: always()
+      # so it still logs when a later step fails. NOTE: a gfx1201 "Initialize
+      # containers" failure happens before any step runs and isn't captured here.
+      - name: Debug (runner identity)
+        if: always()
+        env:
+          MATRIX_NAME: ${{ matrix.name }}
+          MATRIX_GPU_TARGET: ${{ matrix.gpu_target }}
+          MATRIX_RUNS_ON: ${{ matrix.runs_on }}
+        run: "${BENCH_DIR}/scripts/hrx/runner-identity.sh"
+
       - name: Install ROCm build dependencies
         run: "${BENCH_DIR}/scripts/hrx/install-rocm-deps.sh"
 

From bdcd68d6d9fd48dd6edc3c520d2a72efb138ad0f Mon Sep 17 00:00:00 2001
From: AaronStGeorge <aaronstgeorge@gmail.com>
Date: Mon, 15 Jun 2026 05:13:54 +0000
Subject: [PATCH 14/14] cleanup

---
 .github/workflows/build-hrx.yml | 69 +++++----------------------------
 1 file changed, 9 insertions(+), 60 deletions(-)

diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml
index 80f197cde7da..d802d0978593 100644
--- a/.github/workflows/build-hrx.yml
+++ b/.github/workflows/build-hrx.yml
@@ -34,7 +34,7 @@ jobs:
             runs_on: linux-gfx120X-gpu-rocm
             extra_rocm_artifacts: rocwmma_dev_gfx120X-all
     runs-on: ${{ matrix.runs_on }}
-    timeout-minutes: 15
+    timeout-minutes: 10
     defaults:
       run:
         # --noprofile skips loading ~/.bash_profile
@@ -71,23 +71,17 @@ jobs:
       # AaronStGeorge/llamacpp_ci)
       #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced.
       BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci
-      BENCH_REF: 'runner-health'
+      BENCH_REF: 'main'
       # actions/checkout@v6 does auto-cleanup, an in-tree build would be
       # auto-cleaned as well.
       BENCH_DIR: ${{ github.workspace }}/bench
       LLAMA_SRC_DIR: ${{ github.workspace }}/llama-src
-      # Build tree off the bind mount too (source stays under the workspace).
       LLAMA_BUILD_DIR: /work/llama-build
       HRX_EXTRA_ROCM_ARTIFACTS: ${{ matrix.extra_rocm_artifacts }}
       GGML_HRX_AMDGPU_TARGETS: ${{ matrix.gpu_target }}
       CCACHE_COMPILERCHECK: content
 
     steps:
-      - name: Checkout llama.cpp (under test)
-        uses: actions/checkout@v6
-        with:
-          path: llama-src
-
       - name: Checkout bench tooling
         uses: actions/checkout@v6
         with:
@@ -95,18 +89,17 @@ jobs:
           ref: ${{ env.BENCH_REF }}
           path: bench
 
-      # Runner identity dump -- logs RUNNER_NAME so the specific self-hosted
-      # runner behind a failure is greppable from the job log. Runs right after
-      # the bench checkout (earliest a bench script is available); if: always()
-      # so it still logs when a later step fails. NOTE: a gfx1201 "Initialize
-      # containers" failure happens before any step runs and isn't captured here.
-      - name: Debug (runner identity)
-        if: always()
+      - name: Runner info
         env:
           MATRIX_NAME: ${{ matrix.name }}
           MATRIX_GPU_TARGET: ${{ matrix.gpu_target }}
           MATRIX_RUNS_ON: ${{ matrix.runs_on }}
-        run: "${BENCH_DIR}/scripts/hrx/runner-identity.sh"
+        run: "${BENCH_DIR}/scripts/hrx/runner-info.sh"
+
+      - name: Checkout llama.cpp (under test)
+        uses: actions/checkout@v6
+        with:
+          path: llama-src
 
       - name: Install ROCm build dependencies
         run: "${BENCH_DIR}/scripts/hrx/install-rocm-deps.sh"
@@ -124,59 +117,16 @@ jobs:
       - name: Fetch ROCm assets
         run: "${BENCH_DIR}/scripts/hrx/fetch-rocm-assets.sh"
 
-      # Dump GPU/board identity (serial/VBIOS/fw via rocminfo/amd-smi/rocm-smi)
-      # and fail fast on gross breakage (no /dev/kfd, dead rocminfo, expected
-      # gfx arch not enumerated) before the costly HRX/llama build. Runs here
-      # because the diag binaries only exist in the composed prefix after the
-      # fetch step. Does NOT detect the gfx1151 coherence NaN flake.
-      - name: GPU health check
-        timeout-minutes: 3
-        run: "${BENCH_DIR}/scripts/hrx/runner-health.sh"
-
       - name: Build HRX
         run: "${BENCH_DIR}/scripts/hrx/build-hrx.sh"
 
       - name: Validate HRX
-        # hrx-info has been observed hanging on some runners; fail fast.
-        timeout-minutes: 5
         run: "${BENCH_DIR}/scripts/hrx/validate-hrx.sh"
 
       - name: Build llama.cpp with HRX
         run: "${BENCH_DIR}/scripts/hrx/build-llama-hrx.sh"
 
-      # Diagnostic: run the failing MUL_MAT config N times in THIS job to measure
-      # the per-RUNNER failure rate -- does a bad machine fail every iteration or
-      # only some? Advisory (continue-on-error, runs BEFORE the gating step so it
-      # still executes on a failing machine). Correlate the printed count with
-      # RUNNER_NAME (see the Debug step). Tune N via the MULMAT_LOOP_N var.
-      - name: MUL_MAT failure-rate loop (diagnostic)
-        continue-on-error: true
-        timeout-minutes: 20
-        run: |
-          . "${BENCH_DIR}/scripts/hrx/env.sh"
-          . "${BENCH_DIR}/scripts/hrx/runtime-env.sh"
-          set +e   # count pass/fail from the output, don't let a NaN exit abort
-          TB="${LLAMA_BUILD_DIR}/bin/test-backend-ops"
-          TF="${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt"
-          N="${MULMAT_LOOP_N:-50}"
-          pass=0; fail=0; other=0
-          for i in $(seq 1 "$N"); do
-            out="$("$TB" test -o MUL_MAT -b HRX0 --test-file "$TF" 2>&1 | sed 's/\x1b\[[0-9;]*m//g')"
-            if echo "$out" | grep -q 'NaN at index'; then
-              fail=$((fail+1))
-              [ "$fail" -le 3 ] && echo "iter $i FAIL: $(echo "$out" | grep -m1 'NaN at index')"
-            elif echo "$out" | grep -q '1/1 tests passed'; then
-              pass=$((pass+1))
-            else
-              other=$((other+1))
-              [ "$other" -le 2 ] && echo "iter $i OTHER: $(echo "$out" | tail -2 | tr '\n' ' ')"
-            fi
-          done
-          echo "MULMAT_RATE runner=${RUNNER_NAME} fail=${fail} pass=${pass} other=${other} total=${N}"
-          echo "::notice::MUL_MAT rate on ${RUNNER_NAME}: ${fail}/${N} FAIL (pass=${pass}, other=${other})"
-
       - name: Run sample MUL_MAT correctness config on HRX
-        timeout-minutes: 15
         run: |
           . "${BENCH_DIR}/scripts/hrx/env.sh"
           . "${BENCH_DIR}/scripts/hrx/runtime-env.sh"
@@ -188,7 +138,6 @@ jobs:
             --output benchmark-results/sample-mul-mat-f16-hrx-test.jsonl
 
       - name: Run sample MUL_MAT benchmark config on HRX
-        timeout-minutes: 15
         run: |
           . "${BENCH_DIR}/scripts/hrx/env.sh"
           . "${BENCH_DIR}/scripts/hrx/runtime-env.sh"