From 881dcf1215696f19bee699045d0f32f41539b896 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Mon, 2 Feb 2026 12:43:25 -0500
Subject: [PATCH 01/21] Draft buildkite pipeline

We still need base images posted to higherordermethods registry
---
 .buildkite/pipeline.yml | 61 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 .buildkite/pipeline.yml

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
new file mode 100644
index 00000000..eba89b6d
--- /dev/null
+++ b/.buildkite/pipeline.yml
@@ -0,0 +1,61 @@
+steps:
+  - label: ":nvidia: Build & Test V100"
+    command: |
+      srun --container-image=docker://higherordermethods/selfish_x86_sm72:latest \
+           --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
+           --container-workdir=/workspace \
+           bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
+    env:
+      BUILDKITE_SLURM_PARTITION: "main"
+      BUILDKITE_SLURM_GRES: "gpu:v100:4"
+      BUILDKITE_SLURM_TIME: "01:00:00"
+      BUILDKITE_SLURM_NODES: "1"
+      BUILDKITE_SLURM_CPUS_PER_TASK: "8"
+      BUILDKITE_SLURM_NTASKS_PER_NODE: "4"
+
+    agents:
+      queue: "galapagos"
+      slurm: "true"
+
+  - label: ":amd: Build & Test MI210"
+    command: |
+      srun --container-image=docker://higherordermethods/selfish_x86_gfx906:latest \
+           --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
+           --container-workdir=/workspace \
+           bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
+    env:
+      BUILDKITE_SLURM_PARTITION: "main"
+      BUILDKITE_SLURM_GRES: "gpu:mi210:4"
+      BUILDKITE_SLURM_TIME: "01:00:00"
+      BUILDKITE_SLURM_NODES: "1"
+      BUILDKITE_SLURM_CPUS_PER_TASK: "8"
+      BUILDKITE_SLURM_NTASKS_PER_NODE: "4"
+    agents:
+      queue: "galapagos"
+      slurm: "true"
+
+  - label: ":amd: Build & Test MI300A"
+    command: |
+      srun --container-image=docker://higherordermethods/selfish_x86_gfx906:latest \
+           --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
+           --container-workdir=/workspace \
+           bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
+    env:
+      BUILDKITE_SLURM_PARTITION: "main"
+      BUILDKITE_SLURM_GRES: "gpu:mi300a:4"
+      BUILDKITE_SLURM_TIME: "01:00:00"
+      BUILDKITE_SLURM_NODES: "1"
+      BUILDKITE_SLURM_CPUS_PER_TASK: "8"
+      BUILDKITE_SLURM_NTASKS_PER_NODE: "4"
+    agents:
+      queue: "galapagos"
+      slurm: "true"
+
+ # Optional: Upload test results as artifacts
+  - wait
+  
+  - label: ":package: Collect Artifacts"
+    command: |
+      buildkite-agent artifact upload "build/Testing/**/*"
+    agents:
+      queue: "galapagos"

From 43e9b6a76897aae8234b3def6c01719537059d51 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Tue, 3 Feb 2026 14:02:08 -0500
Subject: [PATCH 02/21] Run `srun` directly, assuming no pre-command

---
 .buildkite/pipeline.yml | 54 +++++++----------------------------------
 1 file changed, 9 insertions(+), 45 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index eba89b6d..87d8311f 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,57 +1,21 @@
 steps:
-  - label: ":nvidia: Build & Test V100"
+  - label: ":nvidia: Build & Test MI210"
+    key: "test-mi210"
     command: |
-      srun --container-image=docker://higherordermethods/selfish_x86_sm72:latest \
+      srun --partition=main \
+           --gres=gpu:v100:4 \
+           --time=01:00:00 \
+           --nodes=1 \
+           --ntasks-per-node=4 \
+           --cpus-per-task=8
+           --container-image=docker://higherordermethods/selfish:latest-x86-rocm643-sm72 \
            --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
            --container-workdir=/workspace \
            bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
-    env:
-      BUILDKITE_SLURM_PARTITION: "main"
-      BUILDKITE_SLURM_GRES: "gpu:v100:4"
-      BUILDKITE_SLURM_TIME: "01:00:00"
-      BUILDKITE_SLURM_NODES: "1"
-      BUILDKITE_SLURM_CPUS_PER_TASK: "8"
-      BUILDKITE_SLURM_NTASKS_PER_NODE: "4"
-
-    agents:
-      queue: "galapagos"
-      slurm: "true"
-
-  - label: ":amd: Build & Test MI210"
-    command: |
-      srun --container-image=docker://higherordermethods/selfish_x86_gfx906:latest \
-           --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
-           --container-workdir=/workspace \
-           bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
-    env:
-      BUILDKITE_SLURM_PARTITION: "main"
-      BUILDKITE_SLURM_GRES: "gpu:mi210:4"
-      BUILDKITE_SLURM_TIME: "01:00:00"
-      BUILDKITE_SLURM_NODES: "1"
-      BUILDKITE_SLURM_CPUS_PER_TASK: "8"
-      BUILDKITE_SLURM_NTASKS_PER_NODE: "4"
-    agents:
-      queue: "galapagos"
-      slurm: "true"
-
-  - label: ":amd: Build & Test MI300A"
-    command: |
-      srun --container-image=docker://higherordermethods/selfish_x86_gfx906:latest \
-           --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
-           --container-workdir=/workspace \
-           bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
-    env:
-      BUILDKITE_SLURM_PARTITION: "main"
-      BUILDKITE_SLURM_GRES: "gpu:mi300a:4"
-      BUILDKITE_SLURM_TIME: "01:00:00"
-      BUILDKITE_SLURM_NODES: "1"
-      BUILDKITE_SLURM_CPUS_PER_TASK: "8"
-      BUILDKITE_SLURM_NTASKS_PER_NODE: "4"
     agents:
       queue: "galapagos"
       slurm: "true"
 
- # Optional: Upload test results as artifacts
   - wait
   
   - label: ":package: Collect Artifacts"

From 803ed9e3805a720b55df51bef906c0d24618627c Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Tue, 3 Feb 2026 14:08:17 -0500
Subject: [PATCH 03/21] Fix missing "\"

---
 .buildkite/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 87d8311f..99975fe9 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,5 +1,5 @@
 steps:
-  - label: ":nvidia: Build & Test MI210"
+  - label: ":AMD: Build & Test MI210"
     key: "test-mi210"
     command: |
       srun --partition=main \
@@ -7,7 +7,7 @@ steps:
            --time=01:00:00 \
            --nodes=1 \
            --ntasks-per-node=4 \
-           --cpus-per-task=8
+           --cpus-per-task=8 \
            --container-image=docker://higherordermethods/selfish:latest-x86-rocm643-sm72 \
            --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
            --container-workdir=/workspace \

From c3a4b92d81ff5a53f7fbc09dd5325354f16af145 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Tue, 3 Feb 2026 14:09:10 -0500
Subject: [PATCH 04/21] Change gpu type to mi210

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 99975fe9..5cf3b9cd 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -3,7 +3,7 @@ steps:
     key: "test-mi210"
     command: |
       srun --partition=main \
-           --gres=gpu:v100:4 \
+           --gres=gpu:mi210:4 \
            --time=01:00:00 \
            --nodes=1 \
            --ntasks-per-node=4 \

From 540b7ea57e009d9f2d5bf72974c4115777091fec Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Tue, 3 Feb 2026 15:56:54 -0500
Subject: [PATCH 05/21] Fix tag for mi210 base image

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 5cf3b9cd..bfdddbe6 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -8,7 +8,7 @@ steps:
            --nodes=1 \
            --ntasks-per-node=4 \
            --cpus-per-task=8 \
-           --container-image=docker://higherordermethods/selfish:latest-x86-rocm643-sm72 \
+           --container-image=docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a \
            --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
            --container-workdir=/workspace \
            bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"

From c041016a047e63f8bc667b0a98548dc43fff30f2 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Feb 2026 12:18:54 -0500
Subject: [PATCH 06/21] Adjust to use new pre-command hook for wrapping
 commands in srun

---
 .buildkite/pipeline.yml | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index bfdddbe6..456ea524 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -2,19 +2,18 @@ steps:
   - label: ":AMD: Build & Test MI210"
     key: "test-mi210"
     command: |
-      srun --partition=main \
-           --gres=gpu:mi210:4 \
-           --time=01:00:00 \
-           --nodes=1 \
-           --ntasks-per-node=4 \
-           --cpus-per-task=8 \
-           --container-image=docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a \
-           --container-mounts=${BUILDKITE_BUILD_CHECKOUT_PATH}:/workspace \
-           --container-workdir=/workspace \
-           bash -c "cmake -B build -S . && cmake --build build && ctest --test-dir build --output-on-failure"
+      cmake -B build -S .
+      cmake --build build
+      ctest --test-dir build --output-on-failure
     agents:
       queue: "galapagos"
-      slurm: "true"
+      slurm_partition: "main"
+      slurm_gres: "gpu:mi210:4"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks_per_node: 4
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
 
   - wait
   

From 5fe97330d1bcfa84863b99eb864d97fd6fec123f Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Feb 2026 13:30:13 -0500
Subject: [PATCH 07/21] Make build env and step more verbose

---
 .buildkite/pipeline.yml | 43 ++++++++++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 456ea524..b4fabb40 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -2,23 +2,40 @@ steps:
   - label: ":AMD: Build & Test MI210"
     key: "test-mi210"
     command: |
-      cmake -B build -S .
-      cmake --build build
-      ctest --test-dir build --output-on-failure
+       mkdir -p /workspace/build
+       cd /workspace/build
+       cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
+             -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -DSELF_ENABLE_GPU=${ENABLE_GPU} \
+             -DSELF_ENABLE_MULTITHREADING=${ENABLE_MULTITHREADING} \
+             -DSELF_MULTITHREADING_NTHREADS=${NTHREADS} \
+             -DSELF_ENABLE_DOUBLE_PRECISION=${ENABLE_DOUBLE_PRECISION} \
+             -DSELF_ENABLE_TESTING=${ENABLE_TESTING} \
+             -DCMAKE_HIP_ARCHITECTURES=${GPU_ARCH} \
+             -DSELF_ENABLE_EXAMPLES=${ENABLE_EXAMPLES} \
+             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+             -DSELF_ENABLE_INTERFACE=${ENABLE_INTERFACE} \
+             /workspace/
+        make -j
+        ctest --test-dir /workspace/build --output-on-failure
+    env:
+      BUILD_TYPE: "debug"
+      ENABLE_GPU: "ON"
+      ENABLE_INTERFACE: "OFF"
+      ENABLE_DOUBLE_PRECISION: "ON"
+      ENABLE_MULTITHREADING: "ON"
+      NTHREADS: 8
+      GPU_ARCH: "gfx90a"
+      GCOV: "gcov"
+      RUN_TESTS: "ON"
+      ENABLE_TESTING: "ON"
+      ENABLE_EXAMPLES: "ON"
     agents:
       queue: "galapagos"
       slurm_partition: "main"
-      slurm_gres: "gpu:mi210:4"
+      slurm_gres: "gpu:mi210:2"
       slurm_time: "01:00:00"
       slurm_nodes: 1
-      slurm_ntasks_per_node: 4
+      slurm_ntasks_per_node: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
-
-  - wait
-  
-  - label: ":package: Collect Artifacts"
-    command: |
-      buildkite-agent artifact upload "build/Testing/**/*"
-    agents:
-      queue: "galapagos"

From 1eba659f39bb65538655597fad78f954748135f6 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Feb 2026 13:54:31 -0500
Subject: [PATCH 08/21] Move slurm vars to env

---
 .buildkite/pipeline.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index b4fabb40..e7e71f5d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -19,6 +19,13 @@ steps:
         make -j
         ctest --test-dir /workspace/build --output-on-failure
     env:
+      slurm_partition: "main"
+      slurm_gres: "gpu:mi210:2"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks_per_node: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
       BUILD_TYPE: "debug"
       ENABLE_GPU: "ON"
       ENABLE_INTERFACE: "OFF"
@@ -32,10 +39,3 @@ steps:
       ENABLE_EXAMPLES: "ON"
     agents:
       queue: "galapagos"
-      slurm_partition: "main"
-      slurm_gres: "gpu:mi210:2"
-      slurm_time: "01:00:00"
-      slurm_nodes: 1
-      slurm_ntasks_per_node: 2
-      slurm_cpus_per_task: 8
-      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"

From 6f1861e9e3c99493b9c1851325839c1941904b90 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Tue, 10 Feb 2026 21:04:56 -0500
Subject: [PATCH 09/21] Enable spack environment, specify fortran compiler in
 cmake stage

---
 .buildkite/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index e7e71f5d..86e95386 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -2,9 +2,10 @@ steps:
   - label: ":AMD: Build & Test MI210"
     key: "test-mi210"
     command: |
+       . /opt/spack-environment/activate.sh
        mkdir -p /workspace/build
        cd /workspace/build
-       cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
+       FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
              -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
              -DSELF_ENABLE_GPU=${ENABLE_GPU} \
              -DSELF_ENABLE_MULTITHREADING=${ENABLE_MULTITHREADING} \
@@ -34,7 +35,6 @@ steps:
       NTHREADS: 8
       GPU_ARCH: "gfx90a"
       GCOV: "gcov"
-      RUN_TESTS: "ON"
       ENABLE_TESTING: "ON"
       ENABLE_EXAMPLES: "ON"
     agents:

From 8bbef92a664a82ceafbe3f4d49e7a4702084d22c Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Sun, 1 Mar 2026 18:47:01 -0500
Subject: [PATCH 10/21] Add workspace env variable

This is needed to load the mesh files in tests
---
 .buildkite/pipeline.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 86e95386..3fac6553 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -2,7 +2,8 @@ steps:
   - label: ":AMD: Build & Test MI210"
     key: "test-mi210"
     command: |
-       . /opt/spack-environment/activate.sh
+       source /opt/spack-environment/activate.sh
+       env
        mkdir -p /workspace/build
        cd /workspace/build
        FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
@@ -27,6 +28,7 @@ steps:
       slurm_ntasks_per_node: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+      WORKSPACE: "/workspace"
       BUILD_TYPE: "debug"
       ENABLE_GPU: "ON"
       ENABLE_INTERFACE: "OFF"

From dfb7cf5e9e0e9f2afcbffd11c5cacfe301196cee Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Sun, 1 Mar 2026 21:19:09 -0500
Subject: [PATCH 11/21] Update pipeline assuming buildkite is running as batch
 job in container

---
 .buildkite/pipeline.yml | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 3fac6553..f4baddb3 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,31 +1,30 @@
 steps:
-  - label: ":AMD: Build & Test MI210"
-    key: "test-mi210"
+  - label: ":AMD MI210: Build & Test"
+    key: "build-mi210"
     command: |
-       source /opt/spack-environment/activate.sh
-       env
-       mkdir -p /workspace/build
-       cd /workspace/build
-       FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
-             -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-             -DSELF_ENABLE_GPU=${ENABLE_GPU} \
-             -DSELF_ENABLE_MULTITHREADING=${ENABLE_MULTITHREADING} \
-             -DSELF_MULTITHREADING_NTHREADS=${NTHREADS} \
-             -DSELF_ENABLE_DOUBLE_PRECISION=${ENABLE_DOUBLE_PRECISION} \
-             -DSELF_ENABLE_TESTING=${ENABLE_TESTING} \
-             -DCMAKE_HIP_ARCHITECTURES=${GPU_ARCH} \
-             -DSELF_ENABLE_EXAMPLES=${ENABLE_EXAMPLES} \
-             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-             -DSELF_ENABLE_INTERFACE=${ENABLE_INTERFACE} \
-             /workspace/
-        make -j
-        ctest --test-dir /workspace/build --output-on-failure
+      source /opt/spack-environment/activate.sh
+      mkdir -p /workspace/build
+      cd /workspace/build
+      FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
+            -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+            -DSELF_ENABLE_GPU=${ENABLE_GPU} \
+            -DSELF_ENABLE_MULTITHREADING=${ENABLE_MULTITHREADING} \
+            -DSELF_MULTITHREADING_NTHREADS=${NTHREADS} \
+            -DSELF_ENABLE_DOUBLE_PRECISION=${ENABLE_DOUBLE_PRECISION} \
+            -DSELF_ENABLE_TESTING=${ENABLE_TESTING} \
+            -DCMAKE_HIP_ARCHITECTURES=${GPU_ARCH} \
+            -DSELF_ENABLE_EXAMPLES=${ENABLE_EXAMPLES} \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+            -DSELF_ENABLE_INTERFACE=${ENABLE_INTERFACE} \
+            /workspace/
+      make -j
+      ctest --test-dir /workspace/build --output-on-failure
     env:
       slurm_partition: "main"
       slurm_gres: "gpu:mi210:2"
       slurm_time: "01:00:00"
       slurm_nodes: 1
-      slurm_ntasks_per_node: 2
+      slurm_ntasks: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
       WORKSPACE: "/workspace"

From 5ea6ee090164b3e59fa49cf4277df5a09c7737cb Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Wed, 4 Mar 2026 19:14:52 -0500
Subject: [PATCH 12/21] Report commands to stdout

---
 .buildkite/pipeline.yml | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f4baddb3..533fdd35 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,18 +5,17 @@ steps:
       source /opt/spack-environment/activate.sh
       mkdir -p /workspace/build
       cd /workspace/build
+      set -x
       FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
             -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
             -DSELF_ENABLE_GPU=${ENABLE_GPU} \
-            -DSELF_ENABLE_MULTITHREADING=${ENABLE_MULTITHREADING} \
-            -DSELF_MULTITHREADING_NTHREADS=${NTHREADS} \
-            -DSELF_ENABLE_DOUBLE_PRECISION=${ENABLE_DOUBLE_PRECISION} \
             -DSELF_ENABLE_TESTING=${ENABLE_TESTING} \
-            -DCMAKE_HIP_ARCHITECTURES=${GPU_ARCH} \
+            -DCMAKE_HIP_ARCHITECTURES=${TARGET_GPU_ARCH} \
             -DSELF_ENABLE_EXAMPLES=${ENABLE_EXAMPLES} \
-            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
             -DSELF_ENABLE_INTERFACE=${ENABLE_INTERFACE} \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
             /workspace/
+      set +x
       make -j
       ctest --test-dir /workspace/build --output-on-failure
     env:
@@ -31,10 +30,7 @@ steps:
       BUILD_TYPE: "debug"
       ENABLE_GPU: "ON"
       ENABLE_INTERFACE: "OFF"
-      ENABLE_DOUBLE_PRECISION: "ON"
-      ENABLE_MULTITHREADING: "ON"
-      NTHREADS: 8
-      GPU_ARCH: "gfx90a"
+      TARGET_GPU_ARCH: "gfx90a"
       GCOV: "gcov"
       ENABLE_TESTING: "ON"
       ENABLE_EXAMPLES: "ON"

From 926da9c2ee54316eb7f02271f1f4a20229ddc5ff Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Wed, 4 Mar 2026 19:20:12 -0500
Subject: [PATCH 13/21] Hard-code cmake build options in command

---
 .buildkite/pipeline.yml | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 533fdd35..54f52f61 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -7,12 +7,12 @@ steps:
       cd /workspace/build
       set -x
       FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
-            -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-            -DSELF_ENABLE_GPU=${ENABLE_GPU} \
-            -DSELF_ENABLE_TESTING=${ENABLE_TESTING} \
-            -DCMAKE_HIP_ARCHITECTURES=${TARGET_GPU_ARCH} \
-            -DSELF_ENABLE_EXAMPLES=${ENABLE_EXAMPLES} \
-            -DSELF_ENABLE_INTERFACE=${ENABLE_INTERFACE} \
+            -DCMAKE_BUILD_TYPE="debug" \
+            -DSELF_ENABLE_GPU=ON \
+            -DSELF_ENABLE_TESTING=ON \
+            -DCMAKE_HIP_ARCHITECTURES="gfx90a" \
+            -DSELF_ENABLE_EXAMPLES=ON \
+            -DSELF_ENABLE_INTERFACE=OFF \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
             /workspace/
       set +x
@@ -27,12 +27,5 @@ steps:
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
       WORKSPACE: "/workspace"
-      BUILD_TYPE: "debug"
-      ENABLE_GPU: "ON"
-      ENABLE_INTERFACE: "OFF"
-      TARGET_GPU_ARCH: "gfx90a"
-      GCOV: "gcov"
-      ENABLE_TESTING: "ON"
-      ENABLE_EXAMPLES: "ON"
     agents:
       queue: "galapagos"

From 0f824c2149c6317d87aa5e220c83020a1953c6ed Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Wed, 4 Mar 2026 20:47:38 -0500
Subject: [PATCH 14/21] Clean up cmake and add codecov upload stage to pipeline

---
 .buildkite/pipeline.yml      |  38 ++++++++--
 CMakeLists.txt               | 132 ++++++++++++++++++++++-------------
 cmake/CheckMPIFeatures.cmake |   4 +-
 src/CMakeLists.txt           |   4 +-
 test/CMakeLists.txt          |   4 +-
 5 files changed, 120 insertions(+), 62 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 54f52f61..1591429d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,23 +1,32 @@
 steps:
-  - label: ":AMD MI210: Build & Test"
-    key: "build-mi210"
+  - label: ":AMD MI210: Build & Test (Coverage)"
+    key: "build-test-mi210"
     command: |
       source /opt/spack-environment/activate.sh
+      export WORKSPACE=/workspace
       mkdir -p /workspace/build
       cd /workspace/build
-      set -x
       FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
-            -DCMAKE_BUILD_TYPE="debug" \
+            -DCMAKE_BUILD_TYPE="coverage" \
             -DSELF_ENABLE_GPU=ON \
+            -DSELF_GPU_BACKEND=HIP \
             -DSELF_ENABLE_TESTING=ON \
             -DCMAKE_HIP_ARCHITECTURES="gfx90a" \
+            -DGPU_TARGETS="gfx90a" \
             -DSELF_ENABLE_EXAMPLES=ON \
-            -DSELF_ENABLE_INTERFACE=OFF \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
             /workspace/
-      set +x
       make -j
+      lcov --capture --initial \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/initial.info
       ctest --test-dir /workspace/build --output-on-failure
+      lcov --capture \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/ctest-capture.info
+      lcov --add-tracefile /workspace/initial.info \
+           --add-tracefile /workspace/ctest-capture.info \
+           --output-file /workspace/coverage.info
     env:
       slurm_partition: "main"
       slurm_gres: "gpu:mi210:2"
@@ -26,6 +35,21 @@ steps:
       slurm_ntasks: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
-      WORKSPACE: "/workspace"
+    agents:
+      queue: "galapagos"
+
+  - label: ":codecov: Upload Coverage"
+    key: "upload-coverage"
+    depends_on: "build-test-mi210"
+    command: |
+      curl -Os https://uploader.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      ./codecov -t "$${CODECOV_TOKEN}" \
+                -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
+                -F buildkite-rocm-mi210
+    plugins:
+      - cluster-secrets#v1.0.1:
+          env:
+            CODECOV_TOKEN: CODECOV_TOKEN
     agents:
       queue: "galapagos"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e63bdea3..d65cded3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,7 +36,12 @@ option(SELF_ENABLE_MULTITHREADING "Option to enable CPU multithreading for `do c
 option(SELF_ENABLE_TESTING "Option to enable build of tests. (Default On)"  ON)
 option(SELF_ENABLE_EXAMPLES "Option to enable build of examples. (Default On)"  ON)
 option(SELF_ENABLE_GPU "Option to enable GPU backend. Requires either CUDA or HIP. (Default Off)"  OFF)
-option(SELF_ENABLE_APU "Option to enable APU backend. Requires either CUDA or HIP. (Default Off)"  OFF)
+option(SELF_ENABLE_APU "Option to enable APU backend. (Default Off)"  OFF)
+
+set(SELF_GPU_BACKEND "AUTO" CACHE STRING "GPU backend to use when SELF_ENABLE_GPU is ON: AUTO, CUDA, or HIP. AUTO prefers HIP when available. (Default AUTO)")
+set_property(CACHE SELF_GPU_BACKEND PROPERTY STRINGS AUTO CUDA HIP)
+
+option(SELF_REQUIRE_GPU_AWARE_MPI "Require GPU-aware MPI when building with GPU support. Set OFF to downgrade the check to a warning. (Default ON)"  ON)
 option(SELF_ENABLE_DOUBLE_PRECISION "Option to enable double precision for floating point arithmetic. (Default On)"  ON)
 
 set(SELF_MPIEXEC_NUMPROCS "2" CACHE STRING "The number of MPI ranks to use to launch MPI tests. Only used when launching test programs via ctest.")
@@ -151,76 +156,103 @@ if(SELF_ENABLE_DOUBLE_PRECISION)
 
 endif()
 
+if(SELF_ENABLE_GPU AND SELF_ENABLE_APU)
+    message(FATAL_ERROR "SELF_ENABLE_GPU and SELF_ENABLE_APU are mutually exclusive. Enable only one.")
+endif()
+
 if(SELF_ENABLE_GPU)
-    message("-- SELF Build System : Enabling GPU Support")
+    message("-- SELF Build System : Enabling GPU Support (backend: ${SELF_GPU_BACKEND})")
     set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_GPU" )
     set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_GPU" )
     set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_GPU")
     set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_GPU")
     set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_GPU" )
 
-    # Check MPI for GPU awareness
     # Add SELF's cmake module directory to the search path
     set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-
-    # Include the custom module
     include(CheckMPIFeatures)
     CheckMPIFeatures()
 
-    # HIP (Optional)
-    # IF HIP is found, we prefer HIP. IF HIP is not found, then we fall back to look for CUDA
-    # only. If neither HIP nor CUDA are found, Cmake will error out.
-    find_package(hip)
-    if(hip_FOUND)
-        if(MPI_HAS_QUERY_HIP_SUPPORT)
-            find_package(hipblas REQUIRED)
-            message("-- HIP found. Enabling HIP language.")
-            enable_language(HIP)
-            set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
-            set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
-            set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
-            set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
-            set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
-
-            if(SELF_ENABLE_DOUBLE_PRECISION)
-                set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" )
-                set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
-                set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
-                set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION")
-                set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
-            endif()
-            set( BACKEND_LIBRARIES hip::device roc::hipblas)
+    # Resolve which backend to activate based on SELF_GPU_BACKEND
+    if(SELF_GPU_BACKEND STREQUAL "HIP")
+        find_package(hip REQUIRED)
+        set(_self_use_hip TRUE)
+    elseif(SELF_GPU_BACKEND STREQUAL "CUDA")
+        find_package(CUDAToolkit REQUIRED)
+        set(_self_use_hip FALSE)
+    else() # AUTO: prefer HIP, fall back to CUDA
+        find_package(hip)
+        if(hip_FOUND)
+            set(_self_use_hip TRUE)
         else()
-            message( FATAL_ERROR "MPI installation is not GPU-aware" )
+            find_package(CUDAToolkit REQUIRED)
+            set(_self_use_hip FALSE)
         endif()
-    else()
-        # CUDA
-        find_package(CUDAToolkit REQUIRED)
+    endif()
+
+    if(_self_use_hip)
+        find_package(hipblas REQUIRED)
+        message("-- HIP found. Enabling HIP language.")
+        enable_language(HIP)
+        if(NOT MPI_HAS_QUERY_HIP_SUPPORT)
+            if(SELF_REQUIRE_GPU_AWARE_MPI)
+                message(FATAL_ERROR "MPI installation is not HIP/ROCm-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.")
+            else()
+                message(WARNING "MPI installation is not HIP/ROCm-aware. GPU-direct communication will not be available.")
+            endif()
+        endif()
+        set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
+        set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
+        set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
+        set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
+        set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
+        if(SELF_ENABLE_DOUBLE_PRECISION)
+            set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" )
+            set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
+            set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
+            set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION")
+            set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
+        endif()
+        set( BACKEND_LIBRARIES hip::device roc::hipblas)
+        set( SELF_ACTIVE_GPU_BACKEND "HIP")
+
+    else() # CUDA
         message("-- CUDA found. Enabling CUDA language.")
         enable_language(CUDA)
-        if(MPI_HAS_QUERY_CUDA_SUPPORT)
-            set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
-            set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
-            set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
-            set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
-            set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
-
-            if(SELF_ENABLE_DOUBLE_PRECISION)
-                set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" )
-                set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
-                set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
-                set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION")
-                set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
+        if(NOT MPI_HAS_QUERY_CUDA_SUPPORT)
+            if(SELF_REQUIRE_GPU_AWARE_MPI)
+                message(FATAL_ERROR "MPI installation is not CUDA-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.")
+            else()
+                message(WARNING "MPI installation is not CUDA-aware. GPU-direct communication will not be available.")
             endif()
-
-            set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas)
-
-        else()
-            message( FATAL_ERROR "MPI installation is not GPU-aware" )
         endif()
+        set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
+        set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
+        set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
+        set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
+        set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
+        if(SELF_ENABLE_DOUBLE_PRECISION)
+            set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" )
+            set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
+            set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
+            set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION")
+            set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
+        endif()
+        set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas)
+        set( SELF_ACTIVE_GPU_BACKEND "CUDA")
+
     endif()
 endif()
 
+if(SELF_ENABLE_APU)
+    message("-- SELF Build System : Enabling APU Support")
+    set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_APU" )
+    set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_APU" )
+    set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_APU")
+    set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_APU")
+    set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_APU" )
+endif()
+
 # ----- end dependencies ----- #
 
 # Libraries
diff --git a/cmake/CheckMPIFeatures.cmake b/cmake/CheckMPIFeatures.cmake
index ea555023..672f895b 100644
--- a/cmake/CheckMPIFeatures.cmake
+++ b/cmake/CheckMPIFeatures.cmake
@@ -98,6 +98,4 @@ function(CheckMPIFeatures)
 
     list(REMOVE_ITEM CMAKE_REQUIRED_DEFINITIONS -DHAVE_MPI_EXT)
   endif()
-endfunction()
-
-CheckMPIFeatures()
\ No newline at end of file
+endfunction()
\ No newline at end of file
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6dc5ca79..b4dbf925 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -30,11 +30,13 @@ if(SELF_ENABLE_GPU)
     file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.f*")
     file(GLOB SELF_BACKEND_CPPSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp*")
 
-    if(hip_FOUND)
+    if(SELF_ACTIVE_GPU_BACKEND STREQUAL "HIP")
       set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE HIP)
     else()
       set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE CUDA)
     endif()
+elseif(SELF_ENABLE_APU)
+    file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/apu/*.f*")
 else()
     file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/cpu/*.f*")
 endif()
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index ea5a18e9..19538a1d 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -38,6 +38,7 @@ function (add_fortran_tests)
         target_link_libraries(${TEST_NAME} self)
         target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include)
         add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
+	set_tests_properties(${TEST_NAME} PROPERTIES LABELS "serial")
         install(TARGETS ${TEST_NAME} DESTINATION test)
     endforeach ()
 
@@ -52,6 +53,7 @@ function (add_mpi_fortran_tests)
         target_link_libraries(${TEST_NAME} self)
         target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include)
         add_test(NAME ${TEST_NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SELF_MPIEXEC_NUMPROCS} ${SELF_MPIEXEC_OPTIONS} ${TEST_NAME})
+	set_tests_properties(${TEST_NAME} PROPERTIES LABELS "parallel")
 
         install(TARGETS ${TEST_NAME} DESTINATION test)
     endforeach ()
@@ -146,4 +148,4 @@ add_mpi_fortran_tests( "mappedvectordgdivergence_2d_linear_mpi.f90"
                        "advection_diffusion_2d_rk3_mpi.f90"
                        "advection_diffusion_2d_rk3_pickup_mpi.f90"
                        "advection_diffusion_3d_rk3_mpi.f90"
-                       "advection_diffusion_3d_rk3_pickup_mpi.f90" )
\ No newline at end of file
+                       "advection_diffusion_3d_rk3_pickup_mpi.f90" )

From 256986e683ccff06e17105ed9b889e4f49368603 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Wed, 4 Mar 2026 20:54:39 -0500
Subject: [PATCH 15/21] Add partyparrot emoji in pipeline

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 1591429d..1636549f 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,5 +1,5 @@
 steps:
-  - label: ":AMD MI210: Build & Test (Coverage)"
+  - label: ":partyparrot: Build & Test AMD MI210 (Coverage)"
     key: "build-test-mi210"
     command: |
       source /opt/spack-environment/activate.sh

From 985d6942b8c2c9863ff85c80a9a8c5e1b71fb55b Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Mar 2026 06:55:13 -0500
Subject: [PATCH 16/21] Add call to rocminfo

---
 .buildkite/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 1636549f..5dfaedaa 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,6 +5,7 @@ steps:
       source /opt/spack-environment/activate.sh
       export WORKSPACE=/workspace
       mkdir -p /workspace/build
+      rocminfo
       cd /workspace/build
       FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
             -DCMAKE_BUILD_TYPE="coverage" \

From e366874e136d2a79f52e13a2e53c76f8bfa75a23 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Mar 2026 07:25:39 -0500
Subject: [PATCH 17/21] Add container remap root flag

---
 .buildkite/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 5dfaedaa..0c7492e9 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -36,6 +36,7 @@ steps:
       slurm_ntasks: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+      slrum_container_remap_root: ""
     agents:
       queue: "galapagos"
 

From 22d1e6689948eaac326cc8167f84336c0ddf7336 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Mar 2026 14:36:03 -0500
Subject: [PATCH 18/21] Fix buildkite pipeline typo

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 0c7492e9..76f6d373 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -36,7 +36,7 @@ steps:
       slurm_ntasks: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
-      slrum_container_remap_root: ""
+      slurm_container_remap_root: ""
     agents:
       queue: "galapagos"
 

From 98b919fd3b59881f2a2a936e5e36c910f31f6c60 Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Mar 2026 14:40:02 -0500
Subject: [PATCH 19/21] set container remap root to true to assert boolean flag
 for sbatch

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 76f6d373..33cd1428 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -36,7 +36,7 @@ steps:
       slurm_ntasks: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
-      slurm_container_remap_root: ""
+      slurm_container_remap_root: "true"
     agents:
       queue: "galapagos"
 

From 659ad4ba836fef67a41687a1f25ad5bfaf3b512b Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Mar 2026 14:54:36 -0500
Subject: [PATCH 20/21] Remove container-remap-root

---
 .buildkite/pipeline.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 33cd1428..5dfaedaa 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -36,7 +36,6 @@ steps:
       slurm_ntasks: 2
       slurm_cpus_per_task: 8
       slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
-      slurm_container_remap_root: "true"
     agents:
       queue: "galapagos"
 

From 61ee02d5d3dc5c97e2363596eb5ee48031a37afc Mon Sep 17 00:00:00 2001
From: Joe Schoonover <joe@fluidnumerics.com>
Date: Thu, 5 Mar 2026 16:02:25 -0500
Subject: [PATCH 21/21] Fix secrets syntax for codecov

---
 .buildkite/pipeline.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 5dfaedaa..5a685a2d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -48,9 +48,7 @@ steps:
       ./codecov -t "$${CODECOV_TOKEN}" \
                 -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
                 -F buildkite-rocm-mi210
-    plugins:
-      - cluster-secrets#v1.0.1:
-          env:
-            CODECOV_TOKEN: CODECOV_TOKEN
+    secrets:
+      - CODECOV_TOKEN
     agents:
       queue: "galapagos"