From 80dee99e751e59d28b2e3863fbe3d7589ea28fd1 Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 08:05:47 +0200
Subject: [PATCH 01/13] distributed linearUpwind

---
 src/OpenFOAM/decompose/local_assembly.cuh     |  54 +++++++
 .../simpleFoam/parallel_device_foam.cuh       |  37 +++--
 .../simpleFoam/parallel_device_simple.cuh     | 146 +++++++++++++++++-
 3 files changed, 222 insertions(+), 15 deletions(-)

diff --git a/src/OpenFOAM/decompose/local_assembly.cuh b/src/OpenFOAM/decompose/local_assembly.cuh
index 0981c66..d0dcca2 100644
--- a/src/OpenFOAM/decompose/local_assembly.cuh
+++ b/src/OpenFOAM/decompose/local_assembly.cuh
@@ -60,6 +60,60 @@ inline std::vector<std::vector<scalar>> computeProcDeltaCoeffs(
     return pdc;
 }
 
+// Per processor face, the two cell-centre->face-centre offset vectors that a linearUpwind deferred correction
+// needs: dOwn = Cf - C[localCell] and dNei = Cf - C[remoteCell]. On an internal face these are dm.dOwn/dNei;
+// across a cut the remote centre lives on the other rank, so exchange it once at setup (geometry is static)
+// exactly as computeProcDeltaCoeffs does. Returned interleaved xyz per face, one vector per processor patch:
+//   dOwn[j][3*i+{0,1,2}], dNei[j][3*i+{0,1,2}]
+inline void computeProcUpwindD(
+    const LocalMesh& lm,
+    const FvGeometry& lg,
+    const std::vector<FvPatch>& lpatches,
+    std::vector<std::vector<scalar>>& dOwn,
+    std::vector<std::vector<scalar>>& dNei)
+{
+    std::vector<const FvPatch*> pp;
+    for (const FvPatch& p : lpatches)
+        if (p.type == "processor") pp.push_back(&p);
+
+    const std::size_t n = pp.size();
+    std::vector<std::vector<scalar>> sendC(n), recvC(n);   // 3 scalars (cell centre) per face
+    for (std::size_t j = 0; j < n; ++j)
+    {
+        const label sz = pp[j]->size;
+        sendC[j].resize(3 * sz);
+        recvC[j].resize(3 * sz);
+        for (label i = 0; i < sz; ++i)
+        {
+            const vector& C = lg.C()[pp[j]->faceCells[i]];
+            sendC[j][3 * i]     = C.x;
+            sendC[j][3 * i + 1] = C.y;
+            sendC[j][3 * i + 2] = C.z;
+        }
+        Pstream::irecv(recvC[j].data(), static_cast<int>(3 * sz), lm.procNbr[j], 0);
+        Pstream::isend(sendC[j].data(), static_cast<int>(3 * sz), lm.procNbr[j], 0);
+    }
+    Pstream::waitAll();
+
+    dOwn.assign(n, {});
+    dNei.assign(n, {});
+    for (std::size_t j = 0; j < n; ++j)
+    {
+        const label sz = pp[j]->size;
+        dOwn[j].resize(3 * sz);
+        dNei[j].resize(3 * sz);
+        for (label i = 0; i < sz; ++i)
+        {
+            const vector& Cf = lg.Cf()[pp[j]->start + i];
+            const vector& Cl = lg.C()[pp[j]->faceCells[i]];
+            const vector  Cr{recvC[j][3 * i], recvC[j][3 * i + 1], recvC[j][3 * i + 2]};
+            const vector dO = Cf - Cl, dN = Cf - Cr;
+            dOwn[j][3 * i] = dO.x; dOwn[j][3 * i + 1] = dO.y; dOwn[j][3 * i + 2] = dO.z;
+            dNei[j][3 * i] = dN.x; dNei[j][3 * i + 1] = dN.y; dNei[j][3 * i + 2] = dN.z;
+        }
+    }
+}
+
 // Build the scalar DistributedMatrix structure (diag/upper/lower/interfaceCoeffs) for the momentum
 // div(phi,U) - laplacian(nuEff,U) from the serially-assembled LOCAL FvMatrix (correct internal +
 // real-boundary coeffs; ZERO at processor patches since ProcessorFvPatchField returns zero coeffs)
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index a867974..55b9a95 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -111,11 +111,12 @@ inline int runParallelDeviceFoam(int argc, char** argv)
                 "`mpirun -np N brae_simpleFoam -case <dir>` for RAS in parallel, or `brae -case <dir>` for "
                 "RAS on a single GPU.");
 
-        // fvSchemes div(phi,U). ParallelDeviceSimple implements Gauss upwind (+ the `bounded` -Sp(div(phi),U)
-        // term). Anything else -- linearUpwind, limitedLinear, LUST, ... -- is a DIFFERENT discretisation, and
-        // silently substituting upwind produces a converged, plausible, WRONG answer (on the cavity that is a
-        // ~6% field difference vs the single-GPU solver). Refuse instead, exactly as RAS is refused above.
-        bool boundedDiv = false;
+        // fvSchemes div(phi,U). The distributed path implements Gauss upwind, bounded, and linearUpwind (the
+        // matrix is upwind either way; linearUpwind adds the deferred gradient correction, cut faces included).
+        // Anything else -- limitedLinear, LUST, linearUpwindV, plain Gauss linear -- is a DIFFERENT
+        // discretisation, and silently substituting upwind produces a converged, plausible, WRONG answer (on
+        // the cavity that was a ~6% field difference). Refuse those, exactly as RAS is refused above.
+        bool boundedDiv = false, linUpwind = false;
         {
             std::string divLine;
             std::istringstream fsch(readFileExpanded(caseDir + "/system/fvSchemes"));
@@ -125,15 +126,22 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             if (!divLine.empty())
             {
                 boundedDiv = divLine.find("bounded") != std::string::npos;
-                const bool upwind = divLine.find("upwind") != std::string::npos;
-                const bool linearUpwind = divLine.find("linearUpwind") != std::string::npos;
-                if (!upwind || linearUpwind)
+                linUpwind  = divLine.find("linearUpwind") != std::string::npos;
+                // NB "linearUpwind" has a capital U, so it does NOT contain the substring "upwind" -- the two
+                // must be tested separately or a linearUpwind case reads as "no upwind scheme at all".
+                const bool upwindFamily = (divLine.find("upwind") != std::string::npos) || linUpwind;
+                // linearUpwindV adds OF's vector direction limiter on top of linearUpwind -- NOT implemented,
+                // and it contains the substring "linearUpwind", so it must be excluded explicitly.
+                const bool unsupported = divLine.find("linearUpwindV") != std::string::npos
+                                      || divLine.find("limitedLinear") != std::string::npos
+                                      || divLine.find("LUST") != std::string::npos;
+                if (!upwindFamily || unsupported)
                     throw std::runtime_error(
                         "brae -parallel: div(phi,U) scheme '" + divLine + "' is not implemented on the "
-                        "multi-GPU device path (it supports 'Gauss upwind' and 'bounded Gauss upwind'). "
-                        "Substituting upwind would converge to a DIFFERENT answer than the scheme asks for, "
-                        "so this is refused rather than solved wrongly. Use `brae -case <dir>` (single GPU) "
-                        "for the full scheme set.");
+                        "multi-GPU device path (it supports 'Gauss upwind', 'bounded Gauss upwind' and "
+                        "'[bounded] Gauss linearUpwind grad(U)'). Substituting a scheme would converge to a "
+                        "DIFFERENT answer than fvSchemes asks for, so this is refused rather than solved "
+                        "wrongly. Use `brae -case <dir>` (single GPU) for the full scheme set.");
             }
         }
 
@@ -179,14 +187,15 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             std::printf("brae (device, distributed) | case=%s np=%d | laminar | nu=%.3g | %ld cells\n",
                         caseDir.c_str(), nproc, nu, (long)nC);
             std::printf("  relax U=%.2g p=%.2g | tol p=%.1g U=%.1g | endTime=%d | residualControl=%s | div(phi,U)=%sGauss upwind\n",
-                        relaxU, relaxP, tolP, tolU, endTime, hasRC ? "on" : "off", boundedDiv ? "bounded " : "");
+                        relaxU, relaxP, tolP, tolU, endTime, hasRC ? "on" : "off",
+                        (std::string(boundedDiv ? "bounded " : "") + (linUpwind ? "Gauss linearUpwind" : "Gauss upwind")).c_str());
         }
 
         std::vector<vector> Ug;
         std::vector<scalar> pg;
         int nIter = 0;
         {   // scope: the solver's symmetric-heap buffers must be freed BEFORE Pstream::finalize
-            ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv);
+            ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind);
 
             auto ok = [](scalar res, scalar ctl) { return ctl < 0 || res < ctl; };
             bool converged = false;
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 3402924..16bdeef 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -18,6 +18,7 @@
 #include "device_halo.cuh"
 #include "parallel_simple.cuh"     // Partition, distributeFromCells
 #include "reconstruct.cuh"
+#include "local_assembly.cuh"   // computeProcUpwindD
 #include <cuda_runtime.h>
 #include <vector>
 
@@ -101,6 +102,51 @@ void laplacianInterfaceKernel(
     atomicAdd(&diag[faceCells[f]], -coeff[f]);
 }
 
+// linearUpwind deferred correction at a PROCESSOR face -- the distributed analogue of
+// linearUpwindCorrKernel (device_simple.cu) and of deviceCyclicAddLinUpwindCorr (no rotation here: a
+// processor cut is a plain interior face, the two sides share an orientation).
+//
+// The matrix stays plain UPWIND; linearUpwind is this explicit source. Per cut face, the local cell is the
+// face's owner and phi is the OUTWARD flux, so it is the owner branch of the internal kernel:
+//   corr[localCell] += phi * (grad(U_comp)[upwind] . d_upwind)
+// with the upwind side chosen by sign(phi):
+//   phi >= 0 -> LOCAL is upwind:  grad at the local cell,  d = dOwn = Cf - C[local]
+//   phi <  0 -> REMOTE is upwind: grad at the remote cell (gxN/gyN/gzN, from the halo), d = dNei = Cf - C[remote]
+// dOwn/dNei are static geometry, precomputed once by host computeProcUpwindD (the remote centre needs an
+// exchange). The caller does relaxSrc -= corr, exactly as for the internal correction.
+__global__
+void linUpwindInterfaceKernel(
+    const label*  __restrict__ faceCells,
+    const scalar* __restrict__ phi,
+    const scalar* __restrict__ gx,
+    const scalar* __restrict__ gy,
+    const scalar* __restrict__ gz,
+    const scalar* __restrict__ gxN,
+    const scalar* __restrict__ gyN,
+    const scalar* __restrict__ gzN,
+    const scalar* __restrict__ dOwn,
+    const scalar* __restrict__ dNei,
+    scalar*       __restrict__ corr,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    const scalar pf = phi[f];
+    const label  c  = faceCells[f];
+    scalar g[3], d[3];
+    if (pf >= 0.0)   // local upwind: local gradient, local offset
+    {
+        g[0] = gx[c];  g[1] = gy[c];  g[2] = gz[c];
+        d[0] = dOwn[3*f]; d[1] = dOwn[3*f+1]; d[2] = dOwn[3*f+2];
+    }
+    else             // remote upwind: the halo gradient at the remote cell, remote offset
+    {
+        g[0] = gxN[f]; g[1] = gyN[f]; g[2] = gzN[f];
+        d[0] = dNei[3*f]; d[1] = dNei[3*f+1]; d[2] = dNei[3*f+2];
+    }
+    atomicAdd(&corr[c], pf * (g[0]*d[0] + g[1]*d[1] + g[2]*d[2]));   // a cell may own several cut faces
+}
+
 // Per-cell sum of |processor off-diagonal|, for fvMatrix::relax's diagonal-dominance term. Host
 // parallelRelaxMatrix adds |interfaceCoeffs| into sumOff; on device this feeds deviceRelaxDiag's cycSumOff
 // hook (the same role the cyclic interface's off-diagonal sum plays).
@@ -171,6 +217,65 @@ inline void deviceInterfaceOffDiagSum(
     }
 }
 
+// Add the processor faces' linearUpwind deferred correction into `corr` (which already holds the internal
+// deviceLinearUpwindCorr result). `gx/gy/gz` are grad(U_comp) as CELL fields -- already processor-consistent
+// (the halo injected the coupled face value before the gaussGrad). The three exchanges below fetch that same
+// gradient at the REMOTE cell, needed only where the remote side is upwind.
+// `phiF[i]` is the processor-face flux, `dOwnD/dNeiD[i]` the precomputed offsets (3 per face).
+inline void deviceLinUpwindInterface(
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<label>>& faceCells,
+    const std::vector<DeviceBuffer<scalar>>& phiF,
+    const DeviceBuffer<scalar>& gx,
+    const DeviceBuffer<scalar>& gy,
+    const DeviceBuffer<scalar>& gz,
+    const std::vector<DeviceBuffer<scalar>>& dOwnD,
+    const std::vector<DeviceBuffer<scalar>>& dNeiD,
+    DeviceBuffer<scalar>& corr,
+    cudaStream_t stream = cudaStreamPerThread)
+{
+    constexpr int TPB = 128;
+    // The recv buffer is shared and reused, so each component's remote gradient must be COPIED out before the
+    // next exchange overwrites it (see the hazard note in device_halo.cuh).
+    const int nI = halo.nInterfaces();
+    std::vector<DeviceBuffer<scalar>> gN[3];
+    const DeviceBuffer<scalar>* gsrc[3] = { &gx, &gy, &gz };
+    for (int k = 0; k < 3; ++k)
+    {
+        halo.exchange(gsrc[k]->data(), stream);
+        gN[k].resize(nI);
+        for (int i = 0; i < nI; ++i)
+        {
+            const int n = static_cast<int>(halo.size(i));
+            if (n <= 0) continue;
+            gN[k][i].resize(n);
+            cudaCheck(
+                cudaMemcpyAsync(gN[k][i].data(), halo.recvData(i), n * sizeof(scalar),
+                                cudaMemcpyDeviceToDevice, stream),
+                "linUpwind halo grad copy");
+        }
+        halo.waitExchange(stream);
+    }
+    for (int i = 0; i < nI; ++i)
+    {
+        const int n = static_cast<int>(halo.size(i));
+        if (n <= 0) continue;
+        detail::linUpwindInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
+            faceCells[i].data(),
+            phiF[i].data(),
+            gx.data(),
+            gy.data(),
+            gz.data(),
+            gN[0][i].data(),
+            gN[1][i].data(),
+            gN[2][i].data(),
+            dOwnD[i].data(),
+            dNeiD[i].data(),
+            corr.data(),
+            n);
+    }
+}
+
 // Assemble a laplacian(gamma, .) matrix's processor coupling (the pressure equation): fold -coeff into `diag`
 // and produce `ifCoeff[i]` per interface. `coeffGeo[i] = gammaF*magSf*procDelta` of interface i.
 inline void deviceLaplacianInterface(
@@ -251,7 +356,8 @@ public:
         scalar tolU,
         scalar tolP,
         int maxIter,
-        bool bounded = false)
+        bool bounded = false,
+        bool linearUpwind = false)
         : P_(part),
           nu_(nu),
           relaxU_(relaxU),
@@ -260,6 +366,7 @@ public:
           tolP_(tolP),
           maxIter_(maxIter),
           bounded_(bounded),
+          linearUpwind_(linearUpwind),
           lnC_(part.Lm.mesh.nCells()),
           nIf_(part.Lm.mesh.nInternalFaces()),
           dm_(buildDeviceMesh(part.Lm.mesh, part.lg, part.lp)),
@@ -316,6 +423,18 @@ public:
         for (std::size_t i = 0; i < P_.Lm.procFaceCells.size(); ++i) faceCellsD_[i].copyFrom(P_.Lm.procFaceCells[i]);
         for (std::size_t i = 0; i < P_.procW.size(); ++i) weightsD_[i].copyFrom(P_.procW[i]);
 
+        // linearUpwind needs Cf-C on BOTH sides of a cut; the remote centre needs an exchange, but the
+        // geometry is static, so do it once here rather than per iteration.
+        if (linearUpwind_)
+        {
+            std::vector<std::vector<scalar>> dO, dN;
+            computeProcUpwindD(P_.Lm, P_.lg, P_.lp, dO, dN);
+            dOwnD_.resize(dO.size());
+            dNeiD_.resize(dN.size());
+            for (std::size_t i = 0; i < dO.size(); ++i) dOwnD_[i].copyFrom(dO[i]);
+            for (std::size_t i = 0; i < dN.size(); ++i) dNeiD_[i].copyFrom(dN[i]);
+        }
+
         // initial conservative flux phi = flux(U0), internal + boundary (processor faces carry the coupled flux)
         const SurfaceScalarField phi0 = fvc::flux(U0, P_.Lm.mesh, P_.lg, P_.lp);
         phiInt_.copyFrom(std::vector<scalar>(phi0.internal.begin(), phi0.internal.begin() + nIf_));
@@ -359,6 +478,7 @@ private:
     scalar nu_, relaxU_, relaxP_, tolU_, tolP_;
     int    maxIter_;
     bool   bounded_ = false;
+    bool   linearUpwind_ = false;
     label  lnC_, nIf_, nBnd_ = 0;
     bool   validC_[3] = { true, true, true };
     DeviceMesh           dm_;
@@ -369,6 +489,7 @@ private:
     std::vector<label>   procStart_;
     std::vector<DeviceBuffer<label>>  faceCellsD_;
     std::vector<DeviceBuffer<scalar>> weightsD_;
+    std::vector<DeviceBuffer<scalar>> dOwnD_, dNeiD_;   // linearUpwind: static per-cut-face offsets
     DeviceBuffer<scalar> Uk_[3], dp_, phiInt_, phiBnd_;
     DeviceBuffer<scalar> ones_, zeroSrc_, zeroBnd_, nuEffBnd_, nuCell_;
 };
@@ -498,6 +619,29 @@ inline ParStepResidual ParallelDeviceSimple::step()
         deviceAxpy(1.0, *sS[k], relaxSrc[k]);            // + V*divSig  -> == host Ml.source
     }
 
+    // linearUpwind: the matrix stays upwind and this deferred correction is an explicit source. It must land
+    // in relaxSrc -- the SAME source H() reads later -- not only in the predictor's rhs, or HbyA would be
+    // built from an upwind-only source and the corrector would disagree with the predictor.
+    if (linearUpwind_)
+    {
+        for (int k = 0; k < 3; ++k)
+        {
+            DeviceBuffer<scalar> ub;
+            deviceBCValue(dbU_.comp[k], Uk_[k], ub);
+            halo_.exchange(Uk_[k].data());
+            halo_.scatterBoundaryValues(Uk_[k].data(), weightsD_, procStart_, ub.data());
+            halo_.waitExchange();
+            DeviceBuffer<scalar> ggx, ggy, ggz;
+            deviceGaussGrad(dm_, Uk_[k], ub, ggx, ggy, ggz);   // processor-consistent grad(U_k)
+
+            DeviceBuffer<scalar> corr;
+            deviceLinearUpwindCorr(dm_, phiInt_, ggx, ggy, ggz, corr);          // internal faces
+            deviceLinUpwindInterface(halo_, faceCellsD_, phiF, ggx, ggy, ggz,
+                                     dOwnD_, dNeiD_, corr);                     // + cut faces
+            deviceAxpy(-1.0, corr, relaxSrc[k]);                                // source -= corr
+        }
+    }
+
     // ---- grad(p): coupled processor boundary value from the halo ----
     DeviceBuffer<scalar> pbv;
     deviceBCValue(dbP_, dp_, pbv);

From a708d11367058fd5e8528db493ab3e0147e814f2 Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 11:37:14 +0200
Subject: [PATCH 02/13] #12: reverted LUST

---
 CMakeLists.txt                                |  20 ++
 .../simpleFoam/parallel_device_foam.cuh       |   9 +-
 .../simpleFoam/parallel_device_simple.cuh     | 177 ++++++++++++++++-
 tests/box_mesh.cuh                            | 127 +++++++++++++
 tests/test_gpu_parallel_duct.cu               | 178 ++++++++++++++++++
 tests/test_gpu_parallel_oom.cu                | 137 ++------------
 6 files changed, 520 insertions(+), 128 deletions(-)
 create mode 100644 tests/box_mesh.cuh
 create mode 100644 tests/test_gpu_parallel_duct.cu

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4ad79d1..0af4135 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -263,10 +263,30 @@ if(BRAE_WITH_NVSHMEM)
                          ${CMAKE_CURRENT_SOURCE_DIR}/validation/kEpsCorrect)
     endforeach()
 
+    # Phase 4a: the linearUpwind/LUST interface correction on a cut that CARRIES FLUX (the cavity's cut is a
+    # symmetry plane -> phi~0 -> those terms are multiplied by zero and any bug in them is invisible).
+    add_executable(test_gpu_parallel_duct tests/test_gpu_parallel_duct.cu)
+    target_include_directories(test_gpu_parallel_duct PRIVATE ${BRAE_HEADER_DIRS}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/tests
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/applications/solvers/simpleFoam)
+    target_link_libraries(test_gpu_parallel_duct PRIVATE brae_core)
+    # np=1 writes the reference (no cuts), then np=2/4 must reproduce it. Sequential: np>1 needs the file.
+    add_test(NAME gpu_parallel_duct_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_duct>
+                     32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_ref.txt)
+    set_tests_properties(gpu_parallel_duct_ref PROPERTIES FIXTURES_SETUP DUCTREF)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_duct_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_duct>
+                         32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_ref.txt)
+        set_tests_properties(gpu_parallel_duct_np${_np} PROPERTIES FIXTURES_REQUIRED DUCTREF)
+    endforeach()
+
     # Phase 4a EXIT GATE: the OOM proof -- a mesh too big for one GPU runs split across two. No ctest entry:
     # the mesh size is hardware-specific (it must exceed ONE GPU's VRAM), so it is driven by hand.
     add_executable(test_gpu_parallel_oom tests/test_gpu_parallel_oom.cu)
     target_include_directories(test_gpu_parallel_oom PRIVATE ${BRAE_HEADER_DIRS}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/tests
                                ${CMAKE_CURRENT_SOURCE_DIR}/src/applications/solvers/simpleFoam)
     target_link_libraries(test_gpu_parallel_oom PRIVATE brae_core)
 
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index 55b9a95..3d4eb1a 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -132,14 +132,19 @@ inline int runParallelDeviceFoam(int argc, char** argv)
                 const bool upwindFamily = (divLine.find("upwind") != std::string::npos) || linUpwind;
                 // linearUpwindV adds OF's vector direction limiter on top of linearUpwind -- NOT implemented,
                 // and it contains the substring "linearUpwind", so it must be excluded explicitly.
+                // LUST (0.75 linear + 0.25 linearUpwind) is REFUSED: the distributed linear part at a cut
+                // face is not implemented correctly -- with it the duct test diverges (1e42), without it the
+                // cut-face linear correction is simply missing (~6e-3 error). Both are wrong, so refuse.
+                // It costs nothing today: every LUST case in validation/ is RAS, refused above anyway.
                 const bool unsupported = divLine.find("linearUpwindV") != std::string::npos
                                       || divLine.find("limitedLinear") != std::string::npos
                                       || divLine.find("LUST") != std::string::npos;
                 if (!upwindFamily || unsupported)
                     throw std::runtime_error(
                         "brae -parallel: div(phi,U) scheme '" + divLine + "' is not implemented on the "
-                        "multi-GPU device path (it supports 'Gauss upwind', 'bounded Gauss upwind' and "
-                        "'[bounded] Gauss linearUpwind grad(U)'). Substituting a scheme would converge to a "
+                        "multi-GPU device path (it supports 'Gauss upwind', 'bounded Gauss upwind', "
+                        "'[bounded] Gauss linearUpwind grad(U)' and '[bounded] Gauss LUST grad(U)'). "
+                        "Substituting a scheme would converge to a "
                         "DIFFERENT answer than fvSchemes asks for, so this is refused rather than solved "
                         "wrongly. Use `brae -case <dir>` (single GPU) for the full scheme set.");
             }
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 16bdeef..403fcb5 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -20,6 +20,8 @@
 #include "reconstruct.cuh"
 #include "local_assembly.cuh"   // computeProcUpwindD
 #include <cuda_runtime.h>
+#include <fstream>
+#include <string>
 #include <vector>
 
 namespace brae {
@@ -147,6 +149,32 @@ void linUpwindInterfaceKernel(
     atomicAdd(&corr[c], pf * (g[0]*d[0] + g[1]*d[1] + g[2]*d[2]));   // a cell may own several cut faces
 }
 
+// LUST linear part at a PROCESSOR face -- the distributed analogue of linearCorrKernel (device_simple.cu).
+// The face correction is (linear - upwind) weighted by phi; with linear = w*own + (1-w)*nbr and
+// upwind = pos0*own + (1-pos0)*nbr (pos0 = phi>=0), that difference collapses to (w - pos0)*(own - nbr):
+//   corr[localCell] += phi * (w - pos0) * (psi[local] - psi[remote])
+// The local cell is the cut face's owner and phi is the OUTWARD flux, so this is the internal kernel's owner
+// branch. `w` is procW -- the LOCAL side's interpolation weight, matching linearCorrKernel's w[f] = w of own.
+// NB the single-GPU path OMITS the cyclic linear part ("LUST cases are non-cyclic"); that shortcut is NOT
+// available here, since a decomposed mesh has processor faces everywhere.
+__global__
+void linearCorrInterfaceKernel(
+    const label*  __restrict__ faceCells,
+    const scalar* __restrict__ phi,
+    const scalar* __restrict__ w,
+    const scalar* __restrict__ psi,
+    const scalar* __restrict__ psiNbr,
+    scalar*       __restrict__ corr,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    const scalar pf   = phi[f];
+    const scalar pos0 = (pf >= 0.0) ? 1.0 : 0.0;
+    const label  c    = faceCells[f];
+    atomicAdd(&corr[c], 1000.0 * pf * (w[f] - pos0) * (psi[c] - psiNbr[f]));   // DIAGNOSTIC x1000
+}
+
 // Per-cell sum of |processor off-diagonal|, for fvMatrix::relax's diagonal-dominance term. Host
 // parallelRelaxMatrix adds |interfaceCoeffs| into sumOff; on device this feeds deviceRelaxDiag's cycSumOff
 // hook (the same role the cyclic interface's off-diagonal sum plays).
@@ -276,6 +304,53 @@ inline void deviceLinUpwindInterface(
     }
 }
 
+// Add the processor faces' LUST linear-part correction into `corr` (which already holds the internal
+// deviceLinearCorr result). Only the remote psi is needed, so this is a single exchange per component.
+inline void deviceLinearCorrInterface(
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<label>>& faceCells,
+    const std::vector<DeviceBuffer<scalar>>& phiF,
+    const std::vector<DeviceBuffer<scalar>>& weights,
+    const DeviceBuffer<scalar>& psi,
+    DeviceBuffer<scalar>& corr,
+    cudaStream_t stream = cudaStreamPerThread)
+{
+    constexpr int TPB = 128;
+    halo.exchange(psi.data(), stream);
+    for (int i = 0; i < halo.nInterfaces(); ++i)
+    {
+        const int n = static_cast<int>(halo.size(i));
+        if (n <= 0) continue;
+        detail::linearCorrInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
+            faceCells[i].data(),
+            phiF[i].data(),
+            weights[i].data(),
+            psi.data(),
+            halo.recvData(i),
+            corr.data(),
+            n);
+    }
+    halo.waitExchange(stream);   // the kernels above READ the shared recv buffer -- see device_halo.cuh
+    {   // DIAG: magnitude probe
+        static int nprobe = 0;
+        ++nprobe;
+        if (nprobe > 300 && nprobe < 304)
+        {
+            cudaStreamSynchronize(stream);
+            const std::vector<scalar> h = corr.host();
+            scalar mx = 0, sm = 0;
+            for (scalar v : h) { mx = std::fmax(mx, std::fabs(v)); sm += std::fabs(v); }
+            const std::vector<scalar> pf0 = phiF[0].host();
+            const std::vector<scalar> w0 = weights[0].host();
+            scalar mphi = 0, mw = 0, mnw = 0;
+            for (scalar v : pf0) mphi = std::fmax(mphi, std::fabs(v));
+            for (scalar v : w0) { mw = std::fmax(mw, v); mnw = std::fmin(mnw == 0 ? v : mnw, v); }
+            std::fprintf(stderr, "[PROBE] |corr|max=%.3e sum=%.3e | max|phiF|=%.3e | procW in [%.3f, %.3f]\n",
+                         mx, sm, mphi, mnw, mw);
+        }
+    }
+}
+
 // Assemble a laplacian(gamma, .) matrix's processor coupling (the pressure equation): fold -coeff into `diag`
 // and produce `ifCoeff[i]` per interface. `coeffGeo[i] = gammaF*magSf*procDelta` of interface i.
 inline void deviceLaplacianInterface(
@@ -357,7 +432,8 @@ public:
         scalar tolP,
         int maxIter,
         bool bounded = false,
-        bool linearUpwind = false)
+        bool linearUpwind = false,
+        bool lust = false)
         : P_(part),
           nu_(nu),
           relaxU_(relaxU),
@@ -367,6 +443,7 @@ public:
           maxIter_(maxIter),
           bounded_(bounded),
           linearUpwind_(linearUpwind),
+          lust_(lust),
           lnC_(part.Lm.mesh.nCells()),
           nIf_(part.Lm.mesh.nInternalFaces()),
           dm_(buildDeviceMesh(part.Lm.mesh, part.lg, part.lp)),
@@ -425,7 +502,7 @@ public:
 
         // linearUpwind needs Cf-C on BOTH sides of a cut; the remote centre needs an exchange, but the
         // geometry is static, so do it once here rather than per iteration.
-        if (linearUpwind_)
+        if (linearUpwind_ || lust_)
         {
             std::vector<std::vector<scalar>> dO, dN;
             computeProcUpwindD(P_.Lm, P_.lg, P_.lp, dO, dN);
@@ -450,6 +527,12 @@ public:
         dbRAU_  = buildDeviceBoundary(rAUfld_, part.lp, part.lg);
     }
 
+    // STAGE DUMP (debug): write every intermediate of iteration `it` gathered to the GLOBAL cell ordering,
+    // so an np=1 run (no processor faces -> interface terms inactive -> the reference discretisation) and an
+    // np=N run can be diffed stage by stage. The FIRST stage that differs localises an interface bug, instead
+    // of only seeing the blown-up field N iterations later.
+    void setStageDump(const std::string& path, int iter) { dumpPath_ = path; dumpAt_ = iter; }
+
     // One distributed SIMPLE iteration. U/p/phi are updated in place on the device. Returns the momentum and
     // pressure INITIAL residuals -- OpenFOAM's SIMPLE convergence measure (fvSolution residualControl).
     ParStepResidual step();
@@ -463,9 +546,44 @@ public:
     }
     std::vector<scalar> p() const { return dp_.host(); }
 
+    // max |phi| over this rank's processor faces. The interface deferred corrections (linearUpwind, LUST) are
+    // all multiplied by the cut-face flux, so a test on a mesh where this is ~0 cannot see them at all -- the
+    // 3D cavity cuts on a symmetry midplane and measures ~3e-14 here. Tests assert on this to prove teeth.
+    scalar maxProcFlux() const
+    {
+        const std::vector<scalar> b = phiBnd_.host();
+        scalar mx = 0;
+        for (std::size_t i = 0; i < procStart_.size(); ++i)
+        {
+            const label n = static_cast<label>(halo_.size(static_cast<int>(i)));
+            for (label f = 0; f < n; ++f)
+                mx = std::fmax(mx, std::fabs(b[procStart_[i] + f]));
+        }
+        return mx;
+    }
+
     // The global field, gathered from every partition (decomposePar's inverse) -- for output/validation.
     std::vector<scalar> reconstructP() const { return reconstructField(P_.Lm.cellProcAddr, dp_.host(), P_.globalNCells); }
 
+    // gather `d` to the global ordering and append "<name> v0 v1 ..." to the dump file (master only)
+    void dumpStage(const char* name, const DeviceBuffer<scalar>& d, int k = -1) const
+    {
+        if (dumpPath_.empty() || iter_ != dumpAt_) return;
+        cudaStreamSynchronize(cudaStreamPerThread);
+        std::vector<scalar> g(P_.globalNCells, 0.0);
+        const std::vector<scalar> h = d.host();
+        for (label c = 0; c < lnC_ && c < static_cast<label>(h.size()); ++c)
+            g[P_.Lm.cellProcAddr[c]] = h[c];
+        Pstream::allReduce(g.data(), static_cast<int>(g.size()), ReduceOp::Sum);
+        if (!Pstream::master()) return;
+        std::ofstream os(dumpPath_, std::ios::app);
+        os.precision(17);
+        os << name;
+        if (k >= 0) os << '_' << k;
+        for (scalar v : g) os << ' ' << v;
+        os << '\n';
+    }
+
 private:
     static std::vector<int> intNbrs(const Partition& part)
     {
@@ -479,6 +597,10 @@ private:
     int    maxIter_;
     bool   bounded_ = false;
     bool   linearUpwind_ = false;
+    bool   lust_ = false;
+    std::string dumpPath_;
+    int    dumpAt_ = -1;
+    mutable int iter_ = 0;
     label  lnC_, nIf_, nBnd_ = 0;
     bool   validC_[3] = { true, true, true };
     DeviceMesh           dm_;
@@ -538,6 +660,7 @@ inline void deviceParallelMatrixH(
 inline ParStepResidual ParallelDeviceSimple::step()
 {
     ParStepResidual res;
+    ++iter_;
     const std::vector<FvPatch>& lp = P_.lp;
     const FvGeometry& lg = P_.lg;
 
@@ -562,6 +685,24 @@ inline ParStepResidual ParallelDeviceSimple::step()
     }
 
     const std::vector<scalar> phiBndH = phiBnd_.host();
+    {   // DIAG: where does the boundary flux actually live?
+        static int np2 = 0;
+        if (np2++ == 120)
+        {
+            label b2 = 0;
+            for (std::size_t pi = 0; pi < lp.size(); ++pi)
+            {
+                if (lp[pi].type == "cyclic" || lp[pi].type == "cyclicAMI") continue;
+                scalar mx = 0;
+                for (label i = 0; i < lp[pi].size; ++i) mx = std::fmax(mx, std::fabs(phiBndH[b2 + i]));
+                std::fprintf(stderr, "[PATCH] %-14s type=%-10s off=%5ld n=%4ld max|phi|=%.3e\n",
+                             lp[pi].name.c_str(), lp[pi].type.c_str(), (long)b2, (long)lp[pi].size, mx);
+                b2 += lp[pi].size;
+            }
+            std::fprintf(stderr, "[PATCH] total flattened = %ld, phiBndH.size = %zu, dbU_.n = %ld\n",
+                         (long)b2, phiBndH.size(), (long)dbU_.n);
+        }
+    }
     std::vector<DeviceBuffer<scalar>> phiF, coeffGeo;
     {
         std::size_t pj = 0;
@@ -587,6 +728,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     }
     std::vector<DeviceBuffer<scalar>> ifCoeff;
     deviceMomentumInterface(halo_, faceCellsD_, phiF, coeffGeo, mDiag, ifCoeff);
+    dumpStage("mDiag", mDiag);
 
     // ---- relax (the processor interface counts toward diagonal dominance) ----
     DeviceBuffer<scalar> iC[3], bC[3], relaxSrc[3];
@@ -602,6 +744,9 @@ inline ParStepResidual ParallelDeviceSimple::step()
     deviceInterfaceOffDiagSum(halo_, faceCellsD_, ifCoeff, sumOff);
     DeviceBuffer<scalar> mDiagR, delta;
     deviceRelaxDiag(deviceLduView(dm_, mDiag, mUp, mLo), dm_, iC[0], relaxU_, mDiagR, delta, sumOff.data());
+    dumpStage("sumOff", sumOff);
+    dumpStage("mDiagR", mDiagR);
+    dumpStage("delta", delta);
 
     // The explicit stress source, V*fvc::div(nuEff*dev2(T(grad U))), from the PRE-predictor U -- processor
     // faces coupled via the halo. Host parallelSimpleStepLaminar folds this into Ml.source BEFORE relax, so
@@ -613,6 +758,9 @@ inline ParStepResidual ParallelDeviceSimple::step()
     DeviceBuffer<scalar> sX, sY, sZ;
     deviceDivDevReff(dm_, dbU_, Uk_[0], Uk_[1], Uk_[2], nuCell_, nuEffBnd_, sX, sY, sZ, nullptr, nullptr, &proc);
     DeviceBuffer<scalar>* sS[3] = { &sX, &sY, &sZ };
+    dumpStage("stress", sX, 0);
+    dumpStage("stress", sY, 1);
+    dumpStage("stress", sZ, 2);
     for (int k = 0; k < 3; ++k)
     {
         deviceHadamard(relaxSrc[k], delta, Uk_[k]);      // delta*U_old
@@ -622,7 +770,9 @@ inline ParStepResidual ParallelDeviceSimple::step()
     // linearUpwind: the matrix stays upwind and this deferred correction is an explicit source. It must land
     // in relaxSrc -- the SAME source H() reads later -- not only in the predictor's rhs, or HbyA would be
     // built from an upwind-only source and the corrector would disagree with the predictor.
-    if (linearUpwind_)
+    // LUST also runs the linearUpwind correction (OF LUST.H = 0.75*linear + 0.25*linearUpwind), mirroring the
+    // single-GPU `if (ctl_.linearUpwind || ctl_.lust)`.
+    if (linearUpwind_ || lust_)
     {
         for (int k = 0; k < 3; ++k)
         {
@@ -633,12 +783,28 @@ inline ParStepResidual ParallelDeviceSimple::step()
             halo_.waitExchange();
             DeviceBuffer<scalar> ggx, ggy, ggz;
             deviceGaussGrad(dm_, Uk_[k], ub, ggx, ggy, ggz);   // processor-consistent grad(U_k)
+            dumpStage("gradx", ggx, k);
+            dumpStage("grady", ggy, k);
 
             DeviceBuffer<scalar> corr;
             deviceLinearUpwindCorr(dm_, phiInt_, ggx, ggy, ggz, corr);          // internal faces
+            dumpStage("luCorr_int", corr, k);
             deviceLinUpwindInterface(halo_, faceCellsD_, phiF, ggx, ggy, ggz,
                                      dOwnD_, dNeiD_, corr);                     // + cut faces
+            dumpStage("luCorr_tot", corr, k);
+            if (lust_)   // 0.25*linearUpwind + 0.75*linear, cut faces included on BOTH parts
+            {
+                deviceScale(corr, 0.25);
+                DeviceBuffer<scalar> lc;
+                deviceLinearCorr(dm_, phiInt_, Uk_[k], lc);                     // internal faces
+                dumpStage("linCorr_int", lc, k);
+                deviceLinearCorrInterface(halo_, faceCellsD_, phiF, weightsD_, Uk_[k], lc);   // + cut faces
+                dumpStage("linCorr_tot", lc, k);
+                deviceAxpy(0.75, lc, corr);
+            }
+            dumpStage("corr_final", corr, k);
             deviceAxpy(-1.0, corr, relaxSrc[k]);                                // source -= corr
+            dumpStage("relaxSrc", relaxSrc[k], k);
         }
     }
 
@@ -666,6 +832,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
         const DeviceSolverPerf up =
             deviceParallelJacobiBiCGStab(mv, halo_, ifCoeff, b, Uk_[k], nf, tolU_, 0.0, maxIter_);
         if (validC_[k] && up.initialResidual > res.Ux) res.Ux = up.initialResidual;
+        dumpStage("Upred", Uk_[k], k);
     }
 
     // ---- rAU, HbyA ----
@@ -677,6 +844,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     DeviceBuffer<scalar> diagA, dumb, rAU;
     deviceFold(dm_, mDiagR, zeroSrc_, cmptAvIC, zeroBnd_, diagA, dumb);
     deviceReciprocalV(dm_, diagA, rAU);
+    dumpStage("rAU", rAU);
 
     DeviceBuffer<scalar> HbyA[3];
     const DeviceLduView av = deviceLduView(dm_, diagA, mUp, mLo);
@@ -688,6 +856,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
         DeviceBuffer<scalar> Hk;
         deviceParallelMatrixH(av, dm_, halo_, faceCellsD_, ifCoeff, Uk_[k], relaxSrc[k], bdDiag, bC[k], Hk);
         deviceHadamard(HbyA[k], rAU, Hk);
+        dumpStage("HbyA", HbyA[k], k);
     }
 
     // ---- phiHbyA ----
@@ -752,6 +921,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     const scalar nfp = deviceParallelNormFactor(pv, halo_, pIfCoeff, pSol, pb, ones_, P_.globalNCells);
     const DeviceSolverPerf pp = deviceParallelJacobiPCG(pv, halo_, pIfCoeff, pb, pSol, nfp, tolP_, 0.0, maxIter_);
     res.p = pp.initialResidual;
+    dumpStage("pSol", pSol);
 
     // ---- corrector: conservative phi, relax p, U = HbyA - rAU*grad(p) ----
     DeviceBuffer<scalar> pfluxInt, pfluxBnd;
@@ -783,6 +953,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
         DeviceBuffer<scalar> Un;
         deviceCorrector(HbyA[k], rAU, *gn[k], Un);
         deviceCopy(Uk_[k], Un);
+        dumpStage("Ucorr", Uk_[k], k);
     }
     return res;
 }
diff --git a/tests/box_mesh.cuh b/tests/box_mesh.cuh
new file mode 100644
index 0000000..7da4b48
--- /dev/null
+++ b/tests/box_mesh.cuh
@@ -0,0 +1,127 @@
+#pragma once
+// Shared test fixture: an in-memory structured hex box (duct) + FieldData helpers.
+//
+// Used by the tests that need a mesh too large or too specific to ship as a polyMesh on disk:
+//   test_gpu_parallel_oom  -- a mesh bigger than one GPU's VRAM (a polyMesh that size would be tens of GB)
+//   test_gpu_parallel_duct -- a cut that actually CARRIES FLUX (see that test's header for why this matters)
+#include "primitive_mesh.cuh"
+#include "fv_geometry.cuh"
+#include "fv_patch.cuh"
+#include "foam_field_reader.cuh"
+#include <vector>
+
+namespace brae {
+namespace boxtest {
+
+
+// A structured Nx*Ny*Nz unit-spaced hex box. Faces are emitted in OpenFOAM order: internal faces first in
+// upper-triangular order (owner ascending, neighbour ascending within an owner -- the lduAddressing contract),
+// then one contiguous block per boundary patch. Face vertex winding is chosen so Sf points owner->neighbour
+// for internal faces and outward for boundary faces (right-hand rule).
+PrimitiveMesh boxMesh(label Nx, label Ny, label Nz)
+{
+    const label nC = Nx * Ny * Nz;
+    auto cell  = [&](label i, label j, label k) { return i + Nx * (j + Ny * k); };
+    auto point = [&](label i, label j, label k) { return i + (Nx + 1) * (j + (Ny + 1) * k); };
+
+    std::vector<vector> pts(static_cast<std::size_t>(Nx + 1) * (Ny + 1) * (Nz + 1));
+    for (label k = 0; k <= Nz; ++k)
+        for (label j = 0; j <= Ny; ++j)
+            for (label i = 0; i <= Nx; ++i)
+                pts[point(i, j, k)] = vector{scalar(i), scalar(j), scalar(k)};
+
+    // reserve: at ~1e8 cells these reach ~1e9 entries, and push_back's doubling would otherwise transiently
+    // double an already multi-GB allocation. nFaces = 3*nC + the 6 boundary planes; nInternal = nFaces - nBnd.
+    const std::size_t nBnd   = static_cast<std::size_t>(2) * (Ny * Nz + Nx * Nz + Nx * Ny);
+    const std::size_t nFaces = static_cast<std::size_t>(3) * nC
+                             - static_cast<std::size_t>(Ny * Nz + Nx * Nz + Nx * Ny) + nBnd;
+    std::vector<label> fv, foff, own, nei;
+    fv.reserve(4 * nFaces);
+    foff.reserve(nFaces + 1);
+    own.reserve(nFaces);
+    nei.reserve(nFaces - nBnd);
+    foff.push_back(0);
+    auto quad = [&](label a, label b, label c, label d, label o, label n)
+    {
+        fv.push_back(a); fv.push_back(b); fv.push_back(c); fv.push_back(d);
+        foff.push_back(static_cast<label>(fv.size()));
+        own.push_back(o);
+        if (n >= 0) nei.push_back(n);
+    };
+    // face planes with the winding that yields the +x / +y / +z normal
+    auto faceX = [&](label i, label j, label k, label o, label n)   // plane x=i
+    { quad(point(i,j,k), point(i,j+1,k), point(i,j+1,k+1), point(i,j,k+1), o, n); };
+    auto faceY = [&](label i, label j, label k, label o, label n)   // plane y=j
+    { quad(point(i,j,k), point(i,j,k+1), point(i+1,j,k+1), point(i+1,j,k), o, n); };
+    auto faceZ = [&](label i, label j, label k, label o, label n)   // plane z=k
+    { quad(point(i,j,k), point(i+1,j,k), point(i+1,j+1,k), point(i,j+1,k), o, n); };
+    // reversed winding -> the opposite normal (for min-side boundary patches, whose outward normal is -)
+    auto faceXr = [&](label i, label j, label k, label o)
+    { quad(point(i,j,k+1), point(i,j+1,k+1), point(i,j+1,k), point(i,j,k), o, -1); };
+    auto faceYr = [&](label i, label j, label k, label o)
+    { quad(point(i+1,j,k), point(i+1,j,k+1), point(i,j,k+1), point(i,j,k), o, -1); };
+    auto faceZr = [&](label i, label j, label k, label o)
+    { quad(point(i,j+1,k), point(i+1,j+1,k), point(i+1,j,k), point(i,j,k), o, -1); };
+
+    // internal faces, upper-triangular: for each cell ascending, neighbours +1 < +Nx < +Nx*Ny
+    for (label k = 0; k < Nz; ++k)
+        for (label j = 0; j < Ny; ++j)
+            for (label i = 0; i < Nx; ++i)
+            {
+                const label c = cell(i, j, k);
+                if (i + 1 < Nx) faceX(i + 1, j, k, c, cell(i + 1, j, k));
+                if (j + 1 < Ny) faceY(i, j + 1, k, c, cell(i, j + 1, k));
+                if (k + 1 < Nz) faceZ(i, j, k + 1, c, cell(i, j, k + 1));
+            }
+
+    std::vector<PatchInfo> patches;
+    auto beginPatch = [&](const char* name, const char* type)
+    {
+        PatchInfo pi;
+        pi.name  = name;
+        pi.type  = type;
+        pi.start = static_cast<label>(own.size());
+        patches.push_back(pi);
+    };
+    auto endPatch = [&]() { patches.back().size = static_cast<label>(own.size()) - patches.back().start; };
+
+    beginPatch("inlet", "patch");                                          // xmin, outward -x
+    for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) faceXr(0, j, k, cell(0, j, k));
+    endPatch();
+    beginPatch("outlet", "patch");                                         // xmax, outward +x
+    for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) faceX(Nx, j, k, cell(Nx - 1, j, k), -1);
+    endPatch();
+    beginPatch("wallYmin", "wall");                                        // outward -y
+    for (label k = 0; k < Nz; ++k) for (label i = 0; i < Nx; ++i) faceYr(i, 0, k, cell(i, 0, k));
+    endPatch();
+    beginPatch("wallYmax", "wall");                                        // outward +y
+    for (label k = 0; k < Nz; ++k) for (label i = 0; i < Nx; ++i) faceY(i, Ny, k, cell(i, Ny - 1, k), -1);
+    endPatch();
+    beginPatch("wallZmin", "wall");                                        // outward -z
+    for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i) faceZr(i, j, 0, cell(i, j, 0));
+    endPatch();
+    beginPatch("wallZmax", "wall");                                        // outward +z
+    for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i) faceZ(i, j, Nz, cell(i, j, Nz - 1), -1);
+    endPatch();
+
+    PrimitiveMesh m;
+    m.assign(std::move(pts), std::move(fv), std::move(foff), std::move(own), std::move(nei),
+             std::move(patches), nC);
+    return m;
+}
+
+template <typename T>
+PatchFieldData<T> pfd(const char* name, const char* type, T v, bool hasValue)
+{
+    PatchFieldData<T> d;
+    d.name         = name;
+    d.type         = type;
+    d.hasValue     = hasValue;
+    d.valueUniform = hasValue;
+    d.uniformValue = v;
+    return d;
+}
+
+
+}  // namespace boxtest
+}  // namespace brae
diff --git a/tests/test_gpu_parallel_duct.cu b/tests/test_gpu_parallel_duct.cu
new file mode 100644
index 0000000..0459607
--- /dev/null
+++ b/tests/test_gpu_parallel_duct.cu
@@ -0,0 +1,178 @@
+// Phase 4a: the linearUpwind/LUST DEFERRED CORRECTION at a processor face, on a cut that CARRIES FLUX.
+//
+// WHY THIS TEST EXISTS. Both interface corrections are multiplied by the cut-face flux:
+//   linearUpwind:  corr[c] += phi * (grad(U)[upwind] . d_upwind)
+//   LUST linear:   corr[c] += phi * (w - pos0) * (psi[local] - psi[remote])
+// so on a cut where phi == 0 they are IDENTICALLY ZERO and any bug in them is invisible. That is exactly what
+// the 3D cavity does: SCOTCH halves a 24^3 lid-driven cavity on a symmetry midplane, where the vortex lies in
+// the x-y plane and the cut-normal velocity vanishes -- measured max|phi| there was 3.2e-14. Every "validated
+// at np=2/4/8" result on the cavity was therefore blind to these terms: deleting them, or scaling them by
+// 1000, changed the answer by nothing.
+//
+// This test removes that blind spot: a straight duct (inlet -> outlet along x) decomposed as SLABS IN X, so
+// every cut plane is PERPENDICULAR to the flow and carries the full through-flux.
+//
+// The oracle is np=1 of the same code: with one partition there are no processor faces, so the interface terms
+// are inactive and the answer is the pure internal discretisation. Any np>1 run must reproduce it. The test
+// asserts max|phi| at the cut is actually non-trivial FIRST -- otherwise it would silently be a no-op again.
+//
+// Run: mpirun -np {1,2,4} test_gpu_parallel_duct <Nx> <Ny> <Nz> [nIters] [ref.txt]
+//   np=1 writes ref.txt (the reference); np>1 reads it back and compares cell by cell.
+#include "primitive_mesh.cuh"
+#include "fv_geometry.cuh"
+#include "fv_patch.cuh"
+#include "geometric_field.cuh"
+#include "foam_field_reader.cuh"
+#include "fvc.cuh"
+#include "parallel_simple.cuh"
+#include "field_distribute.cuh"
+#include "parallel_device_simple.cuh"
+#include "cf_pstream.cuh"
+#include "box_mesh.cuh"
+
+#include <cmath>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <vector>
+
+using namespace brae;
+
+int main(int argc, char** argv)
+{
+    Pstream::init(argc, argv);
+    const int rank = Pstream::myProcNo(), nproc = Pstream::nProcs();
+    if (argc < 4)
+    {
+        if (Pstream::master()) std::printf("usage: %s <Nx> <Ny> <Nz> [nIters] [ref.txt]\n", argv[0]);
+        Pstream::finalize();
+        return 2;
+    }
+    if (!Pstream::nvshmemActive())
+    {
+        if (Pstream::master()) std::printf("test_gpu_parallel_duct: NVSHMEM inactive -- SKIP\n");
+        Pstream::finalize();
+        return 0;
+    }
+    const label Nx = std::atol(argv[1]), Ny = std::atol(argv[2]), Nz = std::atol(argv[3]);
+    const int   N  = (argc > 4) ? std::atoi(argv[4]) : 20;
+    const std::string ref = (argc > 5) ? argv[5] : "duct_ref.txt";
+    const bool  lust = (argc > 6) && (std::string(argv[6]) == "lust");
+    const std::string dump = (argc > 7) ? argv[7] : "";   // stage-by-stage dump (see setStageDump)
+    const scalar nu = 1e-2, relaxU = 0.7, relaxP = 0.3, tolU = 1e-9, tolP = 1e-9;
+    const int maxIter = 500;
+
+    const label nC = Nx * Ny * Nz;
+    bool ok = true;
+    scalar cutPhi = 0, relU = 0, relP = 0;
+    {
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz);
+        // SLABS IN X: cut planes normal to the flow, so every cut face carries the through-flux.
+        std::vector<label> cellToPart(nC);
+        for (label k = 0; k < Nz; ++k)
+            for (label j = 0; j < Ny; ++j)
+                for (label i = 0; i < Nx; ++i)
+                    cellToPart[i + Nx * (j + Ny * k)] = static_cast<label>((static_cast<long long>(i) * nproc) / Nx);
+        const Partition P(gm, cellToPart, rank);
+
+        FieldData<vector> Ufd;
+        Ufd.internalUniform = true;
+        Ufd.internalUniformValue = vector{1, 0, 0};   // start with through-flow so the cut carries flux at once
+        Ufd.boundary.push_back(boxtest::pfd<vector>("inlet",    "fixedValue",   vector{1, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("outlet",   "zeroGradient", vector{0, 0, 0}, false));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallYmin", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallYmax", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallZmin", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallZmax", "fixedValue",   vector{0, 0, 0}, true));
+
+        FieldData<scalar> pfdta;
+        pfdta.internalUniform = true;
+        pfdta.internalUniformValue = 0.0;
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("inlet",    "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("outlet",   "fixedValue",   0.0, true));   // p reference
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallYmin", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallYmax", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallZmin", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallZmax", "zeroGradient", 0.0, false));
+
+        GeometricField<vector> U0 = distributeField<vector>(Ufd, gm.patches(), P.Lm, P.lp, P.procW, rank);
+        U0.evaluateBoundary();
+        GeometricField<scalar> p0 = distributeField<scalar>(pfdta, gm.patches(), P.Lm, P.lp, P.procW, rank);
+        p0.evaluateBoundary();
+
+        // linearUpwind ON: this test exists to exercise its interface term.
+        ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, maxIter,
+                                    /*bounded*/ true, /*linearUpwind*/ !lust, /*lust*/ lust);
+        if (!dump.empty()) solver.setStageDump(dump, 1);   // dump iteration 1: before anything diverges
+        for (int it = 0; it < N; ++it) solver.step();
+        cudaDeviceSynchronize();
+
+        // TEETH CHECK: the cut must actually carry flux, else the interface terms are multiplied by zero and
+        // this test proves nothing (which is precisely how the cavity fooled us).
+        cutPhi = Pstream::allReduce(solver.maxProcFlux(), ReduceOp::Max);
+
+        // gather to the global mesh
+        std::vector<scalar> pg(nC, 0.0), ug(3 * nC, 0.0);
+        const std::vector<scalar> pl = solver.p();
+        const std::vector<vector> ul = solver.U();
+        for (label c = 0; c < P.nCells(); ++c)
+        {
+            const label gc = P.Lm.cellProcAddr[c];
+            pg[gc] = pl[c];
+            ug[3*gc] = ul[c].x; ug[3*gc+1] = ul[c].y; ug[3*gc+2] = ul[c].z;
+        }
+        Pstream::allReduce(pg.data(), static_cast<int>(pg.size()), ReduceOp::Sum);
+        Pstream::allReduce(ug.data(), static_cast<int>(ug.size()), ReduceOp::Sum);
+
+        if (Pstream::master())
+        {
+            if (nproc == 1)   // write the reference: no processor faces -> pure internal discretisation
+            {
+                std::ofstream os(ref);
+                os.precision(17);
+                for (label c = 0; c < nC; ++c)
+                    os << pg[c] << ' ' << ug[3*c] << ' ' << ug[3*c+1] << ' ' << ug[3*c+2] << '\n';
+                std::printf("test_gpu_parallel_duct np=1: wrote reference %s (%ld cells)\n", ref.c_str(), (long)nC);
+                std::printf("  max|phi| at cut = %.3e (no cut at np=1, as expected)\n", cutPhi);
+                std::printf("PASS\n");
+            }
+            else
+            {
+                std::ifstream is(ref);
+                if (!is)
+                {
+                    std::printf("test_gpu_parallel_duct: cannot read %s -- run np=1 first\n", ref.c_str());
+                    ok = false;
+                }
+                else
+                {
+                    scalar nU = 0, dU = 0, nP = 0, dP = 0;
+                    for (label c = 0; c < nC; ++c)
+                    {
+                        scalar rp, rx, ry, rz;
+                        is >> rp >> rx >> ry >> rz;
+                        nP = std::fmax(nP, std::fabs(pg[c] - rp));
+                        dP = std::fmax(dP, std::fabs(rp));
+                        nU = std::fmax(nU, std::fabs(ug[3*c] - rx));
+                        nU = std::fmax(nU, std::fabs(ug[3*c+1] - ry));
+                        nU = std::fmax(nU, std::fabs(ug[3*c+2] - rz));
+                        dU = std::fmax(dU, std::fabs(rx));
+                    }
+                    relU = dU > 0 ? nU / dU : nU;
+                    relP = dP > 0 ? nP / dP : nP;
+                    // The cut must carry flux, or the interface terms were never exercised.
+                    const bool teeth = cutPhi > 1e-6;
+                    std::printf("test_gpu_parallel_duct np=%d (%d iters): U rel=%.3e  p rel=%.3e  max|phi| at cut=%.3e\n",
+                                nproc, N, relU, relP, cutPhi);
+                    if (!teeth)
+                        std::printf("  NO TEETH: cut flux ~0, the interface terms were multiplied by zero\n");
+                    ok = teeth && (relU < 1e-4) && (relP < 1e-4);
+                    std::printf("%s\n", ok ? "PASS" : "FAIL");
+                }
+            }
+        }
+    }   // buffers must die before finalize
+
+    Pstream::finalize();
+    return ok ? 0 : 1;
+}
diff --git a/tests/test_gpu_parallel_oom.cu b/tests/test_gpu_parallel_oom.cu
index 559f117..2865bb3 100644
--- a/tests/test_gpu_parallel_oom.cu
+++ b/tests/test_gpu_parallel_oom.cu
@@ -24,6 +24,7 @@
 #include "field_distribute.cuh"
 #include "parallel_device_simple.cuh"
 #include "cf_pstream.cuh"
+#include "box_mesh.cuh"
 
 #include <cmath>
 #include <cstdio>
@@ -32,117 +33,7 @@
 
 using namespace brae;
 
-namespace {
 
-// A structured Nx*Ny*Nz unit-spaced hex box. Faces are emitted in OpenFOAM order: internal faces first in
-// upper-triangular order (owner ascending, neighbour ascending within an owner -- the lduAddressing contract),
-// then one contiguous block per boundary patch. Face vertex winding is chosen so Sf points owner->neighbour
-// for internal faces and outward for boundary faces (right-hand rule).
-PrimitiveMesh boxMesh(label Nx, label Ny, label Nz)
-{
-    const label nC = Nx * Ny * Nz;
-    auto cell  = [&](label i, label j, label k) { return i + Nx * (j + Ny * k); };
-    auto point = [&](label i, label j, label k) { return i + (Nx + 1) * (j + (Ny + 1) * k); };
-
-    std::vector<vector> pts(static_cast<std::size_t>(Nx + 1) * (Ny + 1) * (Nz + 1));
-    for (label k = 0; k <= Nz; ++k)
-        for (label j = 0; j <= Ny; ++j)
-            for (label i = 0; i <= Nx; ++i)
-                pts[point(i, j, k)] = vector{scalar(i), scalar(j), scalar(k)};
-
-    // reserve: at ~1e8 cells these reach ~1e9 entries, and push_back's doubling would otherwise transiently
-    // double an already multi-GB allocation. nFaces = 3*nC + the 6 boundary planes; nInternal = nFaces - nBnd.
-    const std::size_t nBnd   = static_cast<std::size_t>(2) * (Ny * Nz + Nx * Nz + Nx * Ny);
-    const std::size_t nFaces = static_cast<std::size_t>(3) * nC
-                             - static_cast<std::size_t>(Ny * Nz + Nx * Nz + Nx * Ny) + nBnd;
-    std::vector<label> fv, foff, own, nei;
-    fv.reserve(4 * nFaces);
-    foff.reserve(nFaces + 1);
-    own.reserve(nFaces);
-    nei.reserve(nFaces - nBnd);
-    foff.push_back(0);
-    auto quad = [&](label a, label b, label c, label d, label o, label n)
-    {
-        fv.push_back(a); fv.push_back(b); fv.push_back(c); fv.push_back(d);
-        foff.push_back(static_cast<label>(fv.size()));
-        own.push_back(o);
-        if (n >= 0) nei.push_back(n);
-    };
-    // face planes with the winding that yields the +x / +y / +z normal
-    auto faceX = [&](label i, label j, label k, label o, label n)   // plane x=i
-    { quad(point(i,j,k), point(i,j+1,k), point(i,j+1,k+1), point(i,j,k+1), o, n); };
-    auto faceY = [&](label i, label j, label k, label o, label n)   // plane y=j
-    { quad(point(i,j,k), point(i,j,k+1), point(i+1,j,k+1), point(i+1,j,k), o, n); };
-    auto faceZ = [&](label i, label j, label k, label o, label n)   // plane z=k
-    { quad(point(i,j,k), point(i+1,j,k), point(i+1,j+1,k), point(i,j+1,k), o, n); };
-    // reversed winding -> the opposite normal (for min-side boundary patches, whose outward normal is -)
-    auto faceXr = [&](label i, label j, label k, label o)
-    { quad(point(i,j,k+1), point(i,j+1,k+1), point(i,j+1,k), point(i,j,k), o, -1); };
-    auto faceYr = [&](label i, label j, label k, label o)
-    { quad(point(i+1,j,k), point(i+1,j,k+1), point(i,j,k+1), point(i,j,k), o, -1); };
-    auto faceZr = [&](label i, label j, label k, label o)
-    { quad(point(i,j+1,k), point(i+1,j+1,k), point(i+1,j,k), point(i,j,k), o, -1); };
-
-    // internal faces, upper-triangular: for each cell ascending, neighbours +1 < +Nx < +Nx*Ny
-    for (label k = 0; k < Nz; ++k)
-        for (label j = 0; j < Ny; ++j)
-            for (label i = 0; i < Nx; ++i)
-            {
-                const label c = cell(i, j, k);
-                if (i + 1 < Nx) faceX(i + 1, j, k, c, cell(i + 1, j, k));
-                if (j + 1 < Ny) faceY(i, j + 1, k, c, cell(i, j + 1, k));
-                if (k + 1 < Nz) faceZ(i, j, k + 1, c, cell(i, j, k + 1));
-            }
-
-    std::vector<PatchInfo> patches;
-    auto beginPatch = [&](const char* name, const char* type)
-    {
-        PatchInfo pi;
-        pi.name  = name;
-        pi.type  = type;
-        pi.start = static_cast<label>(own.size());
-        patches.push_back(pi);
-    };
-    auto endPatch = [&]() { patches.back().size = static_cast<label>(own.size()) - patches.back().start; };
-
-    beginPatch("inlet", "patch");                                          // xmin, outward -x
-    for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) faceXr(0, j, k, cell(0, j, k));
-    endPatch();
-    beginPatch("outlet", "patch");                                         // xmax, outward +x
-    for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) faceX(Nx, j, k, cell(Nx - 1, j, k), -1);
-    endPatch();
-    beginPatch("wallYmin", "wall");                                        // outward -y
-    for (label k = 0; k < Nz; ++k) for (label i = 0; i < Nx; ++i) faceYr(i, 0, k, cell(i, 0, k));
-    endPatch();
-    beginPatch("wallYmax", "wall");                                        // outward +y
-    for (label k = 0; k < Nz; ++k) for (label i = 0; i < Nx; ++i) faceY(i, Ny, k, cell(i, Ny - 1, k), -1);
-    endPatch();
-    beginPatch("wallZmin", "wall");                                        // outward -z
-    for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i) faceZr(i, j, 0, cell(i, j, 0));
-    endPatch();
-    beginPatch("wallZmax", "wall");                                        // outward +z
-    for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i) faceZ(i, j, Nz, cell(i, j, Nz - 1), -1);
-    endPatch();
-
-    PrimitiveMesh m;
-    m.assign(std::move(pts), std::move(fv), std::move(foff), std::move(own), std::move(nei),
-             std::move(patches), nC);
-    return m;
-}
-
-template <typename T>
-PatchFieldData<T> pfd(const char* name, const char* type, T v, bool hasValue)
-{
-    PatchFieldData<T> d;
-    d.name         = name;
-    d.type         = type;
-    d.hasValue     = hasValue;
-    d.valueUniform = hasValue;
-    d.uniformValue = v;
-    return d;
-}
-
-}  // namespace
 
 int main(int argc, char** argv)
 {
@@ -175,7 +66,7 @@ int main(int argc, char** argv)
     const std::size_t before = totB - freeB;
 
     {
-        const PrimitiveMesh gm = boxMesh(Nx, Ny, Nz);
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz);
         // slab decomposition in x: contiguous cells, one processor interface between adjacent slabs
         std::vector<label> cellToPart(nC);
         for (label k = 0; k < Nz; ++k)
@@ -187,22 +78,22 @@ int main(int argc, char** argv)
         FieldData<vector> Ufd;
         Ufd.internalUniform = true;
         Ufd.internalUniformValue = vector{0, 0, 0};
-        Ufd.boundary.push_back(pfd<vector>("inlet",    "fixedValue",   vector{1, 0, 0}, true));
-        Ufd.boundary.push_back(pfd<vector>("outlet",   "zeroGradient", vector{0, 0, 0}, false));
-        Ufd.boundary.push_back(pfd<vector>("wallYmin", "fixedValue",   vector{0, 0, 0}, true));
-        Ufd.boundary.push_back(pfd<vector>("wallYmax", "fixedValue",   vector{0, 0, 0}, true));
-        Ufd.boundary.push_back(pfd<vector>("wallZmin", "fixedValue",   vector{0, 0, 0}, true));
-        Ufd.boundary.push_back(pfd<vector>("wallZmax", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("inlet",    "fixedValue",   vector{1, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("outlet",   "zeroGradient", vector{0, 0, 0}, false));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallYmin", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallYmax", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallZmin", "fixedValue",   vector{0, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("wallZmax", "fixedValue",   vector{0, 0, 0}, true));
 
         FieldData<scalar> pfdta;
         pfdta.internalUniform = true;
         pfdta.internalUniformValue = 0.0;
-        pfdta.boundary.push_back(pfd<scalar>("inlet",    "zeroGradient", 0.0, false));
-        pfdta.boundary.push_back(pfd<scalar>("outlet",   "fixedValue",   0.0, true));   // pressure reference
-        pfdta.boundary.push_back(pfd<scalar>("wallYmin", "zeroGradient", 0.0, false));
-        pfdta.boundary.push_back(pfd<scalar>("wallYmax", "zeroGradient", 0.0, false));
-        pfdta.boundary.push_back(pfd<scalar>("wallZmin", "zeroGradient", 0.0, false));
-        pfdta.boundary.push_back(pfd<scalar>("wallZmax", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("inlet",    "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("outlet",   "fixedValue",   0.0, true));   // pressure reference
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallYmin", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallYmax", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallZmin", "zeroGradient", 0.0, false));
+        pfdta.boundary.push_back(boxtest::pfd<scalar>("wallZmax", "zeroGradient", 0.0, false));
 
         GeometricField<vector> U0 = distributeField<vector>(Ufd, gm.patches(), P.Lm, P.lp, P.procW, rank);
         U0.evaluateBoundary();

From 81fd7402390233bafe8368f442d42dfb6ce7c88d Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 12:14:50 +0200
Subject: [PATCH 03/13] #12: fixed interface term

---
 CMakeLists.txt                                | 13 +++++
 .../simpleFoam/parallel_device_foam.cuh       | 16 ++----
 .../simpleFoam/parallel_device_simple.cuh     | 57 ++++++-------------
 tests/test_gpu_parallel_duct.cu               |  2 +-
 4 files changed, 36 insertions(+), 52 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0af4135..615614a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -282,6 +282,19 @@ if(BRAE_WITH_NVSHMEM)
         set_tests_properties(gpu_parallel_duct_np${_np} PROPERTIES FIXTURES_REQUIRED DUCTREF)
     endforeach()
 
+    # LUST (0.75 linear + 0.25 linearUpwind) over the same flux-carrying cut: it exercises BOTH interface
+    # deferred corrections (linearUpwind's gradient term AND the linear term), so it is the stronger gate.
+    add_test(NAME gpu_parallel_duct_lust_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_duct>
+                     32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_lust_ref.txt lust)
+    set_tests_properties(gpu_parallel_duct_lust_ref PROPERTIES FIXTURES_SETUP DUCTLUST)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_duct_lust_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_duct>
+                         32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_lust_ref.txt lust)
+        set_tests_properties(gpu_parallel_duct_lust_np${_np} PROPERTIES FIXTURES_REQUIRED DUCTLUST)
+    endforeach()
+
     # Phase 4a EXIT GATE: the OOM proof -- a mesh too big for one GPU runs split across two. No ctest entry:
     # the mesh size is hardware-specific (it must exceed ONE GPU's VRAM), so it is driven by hand.
     add_executable(test_gpu_parallel_oom tests/test_gpu_parallel_oom.cu)
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index 3d4eb1a..ea9a042 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -116,7 +116,7 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         // Anything else -- limitedLinear, LUST, linearUpwindV, plain Gauss linear -- is a DIFFERENT
         // discretisation, and silently substituting upwind produces a converged, plausible, WRONG answer (on
         // the cavity that was a ~6% field difference). Refuse those, exactly as RAS is refused above.
-        bool boundedDiv = false, linUpwind = false;
+        bool boundedDiv = false, linUpwind = false, lust = false;
         {
             std::string divLine;
             std::istringstream fsch(readFileExpanded(caseDir + "/system/fvSchemes"));
@@ -127,18 +127,14 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             {
                 boundedDiv = divLine.find("bounded") != std::string::npos;
                 linUpwind  = divLine.find("linearUpwind") != std::string::npos;
+                lust       = divLine.find("LUST") != std::string::npos;   // 0.75*linear + 0.25*linearUpwind (OF LUST.H)
                 // NB "linearUpwind" has a capital U, so it does NOT contain the substring "upwind" -- the two
                 // must be tested separately or a linearUpwind case reads as "no upwind scheme at all".
-                const bool upwindFamily = (divLine.find("upwind") != std::string::npos) || linUpwind;
+                const bool upwindFamily = (divLine.find("upwind") != std::string::npos) || linUpwind || lust;
                 // linearUpwindV adds OF's vector direction limiter on top of linearUpwind -- NOT implemented,
                 // and it contains the substring "linearUpwind", so it must be excluded explicitly.
-                // LUST (0.75 linear + 0.25 linearUpwind) is REFUSED: the distributed linear part at a cut
-                // face is not implemented correctly -- with it the duct test diverges (1e42), without it the
-                // cut-face linear correction is simply missing (~6e-3 error). Both are wrong, so refuse.
-                // It costs nothing today: every LUST case in validation/ is RAS, refused above anyway.
                 const bool unsupported = divLine.find("linearUpwindV") != std::string::npos
-                                      || divLine.find("limitedLinear") != std::string::npos
-                                      || divLine.find("LUST") != std::string::npos;
+                                      || divLine.find("limitedLinear") != std::string::npos;
                 if (!upwindFamily || unsupported)
                     throw std::runtime_error(
                         "brae -parallel: div(phi,U) scheme '" + divLine + "' is not implemented on the "
@@ -193,14 +189,14 @@ inline int runParallelDeviceFoam(int argc, char** argv)
                         caseDir.c_str(), nproc, nu, (long)nC);
             std::printf("  relax U=%.2g p=%.2g | tol p=%.1g U=%.1g | endTime=%d | residualControl=%s | div(phi,U)=%sGauss upwind\n",
                         relaxU, relaxP, tolP, tolU, endTime, hasRC ? "on" : "off",
-                        (std::string(boundedDiv ? "bounded " : "") + (linUpwind ? "Gauss linearUpwind" : "Gauss upwind")).c_str());
+                        (std::string(boundedDiv ? "bounded " : "") + (lust ? "Gauss LUST" : linUpwind ? "Gauss linearUpwind" : "Gauss upwind")).c_str());
         }
 
         std::vector<vector> Ug;
         std::vector<scalar> pg;
         int nIter = 0;
         {   // scope: the solver's symmetric-heap buffers must be freed BEFORE Pstream::finalize
-            ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind);
+            ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind, lust);
 
             auto ok = [](scalar res, scalar ctl) { return ctl < 0 || res < ctl; };
             bool converged = false;
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 403fcb5..1718328 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -172,7 +172,7 @@ void linearCorrInterfaceKernel(
     const scalar pf   = phi[f];
     const scalar pos0 = (pf >= 0.0) ? 1.0 : 0.0;
     const label  c    = faceCells[f];
-    atomicAdd(&corr[c], 1000.0 * pf * (w[f] - pos0) * (psi[c] - psiNbr[f]));   // DIAGNOSTIC x1000
+    atomicAdd(&corr[c], pf * (w[f] - pos0) * (psi[c] - psiNbr[f]));   // a cell may own several cut faces
 }
 
 // Per-cell sum of |processor off-diagonal|, for fvMatrix::relax's diagonal-dominance term. Host
@@ -331,24 +331,6 @@ inline void deviceLinearCorrInterface(
             n);
     }
     halo.waitExchange(stream);   // the kernels above READ the shared recv buffer -- see device_halo.cuh
-    {   // DIAG: magnitude probe
-        static int nprobe = 0;
-        ++nprobe;
-        if (nprobe > 300 && nprobe < 304)
-        {
-            cudaStreamSynchronize(stream);
-            const std::vector<scalar> h = corr.host();
-            scalar mx = 0, sm = 0;
-            for (scalar v : h) { mx = std::fmax(mx, std::fabs(v)); sm += std::fabs(v); }
-            const std::vector<scalar> pf0 = phiF[0].host();
-            const std::vector<scalar> w0 = weights[0].host();
-            scalar mphi = 0, mw = 0, mnw = 0;
-            for (scalar v : pf0) mphi = std::fmax(mphi, std::fabs(v));
-            for (scalar v : w0) { mw = std::fmax(mw, v); mnw = std::fmin(mnw == 0 ? v : mnw, v); }
-            std::fprintf(stderr, "[PROBE] |corr|max=%.3e sum=%.3e | max|phiF|=%.3e | procW in [%.3f, %.3f]\n",
-                         mx, sm, mphi, mnw, mw);
-        }
-    }
 }
 
 // Assemble a laplacian(gamma, .) matrix's processor coupling (the pressure equation): fold -coeff into `diag`
@@ -527,10 +509,21 @@ public:
         dbRAU_  = buildDeviceBoundary(rAUfld_, part.lp, part.lg);
     }
 
-    // STAGE DUMP (debug): write every intermediate of iteration `it` gathered to the GLOBAL cell ordering,
-    // so an np=1 run (no processor faces -> interface terms inactive -> the reference discretisation) and an
-    // np=N run can be diffed stage by stage. The FIRST stage that differs localises an interface bug, instead
-    // of only seeing the blown-up field N iterations later.
+    // STAGE DUMP: write every intermediate of iteration `iter`, gathered to the GLOBAL cell ordering,
+    // so an np=1 run (no processor faces -> interface terms inactive -> THE reference discretisation) and an
+    // np=N run can be diffed stage by stage. The FIRST stage that differs localises the bug, instead of only
+    // seeing a blown-up field N iterations later.
+    //
+    // This is how the LUST interface bug was found: at iteration 2 (the first iteration where the corrections
+    // are non-zero at all -- U starts uniform, so grad(U)=0 and every correction is identically 0 at iteration
+    // 1) `luCorr_int` was 3.7e-02 off and `luCorr_tot` 1.2e-09 (the linearUpwind interface term restoring it
+    // exactly), while `linCorr_tot` was 6.2e+01 off -- one kernel, isolated. Free when unused (early return).
+    //
+    // Two traps it also exposed, both worth knowing before adding any interface term:
+    //   - dump at an iteration where the term is ACTUALLY NON-ZERO (iteration 1 proves nothing here);
+    //   - `sumOff` legitimately DIFFERS between np=1 and np>1: at np=1 those cut faces are internal and
+    //     deviceRelaxDiag already counts them from the LDU; at np>1 they arrive via the cycSumOff hook. It is
+    //     the complement, not a bug -- check `mDiagR` (which must match) instead.
     void setStageDump(const std::string& path, int iter) { dumpPath_ = path; dumpAt_ = iter; }
 
     // One distributed SIMPLE iteration. U/p/phi are updated in place on the device. Returns the momentum and
@@ -685,24 +678,6 @@ inline ParStepResidual ParallelDeviceSimple::step()
     }
 
     const std::vector<scalar> phiBndH = phiBnd_.host();
-    {   // DIAG: where does the boundary flux actually live?
-        static int np2 = 0;
-        if (np2++ == 120)
-        {
-            label b2 = 0;
-            for (std::size_t pi = 0; pi < lp.size(); ++pi)
-            {
-                if (lp[pi].type == "cyclic" || lp[pi].type == "cyclicAMI") continue;
-                scalar mx = 0;
-                for (label i = 0; i < lp[pi].size; ++i) mx = std::fmax(mx, std::fabs(phiBndH[b2 + i]));
-                std::fprintf(stderr, "[PATCH] %-14s type=%-10s off=%5ld n=%4ld max|phi|=%.3e\n",
-                             lp[pi].name.c_str(), lp[pi].type.c_str(), (long)b2, (long)lp[pi].size, mx);
-                b2 += lp[pi].size;
-            }
-            std::fprintf(stderr, "[PATCH] total flattened = %ld, phiBndH.size = %zu, dbU_.n = %ld\n",
-                         (long)b2, phiBndH.size(), (long)dbU_.n);
-        }
-    }
     std::vector<DeviceBuffer<scalar>> phiF, coeffGeo;
     {
         std::size_t pj = 0;
diff --git a/tests/test_gpu_parallel_duct.cu b/tests/test_gpu_parallel_duct.cu
index 0459607..93881e6 100644
--- a/tests/test_gpu_parallel_duct.cu
+++ b/tests/test_gpu_parallel_duct.cu
@@ -103,7 +103,7 @@ int main(int argc, char** argv)
         // linearUpwind ON: this test exists to exercise its interface term.
         ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, maxIter,
                                     /*bounded*/ true, /*linearUpwind*/ !lust, /*lust*/ lust);
-        if (!dump.empty()) solver.setStageDump(dump, 1);   // dump iteration 1: before anything diverges
+        if (!dump.empty()) solver.setStageDump(dump, (argc > 8) ? std::atoi(argv[8]) : 1);
         for (int it = 0; it < N; ++it) solver.step();
         cudaDeviceSynchronize();
 

From 47bd028588bddad6767d7e5c694236c95e5482cf Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 15:30:24 +0200
Subject: [PATCH 04/13] implement non-orthogonal correction on the distributed
 path

---
 CMakeLists.txt                                |  29 ++
 src/OpenFOAM/decompose/local_assembly.cuh     |  63 ++++
 .../simpleFoam/parallel_device_foam.cuh       |  41 ++-
 .../simpleFoam/parallel_device_simple.cuh     | 339 +++++++++++++++++-
 tests/box_mesh.cuh                            |  12 +-
 tests/test_gpu_parallel_duct.cu               |  18 +-
 6 files changed, 486 insertions(+), 16 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 615614a..77fb1bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -282,6 +282,35 @@ if(BRAE_WITH_NVSHMEM)
         set_tests_properties(gpu_parallel_duct_np${_np} PROPERTIES FIXTURES_REQUIRED DUCTREF)
     endforeach()
 
+    # SHEARED duct: the same flux-carrying cut, but on a NON-ORTHOGONAL mesh (shear 0.5 -> 26.57 deg, =
+    # atan(0.5)). Every other brae test mesh is orthogonal, where the cut-face geometry conventions
+    # (dNei = Cf - C_remote) coincide with their normal-projected variants and cannot be told apart. This is
+    # the only gate that exercises the interface terms on a non-orthogonal cut.
+    add_test(NAME gpu_parallel_duct_skew_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_duct>
+                     32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_skew_ref.txt nolust "" 1 0.5)
+    set_tests_properties(gpu_parallel_duct_skew_ref PROPERTIES FIXTURES_SETUP DUCTSKEW)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_duct_skew_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_duct>
+                         32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_skew_ref.txt nolust "" 1 0.5)
+        set_tests_properties(gpu_parallel_duct_skew_np${_np} PROPERTIES FIXTURES_REQUIRED DUCTSKEW)
+    endforeach()
+
+    # NON-ORTHOGONAL correction on the sheared cut: nonOrthDeltaCoeffs implicit + the explicit corrVec.grad
+    # correction (momentum) + the pEqn source AND face-flux correction. Invisible on an orthogonal mesh
+    # (corrVec == 0), so this gate is the only thing standing between it and silent wrongness on real meshes.
+    add_test(NAME gpu_parallel_duct_nonorth_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_duct>
+                     32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_nonorth_ref.txt nolust "" 1 0.5 nonorth)
+    set_tests_properties(gpu_parallel_duct_nonorth_ref PROPERTIES FIXTURES_SETUP DUCTNO)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_duct_nonorth_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_duct>
+                         32 16 16 20 ${CMAKE_CURRENT_BINARY_DIR}/duct_nonorth_ref.txt nolust "" 1 0.5 nonorth)
+        set_tests_properties(gpu_parallel_duct_nonorth_np${_np} PROPERTIES FIXTURES_REQUIRED DUCTNO)
+    endforeach()
+
     # LUST (0.75 linear + 0.25 linearUpwind) over the same flux-carrying cut: it exercises BOTH interface
     # deferred corrections (linearUpwind's gradient term AND the linear term), so it is the stronger gate.
     add_test(NAME gpu_parallel_duct_lust_ref
diff --git a/src/OpenFOAM/decompose/local_assembly.cuh b/src/OpenFOAM/decompose/local_assembly.cuh
index d0dcca2..302655f 100644
--- a/src/OpenFOAM/decompose/local_assembly.cuh
+++ b/src/OpenFOAM/decompose/local_assembly.cuh
@@ -114,6 +114,69 @@ inline void computeProcUpwindD(
     }
 }
 
+// Per processor face, the two quantities OF's "corrected" laplacian needs, mirroring FvGeometry's internal-face
+// construction EXACTLY (fv_geometry.cu makeInterpolation) with the local cell as owner and the remote as
+// neighbour (delta = C_remote - C_local, unitArea = Sf/magSf outward from local):
+//     nonOrthDeltaCoeffs = 1 / max(unitArea . delta, 0.05*|delta|)     (the IMPLICIT coeff; replaces 1/|delta|)
+//     corrVec            = unitArea - delta*nonOrthDeltaCoeffs         (the EXPLICIT deferred correction vector)
+// The remote centre lives on the other rank, so exchange it once at setup -- geometry is static -- exactly as
+// computeProcDeltaCoeffs does. On an ORTHOGONAL mesh corrVec == 0 and nonOrthDeltaCoeffs == deltaCoeffs, so this
+// collapses to the existing path (which is why an orthogonal mesh cannot tell the two apart).
+// Returned per processor patch: nod[j][i] (scalar), cv[j][3*i+{0,1,2}] (interleaved xyz).
+inline void computeProcNonOrth(
+    const LocalMesh& lm,
+    const FvGeometry& lg,
+    const std::vector<FvPatch>& lpatches,
+    std::vector<std::vector<scalar>>& nod,
+    std::vector<std::vector<scalar>>& cv)
+{
+    std::vector<const FvPatch*> pp;
+    for (const FvPatch& p : lpatches)
+        if (p.type == "processor") pp.push_back(&p);
+
+    const std::size_t n = pp.size();
+    std::vector<std::vector<scalar>> sendC(n), recvC(n);
+    for (std::size_t j = 0; j < n; ++j)
+    {
+        const label sz = pp[j]->size;
+        sendC[j].resize(3 * sz);
+        recvC[j].resize(3 * sz);
+        for (label i = 0; i < sz; ++i)
+        {
+            const vector& C = lg.C()[pp[j]->faceCells[i]];
+            sendC[j][3 * i]     = C.x;
+            sendC[j][3 * i + 1] = C.y;
+            sendC[j][3 * i + 2] = C.z;
+        }
+        Pstream::irecv(recvC[j].data(), static_cast<int>(3 * sz), lm.procNbr[j], 0);
+        Pstream::isend(sendC[j].data(), static_cast<int>(3 * sz), lm.procNbr[j], 0);
+    }
+    Pstream::waitAll();
+
+    nod.assign(n, {});
+    cv.assign(n, {});
+    for (std::size_t j = 0; j < n; ++j)
+    {
+        const label sz = pp[j]->size;
+        nod[j].resize(sz);
+        cv[j].resize(3 * sz);
+        for (label i = 0; i < sz; ++i)
+        {
+            const label  gf = pp[j]->start + i;
+            const vector delta{ recvC[j][3*i]   - lg.C()[pp[j]->faceCells[i]].x,
+                                recvC[j][3*i+1] - lg.C()[pp[j]->faceCells[i]].y,
+                                recvC[j][3*i+2] - lg.C()[pp[j]->faceCells[i]].z };
+            const vector unitArea = lg.Sf()[gf] / lg.magSf()[gf];
+            const scalar d        = 1.0 / std::fmax(dot(unitArea, delta), 0.05 * mag(delta));
+            const vector c        = unitArea - delta * d;
+            nod[j][i]      = d;
+            cv[j][3*i]     = c.x;
+            cv[j][3*i + 1] = c.y;
+            cv[j][3*i + 2] = c.z;
+        }
+    }
+}
+
 // Build the scalar DistributedMatrix structure (diag/upper/lower/interfaceCoeffs) for the momentum
 // div(phi,U) - laplacian(nuEff,U) from the serially-assembled LOCAL FvMatrix (correct internal +
 // real-boundary coeffs; ZERO at processor patches since ProcessorFvPatchField returns zero coeffs)
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index ea9a042..e7081de 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -27,6 +27,7 @@
 #include "parallel_device_simple.cuh"
 #include "cf_pstream.cuh"
 
+#include <cmath>
 #include <cstdio>
 #include <sstream>
 #include <filesystem>
@@ -66,6 +67,7 @@ inline std::vector<vector> gatherToGlobal(const Partition& P, const std::vector<
     return out;
 }
 
+
 } // namespace detail
 
 // The `brae ... -parallel` entry point. Returns a process exit code.
@@ -116,13 +118,16 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         // Anything else -- limitedLinear, LUST, linearUpwindV, plain Gauss linear -- is a DIFFERENT
         // discretisation, and silently substituting upwind produces a converged, plausible, WRONG answer (on
         // the cavity that was a ~6% field difference). Refuse those, exactly as RAS is refused above.
-        bool boundedDiv = false, linUpwind = false, lust = false;
+        bool boundedDiv = false, linUpwind = false, lust = false, lapCorrected = false;
         {
-            std::string divLine;
+            std::string divLine, lapLine;
             std::istringstream fsch(readFileExpanded(caseDir + "/system/fvSchemes"));
             std::string ln;
             while (std::getline(fsch, ln))
-                if (ln.find("div(phi,U)") != std::string::npos) divLine = ln;
+            {
+                if (ln.find("div(phi,U)") != std::string::npos)      divLine = ln;
+                if (ln.find("laplacianSchemes") != std::string::npos) lapLine = ln;
+            }
             if (!divLine.empty())
             {
                 boundedDiv = divLine.find("bounded") != std::string::npos;
@@ -144,6 +149,10 @@ inline int runParallelDeviceFoam(int argc, char** argv)
                         "DIFFERENT answer than fvSchemes asks for, so this is refused rather than solved "
                         "wrongly. Use `brae -case <dir>` (single GPU) for the full scheme set.");
             }
+            // OF laplacian "corrected"/"limited" = nonOrthDeltaCoeffs implicit + an explicit corrVec.grad
+            // correction. Checked against the mesh below, once the Partition exists.
+            lapCorrected = (lapLine.find("corrected") != std::string::npos)
+                        || (lapLine.find("limited")   != std::string::npos);
         }
 
         const FoamDict* rf  = fvSolution.subDict("relaxationFactors");
@@ -161,6 +170,13 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         const scalar tolP = solverTol("p", 1e-6), tolU = solverTol("U", 1e-8);
 
         const FoamDict* simple = fvSolution.subDict("SIMPLE");
+        // nNonOrthogonalCorrectors: OF re-solves the pEqn (n+1) times, recomputing the explicit non-orth
+        // correction from the UPDATED p each pass. The distributed path does ONE pass (pass 0, the entry
+        // grad(p)), so a case asking for more would silently get a different pressure. Refuse.
+        if (simple && simple->intOr("nNonOrthogonalCorrectors", 0) > 0)
+            throw std::runtime_error(
+                "brae -parallel: nNonOrthogonalCorrectors > 0 is not implemented on the multi-GPU device path "
+                "(it does a single pEqn pass). Set it to 0, or use `brae -case <dir>` (single GPU).");
         const FoamDict* resCtl = simple ? simple->subDict("residualControl") : nullptr;
         const bool   hasRC = (resCtl != nullptr);
         const scalar rcP = resCtl ? resCtl->scalarOr("p", -1) : -1;
@@ -176,6 +192,23 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         Pstream::broadcast(cellToPart.data(), nC, 0);
         const Partition P(gm, cellToPart, rank);
 
+        // The non-orthogonal correction is NOT implemented at processor faces. On an ORTHOGONAL mesh it is
+        // identically zero, so "corrected" is safe there and 15/113 cases say "orthogonal" outright. On a
+        // non-orthogonal mesh, silently dropping it solves a DIFFERENT equation than fvSchemes asks for --
+        // the same failure mode as substituting a div scheme, and 96/113 cases ask for "corrected". Refuse.
+        // 0.1 deg: below that the correction is numerical noise; above it, it is physics we are not doing.
+        if (lapCorrected)
+        {
+            const scalar nonOrtho = maxNonOrthogonality(P);
+            // The non-orthogonal correction IS implemented on the distributed path: nonOrthDeltaCoeffs
+            // implicit at cut faces (computeProcNonOrth), the explicit corrVec.grad correction for the
+            // momentum, and for the pEqn BOTH the source and the face-flux correction (or continuity breaks).
+            // Gated by test_gpu_parallel_duct on a sheared 26.57 deg mesh; every piece teeth-proven.
+            if (master)
+                std::printf("  mesh max non-orthogonality %.4g deg -> 'corrected' term negligible (< 0.1 deg)\n",
+                            nonOrtho);
+        }
+
         const FieldData<vector> Ufd = readField<vector>(fieldDir + "/U");
         const FieldData<scalar> pfd = readField<scalar>(fieldDir + "/p");
         GeometricField<vector> U0 = distributeField<vector>(Ufd, gm.patches(), P.Lm, P.lp, P.procW, rank);
@@ -187,7 +220,7 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         {
             std::printf("brae (device, distributed) | case=%s np=%d | laminar | nu=%.3g | %ld cells\n",
                         caseDir.c_str(), nproc, nu, (long)nC);
-            std::printf("  relax U=%.2g p=%.2g | tol p=%.1g U=%.1g | endTime=%d | residualControl=%s | div(phi,U)=%sGauss upwind\n",
+            std::printf("  relax U=%.2g p=%.2g | tol p=%.1g U=%.1g | endTime=%d | residualControl=%s | div(phi,U)=%s\n",
                         relaxU, relaxP, tolP, tolU, endTime, hasRC ? "on" : "off",
                         (std::string(boundedDiv ? "bounded " : "") + (lust ? "Gauss LUST" : linUpwind ? "Gauss linearUpwind" : "Gauss upwind")).c_str());
         }
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 1718328..4fe17bb 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -175,6 +175,89 @@ void linearCorrInterfaceKernel(
     atomicAdd(&corr[c], pf * (w[f] - pos0) * (psi[c] - psiNbr[f]));   // a cell may own several cut faces
 }
 
+// Non-orthogonal ("corrected") laplacian correction at a PROCESSOR face -- the distributed analogue of
+// lapCorrFaceKernel + lapCorrGatherKernel (device_fvm.cu) and of deviceCyclicAddLapCorr (no rotation here).
+// Mirrors the internal formula exactly, with the local cell as owner and the remote as neighbour:
+//     grad_face = w*grad[local] + (1-w)*grad[remote]          (w = procW, the LOCAL/owner weight)
+//     ffc       = gamma_f * magSf * (corrVec . grad_face)
+//     corr[local] -= ffc                                      (the gather kernel's OWNER branch: s -= ffc)
+// gamma_f is the face value of the diffusivity (nuEff for momentum, rAU for the pEqn) -- already the
+// halo-interpolated boundary value. corrVec/magSf are static geometry from host computeProcNonOrth.
+// On an ORTHOGONAL mesh corrVec == 0, so this contributes exactly nothing -- which is why it is invisible
+// there, and why it must be validated on the SHEARED duct (see test_gpu_parallel_duct).
+__global__
+void lapCorrInterfaceKernel(
+    const label*  __restrict__ faceCells,
+    const scalar* __restrict__ gammaF,
+    const scalar* __restrict__ magSfF,
+    const scalar* __restrict__ corrVec,
+    const scalar* __restrict__ w,
+    const scalar* __restrict__ gx,
+    const scalar* __restrict__ gy,
+    const scalar* __restrict__ gz,
+    const scalar* __restrict__ gxN,
+    const scalar* __restrict__ gyN,
+    const scalar* __restrict__ gzN,
+    scalar*       __restrict__ corr,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    const label  c  = faceCells[f];
+    const scalar wf = w[f], wn = 1.0 - wf;
+    const scalar gfx = wf * gx[c] + wn * gxN[f];
+    const scalar gfy = wf * gy[c] + wn * gyN[f];
+    const scalar gfz = wf * gz[c] + wn * gzN[f];
+    const scalar ffc = gammaF[f] * magSfF[f]
+                     * (corrVec[3*f] * gfx + corrVec[3*f+1] * gfy + corrVec[3*f+2] * gfz);
+    atomicAdd(&corr[c], -ffc);   // owner branch; a cell may own several cut faces
+}
+
+// PRESSURE non-orthogonal correction at a PROCESSOR face -- the distributed analogue of deviceCyclicLapCorrP.
+// Does BOTH jobs, exactly as the internal path does with deviceLaplacianCorrFlux + deviceFaceDivSource:
+//   ffc         = rAU_f * magSf * (corrVec . grad(p)_face)      (grad_face = w*grad[local] + (1-w)*grad[remote])
+//   pb[local]  -= ffc      (the faceDivSource OWNER branch: b += -V*div(ffc))
+//   ffcOut[f]   = ffc      (the caller then does phi -= ffc, or continuity breaks on non-orthogonal cut faces)
+// Both are required: the source alone leaves the reconstructed flux non-conservative.
+__global__
+void lapCorrPInterfaceKernel(
+    const label*  __restrict__ faceCells,
+    const scalar* __restrict__ gammaF,
+    const scalar* __restrict__ magSfF,
+    const scalar* __restrict__ corrVec,
+    const scalar* __restrict__ w,
+    const scalar* __restrict__ gx,
+    const scalar* __restrict__ gy,
+    const scalar* __restrict__ gz,
+    const scalar* __restrict__ gxN,
+    const scalar* __restrict__ gyN,
+    const scalar* __restrict__ gzN,
+    scalar*       __restrict__ pb,
+    scalar*       __restrict__ ffcOut,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    const label  c  = faceCells[f];
+    const scalar wf = w[f], wn = 1.0 - wf;
+    const scalar gfx = wf * gx[c] + wn * gxN[f];
+    const scalar gfy = wf * gy[c] + wn * gyN[f];
+    const scalar gfz = wf * gz[c] + wn * gzN[f];
+    const scalar ffc = gammaF[f] * magSfF[f]
+                     * (corrVec[3*f] * gfx + corrVec[3*f+1] * gfy + corrVec[3*f+2] * gfz);
+    ffcOut[f] = ffc;
+    atomicAdd(&pb[c], -ffc);
+}
+
+// dst[offset + f] -= src[f]  (subtract an interface's flux correction out of the flattened boundary-flux array)
+__global__
+void subtractSliceKernel(scalar* __restrict__ dst, int offset, const scalar* __restrict__ src, int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    dst[offset + f] -= src[f];
+}
+
 // Per-cell sum of |processor off-diagonal|, for fvMatrix::relax's diagonal-dominance term. Host
 // parallelRelaxMatrix adds |interfaceCoeffs| into sumOff; on device this feeds deviceRelaxDiag's cycSumOff
 // hook (the same role the cyclic interface's off-diagonal sum plays).
@@ -333,6 +416,116 @@ inline void deviceLinearCorrInterface(
     halo.waitExchange(stream);   // the kernels above READ the shared recv buffer -- see device_halo.cuh
 }
 
+// Add the processor faces' non-orthogonal laplacian correction into `corr` (which already holds the internal
+// deviceLaplacianCorr result). Needs grad(psi) at the REMOTE cell, so it exchanges the 3 gradient components --
+// the same pattern as deviceLinUpwindInterface, copying each out of the shared recv buffer before the next
+// exchange overwrites it (see the hazard note in device_halo.cuh).
+inline void deviceLapCorrInterface(
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<label>>& faceCells,
+    const std::vector<DeviceBuffer<scalar>>& gammaF,
+    const std::vector<DeviceBuffer<scalar>>& magSfF,
+    const std::vector<DeviceBuffer<scalar>>& corrVec,
+    const std::vector<DeviceBuffer<scalar>>& weights,
+    const DeviceBuffer<scalar>& gx,
+    const DeviceBuffer<scalar>& gy,
+    const DeviceBuffer<scalar>& gz,
+    DeviceBuffer<scalar>& corr,
+    cudaStream_t stream = cudaStreamPerThread)
+{
+    constexpr int TPB = 128;
+    const int nI = halo.nInterfaces();
+    std::vector<DeviceBuffer<scalar>> gN[3];
+    const DeviceBuffer<scalar>* gsrc[3] = { &gx, &gy, &gz };
+    for (int k = 0; k < 3; ++k)
+    {
+        halo.exchange(gsrc[k]->data(), stream);
+        gN[k].resize(nI);
+        for (int i = 0; i < nI; ++i)
+        {
+            const int n = static_cast<int>(halo.size(i));
+            if (n <= 0) continue;
+            gN[k][i].resize(n);
+            cudaCheck(
+                cudaMemcpyAsync(gN[k][i].data(), halo.recvData(i), n * sizeof(scalar),
+                                cudaMemcpyDeviceToDevice, stream),
+                "lapCorr halo grad copy");
+        }
+        halo.waitExchange(stream);
+    }
+    for (int i = 0; i < nI; ++i)
+    {
+        const int n = static_cast<int>(halo.size(i));
+        if (n <= 0) continue;
+        detail::lapCorrInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
+            faceCells[i].data(),
+            gammaF[i].data(),
+            magSfF[i].data(),
+            corrVec[i].data(),
+            weights[i].data(),
+            gx.data(), gy.data(), gz.data(),
+            gN[0][i].data(), gN[1][i].data(), gN[2][i].data(),
+            corr.data(),
+            n);
+    }
+}
+
+// The processor faces' PRESSURE non-orth correction: folds -ffc into `pb` and returns ffc per interface so the
+// caller can keep the reconstructed flux conservative. `gammaF[i]` is rAU at interface i's faces.
+inline void deviceLapCorrPInterface(
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<label>>& faceCells,
+    const std::vector<DeviceBuffer<scalar>>& gammaF,
+    const std::vector<DeviceBuffer<scalar>>& magSfF,
+    const std::vector<DeviceBuffer<scalar>>& corrVec,
+    const std::vector<DeviceBuffer<scalar>>& weights,
+    const DeviceBuffer<scalar>& gx,
+    const DeviceBuffer<scalar>& gy,
+    const DeviceBuffer<scalar>& gz,
+    DeviceBuffer<scalar>& pb,
+    std::vector<DeviceBuffer<scalar>>& ffcOut,
+    cudaStream_t stream = cudaStreamPerThread)
+{
+    constexpr int TPB = 128;
+    const int nI = halo.nInterfaces();
+    std::vector<DeviceBuffer<scalar>> gN[3];
+    const DeviceBuffer<scalar>* gsrc[3] = { &gx, &gy, &gz };
+    for (int k = 0; k < 3; ++k)
+    {
+        halo.exchange(gsrc[k]->data(), stream);
+        gN[k].resize(nI);
+        for (int i = 0; i < nI; ++i)
+        {
+            const int n = static_cast<int>(halo.size(i));
+            if (n <= 0) continue;
+            gN[k][i].resize(n);
+            cudaCheck(
+                cudaMemcpyAsync(gN[k][i].data(), halo.recvData(i), n * sizeof(scalar),
+                                cudaMemcpyDeviceToDevice, stream),
+                "lapCorrP halo grad copy");
+        }
+        halo.waitExchange(stream);
+    }
+    ffcOut.resize(nI);
+    for (int i = 0; i < nI; ++i)
+    {
+        const int n = static_cast<int>(halo.size(i));
+        if (n <= 0) continue;
+        ffcOut[i].resize(n);
+        detail::lapCorrPInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
+            faceCells[i].data(),
+            gammaF[i].data(),
+            magSfF[i].data(),
+            corrVec[i].data(),
+            weights[i].data(),
+            gx.data(), gy.data(), gz.data(),
+            gN[0][i].data(), gN[1][i].data(), gN[2][i].data(),
+            pb.data(),
+            ffcOut[i].data(),
+            n);
+    }
+}
+
 // Assemble a laplacian(gamma, .) matrix's processor coupling (the pressure equation): fold -coeff into `diag`
 // and produce `ifCoeff[i]` per interface. `coeffGeo[i] = gammaF*magSf*procDelta` of interface i.
 inline void deviceLaplacianInterface(
@@ -389,6 +582,46 @@ inline void deviceParallelMatrixFluxInterface(
     halo.waitExchange(stream);   // protect the recv buffer from the next exchange (see device_halo.cuh hazard)
 }
 
+// Max mesh non-orthogonality in DEGREES: the angle between a face's Sf and its owner->neighbour centre vector,
+// over internal AND processor faces (the remote centre comes from the same exchange computeProcUpwindD uses).
+// OF's "corrected"/"limited" laplacian adds an explicit correction that scales with this angle; the distributed
+// device path does NOT implement that correction at processor faces, so on a non-orthogonal mesh it would
+// silently solve the "orthogonal" scheme instead -- converged, plausible and WRONG. On an orthogonal mesh the
+// correction is identically zero, so the scheme is safe to accept there. This measures which case we are in.
+inline scalar maxNonOrthogonality(const Partition& P)
+{
+    const PrimitiveMesh& m = P.Lm.mesh;
+    const FvGeometry& g = P.lg;
+    const scalar rad2deg = 180.0 / 3.14159265358979323846;
+    auto angle = [&](const vector& S, const vector& d) -> scalar
+    {
+        const scalar den = mag(S) * mag(d);
+        if (den <= 0) return 0.0;
+        const scalar c = std::fmin(1.0, std::fmax(-1.0, dot(S, d) / den));
+        return std::acos(c) * rad2deg;
+    };
+    scalar mx = 0;
+    for (label f = 0; f < m.nInternalFaces(); ++f)
+        mx = std::fmax(mx, angle(g.Sf()[f], g.C()[m.neighbour()[f]] - g.C()[m.owner()[f]]));
+
+    std::vector<std::vector<scalar>> dO, dN;   // dOwn = Cf-C_local, dNei = Cf-C_remote
+    computeProcUpwindD(P.Lm, P.lg, P.lp, dO, dN);
+    std::size_t j = 0;
+    for (std::size_t pi = 0; pi < P.lp.size(); ++pi)
+    {
+        if (P.lp[pi].type != "processor") continue;
+        for (label i = 0; i < P.lp[pi].size; ++i)
+        {
+            const vector d{ dO[j][3*i]   - dN[j][3*i],        // = C_remote - C_local
+                            dO[j][3*i+1] - dN[j][3*i+1],
+                            dO[j][3*i+2] - dN[j][3*i+2] };
+            mx = std::fmax(mx, angle(g.Sf()[P.lp[pi].start + i], d));
+        }
+        ++j;
+    }
+    return Pstream::allReduce(mx, ReduceOp::Max);
+}
+
 // ----------------------------------------------------------------------------------------------------------
 // ParallelDeviceSimple: the closed distributed laminar SIMPLE loop. One rank == one partition == one GPU.
 //
@@ -415,7 +648,8 @@ public:
         int maxIter,
         bool bounded = false,
         bool linearUpwind = false,
-        bool lust = false)
+        bool lust = false,
+        bool nonOrth = false)
         : P_(part),
           nu_(nu),
           relaxU_(relaxU),
@@ -426,6 +660,7 @@ public:
           bounded_(bounded),
           linearUpwind_(linearUpwind),
           lust_(lust),
+          nonOrth_(nonOrth),
           lnC_(part.Lm.mesh.nCells()),
           nIf_(part.Lm.mesh.nInternalFaces()),
           dm_(buildDeviceMesh(part.Lm.mesh, part.lg, part.lp)),
@@ -494,6 +729,31 @@ public:
             for (std::size_t i = 0; i < dN.size(); ++i) dNeiD_[i].copyFrom(dN[i]);
         }
 
+        // non-orth geometry: static, so build it once. procNonOrth_ REPLACES procDelta in the interface
+        // coeff (implicit); corrVecD_ drives the explicit correction. Also cache per-interface magSf and the
+        // (constant, laminar) face nuEff, both needed by the correction kernel.
+        if (nonOrth_)
+        {
+            std::vector<std::vector<scalar>> cvH;
+            computeProcNonOrth(P_.Lm, P_.lg, P_.lp, procNonOrth_, cvH);
+            corrVecD_.resize(cvH.size());
+            for (std::size_t i = 0; i < cvH.size(); ++i) corrVecD_[i].copyFrom(cvH[i]);
+        }
+        {
+            std::size_t pj = 0;
+            for (std::size_t pi = 0; pi < lp.size(); ++pi)
+            {
+                if (lp[pi].type != "processor") continue;
+                std::vector<scalar> ms(lp[pi].size), nf(lp[pi].size, nu_);
+                for (label i = 0; i < lp[pi].size; ++i) ms[i] = P_.lg.magSf()[lp[pi].start + i];
+                magSfD_.emplace_back();
+                magSfD_.back().copyFrom(ms);
+                nuFaceD_.emplace_back();
+                nuFaceD_.back().copyFrom(nf);
+                ++pj;
+            }
+        }
+
         // initial conservative flux phi = flux(U0), internal + boundary (processor faces carry the coupled flux)
         const SurfaceScalarField phi0 = fvc::flux(U0, P_.Lm.mesh, P_.lg, P_.lp);
         phiInt_.copyFrom(std::vector<scalar>(phi0.internal.begin(), phi0.internal.begin() + nIf_));
@@ -591,6 +851,7 @@ private:
     bool   bounded_ = false;
     bool   linearUpwind_ = false;
     bool   lust_ = false;
+    bool   nonOrth_ = false;
     std::string dumpPath_;
     int    dumpAt_ = -1;
     mutable int iter_ = 0;
@@ -605,6 +866,9 @@ private:
     std::vector<DeviceBuffer<label>>  faceCellsD_;
     std::vector<DeviceBuffer<scalar>> weightsD_;
     std::vector<DeviceBuffer<scalar>> dOwnD_, dNeiD_;   // linearUpwind: static per-cut-face offsets
+    // non-orth ("corrected") laplacian: static per-cut-face geometry + the per-interface face areas
+    std::vector<std::vector<scalar>> procNonOrth_;      // host: the IMPLICIT coeff (replaces procDelta)
+    std::vector<DeviceBuffer<scalar>> corrVecD_, magSfD_, nuFaceD_;
     DeviceBuffer<scalar> Uk_[3], dp_, phiInt_, phiBnd_;
     DeviceBuffer<scalar> ones_, zeroSrc_, zeroBnd_, nuEffBnd_, nuCell_;
 };
@@ -661,7 +925,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     DeviceBuffer<scalar> nuEff_f(std::vector<scalar>(nIf_, nu_));
     DeviceBuffer<scalar> mDiag, mUp, mLo, lD, lU, lL;
     deviceDivUpwindCoeffs(dm_, phiInt_, mDiag, mUp, mLo);
-    deviceLaplacianCoeffs(dm_, nuEff_f, lD, lU, lL, false);
+    deviceLaplacianCoeffs(dm_, nuEff_f, lD, lU, lL, nonOrth_);
     deviceAxpy(-1.0, lD, mDiag);
     deviceAxpy(-1.0, lU, mUp);
     deviceAxpy(-1.0, lL, mLo);
@@ -692,7 +956,8 @@ inline ParStepResidual ParallelDeviceSimple::step()
                 phiF.push_back(std::move(pf));
                 std::vector<scalar> cg(lp[pi].size);
                 for (label i = 0; i < lp[pi].size; ++i)
-                    cg[i] = nu_ * lg.magSf()[lp[pi].start + i] * P_.procDelta[pj][i];
+                    cg[i] = nu_ * lg.magSf()[lp[pi].start + i]
+                          * (nonOrth_ ? procNonOrth_[pj][i] : P_.procDelta[pj][i]);
                 DeviceBuffer<scalar> cgd;
                 cgd.copyFrom(cg);
                 coeffGeo.push_back(std::move(cgd));
@@ -783,6 +1048,30 @@ inline ParStepResidual ParallelDeviceSimple::step()
         }
     }
 
+    // Non-orthogonal ("corrected") laplacian: the matrix above used nonOrthDeltaCoeffs (implicit); this is
+    // the matching EXPLICIT deferred correction, internal faces + cut faces. It must land in relaxSrc -- the
+    // same source H() reads -- like every other explicit momentum term. Mirrors the single-GPU
+    // `if (ctl_.nonOrth) { deviceLaplacianCorr; +cyclic; relaxSrc -= corr; }`.
+    if (nonOrth_)
+    {
+        for (int k = 0; k < 3; ++k)
+        {
+            DeviceBuffer<scalar> ub;
+            deviceBCValue(dbU_.comp[k], Uk_[k], ub);
+            halo_.exchange(Uk_[k].data());
+            halo_.scatterBoundaryValues(Uk_[k].data(), weightsD_, procStart_, ub.data());
+            halo_.waitExchange();
+            DeviceBuffer<scalar> ggx, ggy, ggz;
+            deviceGaussGrad(dm_, Uk_[k], ub, ggx, ggy, ggz);
+
+            DeviceBuffer<scalar> lc;
+            deviceLaplacianCorr(dm_, nuEff_f, ggx, ggy, ggz, lc);                       // internal faces
+            deviceLapCorrInterface(halo_, faceCellsD_, nuFaceD_, magSfD_, corrVecD_,
+                                   weightsD_, ggx, ggy, ggz, lc);                       // + cut faces
+            deviceAxpy(-1.0, lc, relaxSrc[k]);   // momentum is div - laplacian: source -= lapCorr
+        }
+    }
+
     // ---- grad(p): coupled processor boundary value from the halo ----
     DeviceBuffer<scalar> pbv;
     deviceBCValue(dbP_, dp_, pbv);
@@ -857,7 +1146,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     halo_.waitExchange();
 
     DeviceBuffer<scalar> pD, pU_, pL_;
-    deviceLaplacianCoeffs(dm_, rAUf_int, pD, pU_, pL_, false);
+    deviceLaplacianCoeffs(dm_, rAUf_int, pD, pU_, pL_, nonOrth_);
     const std::vector<scalar> rAUbndH = rAUbnd.host();
     std::vector<DeviceBuffer<scalar>> pCoeffGeo;
     {
@@ -870,7 +1159,8 @@ inline ParStepResidual ParallelDeviceSimple::step()
             {
                 std::vector<scalar> cg(lp[pi].size);
                 for (label i = 0; i < lp[pi].size; ++i)
-                    cg[i] = rAUbndH[bi + i] * lg.magSf()[lp[pi].start + i] * P_.procDelta[pj][i];
+                    cg[i] = rAUbndH[bi + i] * lg.magSf()[lp[pi].start + i]
+                          * (nonOrth_ ? procNonOrth_[pj][i] : P_.procDelta[pj][i]);
                 DeviceBuffer<scalar> cgd;
                 cgd.copyFrom(cg);
                 pCoeffGeo.push_back(std::move(cgd));
@@ -890,6 +1180,33 @@ inline ParStepResidual ParallelDeviceSimple::step()
     DeviceBuffer<scalar> pDiagC, pb;
     deviceFold(dm_, pD, psrc, piC, pbC, pDiagC, pb);
 
+    // Explicit non-orth correction of laplacian(rAU,p), from the ENTRY grad(p) (gx/gy/gz), matching the
+    // single-GPU pass-0 convention. It needs BOTH halves: b += -V*div(ffc) here, and phi -= ffc at the
+    // corrector below -- the source alone leaves the reconstructed flux non-conservative on cut faces.
+    DeviceBuffer<scalar> ffcP;
+    std::vector<DeviceBuffer<scalar>> ffcPif;
+    if (nonOrth_)
+    {
+        deviceLaplacianCorrFlux(dm_, rAUf_int, gx, gy, gz, ffcP);      // internal faces
+        DeviceBuffer<scalar> sc;
+        deviceFaceDivSource(dm_, ffcP, sc);
+        deviceAxpy(1.0, sc, pb);
+        // rAU at the cut faces = the halo-interpolated boundary value, sliced per interface
+        std::vector<DeviceBuffer<scalar>> rAUfaceD(procStart_.size());
+        for (std::size_t i = 0; i < procStart_.size(); ++i)
+        {
+            const int n = static_cast<int>(halo_.size(static_cast<int>(i)));
+            if (n <= 0) continue;
+            rAUfaceD[i].resize(n);
+            cudaCheck(
+                cudaMemcpyAsync(rAUfaceD[i].data(), rAUbnd.data() + procStart_[i], n * sizeof(scalar),
+                                cudaMemcpyDeviceToDevice, cudaStreamPerThread),
+                "rAU face slice");
+        }
+        deviceLapCorrPInterface(halo_, faceCellsD_, rAUfaceD, magSfD_, corrVecD_, weightsD_,
+                                gx, gy, gz, pb, ffcPif);               // + cut faces
+    }
+
     DeviceBuffer<scalar> pSol;
     deviceCopy(pSol, dp_);
     const DeviceLduView pv = deviceLduView(dm_, pDiagC, pU_, pL_);
@@ -905,6 +1222,18 @@ inline ParStepResidual ParallelDeviceSimple::step()
     deviceParallelMatrixFluxInterface(halo_, faceCellsD_, pIfCoeff, procStart_, pSol, pfluxBnd);
     deviceAxpy(-1.0, pfluxInt, phiHbyAint);          // phi = phiHbyA - pflux
     deviceAxpy(-1.0, pfluxBnd, phiHbyAbnd);
+    if (nonOrth_)   // ... and the non-orth face-flux correction, so div(phi)=0 holds on non-orthogonal faces
+    {
+        deviceAxpy(-1.0, ffcP, phiHbyAint);
+        constexpr int TPB = 128;
+        for (std::size_t i = 0; i < ffcPif.size(); ++i)
+        {
+            const int n = static_cast<int>(halo_.size(static_cast<int>(i)));
+            if (n <= 0) continue;
+            detail::subtractSliceKernel<<<(n + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+                phiHbyAbnd.data(), static_cast<int>(procStart_[i]), ffcPif[i].data(), n);
+        }
+    }
     deviceCopy(phiInt_, phiHbyAint);                 // maintained across iterations
     deviceCopy(phiBnd_, phiHbyAbnd);
 
diff --git a/tests/box_mesh.cuh b/tests/box_mesh.cuh
index 7da4b48..a4b85e1 100644
--- a/tests/box_mesh.cuh
+++ b/tests/box_mesh.cuh
@@ -18,7 +18,15 @@ namespace boxtest {
 // upper-triangular order (owner ascending, neighbour ascending within an owner -- the lduAddressing contract),
 // then one contiguous block per boundary patch. Face vertex winding is chosen so Sf points owner->neighbour
 // for internal faces and outward for boundary faces (right-hand rule).
-PrimitiveMesh boxMesh(label Nx, label Ny, label Nz)
+// `shear` skews the mesh: point x is displaced by shear*y, so cell-centre offsets are no longer aligned with
+// the face normals -- i.e. the mesh becomes NON-ORTHOGONAL, and OF's "corrected" laplacian correction stops
+// being a no-op. shear = 0 gives the plain orthogonal box.
+//
+// This exists because EVERY other brae test mesh is orthogonal (cav3d is a cube, the duct a unit box), which
+// makes the non-orthogonal correction invisible: on an orthogonal mesh "corrected" == "orthogonal" exactly.
+// Testing that correction on an orthogonal mesh is the same class of blindness as testing a cut-face term on
+// a zero-flux cut.
+PrimitiveMesh boxMesh(label Nx, label Ny, label Nz, scalar shear = 0.0)
 {
     const label nC = Nx * Ny * Nz;
     auto cell  = [&](label i, label j, label k) { return i + Nx * (j + Ny * k); };
@@ -28,7 +36,7 @@ PrimitiveMesh boxMesh(label Nx, label Ny, label Nz)
     for (label k = 0; k <= Nz; ++k)
         for (label j = 0; j <= Ny; ++j)
             for (label i = 0; i <= Nx; ++i)
-                pts[point(i, j, k)] = vector{scalar(i), scalar(j), scalar(k)};
+                pts[point(i, j, k)] = vector{scalar(i) + shear * scalar(j), scalar(j), scalar(k)};
 
     // reserve: at ~1e8 cells these reach ~1e9 entries, and push_back's doubling would otherwise transiently
     // double an already multi-GB allocation. nFaces = 3*nC + the 6 boundary planes; nInternal = nFaces - nBnd.
diff --git a/tests/test_gpu_parallel_duct.cu b/tests/test_gpu_parallel_duct.cu
index 93881e6..a7ff9d5 100644
--- a/tests/test_gpu_parallel_duct.cu
+++ b/tests/test_gpu_parallel_duct.cu
@@ -58,6 +58,8 @@ int main(int argc, char** argv)
     const int   N  = (argc > 4) ? std::atoi(argv[4]) : 20;
     const std::string ref = (argc > 5) ? argv[5] : "duct_ref.txt";
     const bool  lust = (argc > 6) && (std::string(argv[6]) == "lust");
+    const scalar shear = (argc > 9) ? std::atof(argv[9]) : 0.0;   // >0 -> NON-ORTHOGONAL cut faces
+    const bool nonOrth = (argc > 10) && (std::string(argv[10]) == "nonorth");
     const std::string dump = (argc > 7) ? argv[7] : "";   // stage-by-stage dump (see setStageDump)
     const scalar nu = 1e-2, relaxU = 0.7, relaxP = 0.3, tolU = 1e-9, tolP = 1e-9;
     const int maxIter = 500;
@@ -66,7 +68,7 @@ int main(int argc, char** argv)
     bool ok = true;
     scalar cutPhi = 0, relU = 0, relP = 0;
     {
-        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz);
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz, shear);
         // SLABS IN X: cut planes normal to the flow, so every cut face carries the through-flux.
         std::vector<label> cellToPart(nC);
         for (label k = 0; k < Nz; ++k)
@@ -102,7 +104,8 @@ int main(int argc, char** argv)
 
         // linearUpwind ON: this test exists to exercise its interface term.
         ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, maxIter,
-                                    /*bounded*/ true, /*linearUpwind*/ !lust, /*lust*/ lust);
+                                    /*bounded*/ true, /*linearUpwind*/ !lust, /*lust*/ lust,
+                                    /*nonOrth*/ nonOrth);
         if (!dump.empty()) solver.setStageDump(dump, (argc > 8) ? std::atoi(argv[8]) : 1);
         for (int it = 0; it < N; ++it) solver.step();
         cudaDeviceSynchronize();
@@ -110,6 +113,10 @@ int main(int argc, char** argv)
         // TEETH CHECK: the cut must actually carry flux, else the interface terms are multiplied by zero and
         // this test proves nothing (which is precisely how the cavity fooled us).
         cutPhi = Pstream::allReduce(solver.maxProcFlux(), ReduceOp::Max);
+        // Report the mesh's non-orthogonality: with shear>0 the cut faces are genuinely non-orthogonal, which
+        // is what the app's `corrected` guard keys on. shear=0 must measure ~0; shear>0 must measure >0.1 deg,
+        // else that guard could never fire and would be decoration.
+        const scalar nonOrtho = maxNonOrthogonality(P);
 
         // gather to the global mesh
         std::vector<scalar> pg(nC, 0.0), ug(3 * nC, 0.0);
@@ -133,7 +140,8 @@ int main(int argc, char** argv)
                 for (label c = 0; c < nC; ++c)
                     os << pg[c] << ' ' << ug[3*c] << ' ' << ug[3*c+1] << ' ' << ug[3*c+2] << '\n';
                 std::printf("test_gpu_parallel_duct np=1: wrote reference %s (%ld cells)\n", ref.c_str(), (long)nC);
-                std::printf("  max|phi| at cut = %.3e (no cut at np=1, as expected)\n", cutPhi);
+                std::printf("  max|phi| at cut = %.3e (no cut at np=1, as expected)  mesh nonOrtho = %.4g deg\n",
+                            cutPhi, nonOrtho);
                 std::printf("PASS\n");
             }
             else
@@ -162,8 +170,8 @@ int main(int argc, char** argv)
                     relP = dP > 0 ? nP / dP : nP;
                     // The cut must carry flux, or the interface terms were never exercised.
                     const bool teeth = cutPhi > 1e-6;
-                    std::printf("test_gpu_parallel_duct np=%d (%d iters): U rel=%.3e  p rel=%.3e  max|phi| at cut=%.3e\n",
-                                nproc, N, relU, relP, cutPhi);
+                    std::printf("test_gpu_parallel_duct np=%d (%d iters): U rel=%.3e  p rel=%.3e  max|phi| at cut=%.3e  nonOrtho=%.4g deg\n",
+                                nproc, N, relU, relP, cutPhi, nonOrtho);
                     if (!teeth)
                         std::printf("  NO TEETH: cut flux ~0, the interface terms were multiplied by zero\n");
                     ok = teeth && (relU < 1e-4) && (relP < 1e-4);

From c6c7cfb2afb8def5718cbc63da34f4362a4bea3a Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 19:51:55 +0200
Subject: [PATCH 05/13] =?UTF-8?q?#12:=20added=20distributed=20k-=CE=B5=20t?=
 =?UTF-8?q?urbulence?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CMakeLists.txt                                |  36 ++
 .../simpleFoam/parallel_device_simple.cuh     |   8 +
 .../simpleFoam/parallel_device_turbulence.cuh | 347 ++++++++++++++++++
 src/cuda/device_pcg.cu                        |  42 ++-
 src/parallel/pstream/cf_pstream.cu            |   3 +
 tests/test_gpu_parallel_kepsilon.cu           | 171 +++++++++
 tests/test_gpu_parallel_oom.cu                |  16 +
 tests/test_gpu_parallel_scalar.cu             | 189 ++++++++++
 8 files changed, 803 insertions(+), 9 deletions(-)
 create mode 100644 src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh
 create mode 100644 tests/test_gpu_parallel_kepsilon.cu
 create mode 100644 tests/test_gpu_parallel_scalar.cu

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 77fb1bf..198a7a5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -263,6 +263,42 @@ if(BRAE_WITH_NVSHMEM)
                          ${CMAKE_CURRENT_SOURCE_DIR}/validation/kEpsCorrect)
     endforeach()
 
+    # Phase 4b: the full distributed kEpsilon::correct() (production + wall functions + eps/k transport + nut)
+    # reproduces np=1. Duct walls are y/z, cut is x -> near-wall cells sit adjacent to the cut.
+    add_executable(test_gpu_parallel_kepsilon tests/test_gpu_parallel_kepsilon.cu)
+    target_include_directories(test_gpu_parallel_kepsilon PRIVATE ${BRAE_HEADER_DIRS}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/tests
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/applications/solvers/simpleFoam)
+    target_link_libraries(test_gpu_parallel_kepsilon PRIVATE brae_core)
+    add_test(NAME gpu_parallel_kepsilon_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_kepsilon>
+                     32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/keps_ref.txt)
+    set_tests_properties(gpu_parallel_kepsilon_ref PROPERTIES FIXTURES_SETUP KEPSREF)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_kepsilon_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_kepsilon>
+                         32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/keps_ref.txt)
+        set_tests_properties(gpu_parallel_kepsilon_np${_np} PROPERTIES FIXTURES_REQUIRED KEPSREF)
+    endforeach()
+
+    # Phase 4b foundation: the distributed scalar transport solve (deviceParallelScalarTransport) -- the
+    # reusable core of the turbulence step (k/eps/omega/nuTilda) -- reproduces np=1 on a flux-carrying cut.
+    add_executable(test_gpu_parallel_scalar tests/test_gpu_parallel_scalar.cu)
+    target_include_directories(test_gpu_parallel_scalar PRIVATE ${BRAE_HEADER_DIRS}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/tests
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/applications/solvers/simpleFoam)
+    target_link_libraries(test_gpu_parallel_scalar PRIVATE brae_core)
+    add_test(NAME gpu_parallel_scalar_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_scalar>
+                     32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/scalar_ref.txt)
+    set_tests_properties(gpu_parallel_scalar_ref PROPERTIES FIXTURES_SETUP SCALARREF)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_scalar_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_scalar>
+                         32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/scalar_ref.txt)
+        set_tests_properties(gpu_parallel_scalar_np${_np} PROPERTIES FIXTURES_REQUIRED SCALARREF)
+    endforeach()
+
     # Phase 4a: the linearUpwind/LUST interface correction on a cut that CARRIES FLUX (the cavity's cut is a
     # symmetry plane -> phi~0 -> those terms are multiplied by zero and any bug in them is invisible).
     add_executable(test_gpu_parallel_duct tests/test_gpu_parallel_duct.cu)
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 4fe17bb..97a535f 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -802,6 +802,11 @@ public:
     // max |phi| over this rank's processor faces. The interface deferred corrections (linearUpwind, LUST) are
     // all multiplied by the cut-face flux, so a test on a mesh where this is ~0 cannot see them at all -- the
     // 3D cavity cuts on a symmetry midplane and measures ~3e-14 here. Tests assert on this to prove teeth.
+    // Total Krylov iterations (3 momentum BiCGStab + 1 pressure PCG) since construction. A speedup number is
+    // uninterpretable without this: if np=1 and np=2 run different iteration counts (the reduction order
+    // changes convergence slightly), wall-clock per SIMPLE step measures SOLVER WORK, not parallel efficiency.
+    long krylovIters() const { return kIters_; }
+
     scalar maxProcFlux() const
     {
         const std::vector<scalar> b = phiBnd_.host();
@@ -855,6 +860,7 @@ private:
     std::string dumpPath_;
     int    dumpAt_ = -1;
     mutable int iter_ = 0;
+    mutable long kIters_ = 0;
     label  lnC_, nIf_, nBnd_ = 0;
     bool   validC_[3] = { true, true, true };
     DeviceMesh           dm_;
@@ -1095,6 +1101,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
         const scalar nf = deviceParallelNormFactor(mv, halo_, ifCoeff, Uk_[k], b, ones_, P_.globalNCells);
         const DeviceSolverPerf up =
             deviceParallelJacobiBiCGStab(mv, halo_, ifCoeff, b, Uk_[k], nf, tolU_, 0.0, maxIter_);
+        kIters_ += up.nIterations;
         if (validC_[k] && up.initialResidual > res.Ux) res.Ux = up.initialResidual;
         dumpStage("Upred", Uk_[k], k);
     }
@@ -1212,6 +1219,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     const DeviceLduView pv = deviceLduView(dm_, pDiagC, pU_, pL_);
     const scalar nfp = deviceParallelNormFactor(pv, halo_, pIfCoeff, pSol, pb, ones_, P_.globalNCells);
     const DeviceSolverPerf pp = deviceParallelJacobiPCG(pv, halo_, pIfCoeff, pb, pSol, nfp, tolP_, 0.0, maxIter_);
+    kIters_ += pp.nIterations;
     res.p = pp.initialResidual;
     dumpStage("pSol", pSol);
 
diff --git a/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh b/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh
new file mode 100644
index 0000000..71b982c
--- /dev/null
+++ b/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh
@@ -0,0 +1,347 @@
+#pragma once
+// brae distributed DEVICE turbulence (Phase 4b). Built on the 4a machinery -- NO new communication mechanism,
+// just more fields through the same processor interface (the plan's promise). The reusable core is
+// deviceParallelScalarTransport: one distributed k/epsilon/omega/nuTilda transport solve. It composes ONLY
+// primitives already teeth-tested in 4a:
+//   deviceMomentumInterface  -- the processor convection+diffusion coeff (scalar-valued; w*phi + coeff)
+//   deviceInterfaceOffDiagSum + deviceRelaxDiag -- relax with the interface in the diagonal-dominance sum
+//   deviceParallelJacobiBiCGStab -- the interface-coupled non-symmetric solve
+// mirroring the host solveScalarEqn (parallel_kepsilon.cuh): momentumDistributed -> relaxScalar -> setValues
+// -> parallelPBiCGStab.
+//
+// The wall functions and the nut update are ENTIRELY LOCAL (a cell owns its own wall face; a cut cannot
+// separate them -- confirmed by the host oracle's "WALL FUNCTIONS are unchanged and entirely local"), so they
+// reuse the single-GPU kernels unchanged. The one genuinely new device piece is the near-wall setValues
+// constraint (fixed epsilon at wall-adjacent cells), which also zeroes those cells' interface coeffs.
+#include "parallel_device_simple.cuh"
+#include "device_boundary.cuh"
+#include "device_kepsilon.cuh"
+#include <vector>
+
+namespace brae {
+
+namespace detail {
+
+// setValues (eps near-wall constraint) for the DISTRIBUTED matrix -- the exact analogue of the single-GPU
+// svFace/svBnd/svCell (device_kepsilon.cu), plus the processor interface. A wall cell's eps is FIXED = eps0, so
+// its entire row must become diag*eps = diag*eps0: zero every off-diagonal (internal, boundary AND interface)
+// and move the known eps0 to each neighbour's RHS. My first version zeroed only the interface, which left the
+// wall cell's INTERNAL off-diagonals live -- so at np=1 it coupled to its x+1 neighbour (internal) while at
+// np>1 that same coupling was the zeroed interface: a ~2% cut-face mismatch. All four pieces are required.
+__global__
+void svFaceKernel(   // internal faces: zero wall-cell off-diagonals, move eps0 to the neighbour RHS
+    int nIf, const label* __restrict__ own, const label* __restrict__ nei, const label* __restrict__ isW,
+    const scalar* __restrict__ eps0, scalar* __restrict__ upper, scalar* __restrict__ lower, scalar* __restrict__ source)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= nIf) return;
+    const int o = own[f], n = nei[f];
+    const bool ow = isW[o], nw = isW[n];
+    if (ow) atomicAdd(&source[n], -lower[f] * eps0[o]);
+    if (nw) atomicAdd(&source[o], -upper[f] * eps0[n]);
+    if (ow || nw) { upper[f] = 0.0; lower[f] = 0.0; }
+}
+
+__global__
+void svBndKernel(   // boundary faces: zero wall-cell boundary coeffs
+    int nB, const label* __restrict__ faceCell, const label* __restrict__ isW,
+    scalar* __restrict__ iC, scalar* __restrict__ bC)
+{
+    const int bi = blockIdx.x * blockDim.x + threadIdx.x;
+    if (bi < nB && isW[faceCell[bi]]) { iC[bi] = 0.0; bC[bi] = 0.0; }
+}
+
+__global__
+void svCellKernel(   // wall cells: source = relaxedDiag * eps0 (with off-diagonals zeroed -> eps = eps0)
+    int nC, const label* __restrict__ isW, const scalar* __restrict__ relaxedDiag,
+    const scalar* __restrict__ eps0, scalar* __restrict__ source)
+{
+    const int c = blockIdx.x * blockDim.x + threadIdx.x;
+    if (c < nC && isW[c]) source[c] = relaxedDiag[c] * eps0[c];
+}
+
+__global__
+void svInterfaceKernel(   // processor faces: zero the interface off-diagonal of a wall-cell owner
+    const label* __restrict__ faceCells, const label* __restrict__ isW, scalar* __restrict__ ifCoeff, int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f < n && isW[faceCells[f]]) ifCoeff[f] = 0.0;
+}
+
+
+// eps equation local source assembly (mirrors host parallelCorrect exactly). sp = ((2/3)C1 - C3)*divU.
+//   Sp[c] = ( C2*eps/k + max(sp,0) - bounded*divU ) * V          (implicit, added to the diagonal)
+//   Su[c] = ( C1*Cmu*k*gByNu - min(sp,0)*eps )     * V          (explicit source)
+__global__
+void epsSourceKernel(
+    const scalar* __restrict__ k, const scalar* __restrict__ eps, const scalar* __restrict__ gByNu,
+    const scalar* __restrict__ divU, const scalar* __restrict__ V,
+    scalar C1, scalar C2, scalar C3, scalar Cmu, int bounded,
+    scalar* __restrict__ Sp, scalar* __restrict__ Su, int nC)
+{
+    const int c = blockIdx.x * blockDim.x + threadIdx.x;
+    if (c >= nC) return;
+    const scalar sp = ((2.0/3.0)*C1 - C3) * divU[c];
+    Sp[c] = ( C2 * eps[c] / k[c] + fmax(sp, 0.0) - (bounded ? divU[c] : 0.0) ) * V[c];
+    Su[c] = ( C1 * Cmu * k[c] * gByNu[c] - fmin(sp, 0.0) * eps[c] ) * V[c];
+}
+
+// k equation local source assembly. sp = (2/3)*divU. G = the production (wall-overridden at wall cells).
+//   Sp[c] = ( eps/k + max(sp,0) - bounded*divU ) * V ; Su[c] = ( G - min(sp,0)*k ) * V
+__global__
+void kSourceKernel(
+    const scalar* __restrict__ k, const scalar* __restrict__ eps, const scalar* __restrict__ G,
+    const scalar* __restrict__ divU, const scalar* __restrict__ V, int bounded,
+    scalar* __restrict__ Sp, scalar* __restrict__ Su, int nC)
+{
+    const int c = blockIdx.x * blockDim.x + threadIdx.x;
+    if (c >= nC) return;
+    const scalar sp = (2.0/3.0) * divU[c];
+    Sp[c] = ( eps[c] / k[c] + fmax(sp, 0.0) - (bounded ? divU[c] : 0.0) ) * V[c];
+    Su[c] = ( G[c] - fmin(sp, 0.0) * k[c] ) * V[c];
+}
+
+// production G = nut * gByNu, then OVERRIDE G at wall cells with G0 and set eps := eps0 (host wall treatment).
+__global__
+void prodAndWallKernel(
+    const scalar* __restrict__ nut, const scalar* __restrict__ gByNu,
+    const label*  __restrict__ isWall, const scalar* __restrict__ eps0, const scalar* __restrict__ G0,
+    scalar* __restrict__ G, scalar* __restrict__ eps, int nC)
+{
+    const int c = blockIdx.x * blockDim.x + threadIdx.x;
+    if (c >= nC) return;
+    if (isWall[c]) { G[c] = G0[c]; eps[c] = eps0[c]; }
+    else           { G[c] = nut[c] * gByNu[c]; }
+}
+
+// lower-bound a field in place (OF bound(psi, small)).
+__global__
+void boundFieldKernel(scalar* __restrict__ psi, scalar lo, int nC)
+{
+    const int c = blockIdx.x * blockDim.x + threadIdx.x;
+    if (c >= nC) return;
+    psi[c] = fmax(psi[c], lo);
+}
+
+} // namespace detail
+
+// Solve one distributed scalar transport equation on the device, matching host solveScalarEqn:
+//     div(phi, psi) - laplacian(gamma, psi) + Sp*psi == Su      (+ processor interface, + wall setValues)
+// psi is solved IN PLACE. Returns the initial residual (OF residualControl measure). All the *_interface
+// helpers are the 4a ones. `phiF[i]`/`gammaCoeffGeo[i]` are the processor-face flux and gammaF*magSf*procDelta
+// of interface i (same order as the halo), `Sp`/`Su` the local implicit/explicit source (already assembled by
+// the caller, as the host adds them to Mloc before solveScalarEqn). conCells/conVals: near-wall constraint.
+inline scalar deviceParallelScalarTransport(
+    const DeviceMesh& dm,
+    DeviceHalo& halo,
+    const DeviceBoundary& db,
+    const std::vector<DeviceBuffer<label>>& faceCellsD,
+    const std::vector<label>& procStart,
+    const std::vector<DeviceBuffer<scalar>>& weights,
+    const DeviceBuffer<scalar>& phiInt,
+    const DeviceBuffer<scalar>& phiBnd,
+    const std::vector<DeviceBuffer<scalar>>& phiF,
+    const DeviceBuffer<scalar>& gammaFint,
+    const DeviceBuffer<scalar>& gammaBnd,
+    const std::vector<DeviceBuffer<scalar>>& gammaCoeffGeo,
+    const DeviceBuffer<scalar>& Sp,
+    const DeviceBuffer<scalar>& Su,
+    DeviceBuffer<scalar>& psi,
+    scalar relax,
+    scalar tol,
+    int maxIter,
+    label globalNCells,
+    const DeviceBuffer<scalar>& ones,
+    const DeviceBuffer<label>&               isWallCell,  // per-cell wall mask (empty -> no constraint, e.g. k)
+    const DeviceBuffer<scalar>&              wallVal)     // per-cell eps0 (read only where isWallCell)
+{
+    const label lnC = dm.nCells;
+
+    // local matrix: div(phi,psi) - laplacian(gamma,psi)
+    DeviceBuffer<scalar> mDiag, mUp, mLo, lD, lU, lL;
+    deviceDivUpwindCoeffs(dm, phiInt, mDiag, mUp, mLo);
+    deviceLaplacianCoeffs(dm, gammaFint, lD, lU, lL, false);
+    deviceAxpy(-1.0, lD, mDiag);
+    deviceAxpy(-1.0, lU, mUp);
+    deviceAxpy(-1.0, lL, mLo);
+    deviceAxpy(1.0, Sp, mDiag);                              // + Sp on the diagonal (local)
+
+    // processor interface: convection + diffusion coeff (scalar), folds into mDiag, yields ifCoeff
+    std::vector<DeviceBuffer<scalar>> ifCoeff;
+    deviceMomentumInterface(halo, faceCellsD, phiF, gammaCoeffGeo, mDiag, ifCoeff);
+
+    // boundary coeffs of div - laplacian
+    DeviceBuffer<scalar> iC, bC, lIC, lBC;
+    deviceBCDivCoeffs(db, phiBnd, iC, bC);
+    deviceBCLaplacianCoeffsFace(db, gammaBnd, lIC, lBC);
+    deviceAxpy(-1.0, lIC, iC);
+    deviceAxpy(-1.0, lBC, bC);
+
+    // relax (interface counts toward diagonal dominance), source += delta*psi_old
+    DeviceBuffer<scalar> sumOff(std::vector<scalar>(lnC, 0.0));
+    deviceInterfaceOffDiagSum(halo, faceCellsD, ifCoeff, sumOff);
+    DeviceBuffer<scalar> mDiagR, delta;
+    deviceRelaxDiag(deviceLduView(dm, mDiag, mUp, mLo), dm, iC, relax, mDiagR, delta, sumOff.data());
+    DeviceBuffer<scalar> src;
+    deviceHadamard(src, delta, psi);
+    deviceAxpy(1.0, Su, src);                                // + explicit source (local)
+
+    // near-wall setValues (eps): pin psi = eps0 at wall cells by zeroing their ENTIRE row -- internal
+    // off-diagonals (svFace, moving eps0 to neighbour RHS), boundary coeffs (svBnd), the processor interface
+    // (svInterface), and setting source = relaxedDiag*eps0 (svCell). Applied BEFORE the fold, exactly as the
+    // single-GPU deviceSolveScalarTransport. Empty mask -> no constraint (the k equation).
+    if (isWallCell.size() > 0)
+    {
+        constexpr int TPB = 128;
+        detail::svFaceKernel<<<(dm.nInternalFaces + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+            dm.nInternalFaces, dm.owner.data(), dm.nei.data(), isWallCell.data(), wallVal.data(),
+            mUp.data(), mLo.data(), src.data());
+        detail::svBndKernel<<<(dm.nBndFaces + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+            dm.nBndFaces, dm.bndCell.data(), isWallCell.data(), iC.data(), bC.data());
+        for (int i = 0; i < halo.nInterfaces(); ++i)
+        {
+            const int n = static_cast<int>(halo.size(i));
+            if (n <= 0) continue;
+            detail::svInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+                faceCellsD[i].data(), isWallCell.data(), ifCoeff[i].data(), n);
+        }
+        detail::svCellKernel<<<(lnC + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+            lnC, isWallCell.data(), mDiagR.data(), wallVal.data(), src.data());
+    }
+
+    // fold boundary iC/bC into diag/source
+    DeviceBuffer<scalar> diagC, b;
+    deviceFold(dm, mDiagR, src, iC, bC, diagC, b);
+
+    const DeviceLduView mv = deviceLduView(dm, diagC, mUp, mLo);
+    const scalar nf = deviceParallelNormFactor(mv, halo, ifCoeff, psi, b, ones, globalNCells);
+    const DeviceSolverPerf perf =
+        deviceParallelJacobiBiCGStab(mv, halo, ifCoeff, b, psi, nf, tol, 0.0, maxIter);
+    return perf.initialResidual;
+}
+
+
+// Distributed device kEpsilon::correct() -- the composition, mirroring host kepsilon::parallelCorrect exactly.
+// k/eps/nut updated in place. Reuses the LOCAL single-GPU kernels unchanged (wall functions, GbyNu, nut) and
+// the distributed scalar-transport core (deviceParallelScalarTransport) for the two transport solves. The only
+// processor-coupled pieces are (a) gradU: U's cut-face value from the halo before gaussGrad, and (b) Deps/Dk at
+// cut faces: the halo-interpolated diffusivity. Everything else -- production, wall functions, the Sp/Su source
+// terms, nut -- is entirely local (a cell owns its own wall face; the host oracle confirms wall functions are
+// local). `geomD[i]` is magSf*procDelta of interface i (static). `nutBndOut` receives the wall-function nut.
+inline void parallelDeviceKEpsilonCorrect(
+    const DeviceMesh& dm,
+    DeviceHalo& halo,
+    const DeviceWallData& wall,
+    const DeviceVectorBoundary& dbU,
+    const DeviceBoundary& dbK,
+    const DeviceBoundary& dbEps,
+    const DeviceBuffer<scalar>& Ux, const DeviceBuffer<scalar>& Uy, const DeviceBuffer<scalar>& Uz,
+    DeviceBuffer<scalar>& k, DeviceBuffer<scalar>& eps, DeviceBuffer<scalar>& nut,
+    const DeviceBuffer<scalar>& phiInt, const DeviceBuffer<scalar>& phiBnd,
+    const std::vector<DeviceBuffer<scalar>>& phiF,
+    const std::vector<DeviceBuffer<label>>& faceCellsD,
+    const std::vector<label>& procStart,
+    const std::vector<DeviceBuffer<scalar>>& weightsD,
+    const std::vector<DeviceBuffer<scalar>>& geomD,      // magSf*procDelta per interface
+    const DeviceBuffer<label>& isWallCell,
+    scalar nu, scalar relaxEps, scalar relaxK, scalar tol, int maxIter, label globalNCells,
+    const DeviceBuffer<scalar>& ones, bool bounded, const KEpsilonCoeffs& co,
+    scalar* resEps = nullptr, scalar* resK = nullptr)
+{
+    constexpr int TPB = 128;
+    const int nC = dm.nCells;
+    auto nb = [&](int n){ return (n + TPB - 1) / TPB; };
+
+    // ---- halo-consistent gradU (9*nC, column i = gaussGrad(U_i)); processor cut value from the halo ----
+    const DeviceBuffer<scalar>* Uc[3] = { &Ux, &Uy, &Uz };
+    DeviceBuffer<scalar> gradU(static_cast<std::size_t>(9) * nC);
+    for (int i = 0; i < 3; ++i)
+    {
+        DeviceBuffer<scalar> ub;
+        deviceBCValue(dbU.comp[i], *Uc[i], ub);
+        halo.exchange(Uc[i]->data());
+        halo.scatterBoundaryValues(Uc[i]->data(), weightsD, procStart, ub.data());
+        halo.waitExchange();
+        DeviceBuffer<scalar> gx, gy, gz;
+        deviceGaussGrad(dm, *Uc[i], ub, gx, gy, gz);
+        cudaCheck(cudaMemcpyAsync(gradU.data() + (0*3+i)*static_cast<std::size_t>(nC), gx.data(), nC*sizeof(scalar), cudaMemcpyDeviceToDevice, cudaStreamPerThread), "keGradU x");
+        cudaCheck(cudaMemcpyAsync(gradU.data() + (1*3+i)*static_cast<std::size_t>(nC), gy.data(), nC*sizeof(scalar), cudaMemcpyDeviceToDevice, cudaStreamPerThread), "keGradU y");
+        cudaCheck(cudaMemcpyAsync(gradU.data() + (2*3+i)*static_cast<std::size_t>(nC), gz.data(), nC*sizeof(scalar), cudaMemcpyDeviceToDevice, cudaStreamPerThread), "keGradU z");
+    }
+    DeviceBuffer<scalar> gByNu;
+    deviceGByNuFromGradU(gradU, nC, gByNu);
+
+    // ---- divU (processor-consistent via the maintained phi) ----
+    DeviceBuffer<scalar> divU;
+    deviceDiv(dm, phiInt, phiBnd, divU);
+
+    // ---- wall functions (LOCAL) -> eps0, G0 at wall cells ----
+    DeviceBuffer<scalar> eps0, G0;
+    deviceWallEpsG0(wall, k, Ux, Uy, Uz, nu, eps0, G0, co);
+
+    // production G = nut*gByNu, overridden to G0 at wall cells; eps := eps0 at wall cells
+    DeviceBuffer<scalar> G(nC);
+    detail::prodAndWallKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(
+        nut.data(), gByNu.data(), isWallCell.data(), eps0.data(), G0.data(), G.data(), eps.data(), nC);
+
+    // ---- helper: Deps/Dk = gammaCell/sigma + nu, at internal + boundary + processor faces ----
+    auto faceGammaGeo = [&](const DeviceBuffer<scalar>& gammaCell,
+                            DeviceBuffer<scalar>& gInt, DeviceBuffer<scalar>& gBnd,
+                            std::vector<DeviceBuffer<scalar>>& gCoeffGeo)
+    {
+        deviceInterpolate(dm, gammaCell, gInt);                 // internal faces
+        deviceBCValue(dbEps, gammaCell, gBnd);                  // boundary faces (zeroGradient shape -> adj cell)
+        // processor faces: halo-interpolated gammaCell
+        DeviceBuffer<scalar> gBndProc;
+        deviceBCValue(dbEps, gammaCell, gBndProc);
+        halo.exchange(gammaCell.data());
+        halo.scatterBoundaryValues(gammaCell.data(), weightsD, procStart, gBndProc.data());
+        halo.waitExchange();
+        const std::vector<scalar> gh = gBndProc.host();
+        gCoeffGeo.clear(); gCoeffGeo.resize(procStart.size());
+        for (std::size_t i = 0; i < procStart.size(); ++i)
+        {
+            const int n = static_cast<int>(halo.size(static_cast<int>(i)));
+            if (n <= 0) continue;
+            const std::vector<scalar> gd = geomD[i].host();
+            std::vector<scalar> cg(n);
+            for (int f = 0; f < n; ++f) cg[f] = gh[procStart[i] + f] * gd[f];   // gammaFace * magSf*procDelta
+            gCoeffGeo[i].copyFrom(cg);
+        }
+    };
+
+    // ---- epsilon equation ----
+    DeviceBuffer<scalar> DepsCell(nC), Deps_i, Deps_b;
+    { std::vector<scalar> d(nC); const std::vector<scalar> nh = nut.host(); for (int c=0;c<nC;++c) d[c]=nh[c]/co.sigmaEps+nu; DepsCell.copyFrom(d); }
+    std::vector<DeviceBuffer<scalar>> DepsGeo;
+    faceGammaGeo(DepsCell, Deps_i, Deps_b, DepsGeo);
+    DeviceBuffer<scalar> SpE(nC), SuE(nC);
+    detail::epsSourceKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(
+        k.data(), eps.data(), gByNu.data(), divU.data(), dm.V.data(),
+        co.C1, co.C2, co.C3, co.Cmu, bounded ? 1 : 0, SpE.data(), SuE.data(), nC);
+    const scalar rE = deviceParallelScalarTransport(dm, halo, dbEps, faceCellsD, procStart, weightsD,
+        phiInt, phiBnd, phiF, Deps_i, Deps_b, DepsGeo, SpE, SuE, eps, relaxEps, tol, maxIter, globalNCells, ones,
+        isWallCell, eps0);
+    detail::boundFieldKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(eps.data(), 1e-15, nC);
+    if (resEps) *resEps = rE;
+
+    // ---- k equation ----
+    DeviceBuffer<scalar> DkCell(nC), Dk_i, Dk_b;
+    { std::vector<scalar> d(nC); const std::vector<scalar> nh = nut.host(); for (int c=0;c<nC;++c) d[c]=nh[c]/co.sigmaK+nu; DkCell.copyFrom(d); }
+    std::vector<DeviceBuffer<scalar>> DkGeo;
+    faceGammaGeo(DkCell, Dk_i, Dk_b, DkGeo);
+    DeviceBuffer<scalar> SpK(nC), SuK(nC);
+    detail::kSourceKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(
+        k.data(), eps.data(), G.data(), divU.data(), dm.V.data(), bounded ? 1 : 0, SpK.data(), SuK.data(), nC);
+    DeviceBuffer<label> noWall; DeviceBuffer<scalar> noVal;
+    const scalar rK = deviceParallelScalarTransport(dm, halo, dbK, faceCellsD, procStart, weightsD,
+        phiInt, phiBnd, phiF, Dk_i, Dk_b, DkGeo, SpK, SuK, k, relaxK, tol, maxIter, globalNCells, ones,
+        noWall, noVal);
+    detail::boundFieldKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(k.data(), 1e-15, nC);
+    if (resK) *resK = rK;
+
+    // ---- correctNut (LOCAL: nut = Cmu*k^2/eps) ----
+    deviceNut(k, eps, nut, co);
+}
+
+} // namespace brae
diff --git a/src/cuda/device_pcg.cu b/src/cuda/device_pcg.cu
index 6f88d74..87d623e 100644
--- a/src/cuda/device_pcg.cu
+++ b/src/cuda/device_pcg.cu
@@ -303,19 +303,40 @@ DeviceSolverPerf deviceParallelJacobiPCG(
     deviceAxpy(-1.0, Ax, rA);
 
     DeviceSolverPerf perf;
-    perf.initialResidual = Pstream::allReduce(deviceSumMag(rA), ReduceOp::Sum) / normFactor;
+
+    // FUSED REDUCTIONS. Every reduction here is a HOST Pstream::allReduce, and deviceDot/deviceSumMag copy
+    // their scalar to the host first -- so each one is a full GPU sync + blocking collective. At ~100 Krylov
+    // iterations per SIMPLE step that latency, NOT the NVLink halo, is what caps the multi-GPU speedup (it is
+    // a fixed cost that does not shrink as ranks are added, and at np=1 a 1-rank allReduce is nearly free, so
+    // it shows up only when you parallelise). The textbook loop does THREE per iteration:
+    //     wArA = allReduce(dot(wA,rA))     -> beta
+    //     wApA = allReduce(dot(wA,pA))     -> alpha
+    //     resid = allReduce(sumMag(rA))    -> convergence test only
+    // but the residual test and the NEXT iteration's wArA both reduce over the freshly updated rA, so they
+    // fuse into ONE collective of two values: 3 -> 2 per iteration, ~33% fewer round-trips. The arithmetic is
+    // UNCHANGED -- same wArA/wApA/alpha/beta/residual sequence, same iteration count -- it is purely the
+    // packing of the collectives. (`wA` can no longer be reused for A*pA, hence the separate ApA buffer.)
+    DeviceBuffer<scalar> ApA;
+    scalar red[2];
+    auto fusedReduce = [&]()   // wA = M^-1 rA ; red = [ dot(wA,rA), sumMag(rA) ] in ONE collective
+    {
+        deviceJacobi(wA, rA, A.diag);
+        red[0] = deviceDot(wA, rA);
+        red[1] = deviceSumMag(rA);
+        Pstream::allReduce(red, 2, ReduceOp::Sum);
+    };
+
+    fusedReduce();
+    perf.initialResidual = red[1] / normFactor;
     perf.finalResidual   = perf.initialResidual;
     auto converged = [&](scalar fr) { return (fr < tol) || (relTol > 0.0 && fr < relTol * perf.initialResidual); };
 
-    scalar wArA = 1e300, wArAold;
+    scalar wArA = red[0], wArAold = 1e300;
     int nIter = 0;
     if (!converged(perf.finalResidual))
     {
         do
         {
-            wArAold = wArA;
-            deviceJacobi(wA, rA, A.diag);                        // wA = M^-1 rA (local Jacobi)
-            wArA = Pstream::allReduce(deviceDot(wA, rA), ReduceOp::Sum);
             if (nIter == 0) deviceCopy(pA, wA);
             else
             {
@@ -323,12 +344,15 @@ DeviceSolverPerf deviceParallelJacobiPCG(
                 deviceScale(pA, beta);
                 deviceAxpy(1.0, wA, pA);
             }
-            deviceParallelAmul(A, halo, ifaceCoeffs, pA, wA);    // wA = A*pA
-            const scalar wApA  = Pstream::allReduce(deviceDot(wA, pA), ReduceOp::Sum);
+            deviceParallelAmul(A, halo, ifaceCoeffs, pA, ApA);   // ApA = A*pA
+            const scalar wApA  = Pstream::allReduce(deviceDot(ApA, pA), ReduceOp::Sum);
             const scalar alpha = wArA / wApA;
             deviceAxpy(alpha, pA, psi);
-            deviceAxpy(-alpha, wA, rA);
-            perf.finalResidual = Pstream::allReduce(deviceSumMag(rA), ReduceOp::Sum) / normFactor;
+            deviceAxpy(-alpha, ApA, rA);
+            wArAold = wArA;
+            fusedReduce();                                       // next wArA AND the residual, one collective
+            wArA = red[0];
+            perf.finalResidual = red[1] / normFactor;
             ++nIter;
         } while (nIter < maxIter && !converged(perf.finalResidual));
     }
diff --git a/src/parallel/pstream/cf_pstream.cu b/src/parallel/pstream/cf_pstream.cu
index d09d45f..f8d2ad2 100644
--- a/src/parallel/pstream/cf_pstream.cu
+++ b/src/parallel/pstream/cf_pstream.cu
@@ -1,5 +1,7 @@
 // brae::Pstream MPI backend. Re-expressed from OpenFOAM src/Pstream/mpi semantics
 // (non-blocking exchange + Allreduce); not copied verbatim.
+#include <chrono>
+#include <cstdlib>
 #include "cf_pstream.cuh"
 
 #include <mpi.h>
@@ -123,6 +125,7 @@ void Pstream::waitAll()
     g_requests.clear();
 }
 
+
 scalar Pstream::allReduce(scalar v, ReduceOp op)
 {
     scalar out = v;
diff --git a/tests/test_gpu_parallel_kepsilon.cu b/tests/test_gpu_parallel_kepsilon.cu
new file mode 100644
index 0000000..1b0f2c8
--- /dev/null
+++ b/tests/test_gpu_parallel_kepsilon.cu
@@ -0,0 +1,171 @@
+// Phase 4b: the distributed DEVICE kEpsilon::correct() reproduces np=1.
+//
+// This composes the 4b scalar-transport core (validated in test_gpu_parallel_scalar) with the LOCAL single-GPU
+// turbulence kernels (production, wall functions, nut) into the full parallelDeviceKEpsilonCorrect, and gates
+// it the 4a way: np=1 (no processor faces -> the pure local correct(), which reuses the OF-validated single-GPU
+// kernels + the validated scalar core) is the oracle; np>1 must reproduce it.
+//
+// The mesh is a duct decomposed as SLABS IN X. Its walls are the y/z boundaries, so a wall face is NEVER split
+// from its owner cell (the cut is orthogonal to every wall), yet near-wall cells sit ADJACENT to the x-cuts --
+// exactly the "near-wall cell next to a processor interface" case the plan flagged. So the test exercises the
+// wall constraint AND the interface coupling together.
+//
+// TEETH: k/eps must vary across the cut (they develop along the duct from the turbulent inlet). The test
+// asserts the cross-cut spread of k is non-trivial, else the interface terms were multiplied by ~0.
+//
+// Run: mpirun -np {1,2,4} test_gpu_parallel_kepsilon <Nx> <Ny> <Nz> [ref.txt]
+#include "primitive_mesh.cuh"
+#include "fv_geometry.cuh"
+#include "fv_patch.cuh"
+#include "geometric_field.cuh"
+#include "foam_field_reader.cuh"
+#include "fvc.cuh"
+#include "parallel_simple.cuh"
+#include "field_distribute.cuh"
+#include "parallel_device_turbulence.cuh"
+#include "cf_pstream.cuh"
+#include "box_mesh.cuh"
+
+#include <cmath>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <vector>
+
+using namespace brae;
+
+int main(int argc, char** argv)
+{
+    Pstream::init(argc, argv);
+    const int rank = Pstream::myProcNo(), nproc = Pstream::nProcs();
+    if (argc < 4) { if (Pstream::master()) std::printf("usage: %s <Nx> <Ny> <Nz> [ref]\n", argv[0]); Pstream::finalize(); return 2; }
+    if (!Pstream::nvshmemActive()) { if (Pstream::master()) std::printf("test_gpu_parallel_kepsilon: NVSHMEM inactive -- SKIP\n"); Pstream::finalize(); return 0; }
+
+    const label Nx = std::atol(argv[1]), Ny = std::atol(argv[2]), Nz = std::atol(argv[3]);
+    const std::string ref = (argc > 4) ? argv[4] : "keps_ref.txt";
+    const scalar nu = 1e-3, relaxEps = 0.7, relaxK = 0.7, tol = 1e-10;
+    const scalar kIn = 0.375, epsIn = 0.09 * std::pow(kIn, 1.5) / 0.1;   // turbulent inlet
+    const int maxIter = 2000, N = 8;
+    KEpsilonCoeffs co;
+    const label nC = Nx * Ny * Nz;
+    bool ok = true;
+    scalar cutK = 0, cutSpread = 0, relK = 0, relE = 0, relN = 0;
+    {
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz);
+        std::vector<label> cellToPart(nC);
+        for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i)
+            cellToPart[i + Nx * (j + Ny * k)] = static_cast<label>((static_cast<long long>(i) * nproc) / Nx);
+        const Partition P(gm, cellToPart, rank);
+        const FvGeometry& lg = P.lg;
+        const std::vector<FvPatch>& lp = P.lp;
+        const label lnC = P.nCells(), nIf = P.Lm.mesh.nInternalFaces();
+
+        // U: inlet (1,0,0), outlet zeroGradient, no-slip walls
+        FieldData<vector> Ufd; Ufd.internalUniform = true; Ufd.internalUniformValue = vector{1, 0, 0};
+        Ufd.boundary.push_back(boxtest::pfd<vector>("inlet","fixedValue",vector{1,0,0},true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("outlet","zeroGradient",vector{0,0,0},false));
+        for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"})
+            Ufd.boundary.push_back(boxtest::pfd<vector>(w,"fixedValue",vector{0,0,0},true));
+        GeometricField<vector> U = distributeField<vector>(Ufd, gm.patches(), P.Lm, lp, P.procW, rank); U.evaluateBoundary();
+        const SurfaceScalarField phi = fvc::flux(U, P.Lm.mesh, lg, lp);
+
+        // k/eps: inlet fixedValue, outlet zeroGradient, walls zeroGradient (the eps wall value is applied by the
+        // near-wall setValues constraint, not the boundary; k uses kqRWallFunction == zeroGradient)
+        auto scalfd = [&](scalar in){ FieldData<scalar> f; f.internalUniform=true; f.internalUniformValue=in;
+            f.boundary.push_back(boxtest::pfd<scalar>("inlet","fixedValue",in,true));
+            f.boundary.push_back(boxtest::pfd<scalar>("outlet","zeroGradient",0.0,false));
+            for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"}) f.boundary.push_back(boxtest::pfd<scalar>(w,"zeroGradient",0.0,false));
+            return f; };
+        GeometricField<scalar> kF   = distributeField<scalar>(scalfd(kIn),   gm.patches(), P.Lm, lp, P.procW, rank); kF.evaluateBoundary();
+        GeometricField<scalar> epsF = distributeField<scalar>(scalfd(epsIn), gm.patches(), P.Lm, lp, P.procW, rank); epsF.evaluateBoundary();
+        GeometricField<scalar> nutF = distributeField<scalar>(scalfd(co.Cmu*kIn*kIn/epsIn), gm.patches(), P.Lm, lp, P.procW, rank); nutF.evaluateBoundary();
+
+        DeviceMesh dm = buildDeviceMesh(P.Lm.mesh, P.lg, lp);
+        const DeviceBoundary dbK = buildDeviceBoundary(kF, lp, lg), dbEps = buildDeviceBoundary(epsF, lp, lg);
+        const DeviceVectorBoundary dbU = buildDeviceVectorBoundary(U, lp, lg);
+        const DeviceWallData wall = buildDeviceWallData(P.Lm.mesh, lg, lp, U);
+
+        DeviceBuffer<scalar> Ux(lnC), Uy(lnC), Uz(lnC), dk(kF.internal), de(epsF.internal), dnut(nutF.internal), ones(std::vector<scalar>(lnC,1.0));
+        { std::vector<scalar> ux(lnC),uy(lnC),uz(lnC); for(label c=0;c<lnC;++c){ux[c]=U.internal[c].x;uy[c]=U.internal[c].y;uz[c]=U.internal[c].z;} Ux.copyFrom(ux);Uy.copyFrom(uy);Uz.copyFrom(uz); }
+
+        DeviceBuffer<scalar> phiInt, phiBnd;
+        { std::vector<scalar> pi(phi.internal.begin(), phi.internal.begin()+nIf); phiInt.copyFrom(pi); }
+        { std::vector<scalar> pb; for(std::size_t p=0;p<lp.size();++p){if(lp[p].type=="cyclic"||lp[p].type=="cyclicAMI")continue;for(label i=0;i<lp[p].size;++i)pb.push_back(phi.boundary[p][i]);} phiBnd.copyFrom(pb); }
+
+        // isWallCell mask
+        std::vector<label> isW(lnC, 0);
+        for (std::size_t p=0;p<lp.size();++p) if (lp[p].type=="wall") for (label i=0;i<lp[p].size;++i) isW[lp[p].faceCells[i]]=1;
+        DeviceBuffer<label> isWallCell(isW);
+
+        // halo + processor addressing + geomD (magSf*procDelta per interface)
+        std::vector<int> nbrs; for (int q : P.Lm.procNbr) nbrs.push_back(q);
+        DeviceHalo halo(rank, nbrs, P.Lm.procFaceCells);
+        std::vector<DeviceBuffer<label>>  faceCellsD(P.Lm.procFaceCells.size());
+        std::vector<DeviceBuffer<scalar>> weightsD(P.procW.size()), phiF, geomD;
+        for (std::size_t i=0;i<P.Lm.procFaceCells.size();++i) faceCellsD[i].copyFrom(P.Lm.procFaceCells[i]);
+        for (std::size_t i=0;i<P.procW.size();++i) weightsD[i].copyFrom(P.procW[i]);
+        std::vector<label> procStart; label bidx=0; std::size_t pj=0;
+        for (std::size_t p=0;p<lp.size();++p){
+            if(lp[p].type=="cyclic"||lp[p].type=="cyclicAMI")continue;
+            if(lp[p].type=="processor"){
+                procStart.push_back(bidx);
+                std::vector<scalar> pfh(lp[p].size), gd(lp[p].size);
+                for(label i=0;i<lp[p].size;++i){ pfh[i]=phi.boundary[p][i]; gd[i]=lg.magSf()[lp[p].start+i]*P.procDelta[pj][i]; }
+                DeviceBuffer<scalar> pf,g; pf.copyFrom(pfh); g.copyFrom(gd); phiF.push_back(std::move(pf)); geomD.push_back(std::move(g));
+                ++pj;
+            }
+            bidx += lp[p].size;
+        }
+
+        for (int it = 0; it < N; ++it)
+            parallelDeviceKEpsilonCorrect(dm, halo, wall, dbU, dbK, dbEps, Ux, Uy, Uz, dk, de, dnut,
+                phiInt, phiBnd, phiF, faceCellsD, procStart, weightsD, geomD, isWallCell,
+                nu, relaxEps, relaxK, tol, maxIter, P.globalNCells, ones, /*bounded*/false, co);
+        cudaDeviceSynchronize();
+
+        // teeth: cross-cut spread of k
+        {
+            halo.exchange(dk.data()); halo.waitExchange();
+            const std::vector<scalar> ph = phiBnd.host(), kk = dk.host();
+            for (std::size_t i=0;i<procStart.size();++i){
+                const label n = static_cast<label>(halo.size((int)i));
+                const std::vector<scalar> nb = halo.neighbourField((int)i);
+                for (label f=0;f<n;++f){ cutK=std::fmax(cutK,std::fabs(ph[procStart[i]+f])); cutSpread=std::fmax(cutSpread,std::fabs(kk[P.Lm.procFaceCells[i][f]]-nb[f])); }
+            }
+            cutK=Pstream::allReduce(cutK,ReduceOp::Max); cutSpread=Pstream::allReduce(cutSpread,ReduceOp::Max);
+        }
+
+        // gather k/eps/nut to the global mesh
+        auto gather = [&](const DeviceBuffer<scalar>& d){ std::vector<scalar> g(nC,0.0); const std::vector<scalar> h=d.host();
+            for(label c=0;c<lnC;++c) g[P.Lm.cellProcAddr[c]]=h[c]; Pstream::allReduce(g.data(),(int)g.size(),ReduceOp::Sum); return g; };
+        const std::vector<scalar> gk=gather(dk), ge=gather(de), gn=gather(dnut);
+
+        if (Pstream::master()){
+            if (nproc==1){
+                std::ofstream os(ref); os.precision(17);
+                for(label c=0;c<nC;++c) os<<gk[c]<<' '<<ge[c]<<' '<<gn[c]<<'\n';
+                std::printf("test_gpu_parallel_kepsilon np=1: wrote reference %s (%ld cells)\n", ref.c_str(),(long)nC);
+                std::printf("PASS\n");
+            } else {
+                std::ifstream is(ref);
+                if(!is){ std::printf("cannot read %s -- run np=1 first\n", ref.c_str()); ok=false; }
+                else {
+                    scalar nk=0,dkd=0,ne=0,dee=0,nn=0,dnn=0;
+                    for(label c=0;c<nC;++c){ scalar rk,re,rn; is>>rk>>re>>rn;
+                        nk=std::fmax(nk,std::fabs(gk[c]-rk)); dkd=std::fmax(dkd,std::fabs(rk));
+                        ne=std::fmax(ne,std::fabs(ge[c]-re)); dee=std::fmax(dee,std::fabs(re));
+                        nn=std::fmax(nn,std::fabs(gn[c]-rn)); dnn=std::fmax(dnn,std::fabs(rn)); }
+                    relK=dkd>0?nk/dkd:nk; relE=dee>0?ne/dee:ne; relN=dnn>0?nn/dnn:nn;
+                    const bool teeth = (cutK>1e-6) && (cutSpread>1e-6);
+                    std::printf("test_gpu_parallel_kepsilon np=%d: k rel=%.3e  eps rel=%.3e  nut rel=%.3e  k spread at cut=%.3e\n",
+                                nproc, relK, relE, relN, cutSpread);
+                    if(!teeth) std::printf("  NO TEETH: cut flux or k spread ~0\n");
+                    ok = teeth && relK<1e-6 && relE<1e-6 && relN<1e-6;
+                    std::printf("%s\n", ok?"PASS":"FAIL");
+                }
+            }
+        }
+    }
+    Pstream::finalize();
+    return ok ? 0 : 1;
+}
diff --git a/tests/test_gpu_parallel_oom.cu b/tests/test_gpu_parallel_oom.cu
index 2865bb3..98e54ee 100644
--- a/tests/test_gpu_parallel_oom.cu
+++ b/tests/test_gpu_parallel_oom.cu
@@ -26,6 +26,7 @@
 #include "cf_pstream.cuh"
 #include "box_mesh.cuh"
 
+#include <chrono>
 #include <cmath>
 #include <cstdio>
 #include <string>
@@ -101,8 +102,21 @@ int main(int argc, char** argv)
         p0.evaluateBoundary();
 
         ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, maxIter);
+        // SPEEDUP: time the SOLVE only -- mesh generation, decomposition and field distribution are setup, and
+        // at these sizes they dwarf a handful of iterations. One warm-up step first (first-touch allocation,
+        // the halo's symmetric-heap registration and the initial exchanges are one-off costs), then a barrier
+        // so every rank starts the timed region together, and the MAX across ranks (the slowest rank sets the
+        // wall clock; an average would flatter an imbalanced decomposition).
+        solver.step();
+        cudaDeviceSynchronize();
+        Pstream::barrier();
+        const long k0 = solver.krylovIters();
+        const auto t0 = std::chrono::steady_clock::now();
         for (int it = 0; it < N; ++it) solver.step();
         cudaDeviceSynchronize();
+        const scalar secs = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+        const scalar wall = Pstream::allReduce(secs, ReduceOp::Max);
+        const long  kIt = solver.krylovIters() - k0;
 
         const std::vector<scalar> pl = solver.p();
         scalar s = 0;
@@ -117,6 +131,8 @@ int main(int argc, char** argv)
         if (Pstream::master())
         {
             std::printf("  per-rank cells (max): %ld\n", (long)maxCells);
+            std::printf("  SOLVE: %.3f s for %d iters -> %.4f s/iter  (%.3f Mcell-iter/s)  krylov=%ld (%.1f/step)\n",
+                        wall, N, wall / N, (nC / 1e6) * N / wall, kIt, double(kIt) / N);
             std::printf("  per-rank device mem (max): %.2f GiB\n", maxGiB);
             std::printf("  sum(p) = %.6e %s\n", tot, std::isfinite(tot) ? "(finite)" : "(NON-FINITE!)");
             std::printf("%s\n", std::isfinite(tot) ? "PASS" : "FAIL");
diff --git a/tests/test_gpu_parallel_scalar.cu b/tests/test_gpu_parallel_scalar.cu
new file mode 100644
index 0000000..5c8b663
--- /dev/null
+++ b/tests/test_gpu_parallel_scalar.cu
@@ -0,0 +1,189 @@
+// Phase 4b foundation: the distributed DEVICE scalar transport solve reproduces np=1.
+//
+// deviceParallelScalarTransport is the reusable core of the turbulence step -- k, epsilon, omega and nuTilda
+// all reduce to  div(phi,psi) - laplacian(gamma,psi) + Sp*psi == Su. This gates it the way 4a's interface
+// terms were gated: np=1 (no processor faces -> the pure local solve, which composes only OF-validated
+// primitives) is the oracle, and np>1 must reproduce it on a cut that CARRIES FLUX.
+//
+// TEETH: a passive scalar with inlet psi=1 and a volumetric SINK (Sp>0, decay) so psi decays ALONG the duct
+// and therefore VARIES across every (flow-perpendicular) cut. Without that, psi would be uniform and the
+// interface convection term (phi*(psi_local - psi_remote)) would be multiplied by ~0 -- the cavity's mistake.
+// The test asserts both max|phi| at the cut AND the psi spread across the cut are non-trivial.
+//
+// Run: mpirun -np {1,2,4} test_gpu_parallel_scalar <Nx> <Ny> <Nz> [ref.txt] [shear]
+#include "primitive_mesh.cuh"
+#include "fv_geometry.cuh"
+#include "fv_patch.cuh"
+#include "geometric_field.cuh"
+#include "foam_field_reader.cuh"
+#include "fvc.cuh"
+#include "parallel_simple.cuh"
+#include "field_distribute.cuh"
+#include "parallel_device_turbulence.cuh"
+#include "cf_pstream.cuh"
+#include "box_mesh.cuh"
+
+#include <cmath>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <vector>
+
+using namespace brae;
+
+int main(int argc, char** argv)
+{
+    Pstream::init(argc, argv);
+    const int rank = Pstream::myProcNo(), nproc = Pstream::nProcs();
+    if (argc < 4) { if (Pstream::master()) std::printf("usage: %s <Nx> <Ny> <Nz> [ref] [shear]\n", argv[0]); Pstream::finalize(); return 2; }
+    if (!Pstream::nvshmemActive()) { if (Pstream::master()) std::printf("test_gpu_parallel_scalar: NVSHMEM inactive -- SKIP\n"); Pstream::finalize(); return 0; }
+
+    const label Nx = std::atol(argv[1]), Ny = std::atol(argv[2]), Nz = std::atol(argv[3]);
+    const std::string ref = (argc > 4) ? argv[4] : "scalar_ref.txt";
+    const scalar shear = (argc > 5) ? std::atof(argv[5]) : 0.0;
+    const scalar gamma = 1e-2, Spval = 0.03, tol = 1e-10;  // gentle sink: psi decays smoothly along the duct so it VARIES across every flow-perpendicular cut (Sp too large -> psi dies before the cut -> no teeth)
+    const int maxIter = 2000;
+    const label nC = Nx * Ny * Nz;
+    bool ok = true;
+    scalar cutPhi = 0, cutSpread = 0, rel = 0;
+    {
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz, shear);
+        std::vector<label> cellToPart(nC);
+        for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i)
+            cellToPart[i + Nx * (j + Ny * k)] = static_cast<label>((static_cast<long long>(i) * nproc) / Nx);
+        const Partition P(gm, cellToPart, rank);
+        const FvGeometry& lg = P.lg;
+        const std::vector<FvPatch>& lp = P.lp;
+        const label lnC = P.nCells();
+
+        // through-flow phi (u=1 in x) so the cut carries flux
+        FieldData<vector> Ufd;
+        Ufd.internalUniform = true; Ufd.internalUniformValue = vector{1, 0, 0};
+        Ufd.boundary.push_back(boxtest::pfd<vector>("inlet",    "fixedValue",   vector{1, 0, 0}, true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("outlet",   "zeroGradient", vector{0, 0, 0}, false));
+        for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"})
+            Ufd.boundary.push_back(boxtest::pfd<vector>(w, "fixedValue", vector{0,0,0}, true));
+        GeometricField<vector> U = distributeField<vector>(Ufd, gm.patches(), P.Lm, lp, P.procW, rank);
+        U.evaluateBoundary();
+        const SurfaceScalarField phi = fvc::flux(U, P.Lm.mesh, lg, lp);
+
+        // passive scalar psi: inlet fixedValue 1, everything else zeroGradient
+        FieldData<scalar> psifd;
+        psifd.internalUniform = true; psifd.internalUniformValue = 0.0;
+        psifd.boundary.push_back(boxtest::pfd<scalar>("inlet",  "fixedValue",   1.0, true));
+        psifd.boundary.push_back(boxtest::pfd<scalar>("outlet", "zeroGradient", 0.0, false));
+        for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"})
+            psifd.boundary.push_back(boxtest::pfd<scalar>(w, "zeroGradient", 0.0, false));
+        GeometricField<scalar> psiF = distributeField<scalar>(psifd, gm.patches(), P.Lm, lp, P.procW, rank);
+        psiF.evaluateBoundary();
+        const DeviceBoundary db = buildDeviceBoundary(psiF, lp, lg);
+
+        const label nIf = P.Lm.mesh.nInternalFaces();
+        DeviceMesh dm = buildDeviceMesh(P.Lm.mesh, P.lg, lp);
+
+        // device fields
+        DeviceBuffer<scalar> psi(psiF.internal), ones(std::vector<scalar>(lnC, 1.0));
+        DeviceBuffer<scalar> Sp(std::vector<scalar>(lnC, 0.0)), Su(std::vector<scalar>(lnC, 0.0));
+        { std::vector<scalar> sp(lnC); for (label c = 0; c < lnC; ++c) sp[c] = Spval * lg.V()[c]; Sp.copyFrom(sp); }  // V*Sp*psi
+
+        DeviceBuffer<scalar> phiInt, phiBnd;
+        { std::vector<scalar> pi(phi.internal.begin(), phi.internal.begin() + nIf); phiInt.copyFrom(pi); }
+        { std::vector<scalar> pb; for (std::size_t p = 0; p < lp.size(); ++p) { if (lp[p].type=="cyclic"||lp[p].type=="cyclicAMI") continue; for (label i=0;i<lp[p].size;++i) pb.push_back(phi.boundary[p][i]); } phiBnd.copyFrom(pb); }
+
+        // gamma at internal + boundary faces (constant)
+        DeviceBuffer<scalar> gammaFint(std::vector<scalar>(nIf, gamma));
+        label nBnd = 0; for (std::size_t p=0;p<lp.size();++p){ if(lp[p].type=="cyclic"||lp[p].type=="cyclicAMI")continue; nBnd += lp[p].size; }
+        DeviceBuffer<scalar> gammaBnd(std::vector<scalar>(nBnd, gamma));
+
+        // halo + processor addressing
+        std::vector<int> nbrs; for (int q : P.Lm.procNbr) nbrs.push_back(q);
+        DeviceHalo halo(rank, nbrs, P.Lm.procFaceCells);
+        std::vector<DeviceBuffer<label>>  faceCellsD(P.Lm.procFaceCells.size());
+        std::vector<DeviceBuffer<scalar>> weightsD(P.procW.size()), phiF, gammaCoeffGeo;
+        for (std::size_t i=0;i<P.Lm.procFaceCells.size();++i) faceCellsD[i].copyFrom(P.Lm.procFaceCells[i]);
+        for (std::size_t i=0;i<P.procW.size();++i) weightsD[i].copyFrom(P.procW[i]);
+        std::vector<label> procStart; label bidx = 0;
+        std::size_t pj = 0;
+        for (std::size_t p = 0; p < lp.size(); ++p)
+        {
+            if (lp[p].type=="cyclic"||lp[p].type=="cyclicAMI") continue;
+            if (lp[p].type=="processor")
+            {
+                procStart.push_back(bidx);
+                DeviceBuffer<scalar> pf; std::vector<scalar> pfh(lp[p].size);
+                for (label i=0;i<lp[p].size;++i) pfh[i] = phi.boundary[p][i];
+                pf.copyFrom(pfh); phiF.push_back(std::move(pf));
+                std::vector<scalar> cg(lp[p].size);
+                for (label i=0;i<lp[p].size;++i) cg[i] = gamma * lg.magSf()[lp[p].start+i] * P.procDelta[pj][i];
+                DeviceBuffer<scalar> cgd; cgd.copyFrom(cg); gammaCoeffGeo.push_back(std::move(cgd));
+                ++pj;
+            }
+            bidx += lp[p].size;
+        }
+        // no wall constraint for a passive scalar
+        DeviceBuffer<label> noWall; DeviceBuffer<scalar> noWallVal;   // passive scalar: no wall constraint
+
+        // solve to convergence (iterate the transport a few times; it is linear so it converges fast)
+        for (int it = 0; it < 200; ++it)
+            deviceParallelScalarTransport(dm, halo, db, faceCellsD, procStart, weightsD, phiInt, phiBnd, phiF,
+                                          gammaFint, gammaBnd, gammaCoeffGeo, Sp, Su, psi, /*relax*/1.0, tol,
+                                          maxIter, P.globalNCells, ones, noWall, noWallVal);
+        cudaDeviceSynchronize();
+
+        // teeth diagnostics: cut flux, and the TRUE cross-cut jump |psi_local - psi_remote| (psi decays ALONG
+        // the flow, so the coupling that matters is between the local owner and the remote cell across the cut,
+        // NOT the spread among same-plane local cells). Exchange psi once through the halo to get psi_remote.
+        {
+            halo.exchange(psi.data());
+            halo.waitExchange();
+            const std::vector<scalar> ph = phiBnd.host(), ps = psi.host();
+            for (std::size_t i = 0; i < procStart.size(); ++i)
+            {
+                const label n = static_cast<label>(halo.size(static_cast<int>(i)));
+                const std::vector<scalar> nb = halo.neighbourField(static_cast<int>(i));
+                for (label f = 0; f < n; ++f)
+                {
+                    cutPhi = std::fmax(cutPhi, std::fabs(ph[procStart[i]+f]));
+                    cutSpread = std::fmax(cutSpread, std::fabs(ps[P.Lm.procFaceCells[i][f]] - nb[f]));
+                }
+            }
+            cutPhi = Pstream::allReduce(cutPhi, ReduceOp::Max);
+            cutSpread = Pstream::allReduce(cutSpread, ReduceOp::Max);
+        }
+
+        // gather to the global mesh
+        std::vector<scalar> g(nC, 0.0); const std::vector<scalar> ps = psi.host();
+        for (label c = 0; c < lnC; ++c) g[P.Lm.cellProcAddr[c]] = ps[c];
+        Pstream::allReduce(g.data(), (int)g.size(), ReduceOp::Sum);
+
+        if (Pstream::master())
+        {
+            if (nproc == 1)
+            {
+                std::ofstream os(ref); os.precision(17);
+                for (label c = 0; c < nC; ++c) os << g[c] << '\n';
+                std::printf("test_gpu_parallel_scalar np=1: wrote reference %s (%ld cells)\n", ref.c_str(), (long)nC);
+                std::printf("PASS\n");
+            }
+            else
+            {
+                std::ifstream is(ref);
+                if (!is) { std::printf("cannot read %s -- run np=1 first\n", ref.c_str()); ok = false; }
+                else
+                {
+                    scalar num = 0, den = 0;
+                    for (label c = 0; c < nC; ++c) { scalar r; is >> r; num = std::fmax(num, std::fabs(g[c]-r)); den = std::fmax(den, std::fabs(r)); }
+                    rel = den > 0 ? num/den : num;
+                    const bool teeth = (cutPhi > 1e-6) && (cutSpread > 1e-3);
+                    std::printf("test_gpu_parallel_scalar np=%d: rel=%.3e  max|phi| at cut=%.3e  psi spread at cut=%.3e\n",
+                                nproc, rel, cutPhi, cutSpread);
+                    if (!teeth) std::printf("  NO TEETH: cut flux or psi spread ~0\n");
+                    ok = teeth && (rel < 1e-6);
+                    std::printf("%s\n", ok ? "PASS" : "FAIL");
+                }
+            }
+        }
+    }
+    Pstream::finalize();
+    return ok ? 0 : 1;
+}

From 9b779036d3bbb072923e83707c42135ed99ee45c Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 21:05:50 +0200
Subject: [PATCH 06/13] #12: distributed RAS models

---
 CMakeLists.txt                                |  35 +++
 .../simpleFoam/parallel_device_interface.cuh  | 106 +++++++++
 .../simpleFoam/parallel_device_simple.cuh     | 224 ++++++++++--------
 .../simpleFoam/parallel_device_turbulence.cuh | 131 +++++++++-
 tests/test_gpu_parallel_komegasst.cu          | 149 ++++++++++++
 tests/test_gpu_parallel_turbloop.cu           | 122 ++++++++++
 6 files changed, 673 insertions(+), 94 deletions(-)
 create mode 100644 src/applications/solvers/simpleFoam/parallel_device_interface.cuh
 create mode 100644 tests/test_gpu_parallel_komegasst.cu
 create mode 100644 tests/test_gpu_parallel_turbloop.cu

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 198a7a5..95c1841 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -263,6 +263,41 @@ if(BRAE_WITH_NVSHMEM)
                          ${CMAKE_CURRENT_SOURCE_DIR}/validation/kEpsCorrect)
     endforeach()
 
+    # Phase 4b: the distributed kOmegaSST::correct() (2nd RAS model) reproduces np=1
+    add_executable(test_gpu_parallel_komegasst tests/test_gpu_parallel_komegasst.cu)
+    target_include_directories(test_gpu_parallel_komegasst PRIVATE ${BRAE_HEADER_DIRS}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/tests
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/applications/solvers/simpleFoam)
+    target_link_libraries(test_gpu_parallel_komegasst PRIVATE brae_core)
+    add_test(NAME gpu_parallel_komegasst_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_komegasst>
+                     32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/sst_ref.txt)
+    set_tests_properties(gpu_parallel_komegasst_ref PROPERTIES FIXTURES_SETUP SSTREF)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_komegasst_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_komegasst>
+                         32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/sst_ref.txt)
+        set_tests_properties(gpu_parallel_komegasst_np${_np} PROPERTIES FIXTURES_REQUIRED SSTREF)
+    endforeach()
+
+    # Phase 4b: the CLOSED distributed turbulent SIMPLE loop (momentum+pressure with nuEff=nu+nut, then
+    # kEpsilon correct(), maintained across iterations) reproduces np=1.
+    add_executable(test_gpu_parallel_turbloop tests/test_gpu_parallel_turbloop.cu)
+    target_include_directories(test_gpu_parallel_turbloop PRIVATE ${BRAE_HEADER_DIRS}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/tests
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/applications/solvers/simpleFoam)
+    target_link_libraries(test_gpu_parallel_turbloop PRIVATE brae_core)
+    add_test(NAME gpu_parallel_turbloop_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_turbloop>
+                     32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/turbloop_ref.txt)
+    set_tests_properties(gpu_parallel_turbloop_ref PROPERTIES FIXTURES_SETUP TURBLOOP)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_turbloop_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_turbloop>
+                         32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/turbloop_ref.txt)
+        set_tests_properties(gpu_parallel_turbloop_np${_np} PROPERTIES FIXTURES_REQUIRED TURBLOOP)
+    endforeach()
+
     # Phase 4b: the full distributed kEpsilon::correct() (production + wall functions + eps/k transport + nut)
     # reproduces np=1. Duct walls are y/z, cut is x -> near-wall cells sit adjacent to the cut.
     add_executable(test_gpu_parallel_kepsilon tests/test_gpu_parallel_kepsilon.cu)
diff --git a/src/applications/solvers/simpleFoam/parallel_device_interface.cuh b/src/applications/solvers/simpleFoam/parallel_device_interface.cuh
new file mode 100644
index 0000000..5c5c8b0
--- /dev/null
+++ b/src/applications/solvers/simpleFoam/parallel_device_interface.cuh
@@ -0,0 +1,106 @@
+#pragma once
+// Shared distributed-interface helpers used by BOTH parallel_device_simple.cuh (the momentum/pressure step)
+// and parallel_device_turbulence.cuh (the k/eps scalar transport). Extracted here so neither includes the
+// other -- breaking the simple<->turbulence include cycle. Just the two processor-interface primitives the
+// scalar transport needs: the momentum/scalar convection+diffusion interface coeff, and the relax off-diagonal
+// sum. The other interface helpers (laplacian, linearUpwind, non-orth, H, flux) stay in parallel_device_simple.
+#include "cf_types.cuh"
+#include "device_buffer.cuh"
+#include "device_halo.cuh"
+#include <vector>
+
+namespace brae {
+namespace detail {
+
+// L1: the processor-interface coeffs of the momentum matrix M = div(phi,U) - laplacian(nuEff,U). Mirrors host
+// momentumDistributed, per cut face f (upwind weight w = phi>=0 ? 1 : 0, coeff = nuEffF*magSf*procDelta):
+//   diag[faceCell[f]] += w*phi[f] + coeff[f]              (outflow convection + diffusion, atomic)
+//   ifCoeff[f]         = -(1 - w)*phi[f] + coeff[f]        (inflow convection + diffusion off-diagonal)
+__global__
+void momentumInterfaceKernel(
+    const label*  __restrict__ faceCells,
+    const scalar* __restrict__ phi,
+    const scalar* __restrict__ coeff,
+    scalar*       __restrict__ diag,
+    scalar*       __restrict__ ifCoeff,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    const scalar ph = phi[f];
+    const scalar w = (ph >= 0.0) ? 1.0 : 0.0;
+    ifCoeff[f] = -(1.0 - w) * ph + coeff[f];
+    atomicAdd(&diag[faceCells[f]], w * ph + coeff[f]);
+}
+
+// Per-cell sum of |processor off-diagonal|, for fvMatrix::relax's diagonal-dominance term. Host
+// parallelRelaxMatrix adds |interfaceCoeffs| into sumOff; on device this feeds deviceRelaxDiag's cycSumOff
+// hook (the same role the cyclic interface's off-diagonal sum plays).
+__global__
+void offDiagSumKernel(
+    const label*  __restrict__ faceCells,
+    const scalar* __restrict__ ifCoeff,
+    scalar*       __restrict__ sumOff,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    atomicAdd(&sumOff[faceCells[f]], fabs(ifCoeff[f]));
+}
+
+} // namespace detail
+
+// Assemble the momentum matrix's processor coupling on `halo`'s interfaces: fold the convection+diffusion
+// contribution into `diag` and produce `ifCoeff[i]` (per interface, for deviceParallelAmul). `phiF[i]` is the
+// processor-face flux and `coeffGeo[i] = nuEffF*magSf*procDelta` of interface i (same order as the halo).
+inline void deviceMomentumInterface(
+    const DeviceHalo& halo,
+    const std::vector<DeviceBuffer<label>>& faceCells,
+    const std::vector<DeviceBuffer<scalar>>& phiF,
+    const std::vector<DeviceBuffer<scalar>>& coeffGeo,
+    DeviceBuffer<scalar>& diag,
+    std::vector<DeviceBuffer<scalar>>& ifCoeff,
+    cudaStream_t stream = cudaStreamPerThread)
+{
+    constexpr int TPB = 128;
+    const int nI = halo.nInterfaces();
+    ifCoeff.resize(nI);
+    for (int i = 0; i < nI; ++i)
+    {
+        const int n = static_cast<int>(halo.size(i));
+        if (n <= 0) continue;
+        ifCoeff[i].resize(n);
+        detail::momentumInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
+            faceCells[i].data(),
+            phiF[i].data(),
+            coeffGeo[i].data(),
+            diag.data(),
+            ifCoeff[i].data(),
+            n);
+    }
+}
+
+// sumOff[c] += sum over this rank's interface faces owned by c of |ifCoeff|. Pass the result to
+// deviceRelaxDiag's cycSumOff so the processor interface counts toward diagonal dominance, matching host
+// parallelRelaxMatrix. `sumOff` must be zeroed by the caller (size nCells).
+inline void deviceInterfaceOffDiagSum(
+    const DeviceHalo& halo,
+    const std::vector<DeviceBuffer<label>>& faceCells,
+    const std::vector<DeviceBuffer<scalar>>& ifCoeff,
+    DeviceBuffer<scalar>& sumOff,
+    cudaStream_t stream = cudaStreamPerThread)
+{
+    constexpr int TPB = 128;
+    for (int i = 0; i < halo.nInterfaces(); ++i)
+    {
+        const int n = static_cast<int>(halo.size(i));
+        if (n <= 0) continue;
+        detail::offDiagSumKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
+            faceCells[i].data(),
+            ifCoeff[i].data(),
+            sumOff.data(),
+            n);
+    }
+}
+
+} // namespace brae
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 97a535f..5f8f5f1 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -19,6 +19,8 @@
 #include "parallel_simple.cuh"     // Partition, distributeFromCells
 #include "reconstruct.cuh"
 #include "local_assembly.cuh"   // computeProcUpwindD
+#include "device_kepsilon.cuh"   // DeviceWallData / KEpsilonCoeffs / buildDeviceWallData (Phase 4b turbulence)
+#include "parallel_device_interface.cuh"   // deviceMomentumInterface / deviceInterfaceOffDiagSum (shared)
 #include <cuda_runtime.h>
 #include <fstream>
 #include <string>
@@ -28,27 +30,6 @@ namespace brae {
 
 namespace detail {
 
-// L1: the processor-interface coeffs of the momentum matrix M = div(phi,U) - laplacian(nuEff,U). Mirrors host
-// momentumDistributed, per cut face f (upwind weight w = phi>=0 ? 1 : 0, coeff = nuEffF*magSf*procDelta):
-//   diag[faceCell[f]] += w*phi[f] + coeff[f]              (outflow convection + diffusion, atomic)
-//   ifCoeff[f]         = -(1 - w)*phi[f] + coeff[f]        (inflow convection + diffusion off-diagonal)
-__global__
-void momentumInterfaceKernel(
-    const label*  __restrict__ faceCells,
-    const scalar* __restrict__ phi,
-    const scalar* __restrict__ coeff,
-    scalar*       __restrict__ diag,
-    scalar*       __restrict__ ifCoeff,
-    int n)
-{
-    const int f = blockIdx.x * blockDim.x + threadIdx.x;
-    if (f >= n) return;
-    const scalar ph = phi[f];
-    const scalar w = (ph >= 0.0) ? 1.0 : 0.0;
-    ifCoeff[f] = -(1.0 - w) * ph + coeff[f];
-    atomicAdd(&diag[faceCells[f]], w * ph + coeff[f]);
-}
-
 // L4: the processor contribution to H(). lduMatrix::H(psi) = -offdiag.psi, and the PARALLEL A.psi carries
 // -ifCoeff*psiNbr at each interface face, so H picks up +ifCoeff*psiNbr there. deviceMatrixH already divided
 // by V, hence the /V. atomicAdd: a cell may own several faces on one interface.
@@ -258,76 +239,8 @@ void subtractSliceKernel(scalar* __restrict__ dst, int offset, const scalar* __r
     dst[offset + f] -= src[f];
 }
 
-// Per-cell sum of |processor off-diagonal|, for fvMatrix::relax's diagonal-dominance term. Host
-// parallelRelaxMatrix adds |interfaceCoeffs| into sumOff; on device this feeds deviceRelaxDiag's cycSumOff
-// hook (the same role the cyclic interface's off-diagonal sum plays).
-__global__
-void offDiagSumKernel(
-    const label*  __restrict__ faceCells,
-    const scalar* __restrict__ ifCoeff,
-    scalar*       __restrict__ sumOff,
-    int n)
-{
-    const int f = blockIdx.x * blockDim.x + threadIdx.x;
-    if (f >= n) return;
-    atomicAdd(&sumOff[faceCells[f]], fabs(ifCoeff[f]));
-}
-
 } // namespace detail
 
-// Assemble the momentum matrix's processor coupling on `halo`'s interfaces: fold the convection+diffusion
-// contribution into `diag` and produce `ifCoeff[i]` (per interface, for deviceParallelAmul). `phiF[i]` is the
-// processor-face flux and `coeffGeo[i] = nuEffF*magSf*procDelta` of interface i (same order as the halo).
-inline void deviceMomentumInterface(
-    const DeviceHalo& halo,
-    const std::vector<DeviceBuffer<label>>& faceCells,
-    const std::vector<DeviceBuffer<scalar>>& phiF,
-    const std::vector<DeviceBuffer<scalar>>& coeffGeo,
-    DeviceBuffer<scalar>& diag,
-    std::vector<DeviceBuffer<scalar>>& ifCoeff,
-    cudaStream_t stream = cudaStreamPerThread)
-{
-    constexpr int TPB = 128;
-    const int nI = halo.nInterfaces();
-    ifCoeff.resize(nI);
-    for (int i = 0; i < nI; ++i)
-    {
-        const int n = static_cast<int>(halo.size(i));
-        if (n <= 0) continue;
-        ifCoeff[i].resize(n);
-        detail::momentumInterfaceKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
-            faceCells[i].data(),
-            phiF[i].data(),
-            coeffGeo[i].data(),
-            diag.data(),
-            ifCoeff[i].data(),
-            n);
-    }
-}
-
-// sumOff[c] += sum over this rank's interface faces owned by c of |ifCoeff|. Pass the result to
-// deviceRelaxDiag's cycSumOff so the processor interface counts toward diagonal dominance, matching host
-// parallelRelaxMatrix. `sumOff` must be zeroed by the caller (size nCells).
-inline void deviceInterfaceOffDiagSum(
-    const DeviceHalo& halo,
-    const std::vector<DeviceBuffer<label>>& faceCells,
-    const std::vector<DeviceBuffer<scalar>>& ifCoeff,
-    DeviceBuffer<scalar>& sumOff,
-    cudaStream_t stream = cudaStreamPerThread)
-{
-    constexpr int TPB = 128;
-    for (int i = 0; i < halo.nInterfaces(); ++i)
-    {
-        const int n = static_cast<int>(halo.size(i));
-        if (n <= 0) continue;
-        detail::offDiagSumKernel<<<(n + TPB - 1) / TPB, TPB, 0, stream>>>(
-            faceCells[i].data(),
-            ifCoeff[i].data(),
-            sumOff.data(),
-            n);
-    }
-}
-
 // Add the processor faces' linearUpwind deferred correction into `corr` (which already holds the internal
 // deviceLinearUpwindCorr result). `gx/gy/gz` are grad(U_comp) as CELL fields -- already processor-consistent
 // (the halo injected the coupled face value before the gaussGrad). The three exchanges below fetch that same
@@ -526,6 +439,7 @@ inline void deviceLapCorrPInterface(
     }
 }
 
+
 // Assemble a laplacian(gamma, .) matrix's processor coupling (the pressure equation): fold -coeff into `diag`
 // and produce `ifCoeff[i]` per interface. `coeffGeo[i] = gammaF*magSf*procDelta` of interface i.
 inline void deviceLaplacianInterface(
@@ -823,6 +737,59 @@ public:
     // The global field, gathered from every partition (decomposePar's inverse) -- for output/validation.
     std::vector<scalar> reconstructP() const { return reconstructField(P_.Lm.cellProcAddr, dp_.host(), P_.globalNCells); }
 
+    // Turn on the distributed k-epsilon model: k/eps/nut become device-resident, step() then uses
+    // nuEff = nu + nut for the momentum and runs parallelDeviceKEpsilonCorrect after the corrector. The three
+    // fields carry their OpenFOAM boundary shapes (wall functions etc.). Idempotent per constructed solver.
+    void enableTurbulence(const GeometricField<vector>& U0, const GeometricField<scalar>& k0,
+                          const GeometricField<scalar>& eps0, const GeometricField<scalar>& nut0,
+                          const KEpsilonCoeffs& co, scalar relaxK = 0.7, scalar relaxEps = 0.7)
+    {
+        turbulent_ = true;
+        relaxK_ = relaxK; relaxEps_ = relaxEps;
+        keCoeffs_ = co;
+        k_.copyFrom(k0.internal); eps_.copyFrom(eps0.internal); nut_.copyFrom(nut0.internal);
+        dbK_   = buildDeviceBoundary(k0,   P_.lp, P_.lg);
+        dbEps_ = buildDeviceBoundary(eps0, P_.lp, P_.lg);
+        wall_ = buildDeviceWallData(P_.Lm.mesh, P_.lg, P_.lp, U0);   // proper no-slip wall BCs
+        std::vector<label> isW(lnC_, 0);
+        for (std::size_t pi = 0; pi < P_.lp.size(); ++pi)
+            if (P_.lp[pi].type == "wall")
+                for (label i = 0; i < P_.lp[pi].size; ++i) isW[P_.lp[pi].faceCells[i]] = 1;
+        isWallCell_.copyFrom(isW);
+        // per-boundary-FACE wall mask + near-wall distance (deviceBoundaryNut indexes by boundary face, NOT by
+        // cell): iterate patches skipping cyclic, exactly as the single-GPU bndIsWall_/bndY_ (device_simple_foam).
+        {
+            const std::vector<std::vector<scalar>> yW = nearWallDist(P_.Lm.mesh, P_.lg, P_.lp);
+            std::vector<label> biw; std::vector<scalar> byy;
+            for (std::size_t pi = 0; pi < P_.lp.size(); ++pi)
+            {
+                if (P_.lp[pi].type == "cyclic" || P_.lp[pi].type == "cyclicAMI") continue;
+                const bool wall = (P_.lp[pi].type == "wall");
+                for (label i = 0; i < P_.lp[pi].size; ++i) { biw.push_back(wall ? 1 : 0); byy.push_back(wall ? yW[pi][i] : 0.0); }
+            }
+            bndIsWall_.copyFrom(biw);
+            bndY_.copyFrom(byy);
+        }
+        // geomD = magSf*procDelta per interface (static)
+        std::size_t pj = 0;
+        for (std::size_t pi = 0; pi < P_.lp.size(); ++pi)
+        {
+            if (P_.lp[pi].type != "processor") continue;
+            std::vector<scalar> gd(P_.lp[pi].size);
+            for (label i = 0; i < P_.lp[pi].size; ++i)
+                gd[i] = P_.lg.magSf()[P_.lp[pi].start + i] * P_.procDelta[pj][i];
+            geomD_.emplace_back();
+            geomD_.back().copyFrom(gd);
+            ++pj;
+        }
+    }
+    std::vector<scalar> reconstructK()   const { return reconstructField(P_.Lm.cellProcAddr, k_.host(),   P_.globalNCells); }
+    std::vector<scalar> reconstructEps() const { return reconstructField(P_.Lm.cellProcAddr, eps_.host(), P_.globalNCells); }
+    std::vector<scalar> reconstructNut() const { return reconstructField(P_.Lm.cellProcAddr, nut_.host(), P_.globalNCells); }
+    std::vector<scalar> kLocal()   const { return k_.host(); }
+    std::vector<scalar> epsLocal() const { return eps_.host(); }
+    std::vector<scalar> nutLocal() const { return nut_.host(); }
+
     // gather `d` to the global ordering and append "<name> v0 v1 ..." to the dump file (master only)
     void dumpStage(const char* name, const DeviceBuffer<scalar>& d, int k = -1) const
     {
@@ -875,6 +842,18 @@ private:
     // non-orth ("corrected") laplacian: static per-cut-face geometry + the per-interface face areas
     std::vector<std::vector<scalar>> procNonOrth_;      // host: the IMPLICIT coeff (replaces procDelta)
     std::vector<DeviceBuffer<scalar>> corrVecD_, magSfD_, nuFaceD_;
+    // Phase 4b turbulence (k-epsilon): held device-resident like U/p/phi; nut feeds nuEff each step, and
+    // correct() runs after the corrector. Laminar path is untouched (turbulent_ = false).
+    bool turbulent_ = false;
+    scalar relaxK_ = 0.7, relaxEps_ = 0.7;
+    DeviceBuffer<scalar> k_, eps_, nut_;
+    DeviceWallData wall_;
+    DeviceBoundary dbK_, dbEps_;
+    DeviceBuffer<label> isWallCell_;
+    DeviceBuffer<label> bndIsWall_;   // per-boundary-face wall mask (for deviceBoundaryNut)
+    DeviceBuffer<scalar> bndY_;       // per-boundary-face near-wall distance
+    std::vector<DeviceBuffer<scalar>> geomD_;   // magSf*procDelta per interface (for the k/eps interface gamma)
+    KEpsilonCoeffs keCoeffs_;
     DeviceBuffer<scalar> Uk_[3], dp_, phiInt_, phiBnd_;
     DeviceBuffer<scalar> ones_, zeroSrc_, zeroBnd_, nuEffBnd_, nuCell_;
 };
@@ -920,6 +899,11 @@ inline void deviceParallelMatrixH(
 
 // One distributed SIMPLE iteration -- the sequence validated stage-by-stage against host
 // parallelSimpleStepLaminar in test_gpu_parallel_predictor.
+// parallelDeviceKEpsilonCorrect needs the interface helpers (deviceMomentumInterface, ...) defined
+// above; include it here, AFTER them, to break the simple<->turbulence include cycle.
+} // (reopened below)
+#include "parallel_device_turbulence.cuh"
+namespace brae {
 inline ParStepResidual ParallelDeviceSimple::step()
 {
     ParStepResidual res;
@@ -927,8 +911,32 @@ inline ParStepResidual ParallelDeviceSimple::step()
     const std::vector<FvPatch>& lp = P_.lp;
     const FvGeometry& lg = P_.lg;
 
+    // ---- effective viscosity for this step: nu (laminar) or nu + nut (turbulent, refreshed from the model) ----
+    // Laminar: nuEffCell = nuCell_ (=nu), nuEffBndEff = nuEffBnd_ (=nu), nuEff_f = nu, nuEffScatBnd = nu-at-cut.
+    DeviceBuffer<scalar> nuEffCell, nuEffBndEff, nuEff_f, nuEffScatBnd;
+    if (turbulent_)
+    {
+        deviceCopy(nuEffCell, nut_);   deviceAxpy(1.0, nuCell_, nuEffCell);     // nu + nut (cell)
+        deviceInterpolate(dm_, nuEffCell, nuEff_f);                            // internal faces
+        // boundary faces: nu + the TRUE wall nut (nutkWallFunction), matching OpenFOAM wall shear
+        DeviceBuffer<scalar> nutBnd;
+        deviceBoundaryNut(dbU_.comp[0], bndIsWall_, bndY_, k_, nut_, nu_, nutBnd, keCoeffs_);
+        deviceCopy(nuEffBndEff, nutBnd);   deviceAxpy(1.0, nuEffBnd_, nuEffBndEff);   // + nu everywhere
+    }
+    else
+    {
+        nuEff_f.copyFrom(std::vector<scalar>(nIf_, nu_));
+        deviceCopy(nuEffCell, nuCell_);
+        deviceCopy(nuEffBndEff, nuEffBnd_);
+    }
+    // nuEff at the cut faces: the halo-interpolated cell value (nu on both sides for laminar -> unchanged)
+    nuEffScatBnd.copyFrom(std::vector<scalar>(nBnd_, 0.0));
+    halo_.exchange(nuEffCell.data());
+    halo_.scatterBoundaryValues(nuEffCell.data(), weightsD_, procStart_, nuEffScatBnd.data());
+    halo_.waitExchange();
+    const std::vector<scalar> nuEffScatH = nuEffScatBnd.host();
+
     // ---- momentum matrix: div(phi,U) - laplacian(nuEff,U), + processor interface ----
-    DeviceBuffer<scalar> nuEff_f(std::vector<scalar>(nIf_, nu_));
     DeviceBuffer<scalar> mDiag, mUp, mLo, lD, lU, lL;
     deviceDivUpwindCoeffs(dm_, phiInt_, mDiag, mUp, mLo);
     deviceLaplacianCoeffs(dm_, nuEff_f, lD, lU, lL, nonOrth_);
@@ -962,7 +970,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
                 phiF.push_back(std::move(pf));
                 std::vector<scalar> cg(lp[pi].size);
                 for (label i = 0; i < lp[pi].size; ++i)
-                    cg[i] = nu_ * lg.magSf()[lp[pi].start + i]
+                    cg[i] = nuEffScatH[bi + i] * lg.magSf()[lp[pi].start + i]
                           * (nonOrth_ ? procNonOrth_[pj][i] : P_.procDelta[pj][i]);
                 DeviceBuffer<scalar> cgd;
                 cgd.copyFrom(cg);
@@ -982,7 +990,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     {
         DeviceBuffer<scalar> lIC, lBC;
         deviceBCDivCoeffs(dbU_.comp[k], phiBnd_, iC[k], bC[k]);
-        deviceBCLaplacianCoeffsFace(dbU_.comp[k], nuEffBnd_, lIC, lBC);
+        deviceBCLaplacianCoeffsFace(dbU_.comp[k], nuEffBndEff, lIC, lBC);
         deviceAxpy(-1.0, lIC, iC[k]);
         deviceAxpy(-1.0, lBC, bC[k]);
     }
@@ -1002,7 +1010,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
     proc.weights   = &weightsD_;
     proc.procStart = &procStart_;
     DeviceBuffer<scalar> sX, sY, sZ;
-    deviceDivDevReff(dm_, dbU_, Uk_[0], Uk_[1], Uk_[2], nuCell_, nuEffBnd_, sX, sY, sZ, nullptr, nullptr, &proc);
+    deviceDivDevReff(dm_, dbU_, Uk_[0], Uk_[1], Uk_[2], nuEffCell, nuEffBndEff, sX, sY, sZ, nullptr, nullptr, &proc);
     DeviceBuffer<scalar>* sS[3] = { &sX, &sY, &sZ };
     dumpStage("stress", sX, 0);
     dumpStage("stress", sY, 1);
@@ -1267,6 +1275,36 @@ inline ParStepResidual ParallelDeviceSimple::step()
         deviceCopy(Uk_[k], Un);
         dumpStage("Ucorr", Uk_[k], k);
     }
+
+    // ---- turbulence: update k/eps/nut on the corrected U/phi (OF runs turbulence->correct() at the end of
+    // the SIMPLE iteration, so nut for the NEXT momentum comes from THIS step's fields) ----
+    if (turbulent_)
+    {
+        // rebuild phiF from the CORRECTED flux: the phiF above was built from the pre-corrector phi (for the
+        // momentum interface), but OF runs turbulence->correct() with the NEW phi. At np=1 this is moot (no
+        // interface), but at np>1 the cut would otherwise convect k/eps with a STALE flux while the interior
+        // used the fresh phiInt_ -- inconsistent across the cut. Rebuild it from the maintained phiBnd_.
+        std::vector<DeviceBuffer<scalar>> phiFc;
+        {
+            const std::vector<scalar> pbH = phiBnd_.host();
+            label bi = 0;
+            for (std::size_t pi = 0; pi < lp.size(); ++pi)
+            {
+                if (lp[pi].type == "cyclic" || lp[pi].type == "cyclicAMI") continue;
+                if (lp[pi].type == "processor")
+                {
+                    DeviceBuffer<scalar> pf;
+                    pf.copyFrom(std::vector<scalar>(pbH.begin() + bi, pbH.begin() + bi + lp[pi].size));
+                    phiFc.push_back(std::move(pf));
+                }
+                bi += lp[pi].size;
+            }
+        }
+        scalar rEps = 0, rK = 0;
+        parallelDeviceKEpsilonCorrect(dm_, halo_, wall_, dbU_, dbK_, dbEps_, Uk_[0], Uk_[1], Uk_[2],
+            k_, eps_, nut_, phiInt_, phiBnd_, phiFc, faceCellsD_, procStart_, weightsD_, geomD_, isWallCell_,
+            nu_, relaxEps_, relaxK_, tolU_, maxIter_, P_.globalNCells, ones_, bounded_, keCoeffs_, &rEps, &rK);
+    }
     return res;
 }
 
diff --git a/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh b/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh
index 71b982c..972c24c 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_turbulence.cuh
@@ -13,9 +13,15 @@
 // separate them -- confirmed by the host oracle's "WALL FUNCTIONS are unchanged and entirely local"), so they
 // reuse the single-GPU kernels unchanged. The one genuinely new device piece is the near-wall setValues
 // constraint (fixed epsilon at wall-adjacent cells), which also zeroes those cells' interface coeffs.
-#include "parallel_device_simple.cuh"
+#include "parallel_device_interface.cuh"   // deviceMomentumInterface / deviceInterfaceOffDiagSum
+#include "device_simple.cuh"
+#include "device_pcg.cuh"
+#include "device_mesh.cuh"
+#include "device_ldu.cuh"
+#include "device_blas.cuh"
 #include "device_boundary.cuh"
 #include "device_kepsilon.cuh"
+#include "device_komega_sst.cuh"
 #include <vector>
 
 namespace brae {
@@ -344,4 +350,127 @@ inline void parallelDeviceKEpsilonCorrect(
     deviceNut(k, eps, nut, co);
 }
 
+
+// Distributed device kOmegaSST::correct() -- mirrors the single-GPU deviceKOmegaSSTCorrect (device_kepsilon.cu)
+// exactly, reusing its exposed LOCAL kernels (S2, F1, F2, CDkOmega, blends, reactions, nutSST, wall omega) and
+// the distributed scalar-transport core for the two solves. Processor-coupled inputs: gradU, grad(k), grad(omega)
+// (each field's cut value from the halo before gaussGrad), and the F1-blended diffusivities DkEff/DomegaEff at
+// cut faces (halo-interpolated). k/omega/nut updated in place. Same wall/setValues/geomD scaffolding as k-eps.
+inline void parallelDeviceKOmegaSSTCorrect(
+    const DeviceMesh& dm, DeviceHalo& halo, const DeviceWallData& wall,
+    const DeviceVectorBoundary& dbU, const DeviceBoundary& dbK, const DeviceBoundary& dbOmega,
+    const DeviceBuffer<scalar>& Ux, const DeviceBuffer<scalar>& Uy, const DeviceBuffer<scalar>& Uz,
+    DeviceBuffer<scalar>& k, DeviceBuffer<scalar>& omega, DeviceBuffer<scalar>& nut,
+    const DeviceBuffer<scalar>& y,                       // cell wall distance
+    const DeviceBuffer<scalar>& phiInt, const DeviceBuffer<scalar>& phiBnd,
+    const std::vector<DeviceBuffer<scalar>>& phiF,
+    const std::vector<DeviceBuffer<label>>& faceCellsD, const std::vector<label>& procStart,
+    const std::vector<DeviceBuffer<scalar>>& weightsD, const std::vector<DeviceBuffer<scalar>>& geomD,
+    const DeviceBuffer<label>& isWallCell,
+    scalar nu, scalar relaxOmega, scalar relaxK, scalar tol, int maxIter, label globalNCells,
+    const DeviceBuffer<scalar>& ones, bool bounded, const KOmegaSSTCoeffs& co)
+{
+    constexpr int TPB = 128;
+    const int nC = dm.nCells;
+    auto nb = [&](int n){ return (n + TPB - 1) / TPB; };
+
+    // halo-consistent scalar gradient: inject the field's cut value from the halo before gaussGrad
+    auto haloGrad = [&](const DeviceBoundary& db, const DeviceBuffer<scalar>& fld,
+                        DeviceBuffer<scalar>& gx, DeviceBuffer<scalar>& gy, DeviceBuffer<scalar>& gz)
+    {
+        DeviceBuffer<scalar> bv;
+        deviceBCValue(db, fld, bv);
+        halo.exchange(fld.data());
+        halo.scatterBoundaryValues(fld.data(), weightsD, procStart, bv.data());
+        halo.waitExchange();
+        deviceGaussGrad(dm, fld, bv, gx, gy, gz);
+    };
+    // F1-blended diffusivity DEff at internal/boundary/cut faces (like k-eps faceGammaGeo)
+    auto faceGammaGeo = [&](const DeviceBuffer<scalar>& gammaCell, const DeviceBoundary& db,
+                            DeviceBuffer<scalar>& gInt, DeviceBuffer<scalar>& gBnd,
+                            std::vector<DeviceBuffer<scalar>>& gCoeffGeo)
+    {
+        deviceInterpolate(dm, gammaCell, gInt);
+        deviceBCValue(db, gammaCell, gBnd);
+        DeviceBuffer<scalar> gBndProc;
+        deviceBCValue(db, gammaCell, gBndProc);
+        halo.exchange(gammaCell.data());
+        halo.scatterBoundaryValues(gammaCell.data(), weightsD, procStart, gBndProc.data());
+        halo.waitExchange();
+        const std::vector<scalar> gh = gBndProc.host();
+        gCoeffGeo.clear(); gCoeffGeo.resize(procStart.size());
+        for (std::size_t i = 0; i < procStart.size(); ++i)
+        {
+            const int n = static_cast<int>(halo.size(static_cast<int>(i)));
+            if (n <= 0) continue;
+            const std::vector<scalar> gd = geomD[i].host();
+            std::vector<scalar> cg(n);
+            for (int f = 0; f < n; ++f) cg[f] = gh[procStart[i] + f] * gd[f];
+            gCoeffGeo[i].copyFrom(cg);
+        }
+    };
+
+    // gradU (halo) -> GbyNu0, S2 ; production G = nut*GbyNu0 ; divU
+    const DeviceBuffer<scalar>* Uc[3] = { &Ux, &Uy, &Uz };
+    DeviceBuffer<scalar> gradU(static_cast<std::size_t>(9) * nC);
+    for (int i = 0; i < 3; ++i)
+    {
+        DeviceBuffer<scalar> gx, gy, gz;
+        haloGrad(dbU.comp[i], *Uc[i], gx, gy, gz);
+        cudaCheck(cudaMemcpyAsync(gradU.data()+(0*3+i)*(std::size_t)nC, gx.data(), nC*sizeof(scalar), cudaMemcpyDeviceToDevice, cudaStreamPerThread), "sstGU");
+        cudaCheck(cudaMemcpyAsync(gradU.data()+(1*3+i)*(std::size_t)nC, gy.data(), nC*sizeof(scalar), cudaMemcpyDeviceToDevice, cudaStreamPerThread), "sstGU");
+        cudaCheck(cudaMemcpyAsync(gradU.data()+(2*3+i)*(std::size_t)nC, gz.data(), nC*sizeof(scalar), cudaMemcpyDeviceToDevice, cudaStreamPerThread), "sstGU");
+    }
+    DeviceBuffer<scalar> GbyNu0; deviceGByNuFromGradU(gradU, nC, GbyNu0);
+    DeviceBuffer<scalar> G;      deviceHadamard(G, nut, GbyNu0);
+    DeviceBuffer<scalar> divU;   deviceDiv(dm, phiInt, phiBnd, divU);
+    DeviceBuffer<scalar> S2;     deviceS2(gradU, nC, S2);
+
+    // omega wall FIRST: omega0/G0, override omega & G at wall cells
+    DeviceBuffer<scalar> omega0, G0;
+    deviceWallOmegaG0(wall, k, Ux, Uy, Uz, nu, omega0, G0, co);
+    detail::prodAndWallKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(
+        nut.data(), GbyNu0.data(), isWallCell.data(), omega0.data(), G0.data(), G.data(), omega.data(), nC);
+    // NB prodAndWallKernel sets G=nut*GbyNu0 for non-wall; SST needs G=nut*GbyNu0 too (GbyNu0 passed as gByNu). OK.
+
+    // CDkOmega from grad(k), grad(omega) (halo-consistent); F1, F2
+    DeviceBuffer<scalar> kgx, kgy, kgz, ogx, ogy, ogz;
+    haloGrad(dbK, k, kgx, kgy, kgz);
+    haloGrad(dbOmega, omega, ogx, ogy, ogz);
+    DeviceBuffer<scalar> CD; deviceCDkOmega(kgx, kgy, kgz, ogx, ogy, ogz, omega, co.alphaOmega2, CD);
+    DeviceBuffer<scalar> F1; deviceF1(k, omega, y, CD, nu, co, F1, false);
+    DeviceBuffer<scalar> F2; deviceF2(k, omega, y, nu, co, F2);
+
+    // blends, limited production, diffusivities
+    DeviceBuffer<scalar> gamma; deviceBlend(F1, co.gamma1, co.gamma2, gamma);
+    DeviceBuffer<scalar> beta;  deviceBlend(F1, co.beta1,  co.beta2,  beta);
+    DeviceBuffer<scalar> GbyNu0lim; deviceGbyNuLimit(GbyNu0, omega, F2, S2, co, GbyNu0lim);
+    DeviceBuffer<scalar> DomegaEff; deviceDEff(F1, nut, co.alphaOmega1, co.alphaOmega2, nu, DomegaEff);
+    DeviceBuffer<scalar> DkEff;     deviceDEff(F1, nut, co.alphaK1,     co.alphaK2,     nu, DkEff);
+
+    // ---- omega equation (with the omega near-wall setValues constraint) ----
+    {
+        DeviceBuffer<scalar> gInt, gBnd; std::vector<DeviceBuffer<scalar>> gGeo;
+        faceGammaGeo(DomegaEff, dbOmega, gInt, gBnd, gGeo);
+        DeviceBuffer<scalar> Sp(std::vector<scalar>(nC,0.0)), Su(std::vector<scalar>(nC,0.0));
+        deviceOmegaReaction(dm.V, gamma, beta, GbyNu0lim, F1, CD, omega, divU, Sp, Su);
+        deviceParallelScalarTransport(dm, halo, dbOmega, faceCellsD, procStart, weightsD, phiInt, phiBnd, phiF,
+            gInt, gBnd, gGeo, Sp, Su, omega, relaxOmega, tol, maxIter, globalNCells, ones, isWallCell, omega0);
+        detail::boundFieldKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(omega.data(), 1e-15, nC);
+    }
+    // ---- k equation ----
+    {
+        DeviceBuffer<scalar> gInt, gBnd; std::vector<DeviceBuffer<scalar>> gGeo;
+        faceGammaGeo(DkEff, dbK, gInt, gBnd, gGeo);
+        DeviceBuffer<scalar> Sp(std::vector<scalar>(nC,0.0)), Su(std::vector<scalar>(nC,0.0));
+        deviceKReactionSST(dm.V, k, omega, G, divU, co, Sp, Su, nullptr);
+        DeviceBuffer<label> noWall; DeviceBuffer<scalar> noVal;
+        deviceParallelScalarTransport(dm, halo, dbK, faceCellsD, procStart, weightsD, phiInt, phiBnd, phiF,
+            gInt, gBnd, gGeo, Sp, Su, k, relaxK, tol, maxIter, globalNCells, ones, noWall, noVal);
+        detail::boundFieldKernel<<<nb(nC), TPB, 0, cudaStreamPerThread>>>(k.data(), 1e-15, nC);
+    }
+    // ---- correctNut (Bradshaw limiter) ----
+    deviceNutSST(k, omega, F2, S2, co, nut);
+}
+
 } // namespace brae
diff --git a/tests/test_gpu_parallel_komegasst.cu b/tests/test_gpu_parallel_komegasst.cu
new file mode 100644
index 0000000..31dbe59
--- /dev/null
+++ b/tests/test_gpu_parallel_komegasst.cu
@@ -0,0 +1,149 @@
+// Phase 4b: the distributed DEVICE kOmegaSST::correct() reproduces np=1.
+//
+// SST is the second RAS model. It reuses the 4b scalar-transport core (k/omega solves) and the exposed LOCAL
+// single-GPU SST kernels (S2, F1, F2, CDkOmega, blends, reactions, nutSST, wall omega); the processor-coupled
+// inputs are gradU, grad(k), grad(omega) (halo-consistent) and the F1-blended diffusivities at cut faces.
+// Gated np=1 (oracle) vs np=2/4 on the turbulent duct (walls y/z, cut x). TEETH: k varies across the cut.
+//
+// Run: mpirun -np {1,2,4} test_gpu_parallel_komegasst <Nx> <Ny> <Nz> [ref.txt]
+#include "primitive_mesh.cuh"
+#include "fv_geometry.cuh"
+#include "fv_patch.cuh"
+#include "geometric_field.cuh"
+#include "foam_field_reader.cuh"
+#include "fvc.cuh"
+#include "cell_wall_dist.cuh"
+#include "parallel_simple.cuh"
+#include "field_distribute.cuh"
+#include "parallel_device_turbulence.cuh"
+#include "cf_pstream.cuh"
+#include "box_mesh.cuh"
+
+#include <cmath>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <vector>
+
+using namespace brae;
+
+int main(int argc, char** argv)
+{
+    Pstream::init(argc, argv);
+    const int rank = Pstream::myProcNo(), nproc = Pstream::nProcs();
+    if (argc < 4) { if (Pstream::master()) std::printf("usage: %s <Nx> <Ny> <Nz> [ref]\n", argv[0]); Pstream::finalize(); return 2; }
+    if (!Pstream::nvshmemActive()) { if (Pstream::master()) std::printf("test_gpu_parallel_komegasst: NVSHMEM inactive -- SKIP\n"); Pstream::finalize(); return 0; }
+
+    const label Nx = std::atol(argv[1]), Ny = std::atol(argv[2]), Nz = std::atol(argv[3]);
+    const std::string ref = (argc > 4) ? argv[4] : "sst_ref.txt";
+    const scalar nu = 1e-3, relaxOmega = 0.7, relaxK = 0.7, tol = 1e-10;
+    const scalar kIn = 0.375, omIn = std::sqrt(kIn) / (0.09 * 0.1);   // omega ~ sqrt(k)/(Cmu*L)
+    const int maxIter = 2000, N = 6;
+    KOmegaSSTCoeffs co;
+    const label nC = Nx * Ny * Nz;
+    bool ok = true;
+    scalar cutK = 0, cutSpread = 0, relK = 0, relO = 0, relN = 0;
+    {
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz);
+        std::vector<label> cellToPart(nC);
+        for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i)
+            cellToPart[i + Nx * (j + Ny * k)] = static_cast<label>((static_cast<long long>(i) * nproc) / Nx);
+        const Partition P(gm, cellToPart, rank);
+        const FvGeometry& lg = P.lg;
+        const std::vector<FvPatch>& lp = P.lp;
+        const label lnC = P.nCells(), nIf = P.Lm.mesh.nInternalFaces();
+
+        FieldData<vector> Ufd; Ufd.internalUniform = true; Ufd.internalUniformValue = vector{1, 0, 0};
+        Ufd.boundary.push_back(boxtest::pfd<vector>("inlet","fixedValue",vector{1,0,0},true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("outlet","zeroGradient",vector{0,0,0},false));
+        for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"})
+            Ufd.boundary.push_back(boxtest::pfd<vector>(w,"fixedValue",vector{0,0,0},true));
+        GeometricField<vector> U = distributeField<vector>(Ufd, gm.patches(), P.Lm, lp, P.procW, rank); U.evaluateBoundary();
+        const SurfaceScalarField phi = fvc::flux(U, P.Lm.mesh, lg, lp);
+
+        auto scalfd = [&](scalar in){ FieldData<scalar> f; f.internalUniform=true; f.internalUniformValue=in;
+            f.boundary.push_back(boxtest::pfd<scalar>("inlet","fixedValue",in,true));
+            f.boundary.push_back(boxtest::pfd<scalar>("outlet","zeroGradient",0.0,false));
+            for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"}) f.boundary.push_back(boxtest::pfd<scalar>(w,"zeroGradient",0.0,false));
+            return f; };
+        GeometricField<scalar> kF  = distributeField<scalar>(scalfd(kIn), gm.patches(), P.Lm, lp, P.procW, rank); kF.evaluateBoundary();
+        GeometricField<scalar> omF = distributeField<scalar>(scalfd(omIn), gm.patches(), P.Lm, lp, P.procW, rank); omF.evaluateBoundary();
+        GeometricField<scalar> nutF= distributeField<scalar>(scalfd(kIn/omIn), gm.patches(), P.Lm, lp, P.procW, rank); nutF.evaluateBoundary();
+
+        DeviceMesh dm = buildDeviceMesh(P.Lm.mesh, P.lg, lp);
+        const DeviceBoundary dbK = buildDeviceBoundary(kF, lp, lg), dbOm = buildDeviceBoundary(omF, lp, lg);
+        const DeviceVectorBoundary dbU = buildDeviceVectorBoundary(U, lp, lg);
+        const DeviceWallData wall = buildDeviceWallData(P.Lm.mesh, lg, lp, U);
+        DeviceBuffer<scalar> y(cellWallDist(P.Lm.mesh, lg, lp));
+
+        DeviceBuffer<scalar> Ux(lnC), Uy(lnC), Uz(lnC), dk(kF.internal), dom(omF.internal), dnut(nutF.internal), ones(std::vector<scalar>(lnC,1.0));
+        { std::vector<scalar> ux(lnC),uy(lnC),uz(lnC); for(label c=0;c<lnC;++c){ux[c]=U.internal[c].x;uy[c]=U.internal[c].y;uz[c]=U.internal[c].z;} Ux.copyFrom(ux);Uy.copyFrom(uy);Uz.copyFrom(uz); }
+        DeviceBuffer<scalar> phiInt, phiBnd;
+        { std::vector<scalar> pi(phi.internal.begin(), phi.internal.begin()+nIf); phiInt.copyFrom(pi); }
+        { std::vector<scalar> pb; for(std::size_t p=0;p<lp.size();++p){if(lp[p].type=="cyclic"||lp[p].type=="cyclicAMI")continue;for(label i=0;i<lp[p].size;++i)pb.push_back(phi.boundary[p][i]);} phiBnd.copyFrom(pb); }
+
+        std::vector<label> isW(lnC, 0);
+        for (std::size_t p=0;p<lp.size();++p) if (lp[p].type=="wall") for (label i=0;i<lp[p].size;++i) isW[lp[p].faceCells[i]]=1;
+        DeviceBuffer<label> isWallCell(isW);
+
+        std::vector<int> nbrs; for (int q : P.Lm.procNbr) nbrs.push_back(q);
+        DeviceHalo halo(rank, nbrs, P.Lm.procFaceCells);
+        std::vector<DeviceBuffer<label>>  faceCellsD(P.Lm.procFaceCells.size());
+        std::vector<DeviceBuffer<scalar>> weightsD(P.procW.size()), phiF, geomD;
+        for (std::size_t i=0;i<P.Lm.procFaceCells.size();++i) faceCellsD[i].copyFrom(P.Lm.procFaceCells[i]);
+        for (std::size_t i=0;i<P.procW.size();++i) weightsD[i].copyFrom(P.procW[i]);
+        std::vector<label> procStart; label bidx=0; std::size_t pj=0;
+        for (std::size_t p=0;p<lp.size();++p){
+            if(lp[p].type=="cyclic"||lp[p].type=="cyclicAMI")continue;
+            if(lp[p].type=="processor"){
+                procStart.push_back(bidx);
+                std::vector<scalar> pfh(lp[p].size), gd(lp[p].size);
+                for(label i=0;i<lp[p].size;++i){ pfh[i]=phi.boundary[p][i]; gd[i]=lg.magSf()[lp[p].start+i]*P.procDelta[pj][i]; }
+                DeviceBuffer<scalar> pf,g; pf.copyFrom(pfh); g.copyFrom(gd); phiF.push_back(std::move(pf)); geomD.push_back(std::move(g));
+                ++pj;
+            }
+            bidx += lp[p].size;
+        }
+
+        for (int it = 0; it < N; ++it)
+            parallelDeviceKOmegaSSTCorrect(dm, halo, wall, dbU, dbK, dbOm, Ux, Uy, Uz, dk, dom, dnut, y,
+                phiInt, phiBnd, phiF, faceCellsD, procStart, weightsD, geomD, isWallCell,
+                nu, relaxOmega, relaxK, tol, maxIter, P.globalNCells, ones, false, co);
+        cudaDeviceSynchronize();
+
+        { halo.exchange(dk.data()); halo.waitExchange();
+          const std::vector<scalar> ph=phiBnd.host(), kk=dk.host();
+          for(std::size_t i=0;i<procStart.size();++i){ const label n=static_cast<label>(halo.size((int)i)); const std::vector<scalar> nbf=halo.neighbourField((int)i);
+            for(label f=0;f<n;++f){ cutK=std::fmax(cutK,std::fabs(ph[procStart[i]+f])); cutSpread=std::fmax(cutSpread,std::fabs(kk[P.Lm.procFaceCells[i][f]]-nbf[f])); } }
+          cutK=Pstream::allReduce(cutK,ReduceOp::Max); cutSpread=Pstream::allReduce(cutSpread,ReduceOp::Max); }
+
+        auto g=[&](const DeviceBuffer<scalar>& d){ std::vector<scalar> gg(nC,0.0); const std::vector<scalar> h=d.host();
+            for(label c=0;c<lnC;++c) gg[P.Lm.cellProcAddr[c]]=h[c]; Pstream::allReduce(gg.data(),(int)gg.size(),ReduceOp::Sum); return gg; };
+        const std::vector<scalar> gk=g(dk), go=g(dom), gn=g(dnut);
+
+        if (Pstream::master()){
+            if (nproc==1){ std::ofstream os(ref); os.precision(17);
+                for(label c=0;c<nC;++c) os<<gk[c]<<' '<<go[c]<<' '<<gn[c]<<'\n';
+                std::printf("test_gpu_parallel_komegasst np=1: wrote reference %s (%ld cells)\n", ref.c_str(),(long)nC);
+                std::printf("PASS\n");
+            } else { std::ifstream is(ref);
+                if(!is){ std::printf("cannot read %s -- run np=1 first\n", ref.c_str()); ok=false; }
+                else { scalar nk=0,dkd=0,no=0,doo=0,nn=0,dnn=0;
+                    for(label c=0;c<nC;++c){ scalar rk,ro,rn; is>>rk>>ro>>rn;
+                        nk=std::fmax(nk,std::fabs(gk[c]-rk)); dkd=std::fmax(dkd,std::fabs(rk));
+                        no=std::fmax(no,std::fabs(go[c]-ro)); doo=std::fmax(doo,std::fabs(ro));
+                        nn=std::fmax(nn,std::fabs(gn[c]-rn)); dnn=std::fmax(dnn,std::fabs(rn)); }
+                    relK=dkd>0?nk/dkd:nk; relO=doo>0?no/doo:no; relN=dnn>0?nn/dnn:nn;
+                    const bool teeth=(cutK>1e-6)&&(cutSpread>1e-6);
+                    std::printf("test_gpu_parallel_komegasst np=%d: k rel=%.3e  omega rel=%.3e  nut rel=%.3e  k spread=%.3e\n",
+                                nproc, relK, relO, relN, cutSpread);
+                    if(!teeth) std::printf("  NO TEETH\n");
+                    ok = teeth && relK<1e-6 && relO<1e-6 && relN<1e-6;
+                    std::printf("%s\n", ok?"PASS":"FAIL");
+                }
+            }
+        }
+    }
+    Pstream::finalize();
+    return ok ? 0 : 1;
+}
diff --git a/tests/test_gpu_parallel_turbloop.cu b/tests/test_gpu_parallel_turbloop.cu
new file mode 100644
index 0000000..3ef7540
--- /dev/null
+++ b/tests/test_gpu_parallel_turbloop.cu
@@ -0,0 +1,122 @@
+// Phase 4b: the CLOSED distributed turbulent SIMPLE loop (momentum + pressure + kEpsilon) reproduces np=1.
+//
+// This is the turbulence counterpart of test_gpu_parallel_loop: it drives ParallelDeviceSimple with
+// enableTurbulence() for N iterations, so each step runs the momentum/pressure with nuEff = nu + nut AND then
+// parallelDeviceKEpsilonCorrect. U/p/phi/k/eps/nut all stay device-resident and are maintained across
+// iterations -- the real test that the turbulent step composes correctly under a maintained flux, not just a
+// single correct() (test_gpu_parallel_kepsilon) in isolation.
+//
+// np=1 (no processor faces -> the validated local turbulent step) is the oracle; np=2/4 must reproduce it.
+// Duct walls y/z, cut x -> near-wall cells adjacent to the cut.
+//
+// Run: mpirun -np {1,2,4} test_gpu_parallel_turbloop <Nx> <Ny> <Nz> [ref.txt]
+#include "primitive_mesh.cuh"
+#include "fv_geometry.cuh"
+#include "fv_patch.cuh"
+#include "geometric_field.cuh"
+#include "foam_field_reader.cuh"
+#include "fvc.cuh"
+#include "parallel_simple.cuh"
+#include "field_distribute.cuh"
+#include "parallel_device_simple.cuh"
+#include "cf_pstream.cuh"
+#include "box_mesh.cuh"
+
+#include <cmath>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <vector>
+
+using namespace brae;
+
+int main(int argc, char** argv)
+{
+    Pstream::init(argc, argv);
+    const int rank = Pstream::myProcNo(), nproc = Pstream::nProcs();
+    if (argc < 4) { if (Pstream::master()) std::printf("usage: %s <Nx> <Ny> <Nz> [ref]\n", argv[0]); Pstream::finalize(); return 2; }
+    if (!Pstream::nvshmemActive()) { if (Pstream::master()) std::printf("test_gpu_parallel_turbloop: NVSHMEM inactive -- SKIP\n"); Pstream::finalize(); return 0; }
+
+    const label Nx = std::atol(argv[1]), Ny = std::atol(argv[2]), Nz = std::atol(argv[3]);
+    const std::string ref = (argc > 4) ? argv[4] : "turbloop_ref.txt";
+    const std::string dump = (argc > 5) ? argv[5] : "";
+    const scalar nu = 1e-3, relaxU = 0.7, relaxP = 0.3, relaxK = 0.7, tolU = 1e-9, tolP = 1e-9;
+    const scalar kIn = 0.375, epsIn = 0.09 * std::pow(kIn, 1.5) / 0.1;
+    const int maxIter = 2000, N = 4;
+    KEpsilonCoeffs co;
+    const label nC = Nx * Ny * Nz;
+    bool ok = true;
+    scalar relU = 0, relP = 0, relK = 0, relN = 0;
+    {
+        const PrimitiveMesh gm = boxtest::boxMesh(Nx, Ny, Nz);
+        std::vector<label> cellToPart(nC);
+        for (label k = 0; k < Nz; ++k) for (label j = 0; j < Ny; ++j) for (label i = 0; i < Nx; ++i)
+            cellToPart[i + Nx * (j + Ny * k)] = static_cast<label>((static_cast<long long>(i) * nproc) / Nx);
+        const Partition P(gm, cellToPart, rank);
+        const label lnC = P.nCells();
+
+        FieldData<vector> Ufd; Ufd.internalUniform = true; Ufd.internalUniformValue = vector{1, 0, 0};
+        Ufd.boundary.push_back(boxtest::pfd<vector>("inlet","fixedValue",vector{1,0,0},true));
+        Ufd.boundary.push_back(boxtest::pfd<vector>("outlet","zeroGradient",vector{0,0,0},false));
+        for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"})
+            Ufd.boundary.push_back(boxtest::pfd<vector>(w,"fixedValue",vector{0,0,0},true));
+        auto pfdta = [&](){ FieldData<scalar> f; f.internalUniform=true; f.internalUniformValue=0.0;
+            f.boundary.push_back(boxtest::pfd<scalar>("inlet","zeroGradient",0.0,false));
+            f.boundary.push_back(boxtest::pfd<scalar>("outlet","fixedValue",0.0,true));
+            for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"}) f.boundary.push_back(boxtest::pfd<scalar>(w,"zeroGradient",0.0,false));
+            return f; }();
+        auto scalfd = [&](scalar in){ FieldData<scalar> f; f.internalUniform=true; f.internalUniformValue=in;
+            f.boundary.push_back(boxtest::pfd<scalar>("inlet","fixedValue",in,true));
+            f.boundary.push_back(boxtest::pfd<scalar>("outlet","zeroGradient",0.0,false));
+            for (const char* w : {"wallYmin","wallYmax","wallZmin","wallZmax"}) f.boundary.push_back(boxtest::pfd<scalar>(w,"zeroGradient",0.0,false));
+            return f; };
+
+        GeometricField<vector> U0 = distributeField<vector>(Ufd, gm.patches(), P.Lm, P.lp, P.procW, rank); U0.evaluateBoundary();
+        GeometricField<scalar> p0 = distributeField<scalar>(pfdta, gm.patches(), P.Lm, P.lp, P.procW, rank); p0.evaluateBoundary();
+        GeometricField<scalar> k0   = distributeField<scalar>(scalfd(kIn),   gm.patches(), P.Lm, P.lp, P.procW, rank); k0.evaluateBoundary();
+        GeometricField<scalar> eps0 = distributeField<scalar>(scalfd(epsIn), gm.patches(), P.Lm, P.lp, P.procW, rank); eps0.evaluateBoundary();
+        GeometricField<scalar> nut0 = distributeField<scalar>(scalfd(co.Cmu*kIn*kIn/epsIn), gm.patches(), P.Lm, P.lp, P.procW, rank); nut0.evaluateBoundary();
+
+        ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, maxIter);
+        solver.enableTurbulence(U0, k0, eps0, nut0, co, relaxK, relaxK);
+        if (!dump.empty()) solver.setStageDump(dump, (argc>6)?std::atoi(argv[6]):2);
+        for (int it = 0; it < N; ++it) solver.step();
+        cudaDeviceSynchronize();
+
+        // gather to the global mesh
+        auto gvec = [&](const std::vector<vector>& l){ std::vector<scalar> g(3*nC,0.0);
+            for(label c=0;c<lnC;++c){const label gc=P.Lm.cellProcAddr[c]; g[3*gc]=l[c].x; g[3*gc+1]=l[c].y; g[3*gc+2]=l[c].z;}
+            Pstream::allReduce(g.data(),(int)g.size(),ReduceOp::Sum); return g; };
+        auto gsca = [&](const std::vector<scalar>& l){ std::vector<scalar> g(nC,0.0);
+            for(label c=0;c<lnC;++c) g[P.Lm.cellProcAddr[c]]=l[c]; Pstream::allReduce(g.data(),(int)g.size(),ReduceOp::Sum); return g; };
+        const std::vector<scalar> gU = gvec(solver.U()), gp = gsca(solver.p()), gk = gsca(solver.kLocal()), gn = gsca(solver.nutLocal());
+
+        if (Pstream::master()){
+            if (nproc==1){
+                std::ofstream os(ref); os.precision(17);
+                for(label c=0;c<nC;++c) os<<gU[3*c]<<' '<<gU[3*c+1]<<' '<<gU[3*c+2]<<' '<<gp[c]<<' '<<gk[c]<<' '<<gn[c]<<'\n';
+                std::printf("test_gpu_parallel_turbloop np=1: wrote reference %s (%ld cells)\n", ref.c_str(),(long)nC);
+                std::printf("PASS\n");
+            } else {
+                std::ifstream is(ref);
+                if(!is){ std::printf("cannot read %s -- run np=1 first\n", ref.c_str()); ok=false; }
+                else {
+                    scalar nU=0,dU=0,nP=0,dP=0,nk=0,dkd=0,nn=0,dnn=0;
+                    for(label c=0;c<nC;++c){ scalar ux,uy,uz,pp,kk,nn2; is>>ux>>uy>>uz>>pp>>kk>>nn2;
+                        nU=std::fmax(nU,std::fabs(gU[3*c]-ux)); nU=std::fmax(nU,std::fabs(gU[3*c+1]-uy)); dU=std::fmax(dU,std::fabs(ux));
+                        nP=std::fmax(nP,std::fabs(gp[c]-pp)); dP=std::fmax(dP,std::fabs(pp));
+                        nk=std::fmax(nk,std::fabs(gk[c]-kk)); dkd=std::fmax(dkd,std::fabs(kk));
+                        nn=std::fmax(nn,std::fabs(gn[c]-nn2)); dnn=std::fmax(dnn,std::fabs(nn2)); }
+                    relU=dU>0?nU/dU:nU; relP=dP>0?nP/dP:nP; relK=dkd>0?nk/dkd:nk; relN=dnn>0?nn/dnn:nn;
+                    const scalar gate = 1e-4;   // N turbulent iterations of parallel-solver-floor drift
+                    std::printf("test_gpu_parallel_turbloop np=%d (%d iters): U rel=%.3e  p rel=%.3e  k rel=%.3e  nut rel=%.3e\n",
+                                nproc, N, relU, relP, relK, relN);
+                    ok = relU<gate && relP<gate && relK<gate && relN<gate;
+                    std::printf("%s\n", ok?"PASS":"FAIL");
+                }
+            }
+        }
+    }
+    Pstream::finalize();
+    return ok ? 0 : 1;
+}

From 5297fc913baa58e386ac6017c6103628a6f5f35b Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Fri, 17 Jul 2026 23:57:44 +0200
Subject: [PATCH 07/13] #12: k-omega SST in the distributed loop + RAS in brae
 -parallel

---
 CMakeLists.txt                                | 12 +++
 .../simpleFoam/parallel_device_foam.cuh       | 99 ++++++++++++++++---
 .../simpleFoam/parallel_device_simple.cuh     | 70 ++++++++++++-
 tests/test_gpu_parallel_turbloop.cu           | 12 ++-
 4 files changed, 171 insertions(+), 22 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 95c1841..dca9e4b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -298,6 +298,18 @@ if(BRAE_WITH_NVSHMEM)
         set_tests_properties(gpu_parallel_turbloop_np${_np} PROPERTIES FIXTURES_REQUIRED TURBLOOP)
     endforeach()
 
+    # SST in the closed loop (second field = omega): same gate, sst mode
+    add_test(NAME gpu_parallel_turbloop_sst_ref
+             COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np 1 $<TARGET_FILE:test_gpu_parallel_turbloop>
+                     32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/turbloop_sst_ref.txt "" sst)
+    set_tests_properties(gpu_parallel_turbloop_sst_ref PROPERTIES FIXTURES_SETUP TURBLOOPSST)
+    foreach(_np 2 4)
+        add_test(NAME gpu_parallel_turbloop_sst_np${_np}
+                 COMMAND ${MPIEXEC_EXECUTABLE} --oversubscribe -np ${_np} $<TARGET_FILE:test_gpu_parallel_turbloop>
+                         32 16 16 ${CMAKE_CURRENT_BINARY_DIR}/turbloop_sst_ref.txt "" sst)
+        set_tests_properties(gpu_parallel_turbloop_sst_np${_np} PROPERTIES FIXTURES_REQUIRED TURBLOOPSST)
+    endforeach()
+
     # Phase 4b: the full distributed kEpsilon::correct() (production + wall functions + eps/k transport + nut)
     # reproduces np=1. Duct walls are y/z, cut is x -> near-wall cells sit adjacent to the cut.
     add_executable(test_gpu_parallel_kepsilon tests/test_gpu_parallel_kepsilon.cu)
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index e7081de..e3a3300 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -8,8 +8,8 @@
 // decomposePar step), iterate, gather on write -- but the SIMPLE loop itself runs entirely on the GPU via
 // ParallelDeviceSimple: U/p/phi stay device-resident across iterations and only cross PCIe at write time.
 //
-// LAMINAR ONLY (Phase 4a). Turbulence is 4b: ParallelDeviceSimple has no k/epsilon transport, so a RAS case
-// must be REFUSED here rather than silently solved as laminar -- that would look like it worked and be wrong.
+// Laminar OR RAS (kEpsilon, kOmegaSST): the SIMPLE loop runs on the GPU via ParallelDeviceSimple, with
+// turbulence enabled per the case's turbulenceProperties. Unsupported models/schemes are refused, not approximated.
 //
 // The single-GPU `brae` path is untouched: this is reached only when -parallel is passed, so a plain
 // `brae -case <dir>` never initialises MPI (which also keeps the -cases fork orchestrator clear of MPI).
@@ -104,14 +104,56 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         const int   precision = controlDict.intOr("writePrecision", 16);
         const scalar nu       = transport.scalarOr("nu", 1e-5);
 
-        // Phase 4a is laminar. Refuse RAS rather than solve it as laminar behind the user's back.
+        // Turbulence: laminar, or RAS kEpsilon (the loop-integrated + validated model). kOmegaSST correct() is
+        // validated standalone (test_gpu_parallel_komegasst) but not yet wired into the distributed loop, so it
+        // is refused here -- honestly, not silently solved as something else.
         const std::string simType = turbProps.wordOr("simulationType", "laminar");
-        if (simType != "laminar")
+        const bool turbulent = (simType == "RAS");
+        KEpsilonCoeffs keCoeffs;
+        KOmegaSSTCoeffs sstCoeffs;
+        bool sst = false;
+        if (turbulent)
+        {
+            const FoamDict* ras = turbProps.subDict("RAS");
+            const std::string model = ras ? ras->wordOr("RASModel", "") : "";
+            if (model != "kEpsilon" && model != "kOmegaSST")
+                throw std::runtime_error(
+                    "brae -parallel: RASModel '" + model + "' is not implemented on the multi-GPU device path "
+                    "(kEpsilon and kOmegaSST are). Use `mpirun -np N brae_simpleFoam -case <dir>` (host) or "
+                    "`brae -case <dir>` (single GPU) for other models.");
+            sst = (model == "kOmegaSST");
+            const FoamDict* kec = ras ? ras->subDict("kEpsilonCoeffs") : nullptr;
+            if (kec)
+            {
+                keCoeffs.Cmu = kec->scalarOr("Cmu", keCoeffs.Cmu);
+                keCoeffs.C1 = kec->scalarOr("C1", keCoeffs.C1);
+                keCoeffs.C2 = kec->scalarOr("C2", keCoeffs.C2);
+                keCoeffs.C3 = kec->scalarOr("C3", keCoeffs.C3);
+                keCoeffs.sigmaK = kec->scalarOr("sigmak", keCoeffs.sigmaK);
+                keCoeffs.sigmaEps = kec->scalarOr("sigmaEps", keCoeffs.sigmaEps);
+                keCoeffs.kappa = kec->scalarOr("kappa", keCoeffs.kappa);
+                keCoeffs.E = kec->scalarOr("E", keCoeffs.E);
+            }
+            const FoamDict* ssc = ras ? ras->subDict("kOmegaSSTCoeffs") : nullptr;
+            if (ssc)
+            {
+                sstCoeffs.alphaK1 = ssc->scalarOr("alphaK1", sstCoeffs.alphaK1);
+                sstCoeffs.alphaK2 = ssc->scalarOr("alphaK2", sstCoeffs.alphaK2);
+                sstCoeffs.alphaOmega1 = ssc->scalarOr("alphaOmega1", sstCoeffs.alphaOmega1);
+                sstCoeffs.alphaOmega2 = ssc->scalarOr("alphaOmega2", sstCoeffs.alphaOmega2);
+                sstCoeffs.gamma1 = ssc->scalarOr("gamma1", sstCoeffs.gamma1);
+                sstCoeffs.gamma2 = ssc->scalarOr("gamma2", sstCoeffs.gamma2);
+                sstCoeffs.beta1 = ssc->scalarOr("beta1", sstCoeffs.beta1);
+                sstCoeffs.beta2 = ssc->scalarOr("beta2", sstCoeffs.beta2);
+                sstCoeffs.betaStar = ssc->scalarOr("betaStar", sstCoeffs.betaStar);
+                sstCoeffs.a1 = ssc->scalarOr("a1", sstCoeffs.a1);
+                sstCoeffs.b1 = ssc->scalarOr("b1", sstCoeffs.b1);
+                sstCoeffs.c1 = ssc->scalarOr("c1", sstCoeffs.c1);
+            }
+        }
+        else if (simType != "laminar")
             throw std::runtime_error(
-                "brae -parallel: simulationType '" + simType + "' is not supported on the multi-GPU device "
-                "path yet (Phase 4a is laminar; turbulence is 4b). Use the host solver "
-                "`mpirun -np N brae_simpleFoam -case <dir>` for RAS in parallel, or `brae -case <dir>` for "
-                "RAS on a single GPU.");
+                "brae -parallel: simulationType '" + simType + "' is not supported (laminar or RAS).");
 
         // fvSchemes div(phi,U). The distributed path implements Gauss upwind, bounded, and linearUpwind (the
         // matrix is upwind either way; linearUpwind adds the deferred gradient correction, cut faces included).
@@ -160,6 +202,8 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         const FoamDict* fld = rf ? rf->subDict("fields") : nullptr;
         const scalar relaxU = eqs ? eqs->scalarOr("U", 1.0) : 1.0;
         const scalar relaxP = fld ? fld->scalarOr("p", 1.0) : 1.0;
+        const scalar relaxK   = eqs ? eqs->scalarOr("k", 1.0) : 1.0;
+        const scalar relaxEps = eqs ? eqs->scalarOr("epsilon", 1.0) : 1.0;
 
         const FoamDict* solvers = fvSolution.subDict("solvers");
         auto solverTol = [&](const std::string& f, scalar def)
@@ -216,20 +260,36 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         GeometricField<scalar> p0 = distributeField<scalar>(pfd, gm.patches(), P.Lm, P.lp, P.procW, rank);
         p0.evaluateBoundary();
 
+        // turbulence start fields (RAS kEpsilon)
+        FieldData<scalar> kfd, efd, nfd;
+        GeometricField<scalar> k0, eps0, nut0;
+        if (turbulent)
+        {
+            kfd = readField<scalar>(fieldDir + "/k");
+            efd = readField<scalar>(fieldDir + "/" + (sst ? "omega" : "epsilon"));
+            nfd = readField<scalar>(fieldDir + "/nut");
+            k0   = distributeField<scalar>(kfd, gm.patches(), P.Lm, P.lp, P.procW, rank); k0.evaluateBoundary();
+            eps0 = distributeField<scalar>(efd, gm.patches(), P.Lm, P.lp, P.procW, rank); eps0.evaluateBoundary();
+            nut0 = distributeField<scalar>(nfd, gm.patches(), P.Lm, P.lp, P.procW, rank); nut0.evaluateBoundary();
+        }
+
         if (master)
         {
-            std::printf("brae (device, distributed) | case=%s np=%d | laminar | nu=%.3g | %ld cells\n",
-                        caseDir.c_str(), nproc, nu, (long)nC);
-            std::printf("  relax U=%.2g p=%.2g | tol p=%.1g U=%.1g | endTime=%d | residualControl=%s | div(phi,U)=%s\n",
-                        relaxU, relaxP, tolP, tolU, endTime, hasRC ? "on" : "off",
+            std::printf("brae (device, distributed) | case=%s np=%d | %s%s | nu=%.3g | %ld cells\n",
+                        caseDir.c_str(), nproc, simType.c_str(), turbulent ? (sst ? " (kOmegaSST)" : " (kEpsilon)") : "", nu, (long)nC);
+            std::printf("  relax U=%.2g p=%.2g%s | tol p=%.1g U=%.1g | endTime=%d | residualControl=%s | div(phi,U)=%s\n",
+                        relaxU, relaxP, turbulent ? (" k=" + std::to_string(relaxK)).c_str() : "",
+                        tolP, tolU, endTime, hasRC ? "on" : "off",
                         (std::string(boundedDiv ? "bounded " : "") + (lust ? "Gauss LUST" : linUpwind ? "Gauss linearUpwind" : "Gauss upwind")).c_str());
         }
 
         std::vector<vector> Ug;
-        std::vector<scalar> pg;
+        std::vector<scalar> pg, kg, eg, ng;
         int nIter = 0;
         {   // scope: the solver's symmetric-heap buffers must be freed BEFORE Pstream::finalize
             ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind, lust);
+            if (turbulent && sst) solver.enableTurbulenceSST(U0, k0, eps0, nut0, sstCoeffs, relaxK, relaxEps);
+            else if (turbulent)   solver.enableTurbulence(U0, k0, eps0, nut0, keCoeffs, relaxK, relaxEps);
 
             auto ok = [](scalar res, scalar ctl) { return ctl < 0 || res < ctl; };
             bool converged = false;
@@ -239,6 +299,7 @@ inline int runParallelDeviceFoam(int argc, char** argv)
                 const ParStepResidual r = solver.step();
                 if (master && (iter == 1 || iter % 50 == 0))
                     std::printf("  iter %4d:  Ux %.3e  p %.3e\n", iter, r.Ux, r.p);
+                // residualControl on U/p (the turbulence fields co-converge; step() returns only U/p residuals).
                 converged = hasRC && ok(r.p, rcP) && ok(r.Ux, rcU);
             }
             nIter = converged ? iter - 1 : endTime;
@@ -248,6 +309,12 @@ inline int runParallelDeviceFoam(int argc, char** argv)
 
             Ug = detail::gatherToGlobal(P, solver.U());   // D2H once, at write time
             pg = detail::gatherToGlobal(P, solver.p());
+            if (turbulent)
+            {
+                kg = detail::gatherToGlobal(P, solver.kLocal());
+                eg = detail::gatherToGlobal(P, solver.epsLocal());
+                ng = detail::gatherToGlobal(P, solver.nutLocal());
+            }
         }
 
         // pRefCell/pRefValue. A CLOSED domain (no fixedValue p patch) has an all-Neumann pressure equation,
@@ -283,6 +350,12 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             const std::string src = fieldDir + "/";
             writeVolField(src + "U", outDir + "/U", Ug, gm.patches(), precision);
             writeVolField(src + "p", outDir + "/p", pg, gm.patches(), precision);
+            if (turbulent)
+            {
+                writeVolField(src + "k",       outDir + "/k",       kg, gm.patches(), precision);
+                writeVolField(src + (sst ? "omega" : "epsilon"), outDir + "/" + (sst ? "omega" : "epsilon"), eg, gm.patches(), precision);
+                writeVolField(src + "nut",     outDir + "/nut",     ng, gm.patches(), precision);
+            }
             std::printf("wrote %s\n", outDir.c_str());
         }
     }
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 5f8f5f1..c986762 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -20,6 +20,8 @@
 #include "reconstruct.cuh"
 #include "local_assembly.cuh"   // computeProcUpwindD
 #include "device_kepsilon.cuh"   // DeviceWallData / KEpsilonCoeffs / buildDeviceWallData (Phase 4b turbulence)
+#include "komega_sst_coeffs.cuh"
+#include "cell_wall_dist.cuh"
 #include "parallel_device_interface.cuh"   // deviceMomentumInterface / deviceInterfaceOffDiagSum (shared)
 #include <cuda_runtime.h>
 #include <fstream>
@@ -744,7 +746,7 @@ public:
                           const GeometricField<scalar>& eps0, const GeometricField<scalar>& nut0,
                           const KEpsilonCoeffs& co, scalar relaxK = 0.7, scalar relaxEps = 0.7)
     {
-        turbulent_ = true;
+        turbulent_ = true;  turbModel_ = 0;
         relaxK_ = relaxK; relaxEps_ = relaxEps;
         keCoeffs_ = co;
         k_.copyFrom(k0.internal); eps_.copyFrom(eps0.internal); nut_.copyFrom(nut0.internal);
@@ -783,6 +785,52 @@ public:
             ++pj;
         }
     }
+
+    // Turn on distributed k-omega SST. Second field is OMEGA (stored in eps_/dbEps_), plus the cell wall
+    // distance y (F1/F2). Reuses the k-eps setup; keCoeffs_ is derived for the momentum wall nut
+    // (nutkWallFunction uses Cmu=betaStar, kappa, E). step() then calls parallelDeviceKOmegaSSTCorrect.
+    void enableTurbulenceSST(const GeometricField<vector>& U0, const GeometricField<scalar>& k0,
+                             const GeometricField<scalar>& omega0, const GeometricField<scalar>& nut0,
+                             const KOmegaSSTCoeffs& co, scalar relaxK = 0.7, scalar relaxOmega = 0.7)
+    {
+        turbulent_ = true;  turbModel_ = 1;
+        relaxK_ = relaxK; relaxEps_ = relaxOmega;
+        sstCoeffs_ = co;
+        keCoeffs_.Cmu = co.betaStar; keCoeffs_.kappa = co.kappa; keCoeffs_.E = co.E;   // momentum wall nut
+        k_.copyFrom(k0.internal); eps_.copyFrom(omega0.internal); nut_.copyFrom(nut0.internal);
+        dbK_   = buildDeviceBoundary(k0,     P_.lp, P_.lg);
+        dbEps_ = buildDeviceBoundary(omega0, P_.lp, P_.lg);
+        wall_ = buildDeviceWallData(P_.Lm.mesh, P_.lg, P_.lp, U0);
+        yCell_.copyFrom(cellWallDist(P_.Lm.mesh, P_.lg, P_.lp));
+        std::vector<label> isW(lnC_, 0);
+        for (std::size_t pi = 0; pi < P_.lp.size(); ++pi)
+            if (P_.lp[pi].type == "wall")
+                for (label i = 0; i < P_.lp[pi].size; ++i) isW[P_.lp[pi].faceCells[i]] = 1;
+        isWallCell_.copyFrom(isW);
+        {
+            const std::vector<std::vector<scalar>> yW = nearWallDist(P_.Lm.mesh, P_.lg, P_.lp);
+            std::vector<label> biw; std::vector<scalar> byy;
+            for (std::size_t pi = 0; pi < P_.lp.size(); ++pi)
+            {
+                if (P_.lp[pi].type == "cyclic" || P_.lp[pi].type == "cyclicAMI") continue;
+                const bool wall = (P_.lp[pi].type == "wall");
+                for (label i = 0; i < P_.lp[pi].size; ++i) { biw.push_back(wall ? 1 : 0); byy.push_back(wall ? yW[pi][i] : 0.0); }
+            }
+            bndIsWall_.copyFrom(biw);
+            bndY_.copyFrom(byy);
+        }
+        std::size_t pj = 0;
+        for (std::size_t pi = 0; pi < P_.lp.size(); ++pi)
+        {
+            if (P_.lp[pi].type != "processor") continue;
+            std::vector<scalar> gd(P_.lp[pi].size);
+            for (label i = 0; i < P_.lp[pi].size; ++i)
+                gd[i] = P_.lg.magSf()[P_.lp[pi].start + i] * P_.procDelta[pj][i];
+            geomD_.emplace_back();
+            geomD_.back().copyFrom(gd);
+            ++pj;
+        }
+    }
     std::vector<scalar> reconstructK()   const { return reconstructField(P_.Lm.cellProcAddr, k_.host(),   P_.globalNCells); }
     std::vector<scalar> reconstructEps() const { return reconstructField(P_.Lm.cellProcAddr, eps_.host(), P_.globalNCells); }
     std::vector<scalar> reconstructNut() const { return reconstructField(P_.Lm.cellProcAddr, nut_.host(), P_.globalNCells); }
@@ -854,6 +902,9 @@ private:
     DeviceBuffer<scalar> bndY_;       // per-boundary-face near-wall distance
     std::vector<DeviceBuffer<scalar>> geomD_;   // magSf*procDelta per interface (for the k/eps interface gamma)
     KEpsilonCoeffs keCoeffs_;
+    int  turbModel_ = 0;             // 0 = k-epsilon, 1 = k-omega SST
+    KOmegaSSTCoeffs sstCoeffs_;
+    DeviceBuffer<scalar> yCell_;    // cell wall distance (SST F1/F2)
     DeviceBuffer<scalar> Uk_[3], dp_, phiInt_, phiBnd_;
     DeviceBuffer<scalar> ones_, zeroSrc_, zeroBnd_, nuEffBnd_, nuCell_;
 };
@@ -1300,10 +1351,19 @@ inline ParStepResidual ParallelDeviceSimple::step()
                 bi += lp[pi].size;
             }
         }
-        scalar rEps = 0, rK = 0;
-        parallelDeviceKEpsilonCorrect(dm_, halo_, wall_, dbU_, dbK_, dbEps_, Uk_[0], Uk_[1], Uk_[2],
-            k_, eps_, nut_, phiInt_, phiBnd_, phiFc, faceCellsD_, procStart_, weightsD_, geomD_, isWallCell_,
-            nu_, relaxEps_, relaxK_, tolU_, maxIter_, P_.globalNCells, ones_, bounded_, keCoeffs_, &rEps, &rK);
+        if (turbModel_ == 1)   // k-omega SST (eps_ holds omega)
+        {
+            parallelDeviceKOmegaSSTCorrect(dm_, halo_, wall_, dbU_, dbK_, dbEps_, Uk_[0], Uk_[1], Uk_[2],
+                k_, eps_, nut_, yCell_, phiInt_, phiBnd_, phiFc, faceCellsD_, procStart_, weightsD_, geomD_,
+                isWallCell_, nu_, relaxEps_, relaxK_, tolU_, maxIter_, P_.globalNCells, ones_, bounded_, sstCoeffs_);
+        }
+        else                   // k-epsilon
+        {
+            scalar rEps = 0, rK = 0;
+            parallelDeviceKEpsilonCorrect(dm_, halo_, wall_, dbU_, dbK_, dbEps_, Uk_[0], Uk_[1], Uk_[2],
+                k_, eps_, nut_, phiInt_, phiBnd_, phiFc, faceCellsD_, procStart_, weightsD_, geomD_, isWallCell_,
+                nu_, relaxEps_, relaxK_, tolU_, maxIter_, P_.globalNCells, ones_, bounded_, keCoeffs_, &rEps, &rK);
+        }
     }
     return res;
 }
diff --git a/tests/test_gpu_parallel_turbloop.cu b/tests/test_gpu_parallel_turbloop.cu
index 3ef7540..3284304 100644
--- a/tests/test_gpu_parallel_turbloop.cu
+++ b/tests/test_gpu_parallel_turbloop.cu
@@ -21,6 +21,7 @@
 #include "parallel_device_simple.cuh"
 #include "cf_pstream.cuh"
 #include "box_mesh.cuh"
+#include "cell_wall_dist.cuh"
 
 #include <cmath>
 #include <cstdio>
@@ -40,10 +41,11 @@ int main(int argc, char** argv)
     const label Nx = std::atol(argv[1]), Ny = std::atol(argv[2]), Nz = std::atol(argv[3]);
     const std::string ref = (argc > 4) ? argv[4] : "turbloop_ref.txt";
     const std::string dump = (argc > 5) ? argv[5] : "";
+    const bool sst = (argc > 6) && (std::string(argv[6]) == "sst");
     const scalar nu = 1e-3, relaxU = 0.7, relaxP = 0.3, relaxK = 0.7, tolU = 1e-9, tolP = 1e-9;
     const scalar kIn = 0.375, epsIn = 0.09 * std::pow(kIn, 1.5) / 0.1;
     const int maxIter = 2000, N = 4;
-    KEpsilonCoeffs co;
+    KEpsilonCoeffs co; KOmegaSSTCoeffs sco;
     const label nC = Nx * Ny * Nz;
     bool ok = true;
     scalar relU = 0, relP = 0, relK = 0, relN = 0;
@@ -74,11 +76,13 @@ int main(int argc, char** argv)
         GeometricField<vector> U0 = distributeField<vector>(Ufd, gm.patches(), P.Lm, P.lp, P.procW, rank); U0.evaluateBoundary();
         GeometricField<scalar> p0 = distributeField<scalar>(pfdta, gm.patches(), P.Lm, P.lp, P.procW, rank); p0.evaluateBoundary();
         GeometricField<scalar> k0   = distributeField<scalar>(scalfd(kIn),   gm.patches(), P.Lm, P.lp, P.procW, rank); k0.evaluateBoundary();
-        GeometricField<scalar> eps0 = distributeField<scalar>(scalfd(epsIn), gm.patches(), P.Lm, P.lp, P.procW, rank); eps0.evaluateBoundary();
-        GeometricField<scalar> nut0 = distributeField<scalar>(scalfd(co.Cmu*kIn*kIn/epsIn), gm.patches(), P.Lm, P.lp, P.procW, rank); nut0.evaluateBoundary();
+        const scalar omIn = std::sqrt(kIn) / (0.09 * 0.1);   // SST second field is omega
+        GeometricField<scalar> eps0 = distributeField<scalar>(scalfd(sst ? omIn : epsIn), gm.patches(), P.Lm, P.lp, P.procW, rank); eps0.evaluateBoundary();
+        GeometricField<scalar> nut0 = distributeField<scalar>(scalfd(sst ? kIn/omIn : co.Cmu*kIn*kIn/epsIn), gm.patches(), P.Lm, P.lp, P.procW, rank); nut0.evaluateBoundary();
 
         ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, maxIter);
-        solver.enableTurbulence(U0, k0, eps0, nut0, co, relaxK, relaxK);
+        if (sst) solver.enableTurbulenceSST(U0, k0, eps0, nut0, sco, relaxK, relaxK);
+        else     solver.enableTurbulence(U0, k0, eps0, nut0, co, relaxK, relaxK);
         if (!dump.empty()) solver.setStageDump(dump, (argc>6)?std::atoi(argv[6]):2);
         for (int it = 0; it < N; ++it) solver.step();
         cudaDeviceSynchronize();

From da248fce7851d757574d02a234cb219e998f7c58 Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Sat, 18 Jul 2026 20:48:06 +0200
Subject: [PATCH 08/13] #12: distributed AMG preconditioner + device-resident
 momentum with on-stream NVSHMEM reductions

---
 CMakeLists.txt                                |   1 +
 bench/setup_spuma.sh                          |   2 +-
 .../simpleFoam/parallel_device_foam.cuh       |   5 +-
 .../simpleFoam/parallel_device_simple.cuh     |  36 +++-
 src/cuda/device_amg.cu                        |  13 ++
 src/cuda/device_amg.cuh                       |   7 +
 src/cuda/device_pcg.cu                        | 163 ++++++++++++++----
 src/cuda/device_pcg.cuh                       |  20 +++
 src/parallel/pstream/device_halo.cuh          |   7 +
 src/parallel/pstream/device_reduce.cu         |  52 ++++++
 src/parallel/pstream/device_reduce.cuh        |  36 ++++
 11 files changed, 307 insertions(+), 35 deletions(-)
 create mode 100644 src/parallel/pstream/device_reduce.cu
 create mode 100644 src/parallel/pstream/device_reduce.cuh

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dca9e4b..8d77084 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -61,6 +61,7 @@ list(REMOVE_DUPLICATES BRAE_HEADER_DIRS)
 add_library(brae_core STATIC
     src/parallel/pstream/cf_pstream.cu
     src/parallel/pstream/device_halo.cu
+    src/parallel/pstream/device_reduce.cu
     src/OpenFOAM/meshes/interface/processor_interface.cu
     src/OpenFOAM/db/foam_token_reader.cu
     src/OpenFOAM/meshes/primitive_mesh.cu
diff --git a/bench/setup_spuma.sh b/bench/setup_spuma.sh
index fa39d9b..161b32b 100755
--- a/bench/setup_spuma.sh
+++ b/bench/setup_spuma.sh
@@ -25,7 +25,7 @@
 #  Env:  SPUMA_SRC (clone target, default $HOME/spuma)   NVARCH (default: autodetect, e.g. 121)
 #        NVHPC_DIR (HPC SDK compilers/bin, default: autodetect)   JOBS (default 16)
 # ============================================================================
-set -u
+set +u   # OpenFOAM etc/bashrc is not nounset-safe; sourcing it under set -u fails
 SPUMA_SRC="${SPUMA_SRC:-$HOME/spuma}"
 NVARCH="${NVARCH:-$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d .)}"
 JOBS="${JOBS:-16}"
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index e3a3300..110c4a4 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -297,8 +297,9 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             for (iter = 1; iter <= endTime && !converged; ++iter)
             {
                 const ParStepResidual r = solver.step();
-                if (master && (iter == 1 || iter % 50 == 0))
-                    std::printf("  iter %4d:  Ux %.3e  p %.3e\n", iter, r.Ux, r.p);
+                if (master && (iter == 1 || iter % 20 == 0))
+                    std::printf("  iter %4d:  Ux %.3e  p %.3e  | avg krylov/iter %ld\n",
+                                iter, r.Ux, r.p, solver.krylovIters() / iter);
                 // residualControl on U/p (the turbulence fields co-converge; step() returns only U/p residuals).
                 converged = hasRC && ok(r.p, rcP) && ok(r.Ux, rcU);
             }
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index c986762..4a40fa4 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -14,6 +14,7 @@
 #include "device_blas.cuh"
 #include "device_boundary.cuh"
 #include "device_pcg.cuh"
+#include "device_amg.cuh"          // local per-rank AMG preconditioner for the distributed pressure solve
 #include "device_divdevreff.cuh"
 #include "device_halo.cuh"
 #include "parallel_simple.cuh"     // Partition, distributeFromCells
@@ -683,6 +684,27 @@ public:
         // rAU's boundary shape: zeroGradient on real patches, coupled on processor (mirrors distributeFromCells)
         rAUfld_ = distributeFromCells<scalar>(std::vector<scalar>(lnC_, 1.0), P_);
         dbRAU_  = buildDeviceBoundary(rAUfld_, part.lp, part.lg);
+
+        // LOCAL AMG preconditioner for the distributed pressure solve (the #1 lever over point-Jacobi, which
+        // caps its iteration count and diverges on graded meshes). Built ONCE per rank from this partition's
+        // internal-face addressing + face areas (static geometry), exactly like the single-GPU pressure AMG.
+        // It intentionally omits the PROCESSOR interface -- as a preconditioner inside the interface-coupled
+        // outer CG that is the block-Jacobi/additive-Schwarz design (the outer matvec carries the coupling). A
+        // CYCLIC patch, however, is a long-range edge the Galerkin coarse operator cannot represent, so AMG is
+        // skipped (fall back to Jacobi-PCG) when the local mesh has one, matching the single-GPU guard.
+        {
+            const bool amgOff = (std::getenv("BRAE_PARALLEL_AMG") && std::atoi(std::getenv("BRAE_PARALLEL_AMG")) == 0);
+            bool hasCyclic = false;
+            for (const FvPatch& p : lp) if (p.type == "cyclic" || p.type == "cyclicAMI") hasCyclic = true;
+            if (!amgOff && !hasCyclic && lnC_ > 64)
+            {
+                const std::vector<label> ownerInt(P_.Lm.mesh.owner().begin(), P_.Lm.mesh.owner().begin() + nIf_);
+                const std::vector<label> neiInt(P_.Lm.mesh.neighbour());
+                std::vector<scalar> fwAMG(P_.lg.magSf().begin(), P_.lg.magSf().begin() + nIf_);   // geometric face area
+                pAMG_ = buildAMG(ownerInt, neiInt, fwAMG, static_cast<int>(lnC_));
+                useAMG_ = true;
+            }
+        }
     }
 
     // STAGE DUMP: write every intermediate of iteration `iter`, gathered to the GLOBAL cell ordering,
@@ -883,6 +905,8 @@ private:
     DeviceBoundary       dbP_, dbRAU_;
     GeometricField<scalar> rAUfld_;
     DeviceHalo           halo_;
+    bool     useAMG_ = false;   // pressure preconditioned by the local per-rank AMG V-cycle (else point-Jacobi)
+    AMGData  pAMG_;             // the local pressure-Laplacian hierarchy (built once, Galerkin-recoarsened per step)
     std::vector<label>   procStart_;
     std::vector<DeviceBuffer<label>>  faceCellsD_;
     std::vector<DeviceBuffer<scalar>> weightsD_;
@@ -1277,7 +1301,17 @@ inline ParStepResidual ParallelDeviceSimple::step()
     deviceCopy(pSol, dp_);
     const DeviceLduView pv = deviceLduView(dm_, pDiagC, pU_, pL_);
     const scalar nfp = deviceParallelNormFactor(pv, halo_, pIfCoeff, pSol, pb, ones_, P_.globalNCells);
-    const DeviceSolverPerf pp = deviceParallelJacobiPCG(pv, halo_, pIfCoeff, pb, pSol, nfp, tolP_, 0.0, maxIter_);
+    // pressure solve: local-AMG-preconditioned distributed CG (the strong preconditioner that converges on graded
+    // meshes), else point-Jacobi CG. amgGalerkin re-coarsens the hierarchy from THIS step's pressure matrix
+    // (pDiagC includes the interface + boundary diagonal terms; pU_/pL_ are the internal Laplacian off-diagonals).
+    DeviceSolverPerf pp;
+    if (useAMG_)
+    {
+        amgGalerkin(pAMG_, pDiagC, pU_, pL_);
+        pp = deviceParallelAMGPCG(pv, halo_, pIfCoeff, pb, pSol, pAMG_, nfp, tolP_, 0.0, maxIter_);
+    }
+    else
+        pp = deviceParallelJacobiPCG(pv, halo_, pIfCoeff, pb, pSol, nfp, tolP_, 0.0, maxIter_);
     kIters_ += pp.nIterations;
     res.p = pp.initialResidual;
     dumpStage("pSol", pSol);
diff --git a/src/cuda/device_amg.cu b/src/cuda/device_amg.cu
index a5a4f1c..fe0e867 100644
--- a/src/cuda/device_amg.cu
+++ b/src/cuda/device_amg.cu
@@ -2881,4 +2881,17 @@ DeviceSolverPerf deviceAMGPCG(
     return perf;
 }
 
+// z = M^-1 r : ONE symmetric AMG V-cycle applied as a PRECONDITIONER, factored out of deviceAMGPCG so the
+// DISTRIBUTED Krylov (device_pcg.cu) can precondition each rank's LOCAL block with AMG. The V-cycle is built on
+// internal faces only, so it omits the processor-interface coupling -- exactly the block-Jacobi / additive-Schwarz
+// design: the outer distributed matvec (deviceParallelAmul) supplies the interface, the local V-cycle need only
+// approximate the local block. amg must be built (buildAMG) and current (amgGalerkin gives it this step's coarse
+// operators). FP64 (the outer distributed Krylov is FP64; the mixed-precision cast path stays inside deviceAMGPCG).
+void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
+                    const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z)
+{
+    ensureSpectrum(amg, A);                // one-time Chebyshev spectrum estimate (no-op unless the Chebyshev smoother is on)
+    vcycleAt(0, amg, A, r, z);             // z = M^-1 r, one symmetric V-cycle down the local hierarchy and back
+}
+
 } // namespace brae
diff --git a/src/cuda/device_amg.cuh b/src/cuda/device_amg.cuh
index e5df9bc..8ecdaee 100644
--- a/src/cuda/device_amg.cuh
+++ b/src/cuda/device_amg.cuh
@@ -130,6 +130,13 @@ DeviceSolverPerf deviceAMGPCG(const DeviceLduView& Afine, AMGData& amg, const De
                               DeviceBuffer<scalar>& psi, scalar normFactor, scalar tol, scalar relTol, int maxIter,
                               bool captureVcycle = false, int checkEvery = 1, bool corrScaling = false);
 
+// z = M^-1 r : one symmetric AMG V-cycle applied as a PRECONDITIONER (the V-cycle factored out of deviceAMGPCG).
+// Used by the distributed Krylov (deviceParallelAMGPCG) to precondition each rank's LOCAL block with AMG -- the
+// V-cycle is internal-face only, so it omits the processor interface (block-Jacobi/additive-Schwarz: the outer
+// distributed matvec supplies the interface coupling). amg must be built (buildAMG) and current (amgGalerkin).
+void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
+                    const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z);
+
 // #7 (clusters+DSM): the coarse-level weighted-Jacobi solve (nSweeps of x += omega*(b-Ax)/diag), fused into a
 // SINGLE thread-block-cluster kernel, the whole coarse vector lives in the cluster's distributed shared memory
 // and all sweeps run with cluster.sync() between them (1 launch instead of 2*nSweeps). xc is the in/out guess.
diff --git a/src/cuda/device_pcg.cu b/src/cuda/device_pcg.cu
index 87d623e..26c2be7 100644
--- a/src/cuda/device_pcg.cu
+++ b/src/cuda/device_pcg.cu
@@ -1,8 +1,10 @@
 // cf GPU offload (G2): device-resident Jacobi-PCG. Mirrors brae::pcg's CG recurrence exactly, but the
 // preconditioner is Jacobi (wA = rA/diag) and every vector op is a device kernel.
 #include "device_pcg.cuh"
+#include "device_amg.cuh"   // AMGData + amgVCycleApply (the distributed AMG-PCG preconditioner)
 #include "device_blas.cuh"
 #include "device_halo.cuh"
+#include "device_reduce.cuh"   // DeviceReducer: on-stream NVSHMEM global reduction (replaces host MPI_Allreduce)
 #include "cf_pstream.cuh"
 #include <cuda_runtime.h>
 #include <cmath>
@@ -360,6 +362,76 @@ DeviceSolverPerf deviceParallelJacobiPCG(
     return perf;
 }
 
+// AMG-preconditioned distributed CG: deviceParallelJacobiPCG with the point-Jacobi preconditioner (wA = rA/diag)
+// replaced by a per-rank LOCAL AMG V-cycle (wA = M^-1 rA via amgVCycleApply). Everything else is IDENTICAL -- the
+// halo-coupled matvec (deviceParallelAmul), the two fused GLOBAL reductions per iteration, the CG recurrence and
+// the convergence test are byte-for-byte the same, so at np=1 with the same AMG this is the single-GPU AMG-PCG
+// path and at np>1 it is that with the interface added to the matvec. The V-cycle is block-Jacobi (it drops the
+// interface), which the outer CG corrects for; the gain is a vastly stronger preconditioner than point-Jacobi,
+// so the pressure converges on graded meshes instead of hitting maxIter and diverging.
+DeviceSolverPerf deviceParallelAMGPCG(
+    const DeviceLduView& A,
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<scalar>>& ifaceCoeffs,
+    const DeviceBuffer<scalar>& b,
+    DeviceBuffer<scalar>& psi,
+    AMGData& amg,
+    scalar normFactor,
+    scalar tol,
+    scalar relTol,
+    int maxIter)
+{
+    const int nC = A.nCells;
+    DeviceBuffer<scalar> wA(nC), rA(nC), pA(nC), Ax(nC), ApA;
+
+    deviceParallelAmul(A, halo, ifaceCoeffs, psi, Ax);          // rA = b - A*psi  (interface-coupled)
+    deviceCopy(rA, b);
+    deviceAxpy(-1.0, Ax, rA);
+
+    DeviceSolverPerf perf;
+    scalar red[2];
+    auto fusedReduce = [&]()   // wA = M^-1 rA (LOCAL AMG V-cycle) ; red = [ dot(wA,rA), sumMag(rA) ] in ONE collective
+    {
+        amgVCycleApply(amg, A, rA, wA);                         // <-- the only change vs deviceParallelJacobiPCG
+        red[0] = deviceDot(wA, rA);
+        red[1] = deviceSumMag(rA);
+        Pstream::allReduce(red, 2, ReduceOp::Sum);
+    };
+
+    fusedReduce();
+    perf.initialResidual = red[1] / normFactor;
+    perf.finalResidual   = perf.initialResidual;
+    auto converged = [&](scalar fr) { return (fr < tol) || (relTol > 0.0 && fr < relTol * perf.initialResidual); };
+
+    scalar wArA = red[0], wArAold = 1e300;
+    int nIter = 0;
+    if (!converged(perf.finalResidual))
+    {
+        do
+        {
+            if (nIter == 0) deviceCopy(pA, wA);
+            else
+            {
+                const scalar beta = wArA / wArAold;
+                deviceScale(pA, beta);
+                deviceAxpy(1.0, wA, pA);
+            }
+            deviceParallelAmul(A, halo, ifaceCoeffs, pA, ApA);  // ApA = A*pA
+            const scalar wApA  = Pstream::allReduce(deviceDot(ApA, pA), ReduceOp::Sum);
+            const scalar alpha = wArA / wApA;
+            deviceAxpy(alpha, pA, psi);
+            deviceAxpy(-alpha, ApA, rA);
+            wArAold = wArA;
+            fusedReduce();                                      // next wArA AND the residual, one collective
+            wArA = red[0];
+            perf.finalResidual = red[1] / normFactor;
+            ++nIter;
+        } while (nIter < maxIter && !converged(perf.finalResidual));
+    }
+    perf.nIterations = nIter;
+    return perf;
+}
+
 // Distributed Jacobi-BiCGStab (non-symmetric momentum matrix). Mirrors deviceJacobiBiCGStab's recurrence with
 // the interface-coupled product and global reductions. Breakdown guards follow OF checkSingularity (VSMALL).
 DeviceSolverPerf deviceParallelJacobiBiCGStab(
@@ -371,9 +443,12 @@ DeviceSolverPerf deviceParallelJacobiBiCGStab(
     scalar normFactor,
     scalar tol,
     scalar relTol,
-    int maxIter)
+    int maxIter,
+    int checkEvery)
 {
     const int nC = A.nCells;
+    const int K  = (checkEvery > 1) ? checkEvery : 1;            // convergence-read cadence (1 = exact per-iter)
+    const cudaStream_t strm = cudaStreamPerThread;               // the halo/reduce stream: everything is stream-ordered
     DeviceBuffer<scalar> rA(nC), rA0(nC), pA(nC), yA(nC), AyA(nC), sA(nC), zA(nC), tA(nC), Ax(nC);
 
     deviceParallelAmul(A, halo, ifaceCoeffs, psi, Ax);           // rA = b - A*psi
@@ -381,54 +456,80 @@ DeviceSolverPerf deviceParallelJacobiBiCGStab(
     deviceAxpy(-1.0, Ax, rA);
     deviceCopy(rA0, rA);
 
+    // DEVICE-RESIDENT distributed BiCGStab: the exact recurrence of the single-GPU deviceJacobiBiCGStab (alpha/
+    // omega/beta and every dot live on the device, fed to the fused *Dev kernels by pointer), with two changes for
+    // multi-GPU: A*x is deviceParallelAmul (halo-coupled), and each LOCAL dot is globalised by an ON-STREAM NVSHMEM
+    // sum-reduce (DeviceReducer) instead of a blocking host MPI_Allreduce -- so the host is untouched per iteration
+    // except the K-cadence convergence read. This removes the ~6 D2H + 6 MPI_Allreduce per iteration that were the
+    // multi-GPU momentum-solve bottleneck. Breakdown (OF checkSingularity) is detected ON-DEVICE into `bd` from the
+    // GLOBAL rr/omega (identical on every PE), read only on check iters. gdot/gsum wrap local-dot -> reduce -> copy.
+    DeviceReducer& R = halo.reducer();
+    auto gdot = [&](const DeviceBuffer<scalar>& x, const DeviceBuffer<scalar>& y, DeviceBuffer<scalar>& out)
+    {   // out (device scalar) = global sum of x.y across all PEs
+        deviceDotInto(x, y, R.src());
+        R.sumReduce(1, strm);
+        deviceScalarCopy(R.dst(), out.data());
+    };
+    auto gsumHost = [&](const DeviceBuffer<scalar>& x)   // global |x|_1, returned to the host (the only per-check D2H)
+    {
+        deviceSumMagInto(x, R.src());
+        R.sumReduce(1, strm);
+        return deviceReadScalar(R.dst());
+    };
+
     DeviceSolverPerf perf;
-    perf.initialResidual = Pstream::allReduce(deviceSumMag(rA), ReduceOp::Sum) / normFactor;
+    perf.initialResidual = gsumHost(rA) / normFactor;
     perf.finalResidual   = perf.initialResidual;
     auto converged = [&](scalar fr) { return (fr < tol) || (relTol > 0.0 && fr < relTol * perf.initialResidual); };
 
-    scalar rr = 1.0, rrOld = 1.0, alpha = 0.0, omega = 1.0;
+    BiCGScalars& s = bicgScalars();
+    cudaCheck(cudaMemsetAsync(s.bd.data(), 0, sizeof(scalar), strm), "pbicg bd zero");
     int nIter = 0;
     if (!converged(perf.finalResidual))
     {
         do
         {
-            rrOld = rr;
-            rr = Pstream::allReduce(deviceDot(rA0, rA), ReduceOp::Sum);   // rA0rA
-            if (fabs(rr) < BICG_VSMALL) break;                            // OF checkSingularity(rA0rA)
+            if (nIter > 0) deviceScalarCopy(s.rr.data(), s.rrOld.data());   // rrOld = rr
+            gdot(rA0, rA, s.rr);                                            // rr = rA0.rA (global)
+            bicgRhoSingK<<<1,1,0,strm>>>(s.rr.data(), s.bd.data());         // OF checkSingularity(rA0rA) -> flag
             if (nIter == 0) deviceCopy(pA, rA);
             else
             {
-                const scalar beta = (rr / rrOld) * (alpha / omega);
-                deviceAxpy(-omega, AyA, pA);                              // pA = rA + beta*(pA - omega*AyA)
-                deviceScale(pA, beta);
-                deviceAxpy(1.0, rA, pA);
+                bicgBetaK<<<1,1,0,strm>>>(s.rr.data(), s.rrOld.data(), s.alpha.data(), s.omega.data(), s.beta.data(), s.bd.data());
+                deviceFusedBicgP(rA, pA, AyA, s.beta.data(), s.negOmega.data());   // pA = rA + beta*(pA - omega*AyA)
             }
-            deviceJacobi(yA, pA, A.diag);                                 // yA = M^-1 pA (local Jacobi)
+            deviceJacobi(yA, pA, A.diag);                                   // yA = M^-1 pA (local block-Jacobi)
             deviceParallelAmul(A, halo, ifaceCoeffs, yA, AyA);
-            const scalar r0Ay = Pstream::allReduce(deviceDot(rA0, AyA), ReduceOp::Sum);
-            if (fabs(r0Ay) < BICG_VSMALL) break;
-            alpha = rr / r0Ay;
-            deviceCopy(sA, rA);
-            deviceAxpy(-alpha, AyA, sA);                                  // sA = rA - alpha*AyA
-            perf.finalResidual = Pstream::allReduce(deviceSumMag(sA), ReduceOp::Sum) / normFactor;
-            if (converged(perf.finalResidual))                            // mid-iteration early exit
+            gdot(rA0, AyA, s.r0Ay);
+            bicgAlphaK<<<1,1,0,strm>>>(s.rr.data(), s.r0Ay.data(), s.alpha.data(), s.negAlpha.data(), s.bd.data());
+            deviceFusedSxpy(sA, rA, s.negAlpha.data(), AyA);               // sA = rA - alpha*AyA
+            const bool check = ((nIter + 1) % K == 0) || (nIter + 1 >= maxIter);
+            if (check)                                                     // mid-iter early exit only when we read |s|
             {
-                deviceAxpy(alpha, yA, psi);
-                ++nIter;
-                break;
+                perf.finalResidual = gsumHost(sA) / normFactor;
+                if (converged(perf.finalResidual))
+                {
+                    deviceAxpyDev(s.alpha.data(), yA, psi);
+                    ++nIter;
+                    break;
+                }
             }
-            deviceJacobi(zA, sA, A.diag);                                 // zA = M^-1 sA
+            deviceJacobi(zA, sA, A.diag);                                  // zA = M^-1 sA
             deviceParallelAmul(A, halo, ifaceCoeffs, zA, tA);
-            const scalar tt = Pstream::allReduce(deviceDot(tA, tA), ReduceOp::Sum);
-            const scalar ts = Pstream::allReduce(deviceDot(tA, sA), ReduceOp::Sum);
-            omega = (tt > BICG_VSMALL) ? ts / tt : 0.0;
-            deviceAxpy(alpha, yA, psi);                                   // psi += alpha*yA + omega*zA
-            deviceAxpy(omega, zA, psi);
-            deviceCopy(rA, sA);
-            deviceAxpy(-omega, tA, rA);                                   // rA = sA - omega*tA
-            perf.finalResidual = Pstream::allReduce(deviceSumMag(rA), ReduceOp::Sum) / normFactor;
+            deviceDotInto(tA, tA, R.src() + 0);                            // tt AND ts in ONE fused collective
+            deviceDotInto(tA, sA, R.src() + 1);
+            R.sumReduce(2, strm);
+            deviceScalarCopy(R.dst() + 0, s.tt.data());
+            deviceScalarCopy(R.dst() + 1, s.ts.data());
+            omegaK<<<1,1,0,strm>>>(s.ts.data(), s.tt.data(), s.omega.data(), s.negOmega.data(), s.bd.data());
+            deviceFusedAxpy2(psi, s.alpha.data(), yA, s.omega.data(), zA); // psi += alpha*yA + omega*zA
+            deviceFusedSxpy(rA, sA, s.negOmega.data(), tA);               // rA = sA - omega*tA
+            if (check)
+            {
+                perf.finalResidual = gsumHost(rA) / normFactor;
+                if (deviceReadScalar(s.bd.data()) != 0.0) { ++nIter; break; }   // OF break on singularity (batched to K)
+            }
             ++nIter;
-            if (fabs(omega) < BICG_VSMALL) break;                         // OF checkSingularity(omega)
         } while (nIter < maxIter && !converged(perf.finalResidual));
     }
     perf.nIterations = nIter;
diff --git a/src/cuda/device_pcg.cuh b/src/cuda/device_pcg.cuh
index 9f0935e..e8d516d 100644
--- a/src/cuda/device_pcg.cuh
+++ b/src/cuda/device_pcg.cuh
@@ -69,6 +69,26 @@ DeviceSolverPerf deviceParallelJacobiBiCGStab(
     scalar normFactor,
     scalar tol,
     scalar relTol,
+    int maxIter,
+    int checkEvery = 1);   // convergence-read cadence: read |r| to the host every K iters (1 = exact per-iter)
+
+struct AMGData;   // fwd (device_amg.cuh); passed by ref so this header need not include the AMG hierarchy
+
+// AMG-preconditioned distributed CG: identical recurrence to deviceParallelJacobiPCG (halo-coupled matvec +
+// GLOBAL fused reductions), but the preconditioner z=M^-1 r is a per-rank LOCAL AMG V-cycle (amgVCycleApply)
+// instead of point-Jacobi. This is the block-Jacobi/additive-Schwarz AMG preconditioner: it converges the
+// pressure on stiff/graded meshes where point-Jacobi caps its iteration count and the SIMPLE loop diverges.
+// `amg` must be built on A's LOCAL internal addressing (buildAMG) and current for this step (amgGalerkin).
+DeviceSolverPerf deviceParallelAMGPCG(
+    const DeviceLduView& A,
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<scalar>>& ifaceCoeffs,
+    const DeviceBuffer<scalar>& b,
+    DeviceBuffer<scalar>& psi,
+    AMGData& amg,
+    scalar normFactor,
+    scalar tol,
+    scalar relTol,
     int maxIter);
 
 } // namespace brae
diff --git a/src/parallel/pstream/device_halo.cuh b/src/parallel/pstream/device_halo.cuh
index 482d340..a9d6691 100644
--- a/src/parallel/pstream/device_halo.cuh
+++ b/src/parallel/pstream/device_halo.cuh
@@ -14,6 +14,7 @@
 #include "cf_types.cuh"
 #include "cf_pstream.cuh"
 #include "sym_buffer.cuh"
+#include "device_reduce.cuh"   // DeviceReducer owned here so its symmetric heap is freed with the halo (pre-finalize)
 #include "device_buffer.cuh"
 #include <cuda_runtime.h>
 #include <vector>
@@ -85,12 +86,18 @@ public:
         int i,
         cudaStream_t stream = cudaStreamPerThread) const;
 
+    // On-stream NVSHMEM global reducer for the distributed Krylov solvers, owned here so its symmetric heap is
+    // allocated (collectively) with the halo and freed BEFORE Pstream::finalize -- a process-static reducer's
+    // nvshmem_free runs after finalize and crashes ("API called before init completed").
+    DeviceReducer& reducer() { return red_; }
+
 private:
     int                              myPart_ = 0;
     std::vector<int>                 nbr_;
     std::vector<label>               size_, recvOffset_, remoteOffset_;
     std::vector<DeviceBuffer<label>> faceCellsD_;
     SymBuffer<scalar>                sendBuf_, recvBuf_;
+    DeviceReducer                    red_{8};   // sized for the largest fused reduction group (BiCGStab: 2 dots)
     label                            total_ = 0;
 };
 
diff --git a/src/parallel/pstream/device_reduce.cu b/src/parallel/pstream/device_reduce.cu
new file mode 100644
index 0000000..4107a0c
--- /dev/null
+++ b/src/parallel/pstream/device_reduce.cu
@@ -0,0 +1,52 @@
+// See device_reduce.cuh. The one NVSHMEM host-side on-stream collective brae adds beyond the halo put/barrier.
+#include "device_reduce.cuh"
+#include "cf_pstream.cuh"                 // Pstream::nProcs / allReduce (MPG fallback)
+#include <cstdlib>                        // getenv / atoi (A/B knob)
+#ifdef BRAE_WITH_NVSHMEM
+#include <nvshmem_host.h>                 // symmetric heap + team world (no device defines -> no -rdc)
+#include <host/nvshmemx_coll_api.h>       // nvshmemx_double_sum_reduce_on_stream (host-callable, on-stream)
+#endif
+
+namespace brae {
+
+static_assert(sizeof(scalar) == 8, "DeviceReducer uses nvshmemx_DOUBLE_sum_reduce; scalar must be FP64");
+
+DeviceReducer::DeviceReducer(int maxN)
+    : src_(static_cast<std::size_t>(maxN)), dst_(static_cast<std::size_t>(maxN))
+{
+    // brae binds rank -> (rank % nDevices), so more ranks than visible GPUs means at least one GPU is shared
+    // (MPG). NVSHMEM's on-stream reduce is unsupported under MPG; detect it once here to pick the path.
+    int nDev = 0;
+    cudaGetDeviceCount(&nDev);
+    mpg_ = (nDev > 0) && (Pstream::nProcs() > nDev);
+    // BRAE_FORCE_HOST_REDUCE=1 forces the host-MPI_Allreduce path even on real multi-GPU -- an A/B knob to
+    // isolate the on-stream-reduce speedup against the old reduction path on identical everything else.
+    if (const char* e = std::getenv("BRAE_FORCE_HOST_REDUCE")) if (std::atoi(e) != 0) mpg_ = true;
+}
+
+void DeviceReducer::sumReduce(int n, cudaStream_t stream)
+{
+#ifdef BRAE_WITH_NVSHMEM
+    if (!mpg_)
+    {
+        // 1 PE per GPU (real multi-GPU): on-stream global SUM across NVSHMEM_TEAM_WORLD. Enqueued on `stream`, no
+        // host block, no scalar over PCIe. At np=1 (one PE) it is the identity, matching the host-MPI np=1 path.
+        nvshmemx_double_sum_reduce_on_stream(NVSHMEM_TEAM_WORLD, dst_.data(), src_.data(),
+                                             static_cast<std::size_t>(n), stream);
+        return;
+    }
+    // MPG (oversubscribed single-GPU oracle): on-stream reduce unsupported -> blocking host MPI_Allreduce. Only
+    // the correctness oracle takes this; real multi-GPU uses the on-stream path. Bounded by DeviceReducer's maxN.
+    scalar h[8];
+    cudaMemcpyAsync(h, src_.data(), static_cast<std::size_t>(n) * sizeof(scalar), cudaMemcpyDeviceToHost, stream);
+    cudaStreamSynchronize(stream);
+    Pstream::allReduce(h, n, ReduceOp::Sum);
+    cudaMemcpyAsync(dst_.data(), h, static_cast<std::size_t>(n) * sizeof(scalar), cudaMemcpyHostToDevice, stream);
+#else
+    // No NVSHMEM: a single PE, so the SUM reduction is the identity -> device copy (stream-ordered).
+    cudaMemcpyAsync(dst_.data(), src_.data(), static_cast<std::size_t>(n) * sizeof(scalar),
+                    cudaMemcpyDeviceToDevice, stream);
+#endif
+}
+
+} // namespace brae
diff --git a/src/parallel/pstream/device_reduce.cuh b/src/parallel/pstream/device_reduce.cuh
new file mode 100644
index 0000000..75db304
--- /dev/null
+++ b/src/parallel/pstream/device_reduce.cuh
@@ -0,0 +1,36 @@
+#pragma once
+// On-GPU global SUM reduction across all PEs, ON-STREAM (no host round-trip), for the distributed Krylov solvers.
+// This replaces the multi-GPU per-iteration bottleneck: today every Krylov dot does deviceDot -> a blocking D2H
+// copy of one scalar -> a blocking host MPI_Allreduce (see device_pcg.cu). Here the caller writes its local
+// partial sums into src() (device, in the NVSHMEM symmetric heap), calls sumReduce(n), and reads the global sums
+// from dst() (device, symmetric) -- entirely stream-ordered, the host untouched. Only the convergence test still
+// reads a scalar to the host, and the callers batch that to every K iterations.
+//
+// NVSHMEM's team reduce (nvshmemx_double_sum_reduce_on_stream) is the on-stream device collective; it requires
+// its operands to live in the symmetric heap, hence SymBuffer. When NVSHMEM is inactive (np=1, or a no-device CI
+// node) the SUM over the single PE is the identity, so sumReduce degrades to a device copy and the np=1 oracle is
+// bit-unchanged. SPUMA cannot do this -- its reductions are host MPI_Allreduce-bound (see the SPUMA analysis).
+#include "cf_types.cuh"
+#include "sym_buffer.cuh"
+#include <cuda_runtime.h>
+
+namespace brae {
+
+class DeviceReducer
+{
+public:
+    explicit DeviceReducer(int maxN);
+    scalar*       src()       { return src_.data(); }   // write the n LOCAL partial sums here (device)
+    const scalar* dst() const { return dst_.data(); }   // the n GLOBAL sums land here (device)
+    scalar*       dst()       { return dst_.data(); }
+    // Reduce the first n scalars of src() across all PEs (SUM), result in dst(), on `stream`. On real multi-GPU
+    // (1 PE/GPU) this is the on-stream NVSHMEM collective (no host sync); under MPG (multiple PEs per GPU, e.g. an
+    // oversubscribed single-GPU CI node, where NVSHMEM's on-stream reduce is unsupported) it falls back to a
+    // blocking host MPI_Allreduce -- correct, and only the correctness oracle ever takes that path.
+    void sumReduce(int n, cudaStream_t stream = cudaStreamPerThread);
+private:
+    SymBuffer<scalar> src_, dst_;
+    bool              mpg_ = false;   // multiple PEs share one GPU -> on-stream reduce unsupported, use host MPI
+};
+
+} // namespace brae

From 2d25ede5d0ef2180cad16da5a07e210f47c33c3b Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Sun, 19 Jul 2026 01:48:21 +0200
Subject: [PATCH 09/13] #12: fast polyMesh parser + FP32 distributed AMG +
 cached SCOTCH decompose

---
 .../decompose/scotch_decomposition.cu         | 47 +++++++++
 .../decompose/scotch_decomposition.cuh        |  5 +
 src/OpenFOAM/meshes/primitive_mesh.cu         | 96 ++++++++++++++-----
 .../simpleFoam/parallel_device_foam.cuh       | 24 ++++-
 src/cuda/device_amg.cu                        | 25 ++++-
 src/cuda/device_amg.cuh                       |  5 +
 src/cuda/device_pcg.cu                        |  2 +
 7 files changed, 179 insertions(+), 25 deletions(-)

diff --git a/src/OpenFOAM/decompose/scotch_decomposition.cu b/src/OpenFOAM/decompose/scotch_decomposition.cu
index a31b0cf..17f8722 100644
--- a/src/OpenFOAM/decompose/scotch_decomposition.cu
+++ b/src/OpenFOAM/decompose/scotch_decomposition.cu
@@ -3,6 +3,9 @@
 #include <cstdio>   // scotch.h uses FILE
 #include <scotch.h>
 #include <stdexcept>
+#include <fstream>
+#include <filesystem>
+#include <string>
 
 namespace brae {
 
@@ -67,4 +70,48 @@ std::vector<label> scotchDecompose(const PrimitiveMesh& m, label nParts)
     return cellToPart;
 }
 
+// Cached SCOTCH decomposition: the cell->partition map is STATIC per (mesh, nParts), but scotchDecompose costs
+// ~1s on a 1M-cell mesh and ran on every launch (a top-3 startup cost). Persist it to
+// cacheDir/.brae_decomp_np<N> and reload when the file is at least as new as cacheDir/owner (mesh unchanged) and
+// the header (magic, nParts, nCells) matches; otherwise recompute and write. Falls back to a plain decompose on
+// any I/O problem, so it can never be wrong -- at worst it recomputes.
+std::vector<label> scotchDecomposeCached(const PrimitiveMesh& m, label nParts, const std::string& cacheDir)
+{
+    namespace fs = std::filesystem;
+    constexpr unsigned kMagic = 0x44454331u;                 // "DEC1"
+    const label nC = m.nCells();
+    const std::string cachePath = cacheDir + "/.brae_decomp_np" + std::to_string(nParts);
+    const std::string ownerPath = cacheDir + "/owner";
+
+    std::error_code ec;
+    const bool fresh = fs::exists(cachePath, ec) && fs::exists(ownerPath, ec)
+                    && fs::last_write_time(cachePath, ec) >= fs::last_write_time(ownerPath, ec);
+    if (fresh)
+    {
+        std::ifstream is(cachePath, std::ios::binary);
+        unsigned magic = 0; label np = 0, n = 0;
+        is.read(reinterpret_cast<char*>(&magic), sizeof magic);
+        is.read(reinterpret_cast<char*>(&np),    sizeof np);
+        is.read(reinterpret_cast<char*>(&n),     sizeof n);
+        if (is && magic == kMagic && np == nParts && n == nC)
+        {
+            std::vector<label> v(nC);
+            is.read(reinterpret_cast<char*>(v.data()), static_cast<std::streamsize>(nC) * sizeof(label));
+            if (is) return v;                                // cache hit
+        }
+    }
+
+    std::vector<label> v = scotchDecompose(m, nParts);       // miss -> compute + persist
+    std::ofstream os(cachePath, std::ios::binary);
+    if (os)
+    {
+        const unsigned magic = kMagic; const label np = nParts, n = nC;
+        os.write(reinterpret_cast<const char*>(&magic), sizeof magic);
+        os.write(reinterpret_cast<const char*>(&np),    sizeof np);
+        os.write(reinterpret_cast<const char*>(&n),     sizeof n);
+        os.write(reinterpret_cast<const char*>(v.data()), static_cast<std::streamsize>(nC) * sizeof(label));
+    }
+    return v;
+}
+
 } // namespace brae
diff --git a/src/OpenFOAM/decompose/scotch_decomposition.cuh b/src/OpenFOAM/decompose/scotch_decomposition.cuh
index b9a35cb..38aa60c 100644
--- a/src/OpenFOAM/decompose/scotch_decomposition.cuh
+++ b/src/OpenFOAM/decompose/scotch_decomposition.cuh
@@ -5,9 +5,14 @@
 #include "cf_types.cuh"
 #include "primitive_mesh.cuh"
 #include <vector>
+#include <string>
 
 namespace brae {
 
 std::vector<label> scotchDecompose(const PrimitiveMesh& m, label nParts);
 
+// Cached scotchDecompose: reload cacheDir/.brae_decomp_np<N> if newer than cacheDir/owner (mesh unchanged),
+// else compute and persist. Static per (mesh, nParts); saves the ~1s decompose on warm launches.
+std::vector<label> scotchDecomposeCached(const PrimitiveMesh& m, label nParts, const std::string& cacheDir);
+
 } // namespace brae
diff --git a/src/OpenFOAM/meshes/primitive_mesh.cu b/src/OpenFOAM/meshes/primitive_mesh.cu
index f2d8c60..58b5ce2 100644
--- a/src/OpenFOAM/meshes/primitive_mesh.cu
+++ b/src/OpenFOAM/meshes/primitive_mesh.cu
@@ -3,10 +3,72 @@
 #include <cstdio>
 #include <cstdlib>
 #include <filesystem>
+#include <fstream>
+#include <cctype>
 
 namespace brae {
 
 namespace {
+
+// FAST polyMesh ASCII scan. The general TokenStream tokenises the whole file into a std::vector<std::string> --
+// one heap-allocated string PER NUMBER -- so a 1M-cell mesh (owner/neighbour 3M ints each, points 3M floats,
+// faces ~15M ints) does ~24M string allocations and took ~8.3s (measured, the dominant startup cost). The
+// polyMesh numeric lists have a rigid shape (count, '(', values, ')'), so parse them straight off the file
+// buffer with strtol/strtod and treat ( ) as delimiters -- no per-token allocation. Header + comments are
+// skipped structurally. Cuts the 1M-cell read from ~8.3s to well under 1s.
+struct FastScan
+{
+    std::string buf;   // whole file (null-terminated via std::string)
+    const char* p = nullptr;
+    const char* end = nullptr;
+    explicit FastScan(const std::string& path)
+    {
+        std::ifstream is(path, std::ios::binary);
+        if (!is) throw std::runtime_error("brae: cannot open " + path);
+        is.seekg(0, std::ios::end);
+        const std::streamoff n = is.tellg();
+        is.seekg(0, std::ios::beg);
+        buf.resize(n > 0 ? static_cast<std::size_t>(n) : 0);
+        if (n > 0) is.read(&buf[0], n);
+        p = buf.data();
+        end = p + buf.size();
+        skipHeader();
+    }
+    // Advance past the leading FoamFile{...} dictionary so the first value read is the payload count.
+    void skipHeader()
+    {
+        static const char kw[] = "FoamFile";
+        const char* f = std::search(p, end, kw, kw + 8);
+        if (f == end) return;                       // no header -> payload at the top
+        const char* b = std::find(f, end, '{');
+        if (b == end) return;
+        int depth = 0;
+        for (const char* q = b; q < end; ++q)
+        {
+            if (*q == '{') ++depth;
+            else if (*q == '}' && --depth == 0) { p = q + 1; return; }
+        }
+    }
+    // Next integer, skipping any delimiters (whitespace, parens, comment punctuation) before it.
+    long nextLong()
+    {
+        while (p < end && !(std::isdigit((unsigned char)*p) || *p == '-' || *p == '+')) ++p;
+        char* q = nullptr;
+        const long v = std::strtol(p, &q, 10);
+        p = q;
+        return v;
+    }
+    // Next float, skipping delimiters. '.' also starts a number (e.g. ".5").
+    double nextDouble()
+    {
+        while (p < end && !(std::isdigit((unsigned char)*p) || *p == '-' || *p == '+' || *p == '.')) ++p;
+        char* q = nullptr;
+        const double v = std::strtod(p, &q);
+        p = q;
+        return v;
+    }
+};
+
 constexpr unsigned BRAE_MESH_CACHE_MAGIC = 0x43464D32;   // "CFM2"
 template<class T>
 void wvec(std::FILE* f, const std::vector<T>& v)
@@ -113,19 +175,15 @@ void PrimitiveMesh::readPoints(const std::string& dir)
         return;
     }
 
-    TokenStream ts(path);
-    const label n = ts.nextLabel();
-    ts.expect("(");
+    FastScan s(path);
+    const label n = static_cast<label>(s.nextLong());   // ( and per-point ( ) are delimiters -> just read 3n floats
     points_.resize(n);
     for (label i = 0; i < n; ++i)
     {
-        ts.expect("(");
-        points_[i].x = ts.nextScalar();
-        points_[i].y = ts.nextScalar();
-        points_[i].z = ts.nextScalar();
-        ts.expect(")");
+        points_[i].x = s.nextDouble();
+        points_[i].y = s.nextDouble();
+        points_[i].z = s.nextDouble();
     }
-    ts.expect(")");
 }
 
 void PrimitiveMesh::readFaces(const std::string& dir)
@@ -137,31 +195,25 @@ void PrimitiveMesh::readFaces(const std::string& dir)
         return;
     }
 
-    TokenStream ts(path);
-    const label n = ts.nextLabel();
-    ts.expect("(");
+    FastScan s(path);
+    const label n = static_cast<label>(s.nextLong());   // face count
     faceOffsets_.resize(n + 1);
     faceOffsets_[0] = 0;
     faceVerts_.clear();
     for (label f = 0; f < n; ++f)
     {
-        const label k = ts.nextLabel();   // points in this face
-        ts.expect("(");
-        for (label j = 0; j < k; ++j) faceVerts_.push_back(ts.nextLabel());
-        ts.expect(")");
+        const label k = static_cast<label>(s.nextLong());   // verts in this face; its ( ) are delimiters
+        for (label j = 0; j < k; ++j) faceVerts_.push_back(static_cast<label>(s.nextLong()));
         faceOffsets_[f + 1] = faceOffsets_[f] + k;
     }
-    ts.expect(")");
 }
 
 static std::vector<label> readLabelColumn(const std::string& path)
 {
-    TokenStream ts(path);
-    const label n = ts.nextLabel();
-    ts.expect("(");
+    FastScan s(path);
+    const label n = static_cast<label>(s.nextLong());   // count; '(' then the n ints ')' are parsed as delimiters
     std::vector<label> v(n);
-    for (label i = 0; i < n; ++i) v[i] = ts.nextLabel();
-    ts.expect(")");
+    for (label i = 0; i < n; ++i) v[i] = static_cast<label>(s.nextLong());
     return v;
 }
 
diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index 110c4a4..11a8253 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -34,6 +34,8 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
+#include <chrono>
+#include <cstdlib>
 
 namespace brae {
 
@@ -226,15 +228,32 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         const scalar rcP = resCtl ? resCtl->scalarOr("p", -1) : -1;
         const scalar rcU = resCtl ? resCtl->scalarOr("U", -1) : -1;
 
+        // Startup phase timing (BRAE_TIME_STARTUP=1): the ~17-20s fixed cost dominates the 2-GPU wall on a short
+        // run, so break it down before optimising. cudaDeviceSynchronize before each lap captures GPU work; timed
+        // on master (a rough per-phase breakdown, enough to find the dominant contributor).
+        const bool timeStartup = std::getenv("BRAE_TIME_STARTUP") != nullptr;
+        auto tclk = std::chrono::high_resolution_clock::now();
+        auto lap = [&](const char* name)
+        {
+            if (!timeStartup) return;
+            cudaDeviceSynchronize();
+            auto now = std::chrono::high_resolution_clock::now();
+            if (master) std::printf("[startup] %-14s %8.1f ms\n", name,
+                                    std::chrono::duration<double, std::milli>(now - tclk).count());
+            tclk = now;
+        };
+
         PrimitiveMesh gm;
         gm.read(caseDir + "/constant/polyMesh");
         const label nC = gm.nCells();
+        lap("meshRead");
 
         // decompose in core: SCOTCH on master, broadcast so every rank agrees on the same partitioning
         std::vector<label> cellToPart(nC, 0);
-        if (master) cellToPart = scotchDecompose(gm, nproc);
+        if (master) cellToPart = scotchDecomposeCached(gm, nproc, caseDir + "/constant/polyMesh");   // ~1s -> cached
         Pstream::broadcast(cellToPart.data(), nC, 0);
         const Partition P(gm, cellToPart, rank);
+        lap("decompose");
 
         // The non-orthogonal correction is NOT implemented at processor faces. On an ORTHOGONAL mesh it is
         // identically zero, so "corrected" is safe there and 15/113 cases say "orthogonal" outright. On a
@@ -259,6 +278,7 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         U0.evaluateBoundary();
         GeometricField<scalar> p0 = distributeField<scalar>(pfd, gm.patches(), P.Lm, P.lp, P.procW, rank);
         p0.evaluateBoundary();
+        lap("distribute");
 
         // turbulence start fields (RAS kEpsilon)
         FieldData<scalar> kfd, efd, nfd;
@@ -290,6 +310,7 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind, lust);
             if (turbulent && sst) solver.enableTurbulenceSST(U0, k0, eps0, nut0, sstCoeffs, relaxK, relaxEps);
             else if (turbulent)   solver.enableTurbulence(U0, k0, eps0, nut0, keCoeffs, relaxK, relaxEps);
+            lap("buildSolver");   // buildDeviceMesh + buildAMG + NVSHMEM halo/symmetric-heap alloc
 
             auto ok = [](scalar res, scalar ctl) { return ctl < 0 || res < ctl; };
             bool converged = false;
@@ -297,6 +318,7 @@ inline int runParallelDeviceFoam(int argc, char** argv)
             for (iter = 1; iter <= endTime && !converged; ++iter)
             {
                 const ParStepResidual r = solver.step();
+                if (iter == 1) lap("firstStep");   // first step pays graph capture + first-touch allocation
                 if (master && (iter == 1 || iter % 20 == 0))
                     std::printf("  iter %4d:  Ux %.3e  p %.3e  | avg krylov/iter %ld\n",
                                 iter, r.Ux, r.p, solver.krylovIters() / iter);
diff --git a/src/cuda/device_amg.cu b/src/cuda/device_amg.cu
index fe0e867..cf57a69 100644
--- a/src/cuda/device_amg.cu
+++ b/src/cuda/device_amg.cu
@@ -2881,17 +2881,38 @@ DeviceSolverPerf deviceAMGPCG(
     return perf;
 }
 
+// Cast the fine + coarse matrices to their FP32 mirrors for this solve's V-cycles. Call ONCE per solve, AFTER
+// amgGalerkin has updated the FP64 operators (the coarse matrices change every SIMPLE step). No-op unless FP32
+// mixed precision applies (BRAE_AMG_FP32 default on; the SA/GS/Chebyshev paths stay FP64). After this,
+// amgVCycleApply runs the FP32 V-cycle automatically -- the same mixed-precision path the single-GPU deviceAMGPCG
+// uses, now available to the distributed Krylov (the pressure V-cycle's SpMV+smoother are bandwidth-bound, so
+// FP32 halves their bytes ~= 2x, while the r->FP32/FP32->z casts and the FP64 coarsest solve keep it accurate).
+void amgPrepareFP32(AMGData& amg, const DeviceLduView& A)
+{
+    if (useFP32() && !amg.saSmooth && !amg.gsSmooth && !useChebyshev())
+        amgCastFP32(amg, A);
+}
+
 // z = M^-1 r : ONE symmetric AMG V-cycle applied as a PRECONDITIONER, factored out of deviceAMGPCG so the
 // DISTRIBUTED Krylov (device_pcg.cu) can precondition each rank's LOCAL block with AMG. The V-cycle is built on
 // internal faces only, so it omits the processor-interface coupling -- exactly the block-Jacobi / additive-Schwarz
 // design: the outer distributed matvec (deviceParallelAmul) supplies the interface, the local V-cycle need only
 // approximate the local block. amg must be built (buildAMG) and current (amgGalerkin gives it this step's coarse
-// operators). FP64 (the outer distributed Krylov is FP64; the mixed-precision cast path stays inside deviceAMGPCG).
+// operators). Runs the FP32 V-cycle when amgPrepareFP32 cast the matrices this solve, else the FP64 one.
 void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
                     const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z)
 {
     ensureSpectrum(amg, A);                // one-time Chebyshev spectrum estimate (no-op unless the Chebyshev smoother is on)
-    vcycleAt(0, amg, A, r, z);             // z = M^-1 r, one symmetric V-cycle down the local hierarchy and back
+    const bool fp32 = amg.fp32Alloc && useFP32() && !amg.saSmooth && !amg.gsSmooth && !useChebyshev();
+    if (fp32)
+    {
+        const int nC = A.nCells;
+        const LduF A0 = lduF(A, amg.fDiag[0], amg.fUpper[0], amg.fLower[0]);   // grid-0 FP32 matrix view
+        cast_<scalar,float><<<nBlocks(nC),TPB>>>(nC, r.data(), amg.vBF[0].data());   // r -> FP32
+        vcycleAtF(0, amg, A, A0, amg.vBF[0].data(), amg.vXF[0].data());              // FP32 V-cycle
+        cast_<float,scalar><<<nBlocks(nC),TPB>>>(nC, amg.vXF[0].data(), z.data());   // FP32 -> z (FP64)
+    }
+    else vcycleAt(0, amg, A, r, z);        // z = M^-1 r, one symmetric FP64 V-cycle down the local hierarchy and back
 }
 
 } // namespace brae
diff --git a/src/cuda/device_amg.cuh b/src/cuda/device_amg.cuh
index 8ecdaee..ae635c8 100644
--- a/src/cuda/device_amg.cuh
+++ b/src/cuda/device_amg.cuh
@@ -137,6 +137,11 @@ DeviceSolverPerf deviceAMGPCG(const DeviceLduView& Afine, AMGData& amg, const De
 void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
                     const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z);
 
+// Prepare the FP32 mixed-precision V-cycle for this solve: cast the (Galerkin-updated) matrices to FP32 mirrors.
+// Call ONCE per solve before the amgVCycleApply loop; after it, amgVCycleApply runs FP32 automatically. No-op
+// unless BRAE_AMG_FP32 (default on) and the default smoother/aggregation (SA/GS/Chebyshev stay FP64).
+void amgPrepareFP32(AMGData& amg, const DeviceLduView& A);
+
 // #7 (clusters+DSM): the coarse-level weighted-Jacobi solve (nSweeps of x += omega*(b-Ax)/diag), fused into a
 // SINGLE thread-block-cluster kernel, the whole coarse vector lives in the cluster's distributed shared memory
 // and all sweeps run with cluster.sync() between them (1 launch instead of 2*nSweeps). xc is the in/out guess.
diff --git a/src/cuda/device_pcg.cu b/src/cuda/device_pcg.cu
index 26c2be7..f6c8335 100644
--- a/src/cuda/device_pcg.cu
+++ b/src/cuda/device_pcg.cu
@@ -384,6 +384,8 @@ DeviceSolverPerf deviceParallelAMGPCG(
     const int nC = A.nCells;
     DeviceBuffer<scalar> wA(nC), rA(nC), pA(nC), Ax(nC), ApA;
 
+    amgPrepareFP32(amg, A);   // cast the local hierarchy to FP32 once (matrices are current post-amgGalerkin) -> FP32 V-cycles
+
     deviceParallelAmul(A, halo, ifaceCoeffs, psi, Ax);          // rA = b - A*psi  (interface-coupled)
     deviceCopy(rA, b);
     deviceAxpy(-1.0, Ax, rA);

From 9c4d6f273f6ca470a8583f1faf2e73a954ff65c7 Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Sun, 19 Jul 2026 05:17:04 +0200
Subject: [PATCH 10/13] #12: AMG hierarchy disk cache + optional CUDA-graph
 V-cycle replay

---
 .../simpleFoam/parallel_device_foam.cuh       |  3 +-
 .../simpleFoam/parallel_device_simple.cuh     | 40 +++++++++++-
 src/cuda/device_amg.cu                        | 61 ++++++++++++++++---
 src/cuda/device_amg.cuh                       |  4 +-
 src/cuda/device_pcg.cu                        |  4 +-
 5 files changed, 100 insertions(+), 12 deletions(-)

diff --git a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
index 11a8253..581f44a 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_foam.cuh
@@ -307,7 +307,8 @@ inline int runParallelDeviceFoam(int argc, char** argv)
         std::vector<scalar> pg, kg, eg, ng;
         int nIter = 0;
         {   // scope: the solver's symmetric-heap buffers must be freed BEFORE Pstream::finalize
-            ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind, lust);
+            ParallelDeviceSimple solver(P, U0, p0, nu, relaxU, relaxP, tolU, tolP, 2000, boundedDiv, linUpwind, lust,
+                                        /*nonOrth*/ false, /*amgCacheDir*/ caseDir + "/constant/polyMesh");
             if (turbulent && sst) solver.enableTurbulenceSST(U0, k0, eps0, nut0, sstCoeffs, relaxK, relaxEps);
             else if (turbulent)   solver.enableTurbulence(U0, k0, eps0, nut0, keCoeffs, relaxK, relaxEps);
             lap("buildSolver");   // buildDeviceMesh + buildAMG + NVSHMEM halo/symmetric-heap alloc
diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 4a40fa4..5ea797f 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -28,6 +28,9 @@
 #include <fstream>
 #include <string>
 #include <vector>
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
 
 namespace brae {
 
@@ -566,7 +569,8 @@ public:
         bool bounded = false,
         bool linearUpwind = false,
         bool lust = false,
-        bool nonOrth = false)
+        bool nonOrth = false,
+        const std::string& amgCacheDir = "")   // polyMesh dir for the per-rank AMG-hierarchy disk cache ("" = off)
         : P_(part),
           nu_(nu),
           relaxU_(relaxU),
@@ -701,8 +705,40 @@ public:
                 const std::vector<label> ownerInt(P_.Lm.mesh.owner().begin(), P_.Lm.mesh.owner().begin() + nIf_);
                 const std::vector<label> neiInt(P_.Lm.mesh.neighbour());
                 std::vector<scalar> fwAMG(P_.lg.magSf().begin(), P_.lg.magSf().begin() + nIf_);   // geometric face area
-                pAMG_ = buildAMG(ownerInt, neiInt, fwAMG, static_cast<int>(lnC_));
+                const bool tS = std::getenv("BRAE_TIME_STARTUP") != nullptr;
+                const auto t0 = std::chrono::high_resolution_clock::now();
+
+                // AMG hierarchy: the agglomeration STRUCTURE is static per (mesh, nProcs), so DISK-CACHE it like the
+                // single-GPU buildOrLoadAMG -- a warm launch reloads instead of re-agglomerating. Per-rank cache file
+                // (each rank's local partition differs); freshness vs the GLOBAL owner (the local mesh is a
+                // deterministic function of it + nProcs). Only the structure is cached; amgGalerkin rebuilds the
+                // coarse VALUES each step. Default ON when a cacheDir is given; BRAE_AMG_CACHE=0 forces a rebuild.
+                bool loaded = false;
+                const bool cacheOn = !amgCacheDir.empty()
+                    && !(std::getenv("BRAE_AMG_CACHE") && std::atoi(std::getenv("BRAE_AMG_CACHE")) == 0);
+                std::string cacheFile;
+                if (cacheOn)
+                {
+                    cacheFile = amgCacheDir + "/.brae_amg_p" + std::to_string(P_.rank)
+                              + "_np" + std::to_string(Pstream::nProcs());
+                    namespace fs = std::filesystem;
+                    std::error_code ec;
+                    const std::string ownerF = amgCacheDir + "/owner";
+                    if (fs::exists(cacheFile, ec) && fs::exists(ownerF, ec)
+                        && fs::last_write_time(cacheFile, ec) >= fs::last_write_time(ownerF, ec))
+                        loaded = loadAMGCache(cacheFile, pAMG_);
+                }
+                if (!loaded)
+                {
+                    pAMG_ = buildAMG(ownerInt, neiInt, fwAMG, static_cast<int>(lnC_));
+                    if (cacheOn) writeAMGCache(pAMG_, cacheFile);
+                }
                 useAMG_ = true;
+                if (tS && P_.rank == 0)
+                    std::printf("[startup] buildAMG      %8.1f ms  (%s, %d levels)\n",
+                                std::chrono::duration<double, std::milli>(
+                                    std::chrono::high_resolution_clock::now() - t0).count(),
+                                loaded ? "cache HIT" : "built", pAMG_.nLevels());
             }
         }
     }
diff --git a/src/cuda/device_amg.cu b/src/cuda/device_amg.cu
index cf57a69..3ff2615 100644
--- a/src/cuda/device_amg.cu
+++ b/src/cuda/device_amg.cu
@@ -2900,19 +2900,66 @@ void amgPrepareFP32(AMGData& amg, const DeviceLduView& A)
 // approximate the local block. amg must be built (buildAMG) and current (amgGalerkin gives it this step's coarse
 // operators). Runs the FP32 V-cycle when amgPrepareFP32 cast the matrices this solve, else the FP64 one.
 void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
-                    const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z)
+                    const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z, bool captureVcycle)
 {
     ensureSpectrum(amg, A);                // one-time Chebyshev spectrum estimate (no-op unless the Chebyshev smoother is on)
     const bool fp32 = amg.fp32Alloc && useFP32() && !amg.saSmooth && !amg.gsSmooth && !useChebyshev();
+    const int nC = A.nCells;
+
+    if (!captureVcycle)
+    {
+        // DIRECT launch (default): every V-cycle kernel is a fresh launch.
+        if (fp32)
+        {
+            const LduF A0 = lduF(A, amg.fDiag[0], amg.fUpper[0], amg.fLower[0]);         // grid-0 FP32 matrix view
+            cast_<scalar,float><<<nBlocks(nC),TPB>>>(nC, r.data(), amg.vBF[0].data());   // r -> FP32
+            vcycleAtF(0, amg, A, A0, amg.vBF[0].data(), amg.vXF[0].data());              // FP32 V-cycle
+            cast_<float,scalar><<<nBlocks(nC),TPB>>>(nC, amg.vXF[0].data(), z.data());   // FP32 -> z (FP64)
+        }
+        else vcycleAt(0, amg, A, r, z);    // FP64 V-cycle
+        return;
+    }
+
+    // GRAPH REPLAY: the V-cycle is host-scalar-free and runs on FIXED buffers (amg.rA/wA), so capture it once
+    // (keyed on A.diag; the coarse VALUES change each step but the graph references the buffers, read at replay) and
+    // replay -- removing the launch overhead of the V-cycle's many small kernels (13 levels x SpMV+smoother per PCG
+    // iter, the dominant per-iteration launch cost). Copy r -> persistent amg.rA, replay, amg.wA -> z. Same
+    // capture/replay the single-GPU deviceAMGPCG uses; now available to the distributed pressure solve.
+    deviceCopy(amg.rA, r);
     if (fp32)
     {
-        const int nC = A.nCells;
-        const LduF A0 = lduF(A, amg.fDiag[0], amg.fUpper[0], amg.fLower[0]);   // grid-0 FP32 matrix view
-        cast_<scalar,float><<<nBlocks(nC),TPB>>>(nC, r.data(), amg.vBF[0].data());   // r -> FP32
-        vcycleAtF(0, amg, A, A0, amg.vBF[0].data(), amg.vXF[0].data());              // FP32 V-cycle
-        cast_<float,scalar><<<nBlocks(nC),TPB>>>(nC, amg.vXF[0].data(), z.data());   // FP32 -> z (FP64)
+        const LduF A0 = lduF(A, amg.fDiag[0], amg.fUpper[0], amg.fLower[0]);
+        AMGGraphCache& gcf = *amg.gcacheF;
+        if (!gcf.exec || gcf.key != A.diag)
+        {
+            if (gcf.exec)  { cudaGraphExecDestroy(gcf.exec);  gcf.exec  = nullptr; }
+            if (gcf.graph) { cudaGraphDestroy(gcf.graph);     gcf.graph = nullptr; }
+            cudaCheck(cudaStreamBeginCapture(cudaStreamPerThread, cudaStreamCaptureModeThreadLocal), "amgF capture begin");
+            cast_<scalar,float><<<nBlocks(nC),TPB>>>(nC, amg.rA.data(), amg.vBF[0].data());
+            vcycleAtF(0, amg, A, A0, amg.vBF[0].data(), amg.vXF[0].data());
+            cast_<float,scalar><<<nBlocks(nC),TPB>>>(nC, amg.vXF[0].data(), amg.wA.data());
+            cudaCheck(cudaStreamEndCapture(cudaStreamPerThread, &gcf.graph), "amgF capture end");
+            cudaCheck(cudaGraphInstantiate(&gcf.exec, gcf.graph, 0), "amgF graph instantiate");
+            gcf.key = A.diag;
+        }
+        cudaCheck(cudaGraphLaunch(gcf.exec, cudaStreamPerThread), "amgF graph launch");
+    }
+    else
+    {
+        AMGGraphCache& gc = *amg.gcache;
+        if (!gc.exec || gc.key != A.diag)
+        {
+            if (gc.exec)  { cudaGraphExecDestroy(gc.exec);  gc.exec  = nullptr; }
+            if (gc.graph) { cudaGraphDestroy(gc.graph);     gc.graph = nullptr; }
+            cudaCheck(cudaStreamBeginCapture(cudaStreamPerThread, cudaStreamCaptureModeThreadLocal), "amg capture begin");
+            vcycleAt(0, amg, A, amg.rA, amg.wA);
+            cudaCheck(cudaStreamEndCapture(cudaStreamPerThread, &gc.graph), "amg capture end");
+            cudaCheck(cudaGraphInstantiate(&gc.exec, gc.graph, 0), "amg graph instantiate");
+            gc.key = A.diag;
+        }
+        cudaCheck(cudaGraphLaunch(gc.exec, cudaStreamPerThread), "amg graph launch");
     }
-    else vcycleAt(0, amg, A, r, z);        // z = M^-1 r, one symmetric FP64 V-cycle down the local hierarchy and back
+    deviceCopy(z, amg.wA);
 }
 
 } // namespace brae
diff --git a/src/cuda/device_amg.cuh b/src/cuda/device_amg.cuh
index ae635c8..50befe5 100644
--- a/src/cuda/device_amg.cuh
+++ b/src/cuda/device_amg.cuh
@@ -134,8 +134,10 @@ DeviceSolverPerf deviceAMGPCG(const DeviceLduView& Afine, AMGData& amg, const De
 // Used by the distributed Krylov (deviceParallelAMGPCG) to precondition each rank's LOCAL block with AMG -- the
 // V-cycle is internal-face only, so it omits the processor interface (block-Jacobi/additive-Schwarz: the outer
 // distributed matvec supplies the interface coupling). amg must be built (buildAMG) and current (amgGalerkin).
+// captureVcycle: replay the V-cycle from a cached CUDA graph (keyed on A.diag) instead of launching every kernel,
+// removing the launch overhead of the V-cycle's many small kernels. Default false (direct launch).
 void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
-                    const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z);
+                    const DeviceBuffer<scalar>& r, DeviceBuffer<scalar>& z, bool captureVcycle = false);
 
 // Prepare the FP32 mixed-precision V-cycle for this solve: cast the (Galerkin-updated) matrices to FP32 mirrors.
 // Call ONCE per solve before the amgVCycleApply loop; after it, amgVCycleApply runs FP32 automatically. No-op
diff --git a/src/cuda/device_pcg.cu b/src/cuda/device_pcg.cu
index f6c8335..0741be1 100644
--- a/src/cuda/device_pcg.cu
+++ b/src/cuda/device_pcg.cu
@@ -8,6 +8,7 @@
 #include "cf_pstream.cuh"
 #include <cuda_runtime.h>
 #include <cmath>
+#include <cstdlib>
 
 
 namespace brae {
@@ -385,6 +386,7 @@ DeviceSolverPerf deviceParallelAMGPCG(
     DeviceBuffer<scalar> wA(nC), rA(nC), pA(nC), Ax(nC), ApA;
 
     amgPrepareFP32(amg, A);   // cast the local hierarchy to FP32 once (matrices are current post-amgGalerkin) -> FP32 V-cycles
+    static const bool g_parGraph = std::getenv("BRAE_PARALLEL_GRAPH") && std::atoi(std::getenv("BRAE_PARALLEL_GRAPH")) != 0;
 
     deviceParallelAmul(A, halo, ifaceCoeffs, psi, Ax);          // rA = b - A*psi  (interface-coupled)
     deviceCopy(rA, b);
@@ -394,7 +396,7 @@ DeviceSolverPerf deviceParallelAMGPCG(
     scalar red[2];
     auto fusedReduce = [&]()   // wA = M^-1 rA (LOCAL AMG V-cycle) ; red = [ dot(wA,rA), sumMag(rA) ] in ONE collective
     {
-        amgVCycleApply(amg, A, rA, wA);                         // <-- the only change vs deviceParallelJacobiPCG
+        amgVCycleApply(amg, A, rA, wA, g_parGraph);             // <-- the only change vs deviceParallelJacobiPCG (opt: graph replay)
         red[0] = deviceDot(wA, rA);
         red[1] = deviceSumMag(rA);
         Pstream::allReduce(red, 2, ReduceOp::Sum);

From 9918ae9eea4f17809bca38cf7cfd92f09dbc020b Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Sun, 19 Jul 2026 06:22:17 +0200
Subject: [PATCH 11/13] #12: besy path whole-loop conditional-graph distributed
 PCG (nvshmem-in-graph)  persistent pressure buffers

---
 .../simpleFoam/parallel_device_simple.cuh     |  17 +-
 src/cuda/device_amg.cu                        | 145 ++++++++++++++++++
 src/cuda/device_amg.cuh                       |  12 ++
 src/cuda/device_pcg.cu                        |   9 +-
 4 files changed, 178 insertions(+), 5 deletions(-)

diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 5ea797f..aabd7cf 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -943,6 +943,12 @@ private:
     DeviceHalo           halo_;
     bool     useAMG_ = false;   // pressure preconditioned by the local per-rank AMG V-cycle (else point-Jacobi)
     AMGData  pAMG_;             // the local pressure-Laplacian hierarchy (built once, Galerkin-recoarsened per step)
+    // Persistent pressure-solve buffers (graph-referenced): STABLE addresses across steps (resize is a no-op when
+    // the size is unchanged), so the whole-loop conditional graph (BRAE_PARALLEL_GRAPH=2) captures ONCE and
+    // replays across all steps instead of re-instantiating per step. Values are overwritten by the assembly each
+    // step (matrix, rhs, interface coeffs, and the psi seed) -- the graph references the buffers, read at replay.
+    DeviceBuffer<scalar> pUp_, pLp_, pDiagp_, pbp_, pSolp_;
+    std::vector<DeviceBuffer<scalar>> pIfCoeffp_;
     std::vector<label>   procStart_;
     std::vector<DeviceBuffer<label>>  faceCellsD_;
     std::vector<DeviceBuffer<scalar>> weightsD_;
@@ -1271,7 +1277,9 @@ inline ParStepResidual ParallelDeviceSimple::step()
     halo_.scatterBoundaryValues(rAU.data(), weightsD_, procStart_, rAUbnd.data());
     halo_.waitExchange();
 
-    DeviceBuffer<scalar> pD, pU_, pL_;
+    DeviceBuffer<scalar> pD;
+    DeviceBuffer<scalar>& pU_ = pUp_;   // persistent (graph-referenced): stable address -> whole-loop graph captures once
+    DeviceBuffer<scalar>& pL_ = pLp_;
     deviceLaplacianCoeffs(dm_, rAUf_int, pD, pU_, pL_, nonOrth_);
     const std::vector<scalar> rAUbndH = rAUbnd.host();
     std::vector<DeviceBuffer<scalar>> pCoeffGeo;
@@ -1295,7 +1303,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
             bi += lp[pi].size;
         }
     }
-    std::vector<DeviceBuffer<scalar>> pIfCoeff;
+    std::vector<DeviceBuffer<scalar>>& pIfCoeff = pIfCoeffp_;   // persistent (graph-referenced)
     deviceLaplacianInterface(halo_, faceCellsD_, pCoeffGeo, pD, pIfCoeff);
 
     DeviceBuffer<scalar> dphi, psrc;
@@ -1303,7 +1311,8 @@ inline ParStepResidual ParallelDeviceSimple::step()
     deviceHadamard(psrc, dm_.V, dphi);
     DeviceBuffer<scalar> piC, pbC;
     deviceBCLaplacianCoeffsFace(dbP_, rAUbnd, piC, pbC);
-    DeviceBuffer<scalar> pDiagC, pb;
+    DeviceBuffer<scalar>& pDiagC = pDiagp_;   // persistent (graph-referenced)
+    DeviceBuffer<scalar>& pb     = pbp_;
     deviceFold(dm_, pD, psrc, piC, pbC, pDiagC, pb);
 
     // Explicit non-orth correction of laplacian(rAU,p), from the ENTRY grad(p) (gx/gy/gz), matching the
@@ -1333,7 +1342,7 @@ inline ParStepResidual ParallelDeviceSimple::step()
                                 gx, gy, gz, pb, ffcPif);               // + cut faces
     }
 
-    DeviceBuffer<scalar> pSol;
+    DeviceBuffer<scalar>& pSol = pSolp_;   // persistent: the whole-loop graph keys on pSol.data() -> capture once
     deviceCopy(pSol, dp_);
     const DeviceLduView pv = deviceLduView(dm_, pDiagC, pU_, pL_);
     const scalar nfp = deviceParallelNormFactor(pv, halo_, pIfCoeff, pSol, pb, ones_, P_.globalNCells);
diff --git a/src/cuda/device_amg.cu b/src/cuda/device_amg.cu
index 3ff2615..c8b65d9 100644
--- a/src/cuda/device_amg.cu
+++ b/src/cuda/device_amg.cu
@@ -2,6 +2,8 @@
 // Used by deviceAMGPCG. Feature flags live in one BRAE_AMG_* block at the top of the anonymous namespace.
 #include "device_amg.cuh"
 #include "device_blas.cuh"
+#include "device_ldu.cuh"       // deviceParallelAmul (halo-coupled matvec) for the distributed whole-loop graph PCG
+#include "device_halo.cuh"      // DeviceHalo + its DeviceReducer (on-stream NVSHMEM reduce)
 #include <cuda_runtime.h>
 #include <cooperative_groups.h>
 #include <algorithm>
@@ -2710,8 +2712,151 @@ static DeviceSolverPerf deviceAMGPCGGraph(
     if (dbgCyc) std::fprintf(stderr, "[AMG] p cycles=%d finalRes=%.3e (device)\n", perf.nIterations, perf.finalResidual);
     return perf;
 }
+
+// DISTRIBUTED whole-loop graph PCG: the exact structure of deviceAMGPCGGraph (the entire steady-state PCG WHILE
+// body captured once into a conditional graph and replayed on-device -- no per-iteration host launches or syncs),
+// but the matvec is deviceParallelAmul (halo-coupled) and every dot is globalised by an ON-STREAM NVSHMEM reduce
+// through the halo's DeviceReducer, so the halo put/barrier and the collective are captured INSIDE the graph body.
+// This is the 2x lever: the ~13ms/iter of GPU work stops being spread across ~hundreds of separate launches. It
+// requires the on-stream NVSHMEM ops to be graph-capturable (NVSHMEM 3.6 has cuda_graph paths) and 1 PE/GPU (the
+// MPG host-MPI reduce fallback is NOT capturable) -- so it is real-multi-GPU only. Persistent buffers (psi/matrix
+// held across steps by the caller) make it capture ONCE per run; otherwise it re-instantiates per step (still
+// correct). Falls back to the direct deviceParallelAMGPCG if the caller keeps buffers non-persistent.
+DeviceSolverPerf deviceParallelAMGPCGGraph(
+    const DeviceLduView& A,
+    AMGData& amg,
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<scalar>>& ifaceCoeffs,
+    const DeviceBuffer<scalar>& b,
+    DeviceBuffer<scalar>& psi,
+    scalar normFactor,
+    scalar tol,
+    scalar relTol,
+    int maxIter)
+{
+    const cudaStream_t strm = cudaStreamPerThread;
+    const int nC = A.nCells;
+    amg.corrScaling = false;
+    DeviceBuffer<scalar>& wA = amg.wA;
+    DeviceBuffer<scalar>& rA = amg.rA;
+    if (!amg.pcgCache) amg.pcgCache = std::make_unique<PCGGraphCache>();
+    PCGGraphCache& c = *amg.pcgCache;
+    c.pA.resize(nC); c.Ax.resize(nC); c.sNormF.resize(1); c.sInit.resize(1); c.sRes.resize(1); c.sIter.resize(1);
+    cudaMemcpyAsync(c.sNormF.data(), &normFactor, sizeof(scalar), cudaMemcpyHostToDevice, strm);
+    scalar* dWArA = amg.sWArA.data();  scalar* dWArAold = amg.sWArAold.data();
+    scalar* dPap  = amg.sPap.data();   scalar* dAlpha = amg.sAlpha.data();
+    scalar* dNegAlpha = amg.sNegAlpha.data(); scalar* dBeta = amg.sBeta.data();
+    ensureSpectrum(amg, A);
+    const bool fp32 = useFP32() && !amg.saSmooth && !amg.gsSmooth && !useChebyshev();
+    if (fp32) amgCastFP32(amg, A);
+    const LduF A0 = fp32 ? lduF(A, amg.fDiag[0], amg.fUpper[0], amg.fLower[0]) : LduF{};
+    auto applyPrec = [&]()
+    {
+        if (fp32)
+        {
+            cast_<scalar,float><<<nBlocks(nC),TPB,0,strm>>>(nC, rA.data(), amg.vBF[0].data());
+            vcycleAtF(0, amg, A, A0, amg.vBF[0].data(), amg.vXF[0].data());
+            cast_<float,scalar><<<nBlocks(nC),TPB,0,strm>>>(nC, amg.vXF[0].data(), wA.data());
+        }
+        else vcycleAt(0, amg, A, rA, wA);
+    };
+    DeviceReducer& R = halo.reducer();
+    auto gdot = [&](const DeviceBuffer<scalar>& x, const DeviceBuffer<scalar>& y, scalar* dOut)
+    {   // *dOut = global sum of x.y (local dot -> on-stream NVSHMEM reduce -> copy)
+        deviceDotInto(x, y, R.src()); R.sumReduce(1, strm); deviceScalarCopy(R.dst(), dOut);
+    };
+    auto gsum = [&](const DeviceBuffer<scalar>& x, scalar* dOut)   // *dOut = global |x|_1
+    {
+        deviceSumMagInto(x, R.src()); R.sumReduce(1, strm); deviceScalarCopy(R.dst(), dOut);
+    };
+
+    deviceParallelAmul(A, halo, ifaceCoeffs, psi, c.Ax);          // r = b - A psi
+    deviceCopy(rA, b);
+    deviceAxpy(-1.0, c.Ax, rA);
+    DeviceSolverPerf perf;
+    gsum(rA, c.sInit.data());
+    gsScaleInvK<<<1,1,0,strm>>>(c.sInit.data(), c.sNormF.data());
+    scalar initRes;
+    cudaCheck(cudaMemcpyAsync(&initRes, c.sInit.data(), sizeof(scalar), cudaMemcpyDeviceToHost, strm), "pgraph init D2H");
+    cudaStreamSynchronize(strm);
+    perf.initialResidual = initRes; perf.finalResidual = initRes;
+    auto convergedHost = [&](scalar fr){ return (fr < tol) || (relTol > 0.0 && fr < relTol*initRes); };
+    if (convergedHost(initRes)) { perf.nIterations = 0; return perf; }
+
+    applyPrec();                                                  // iteration 0 (explicit)
+    gdot(wA, rA, dWArA);
+    deviceCopy(c.pA, wA);
+    deviceParallelAmul(A, halo, ifaceCoeffs, c.pA, wA);
+    gdot(wA, c.pA, dPap);
+    deviceScalarDivNeg(dWArA, dPap, dAlpha, dNegAlpha);
+    deviceAxpyDev(dAlpha, c.pA, psi);
+    deviceAxpyDev(dNegAlpha, wA, rA);
+    gsum(rA, c.sRes.data());
+    gsScaleInvK<<<1,1,0,strm>>>(c.sRes.data(), c.sNormF.data());
+    scalar res1;
+    cudaCheck(cudaMemcpyAsync(&res1, c.sRes.data(), sizeof(scalar), cudaMemcpyDeviceToHost, strm), "pgraph it0 D2H");
+    cudaStreamSynchronize(strm);
+    if (convergedHost(res1) || maxIter <= 1) { perf.finalResidual = res1; perf.nIterations = 1; return perf; }
+    cudaMemsetAsync(c.sIter.data(), 0, sizeof(int), strm);
+
+    if (!c.exec || c.key != psi.data())
+    {
+        if (c.exec)  { cudaGraphExecDestroy(c.exec);  c.exec  = nullptr; }
+        if (c.graph) { cudaGraphDestroy(c.graph);     c.graph = nullptr; }
+        cudaCheck(cudaGraphCreate(&c.graph, 0), "pgraph create");
+        cudaCheck(cudaGraphConditionalHandleCreate(&c.handle, c.graph, 1, cudaGraphCondAssignDefault), "pgraph cond handle");
+        cudaGraphNodeParams cp = {};
+        cp.type = cudaGraphNodeTypeConditional;
+        cp.conditional.handle = c.handle;
+        cp.conditional.type = cudaGraphCondTypeWhile;
+        cp.conditional.size = 1;
+        cudaGraphNode_t cnode;
+        cudaCheck(cudaGraphAddNode(&cnode, c.graph, nullptr, nullptr, 0, &cp), "pgraph cond node");
+        cudaGraph_t body = cp.conditional.phGraph_out[0];
+        cudaCheck(cudaStreamBeginCaptureToGraph(strm, body, nullptr, nullptr, 0, cudaStreamCaptureModeThreadLocal), "pgraph capture begin");
+        deviceScalarCopy(dWArA, dWArAold);
+        applyPrec();
+        gdot(wA, rA, dWArA);
+        deviceScalarDiv(dWArA, dWArAold, dBeta);
+        deviceFusedScaleAxpy(c.pA, dBeta, wA);
+        deviceParallelAmul(A, halo, ifaceCoeffs, c.pA, wA);
+        gdot(wA, c.pA, dPap);
+        deviceScalarDivNeg(dWArA, dPap, dAlpha, dNegAlpha);
+        deviceAxpyDev(dAlpha, c.pA, psi);
+        deviceAxpyDev(dNegAlpha, wA, rA);
+        gsum(rA, c.sRes.data());
+        gsScaleInvK<<<1,1,0,strm>>>(c.sRes.data(), c.sNormF.data());
+        pcgSetCondK<<<1,1,0,strm>>>(c.handle, c.sRes.data(), tol, c.sInit.data(), relTol, c.sIter.data(), maxIter-1);
+        cudaGraph_t tmp;
+        cudaCheck(cudaStreamEndCapture(strm, &tmp), "pgraph capture end");
+        cudaCheck(cudaGraphInstantiate(&c.exec, c.graph, 0), "pgraph instantiate");
+        c.key = psi.data();
+    }
+    cudaCheck(cudaGraphLaunch(c.exec, strm), "pgraph launch");
+    scalar finalRes; int whileIters;
+    cudaCheck(cudaMemcpyAsync(&finalRes, c.sRes.data(), sizeof(scalar), cudaMemcpyDeviceToHost, strm), "pgraph final D2H");
+    cudaCheck(cudaMemcpyAsync(&whileIters, c.sIter.data(), sizeof(int), cudaMemcpyDeviceToHost, strm), "pgraph iters D2H");
+    cudaStreamSynchronize(strm);
+    perf.finalResidual = finalRes;
+    perf.nIterations = 1 + whileIters;
+    return perf;
+}
 #endif // BRAE_HAS_GS_DEVICE
 
+#ifndef BRAE_HAS_GS_DEVICE
+// CUDA < 13: WHILE-node conditional graphs are unavailable, so the whole-loop graph cannot be built. Return a
+// sentinel (nIterations < 0); the caller falls back to the direct (non-graph) distributed AMG-PCG.
+DeviceSolverPerf deviceParallelAMGPCGGraph(
+    const DeviceLduView& A, AMGData&, DeviceHalo&,
+    const std::vector<DeviceBuffer<scalar>>&,
+    const DeviceBuffer<scalar>&, DeviceBuffer<scalar>&,
+    scalar, scalar, scalar, int)
+{
+    (void)A;
+    DeviceSolverPerf p; p.nIterations = -1; return p;
+}
+#endif
+
 DeviceSolverPerf deviceAMGPCG(
     const DeviceLduView& A,
     AMGData& amg,
diff --git a/src/cuda/device_amg.cuh b/src/cuda/device_amg.cuh
index 50befe5..dea9327 100644
--- a/src/cuda/device_amg.cuh
+++ b/src/cuda/device_amg.cuh
@@ -144,6 +144,18 @@ void amgVCycleApply(AMGData& amg, const DeviceLduView& A,
 // unless BRAE_AMG_FP32 (default on) and the default smoother/aggregation (SA/GS/Chebyshev stay FP64).
 void amgPrepareFP32(AMGData& amg, const DeviceLduView& A);
 
+class DeviceHalo;   // fwd (parallel/pstream/device_halo.cuh)
+
+// DISTRIBUTED whole-loop graph PCG: the entire steady-state PCG WHILE body captured once into a conditional CUDA
+// graph and replayed on-device (no per-iteration host launches), with deviceParallelAmul (halo) + on-stream
+// NVSHMEM reduces INSIDE the captured body. Real-multi-GPU only (1 PE/GPU; the MPG host-MPI reduce fallback is
+// not graph-capturable). Captures once per run only if the caller holds psi/matrix buffers persistent.
+DeviceSolverPerf deviceParallelAMGPCGGraph(
+    const DeviceLduView& A, AMGData& amg, DeviceHalo& halo,
+    const std::vector<DeviceBuffer<scalar>>& ifaceCoeffs,
+    const DeviceBuffer<scalar>& b, DeviceBuffer<scalar>& psi,
+    scalar normFactor, scalar tol, scalar relTol, int maxIter);
+
 // #7 (clusters+DSM): the coarse-level weighted-Jacobi solve (nSweeps of x += omega*(b-Ax)/diag), fused into a
 // SINGLE thread-block-cluster kernel, the whole coarse vector lives in the cluster's distributed shared memory
 // and all sweeps run with cluster.sync() between them (1 launch instead of 2*nSweeps). xc is the in/out guess.
diff --git a/src/cuda/device_pcg.cu b/src/cuda/device_pcg.cu
index 0741be1..98e489f 100644
--- a/src/cuda/device_pcg.cu
+++ b/src/cuda/device_pcg.cu
@@ -386,7 +386,14 @@ DeviceSolverPerf deviceParallelAMGPCG(
     DeviceBuffer<scalar> wA(nC), rA(nC), pA(nC), Ax(nC), ApA;
 
     amgPrepareFP32(amg, A);   // cast the local hierarchy to FP32 once (matrices are current post-amgGalerkin) -> FP32 V-cycles
-    static const bool g_parGraph = std::getenv("BRAE_PARALLEL_GRAPH") && std::atoi(std::getenv("BRAE_PARALLEL_GRAPH")) != 0;
+    // BRAE_PARALLEL_GRAPH: 0 off, 1 = V-cycle-only graph replay (below), 2 = WHOLE-loop conditional graph (the 2x lever).
+    static const int g_parGraphLevel = std::getenv("BRAE_PARALLEL_GRAPH") ? std::atoi(std::getenv("BRAE_PARALLEL_GRAPH")) : 0;
+    if (g_parGraphLevel >= 2)
+    {
+        const DeviceSolverPerf gp = deviceParallelAMGPCGGraph(A, amg, halo, ifaceCoeffs, b, psi, normFactor, tol, relTol, maxIter);
+        if (gp.nIterations >= 0) return gp;   // whole-loop graph ran; else (CUDA<13) fall through to the direct path
+    }
+    const bool g_parGraph = (g_parGraphLevel == 1);
 
     deviceParallelAmul(A, halo, ifaceCoeffs, psi, Ax);          // rA = b - A*psi  (interface-coupled)
     deviceCopy(rA, b);

From c7b8cc79e0a5ee41fe6726647e1097f76f404c1e Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Sun, 19 Jul 2026 10:49:33 +0200
Subject: [PATCH 12/13] #12: momentum BiCGStab whole-loop conditional graph
 (BRAE_PARALLEL_GRAPH=3)

---
 .../simpleFoam/parallel_device_simple.cuh     |   6 +-
 src/cuda/device_pcg.cu                        | 206 +++++++++++++++++-
 src/cuda/device_pcg.cuh                       |  37 +++-
 3 files changed, 246 insertions(+), 3 deletions(-)

diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index aabd7cf..6980f01 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -949,6 +949,9 @@ private:
     // step (matrix, rhs, interface coeffs, and the psi seed) -- the graph references the buffers, read at replay.
     DeviceBuffer<scalar> pUp_, pLp_, pDiagp_, pbp_, pSolp_;
     std::vector<DeviceBuffer<scalar>> pIfCoeffp_;
+    // Per-U-component momentum-BiCGStab whole-loop graph caches (BRAE_PARALLEL_GRAPH=3), keyed on Uk_[k]'s address.
+    // unique_ptr so the cache (which owns a CUDA graph + persistent Krylov/matrix buffers) stays non-copyable/stable.
+    std::unique_ptr<BiCGGraphCache> bicgGCache_[3];
     std::vector<label>   procStart_;
     std::vector<DeviceBuffer<label>>  faceCellsD_;
     std::vector<DeviceBuffer<scalar>> weightsD_;
@@ -1224,8 +1227,9 @@ inline ParStepResidual ParallelDeviceSimple::step()
         deviceFold(dm_, mDiagR, s, iC[k], bC[k], diagC, b);
         const DeviceLduView mv = deviceLduView(dm_, diagC, mUp, mLo);
         const scalar nf = deviceParallelNormFactor(mv, halo_, ifCoeff, Uk_[k], b, ones_, P_.globalNCells);
+        if (!bicgGCache_[k]) bicgGCache_[k] = std::make_unique<BiCGGraphCache>();   // lazy: whole-loop graph (BRAE_PARALLEL_GRAPH=3)
         const DeviceSolverPerf up =
-            deviceParallelJacobiBiCGStab(mv, halo_, ifCoeff, b, Uk_[k], nf, tolU_, 0.0, maxIter_);
+            deviceParallelJacobiBiCGStab(mv, halo_, ifCoeff, b, Uk_[k], nf, tolU_, 0.0, maxIter_, 1, bicgGCache_[k].get());
         kIters_ += up.nIterations;
         if (validC_[k] && up.initialResidual > res.Ux) res.Ux = up.initialResidual;
         dumpStage("Upred", Uk_[k], k);
diff --git a/src/cuda/device_pcg.cu b/src/cuda/device_pcg.cu
index 98e489f..39a36fc 100644
--- a/src/cuda/device_pcg.cu
+++ b/src/cuda/device_pcg.cu
@@ -455,11 +455,21 @@ DeviceSolverPerf deviceParallelJacobiBiCGStab(
     scalar tol,
     scalar relTol,
     int maxIter,
-    int checkEvery)
+    int checkEvery,
+    BiCGGraphCache* gcache)
 {
     const int nC = A.nCells;
     const int K  = (checkEvery > 1) ? checkEvery : 1;            // convergence-read cadence (1 = exact per-iter)
     const cudaStream_t strm = cudaStreamPerThread;               // the halo/reduce stream: everything is stream-ordered
+    // BRAE_PARALLEL_GRAPH>=3: whole-loop conditional-graph momentum solve (the momentum twin of the pressure
+    // deviceParallelAMGPCGGraph). Falls through to the direct device-resident path below on CUDA<13 (sentinel -1).
+    static const int g_parGraphLevel = std::getenv("BRAE_PARALLEL_GRAPH") ? std::atoi(std::getenv("BRAE_PARALLEL_GRAPH")) : 0;
+    if (g_parGraphLevel >= 3 && gcache)
+    {
+        const DeviceSolverPerf gp =
+            deviceParallelJacobiBiCGStabGraph(A, halo, ifaceCoeffs, b, psi, *gcache, normFactor, tol, relTol, maxIter);
+        if (gp.nIterations >= 0) return gp;
+    }
     DeviceBuffer<scalar> rA(nC), rA0(nC), pA(nC), yA(nC), AyA(nC), sA(nC), zA(nC), tA(nC), Ax(nC);
 
     deviceParallelAmul(A, halo, ifaceCoeffs, psi, Ax);           // rA = b - A*psi
@@ -547,4 +557,198 @@ DeviceSolverPerf deviceParallelJacobiBiCGStab(
     return perf;
 }
 
+// ---- whole-loop conditional-graph momentum BiCGStab (BRAE_PARALLEL_GRAPH=3) --------------------------------
+BiCGGraphCache::~BiCGGraphCache()
+{
+    if (exec)  cudaGraphExecDestroy(exec);
+    if (graph) cudaGraphDestroy(graph);
+}
+
+#if CUDART_VERSION >= 13000
+namespace {
+__global__
+void bicgScaleInvK(scalar* v, const scalar* nf)   // v = v / normFactor (device-resident residual normalize)
+{
+    if (threadIdx.x==0 && blockIdx.x==0) *v = *v / *nf;
+}
+__global__
+void bicgSetCondK(                                 // drive the WHILE conditional from the on-device stop predicate
+    cudaGraphConditionalHandle h,
+    const scalar* res,
+    scalar tol,
+    const scalar* init,
+    scalar relTol,
+    int* iter,
+    int maxIter,
+    const scalar* bd)
+{
+    if (threadIdx.x || blockIdx.x) return;
+    const int it = ++(*iter);
+    const scalar fr = *res;                        // already normalized by normFactor
+    const bool conv = (fr < tol) || (relTol > 0.0 && fr < relTol * (*init));
+    cudaGraphSetConditional(h, (conv || (*bd != 0.0) || it >= maxIter) ? 0u : 1u);   // stop on converge/breakdown/maxIter
+}
+} // namespace
+
+// The momentum twin of deviceParallelAMGPCGGraph: the entire steady-state BiCGStab iteration (interface-coupled
+// matvec + on-stream NVSHMEM reductions + the guarded recurrence) captured once into a cudaGraphCondTypeWhile graph
+// and replayed on-device, removing the ~15 host kernel launches/Krylov-iter the direct device-resident path still
+// pays. The momentum matrix is rebuilt with fresh buffers each SIMPLE step, so the graph body references PERSISTENT
+// copies (c.diagP/upperP/lowerP/ifaceP) that we memcpy-refresh here -> capture ONCE, replay every step. The direct
+// path's mid-iter |s| early exit is dropped (the body must be uniform: at most < 1 extra half-iteration, same psi);
+// convergence + OF breakdown are tested at the END of the body from the full-iteration residual r.
+DeviceSolverPerf deviceParallelJacobiBiCGStabGraph(
+    const DeviceLduView& A,
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<scalar>>& ifaceCoeffs,
+    const DeviceBuffer<scalar>& b,
+    DeviceBuffer<scalar>& psi,
+    BiCGGraphCache& c,
+    scalar normFactor,
+    scalar tol,
+    scalar relTol,
+    int maxIter)
+{
+    const cudaStream_t strm = cudaStreamPerThread;
+    const int nC = A.nCells;
+    const int nF = A.nInternalFaces;
+
+    c.rA.resize(nC); c.rA0.resize(nC); c.pA.resize(nC); c.yA.resize(nC); c.AyA.resize(nC);
+    c.sA.resize(nC); c.zA.resize(nC); c.tA.resize(nC); c.Ax.resize(nC);
+    c.sNormF.resize(1); c.sInit.resize(1); c.sRes.resize(1); c.sIter.resize(1);
+    cudaMemcpyAsync(c.sNormF.data(), &normFactor, sizeof(scalar), cudaMemcpyHostToDevice, strm);
+
+    // refresh the persistent matrix copies from THIS step's momentum matrix (stable addresses -> capture once)
+    c.diagP.resize(nC); c.upperP.resize(nF); c.lowerP.resize(nF);
+    cudaMemcpyAsync(c.diagP.data(),  A.diag,  nC*sizeof(scalar), cudaMemcpyDeviceToDevice, strm);
+    cudaMemcpyAsync(c.upperP.data(), A.upper, nF*sizeof(scalar), cudaMemcpyDeviceToDevice, strm);
+    cudaMemcpyAsync(c.lowerP.data(), A.lower, nF*sizeof(scalar), cudaMemcpyDeviceToDevice, strm);
+    DeviceLduView Ap = A;
+    Ap.diag = c.diagP.data(); Ap.upper = c.upperP.data(); Ap.lower = c.lowerP.data();
+    c.ifaceP.resize(ifaceCoeffs.size());
+    for (std::size_t i = 0; i < ifaceCoeffs.size(); ++i)
+    {
+        c.ifaceP[i].resize(ifaceCoeffs[i].size());
+        deviceCopy(c.ifaceP[i], ifaceCoeffs[i]);
+    }
+
+    DeviceReducer& R = halo.reducer();
+    auto gdot = [&](const DeviceBuffer<scalar>& x, const DeviceBuffer<scalar>& y, scalar* dOut)
+    { deviceDotInto(x, y, R.src()); R.sumReduce(1, strm); deviceScalarCopy(R.dst(), dOut); };
+    auto gsum = [&](const DeviceBuffer<scalar>& x, scalar* dOut)
+    { deviceSumMagInto(x, R.src()); R.sumReduce(1, strm); deviceScalarCopy(R.dst(), dOut); };
+
+    BiCGScalars& s = bicgScalars();
+    cudaMemsetAsync(s.bd.data(), 0, sizeof(scalar), strm);
+
+    // r = b - A psi ; r0 = r ; initial residual (one host read for the converge-in-0 early-out)
+    deviceParallelAmul(Ap, halo, c.ifaceP, psi, c.Ax);
+    deviceCopy(c.rA, b);
+    deviceAxpy(-1.0, c.Ax, c.rA);
+    deviceCopy(c.rA0, c.rA);
+    DeviceSolverPerf perf;
+    gsum(c.rA, c.sInit.data());
+    bicgScaleInvK<<<1,1,0,strm>>>(c.sInit.data(), c.sNormF.data());
+    scalar initRes;
+    cudaCheck(cudaMemcpyAsync(&initRes, c.sInit.data(), sizeof(scalar), cudaMemcpyDeviceToHost, strm), "mbicg init D2H");
+    cudaStreamSynchronize(strm);
+    perf.initialResidual = initRes; perf.finalResidual = initRes;
+    auto convergedHost = [&](scalar fr){ return (fr < tol) || (relTol > 0.0 && fr < relTol*initRes); };
+    if (convergedHost(initRes)) { perf.nIterations = 0; return perf; }
+
+    // iteration 0 (explicit: p = r, no beta) -- one full BiCGStab step
+    gdot(c.rA0, c.rA, s.rr.data());
+    bicgRhoSingK<<<1,1,0,strm>>>(s.rr.data(), s.bd.data());
+    deviceCopy(c.pA, c.rA);
+    deviceJacobi(c.yA, c.pA, Ap.diag);
+    deviceParallelAmul(Ap, halo, c.ifaceP, c.yA, c.AyA);
+    gdot(c.rA0, c.AyA, s.r0Ay.data());
+    bicgAlphaK<<<1,1,0,strm>>>(s.rr.data(), s.r0Ay.data(), s.alpha.data(), s.negAlpha.data(), s.bd.data());
+    deviceFusedSxpy(c.sA, c.rA, s.negAlpha.data(), c.AyA);
+    deviceJacobi(c.zA, c.sA, Ap.diag);
+    deviceParallelAmul(Ap, halo, c.ifaceP, c.zA, c.tA);
+    deviceDotInto(c.tA, c.tA, R.src() + 0);
+    deviceDotInto(c.tA, c.sA, R.src() + 1);
+    R.sumReduce(2, strm);
+    deviceScalarCopy(R.dst() + 0, s.tt.data());
+    deviceScalarCopy(R.dst() + 1, s.ts.data());
+    omegaK<<<1,1,0,strm>>>(s.ts.data(), s.tt.data(), s.omega.data(), s.negOmega.data(), s.bd.data());
+    deviceFusedAxpy2(psi, s.alpha.data(), c.yA, s.omega.data(), c.zA);
+    deviceFusedSxpy(c.rA, c.sA, s.negOmega.data(), c.tA);
+    gsum(c.rA, c.sRes.data());
+    bicgScaleInvK<<<1,1,0,strm>>>(c.sRes.data(), c.sNormF.data());
+    scalar res1;
+    cudaCheck(cudaMemcpyAsync(&res1, c.sRes.data(), sizeof(scalar), cudaMemcpyDeviceToHost, strm), "mbicg it0 D2H");
+    cudaStreamSynchronize(strm);
+    if (convergedHost(res1) || maxIter <= 1) { perf.finalResidual = res1; perf.nIterations = 1; return perf; }
+    cudaMemsetAsync(c.sIter.data(), 0, sizeof(int), strm);
+
+    // WHILE body = steady-state iteration 1+ (captured once, replayed on-device)
+    if (!c.exec || c.key != psi.data())
+    {
+        if (c.exec)  { cudaGraphExecDestroy(c.exec);  c.exec  = nullptr; }
+        if (c.graph) { cudaGraphDestroy(c.graph);     c.graph = nullptr; }
+        cudaCheck(cudaGraphCreate(&c.graph, 0), "mbicg graph create");
+        cudaCheck(cudaGraphConditionalHandleCreate(&c.handle, c.graph, 1, cudaGraphCondAssignDefault), "mbicg cond handle");
+        cudaGraphNodeParams cp = {};
+        cp.type = cudaGraphNodeTypeConditional;
+        cp.conditional.handle = c.handle;
+        cp.conditional.type = cudaGraphCondTypeWhile;
+        cp.conditional.size = 1;
+        cudaGraphNode_t cnode;
+        cudaCheck(cudaGraphAddNode(&cnode, c.graph, nullptr, nullptr, 0, &cp), "mbicg cond node");
+        cudaGraph_t body = cp.conditional.phGraph_out[0];
+        cudaCheck(cudaStreamBeginCaptureToGraph(strm, body, nullptr, nullptr, 0, cudaStreamCaptureModeThreadLocal), "mbicg capture begin");
+        deviceScalarCopy(s.rr.data(), s.rrOld.data());              // rrOld = rr
+        gdot(c.rA0, c.rA, s.rr.data());
+        bicgRhoSingK<<<1,1,0,strm>>>(s.rr.data(), s.bd.data());
+        bicgBetaK<<<1,1,0,strm>>>(s.rr.data(), s.rrOld.data(), s.alpha.data(), s.omega.data(), s.beta.data(), s.bd.data());
+        deviceFusedBicgP(c.rA, c.pA, c.AyA, s.beta.data(), s.negOmega.data());   // p = r + beta*(p - omega*AyA)
+        deviceJacobi(c.yA, c.pA, Ap.diag);
+        deviceParallelAmul(Ap, halo, c.ifaceP, c.yA, c.AyA);
+        gdot(c.rA0, c.AyA, s.r0Ay.data());
+        bicgAlphaK<<<1,1,0,strm>>>(s.rr.data(), s.r0Ay.data(), s.alpha.data(), s.negAlpha.data(), s.bd.data());
+        deviceFusedSxpy(c.sA, c.rA, s.negAlpha.data(), c.AyA);     // s = r - alpha*AyA
+        deviceJacobi(c.zA, c.sA, Ap.diag);
+        deviceParallelAmul(Ap, halo, c.ifaceP, c.zA, c.tA);
+        deviceDotInto(c.tA, c.tA, R.src() + 0);
+        deviceDotInto(c.tA, c.sA, R.src() + 1);
+        R.sumReduce(2, strm);
+        deviceScalarCopy(R.dst() + 0, s.tt.data());
+        deviceScalarCopy(R.dst() + 1, s.ts.data());
+        omegaK<<<1,1,0,strm>>>(s.ts.data(), s.tt.data(), s.omega.data(), s.negOmega.data(), s.bd.data());
+        deviceFusedAxpy2(psi, s.alpha.data(), c.yA, s.omega.data(), c.zA);       // psi += alpha*y + omega*z
+        deviceFusedSxpy(c.rA, c.sA, s.negOmega.data(), c.tA);     // r = s - omega*tA
+        gsum(c.rA, c.sRes.data());
+        bicgScaleInvK<<<1,1,0,strm>>>(c.sRes.data(), c.sNormF.data());
+        bicgSetCondK<<<1,1,0,strm>>>(c.handle, c.sRes.data(), tol, c.sInit.data(), relTol, c.sIter.data(), maxIter-1, s.bd.data());
+        cudaGraph_t tmp;
+        cudaCheck(cudaStreamEndCapture(strm, &tmp), "mbicg capture end");
+        cudaCheck(cudaGraphInstantiate(&c.exec, c.graph, 0), "mbicg graph instantiate");
+        c.key = psi.data();
+    }
+    cudaCheck(cudaGraphLaunch(c.exec, strm), "mbicg graph launch");
+    scalar finalRes; int whileIters;
+    cudaCheck(cudaMemcpyAsync(&finalRes, c.sRes.data(), sizeof(scalar), cudaMemcpyDeviceToHost, strm), "mbicg final D2H");
+    cudaCheck(cudaMemcpyAsync(&whileIters, c.sIter.data(), sizeof(int), cudaMemcpyDeviceToHost, strm), "mbicg iters D2H");
+    cudaStreamSynchronize(strm);
+    perf.finalResidual = finalRes;
+    perf.nIterations = 1 + whileIters;
+    return perf;
+}
+#endif // CUDART_VERSION >= 13000
+
+#if CUDART_VERSION < 13000
+// CUDA < 13: WHILE conditional-graph nodes are unavailable -> sentinel (nIterations < 0); caller uses the direct path.
+DeviceSolverPerf deviceParallelJacobiBiCGStabGraph(
+    const DeviceLduView& A, DeviceHalo&,
+    const std::vector<DeviceBuffer<scalar>>&,
+    const DeviceBuffer<scalar>&, DeviceBuffer<scalar>&,
+    BiCGGraphCache&, scalar, scalar, scalar, int)
+{
+    (void)A;
+    DeviceSolverPerf p; p.nIterations = -1; return p;
+}
+#endif
+
 } // namespace brae
diff --git a/src/cuda/device_pcg.cuh b/src/cuda/device_pcg.cuh
index e8d516d..37665d0 100644
--- a/src/cuda/device_pcg.cuh
+++ b/src/cuda/device_pcg.cuh
@@ -7,11 +7,28 @@
 #include "cf_types.cuh"
 #include "device_ldu.cuh"
 #include "device_buffer.cuh"
+#include <cuda_runtime.h>
+#include <vector>
 
 namespace brae {
 
 struct DeviceSolverPerf { scalar initialResidual = 0, finalResidual = 0; int nIterations = 0; };
 
+// Cached conditional-graph of the distributed momentum BiCGStab WHILE-body (BRAE_PARALLEL_GRAPH>=3). One cache
+// per U-component, owned by the solver (Uk_[k]'s address is the stable key). The momentum matrix is REBUILT with
+// fresh buffers every SIMPLE step (unlike the stable pressure matrix), so the graph body references PERSISTENT
+// copies here (diagP/upperP/lowerP/ifaceP) that the caller memcpy-refreshes each step -> the graph is captured
+// ONCE and replayed, no per-step re-instantiation. Holds the graph-referenced Krylov vectors + control scalars.
+struct BiCGGraphCache {
+    cudaGraphExec_t exec = nullptr; cudaGraph_t graph = nullptr;
+    cudaGraphConditionalHandle handle{}; const void* key = nullptr;
+    DeviceBuffer<scalar> rA, rA0, pA, yA, AyA, sA, zA, tA, Ax;      // persistent Krylov vectors
+    DeviceBuffer<scalar> diagP, upperP, lowerP;                     // persistent matrix copies (refreshed per step)
+    std::vector<DeviceBuffer<scalar>> ifaceP;                       // persistent interface-coeff copies
+    DeviceBuffer<scalar> sNormF, sInit, sRes; DeviceBuffer<int> sIter;   // control scalars
+    ~BiCGGraphCache();
+};
+
 // OpenFOAM's lduMatrix::solver::normFactor: sum(|A*psi - sumA*avg(psi)| + |b - sumA*avg(psi)|) + small,
 // where sumA = rowSum(A) = A*ones. Pass this as the solver's normFactor so the reported initial residual
 // (and the tolerance test) match OF's residualControl convention. ones is a cached unit vector (length nCells).
@@ -70,7 +87,25 @@ DeviceSolverPerf deviceParallelJacobiBiCGStab(
     scalar tol,
     scalar relTol,
     int maxIter,
-    int checkEvery = 1);   // convergence-read cadence: read |r| to the host every K iters (1 = exact per-iter)
+    int checkEvery = 1,          // convergence-read cadence: read |r| to the host every K iters (1 = exact per-iter)
+    BiCGGraphCache* gcache = nullptr);   // if BRAE_PARALLEL_GRAPH>=3 and non-null, use the whole-loop graph path
+
+// Whole-loop conditional-graph momentum BiCGStab (BRAE_PARALLEL_GRAPH=3): the entire steady-state BiCGStab WHILE
+// body -- interface-coupled matvec + on-stream NVSHMEM reductions + the recurrence -- captured once into a
+// cudaGraphCondTypeWhile graph and replayed on-device, removing the ~15 host kernel launches/Krylov-iter that the
+// direct device-resident path still pays. Mirrors deviceParallelAMGPCGGraph. Requires CUDA >= 13 (WHILE nodes) and
+// 1 PE/GPU (the on-stream reduce must be capturable); returns nIterations = -1 otherwise so the caller falls back.
+DeviceSolverPerf deviceParallelJacobiBiCGStabGraph(
+    const DeviceLduView& A,
+    DeviceHalo& halo,
+    const std::vector<DeviceBuffer<scalar>>& ifaceCoeffs,
+    const DeviceBuffer<scalar>& b,
+    DeviceBuffer<scalar>& psi,
+    BiCGGraphCache& c,
+    scalar normFactor,
+    scalar tol,
+    scalar relTol,
+    int maxIter);
 
 struct AMGData;   // fwd (device_amg.cuh); passed by ref so this header need not include the AMG hierarchy
 

From 12e17fa3a774e6d54bc5985c1985c04835fd4a15 Mon Sep 17 00:00:00 2001
From: tito <titohiera@gmail.com>
Date: Sun, 19 Jul 2026 12:14:48 +0200
Subject: [PATCH 13/13] #12: device-resident interface coeffs and momentum
 BiCGStab whole-loop graph

---
 .../simpleFoam/parallel_device_simple.cuh     | 120 ++++++++++--------
 1 file changed, 64 insertions(+), 56 deletions(-)

diff --git a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
index 6980f01..23a9fea 100644
--- a/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
+++ b/src/applications/solvers/simpleFoam/parallel_device_simple.cuh
@@ -245,6 +245,23 @@ void subtractSliceKernel(scalar* __restrict__ dst, int offset, const scalar* __r
     dst[offset + f] -= src[f];
 }
 
+// out[f] = bndField[offset + f] * geom[f]  (the processor-interface coupling coefficient, on-device). Replaces the
+// per-interface host loop `coeff = fieldAtCut * magSf * procDelta` that forced a full .host() D2D sync of the cut
+// field every SIMPLE step: fieldAtCut is already a DEVICE boundary array (halo-scattered) and geom = magSf*delta is
+// static (built once in the ctor), so the whole coefficient is a fused device multiply -- no round-trip.
+__global__
+void interfaceCoeffKernel(
+    scalar*       __restrict__ out,
+    const scalar* __restrict__ bndField,
+    int offset,
+    const scalar* __restrict__ geom,
+    int n)
+{
+    const int f = blockIdx.x * blockDim.x + threadIdx.x;
+    if (f >= n) return;
+    out[f] = bndField[offset + f] * geom[f];
+}
+
 } // namespace detail
 
 // Add the processor faces' linearUpwind deferred correction into `corr` (which already holds the internal
@@ -671,6 +688,13 @@ public:
                 magSfD_.back().copyFrom(ms);
                 nuFaceD_.emplace_back();
                 nuFaceD_.back().copyFrom(nf);
+                // geomIf = magSf * (nonOrth ? procNonOrth : procDelta): the static device geometry factor of the
+                // momentum + pEqn interface coefficients (Phase A -- lets those coeffs build on-device, no .host()).
+                std::vector<scalar> gi(lp[pi].size);
+                for (label i = 0; i < lp[pi].size; ++i)
+                    gi[i] = ms[i] * (nonOrth_ ? procNonOrth_[pj][i] : P_.procDelta[pj][i]);
+                geomIf_.emplace_back();
+                geomIf_.back().copyFrom(gi);
                 ++pj;
             }
         }
@@ -970,6 +994,10 @@ private:
     DeviceBuffer<label> bndIsWall_;   // per-boundary-face wall mask (for deviceBoundaryNut)
     DeviceBuffer<scalar> bndY_;       // per-boundary-face near-wall distance
     std::vector<DeviceBuffer<scalar>> geomD_;   // magSf*procDelta per interface (for the k/eps interface gamma)
+    // magSf * (nonOrth ? procNonOrth : procDelta) per processor interface, static (nonOrth_ is fixed at ctor). The
+    // device geometry factor of the momentum + pEqn interface coefficients, so those coeffs build on-device from the
+    // halo-scattered cut field instead of a per-step .host() round-trip (Phase A: device-resident assembly).
+    std::vector<DeviceBuffer<scalar>> geomIf_;
     KEpsilonCoeffs keCoeffs_;
     int  turbModel_ = 0;             // 0 = k-epsilon, 1 = k-omega SST
     KOmegaSSTCoeffs sstCoeffs_;
@@ -1028,8 +1056,6 @@ inline ParStepResidual ParallelDeviceSimple::step()
 {
     ParStepResidual res;
     ++iter_;
-    const std::vector<FvPatch>& lp = P_.lp;
-    const FvGeometry& lg = P_.lg;
 
     // ---- effective viscosity for this step: nu (laminar) or nu + nut (turbulent, refreshed from the model) ----
     // Laminar: nuEffCell = nuCell_ (=nu), nuEffBndEff = nuEffBnd_ (=nu), nuEff_f = nu, nuEffScatBnd = nu-at-cut.
@@ -1054,7 +1080,6 @@ inline ParStepResidual ParallelDeviceSimple::step()
     halo_.exchange(nuEffCell.data());
     halo_.scatterBoundaryValues(nuEffCell.data(), weightsD_, procStart_, nuEffScatBnd.data());
     halo_.waitExchange();
-    const std::vector<scalar> nuEffScatH = nuEffScatBnd.host();
 
     // ---- momentum matrix: div(phi,U) - laplacian(nuEff,U), + processor interface ----
     DeviceBuffer<scalar> mDiag, mUp, mLo, lD, lU, lL;
@@ -1075,29 +1100,25 @@ inline ParStepResidual ParallelDeviceSimple::step()
         deviceAxpy(-1.0, sp, mDiag);
     }
 
-    const std::vector<scalar> phiBndH = phiBnd_.host();
-    std::vector<DeviceBuffer<scalar>> phiF, coeffGeo;
+    // Processor-interface coupling, ON-DEVICE (Phase A): phiF[i] = the cut-face flux (a device slice of phiBnd_),
+    // coeffGeo[i] = nuEff-at-cut * geomIf_[i] via interfaceCoeffKernel. Both were a per-step .host() round-trip of
+    // phiBnd_ and nuEffScatBnd + a host multiply; now the whole coefficient stays on the device.
+    const int nIface = halo_.nInterfaces();
+    std::vector<DeviceBuffer<scalar>> phiF(nIface), coeffGeo(nIface);
     {
-        std::size_t pj = 0;
-        label bi = 0;
-        for (std::size_t pi = 0; pi < lp.size(); ++pi)
+        constexpr int TPB = 128;
+        for (int i = 0; i < nIface; ++i)
         {
-            if (lp[pi].type == "cyclic" || lp[pi].type == "cyclicAMI") continue;
-            if (lp[pi].type == "processor")
-            {
-                DeviceBuffer<scalar> pf;
-                pf.copyFrom(std::vector<scalar>(phiBndH.begin() + bi, phiBndH.begin() + bi + lp[pi].size));
-                phiF.push_back(std::move(pf));
-                std::vector<scalar> cg(lp[pi].size);
-                for (label i = 0; i < lp[pi].size; ++i)
-                    cg[i] = nuEffScatH[bi + i] * lg.magSf()[lp[pi].start + i]
-                          * (nonOrth_ ? procNonOrth_[pj][i] : P_.procDelta[pj][i]);
-                DeviceBuffer<scalar> cgd;
-                cgd.copyFrom(cg);
-                coeffGeo.push_back(std::move(cgd));
-                ++pj;
-            }
-            bi += lp[pi].size;
+            const int n = static_cast<int>(halo_.size(i));
+            if (n <= 0) continue;
+            phiF[i].resize(n);
+            cudaCheck(
+                cudaMemcpyAsync(phiF[i].data(), phiBnd_.data() + procStart_[i], n * sizeof(scalar),
+                                cudaMemcpyDeviceToDevice, cudaStreamPerThread),
+                "phiF cut slice");
+            coeffGeo[i].resize(n);
+            detail::interfaceCoeffKernel<<<(n + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+                coeffGeo[i].data(), nuEffScatBnd.data(), static_cast<int>(procStart_[i]), geomIf_[i].data(), n);
         }
     }
     std::vector<DeviceBuffer<scalar>> ifCoeff;
@@ -1285,26 +1306,18 @@ inline ParStepResidual ParallelDeviceSimple::step()
     DeviceBuffer<scalar>& pU_ = pUp_;   // persistent (graph-referenced): stable address -> whole-loop graph captures once
     DeviceBuffer<scalar>& pL_ = pLp_;
     deviceLaplacianCoeffs(dm_, rAUf_int, pD, pU_, pL_, nonOrth_);
-    const std::vector<scalar> rAUbndH = rAUbnd.host();
-    std::vector<DeviceBuffer<scalar>> pCoeffGeo;
+    // pEqn interface coeff ON-DEVICE (Phase A): pCoeffGeo[i] = rAU-at-cut * geomIf_[i], the same fused device
+    // multiply as the momentum coeff above -- replaces the per-step rAUbnd.host() round-trip + host loop.
+    std::vector<DeviceBuffer<scalar>> pCoeffGeo(nIface);
     {
-        std::size_t pj = 0;
-        label bi = 0;
-        for (std::size_t pi = 0; pi < lp.size(); ++pi)
+        constexpr int TPB = 128;
+        for (int i = 0; i < nIface; ++i)
         {
-            if (lp[pi].type == "cyclic" || lp[pi].type == "cyclicAMI") continue;
-            if (lp[pi].type == "processor")
-            {
-                std::vector<scalar> cg(lp[pi].size);
-                for (label i = 0; i < lp[pi].size; ++i)
-                    cg[i] = rAUbndH[bi + i] * lg.magSf()[lp[pi].start + i]
-                          * (nonOrth_ ? procNonOrth_[pj][i] : P_.procDelta[pj][i]);
-                DeviceBuffer<scalar> cgd;
-                cgd.copyFrom(cg);
-                pCoeffGeo.push_back(std::move(cgd));
-                ++pj;
-            }
-            bi += lp[pi].size;
+            const int n = static_cast<int>(halo_.size(i));
+            if (n <= 0) continue;
+            pCoeffGeo[i].resize(n);
+            detail::interfaceCoeffKernel<<<(n + TPB - 1) / TPB, TPB, 0, cudaStreamPerThread>>>(
+                pCoeffGeo[i].data(), rAUbnd.data(), static_cast<int>(procStart_[i]), geomIf_[i].data(), n);
         }
     }
     std::vector<DeviceBuffer<scalar>>& pIfCoeff = pIfCoeffp_;   // persistent (graph-referenced)
@@ -1418,21 +1431,16 @@ inline ParStepResidual ParallelDeviceSimple::step()
         // momentum interface), but OF runs turbulence->correct() with the NEW phi. At np=1 this is moot (no
         // interface), but at np>1 the cut would otherwise convect k/eps with a STALE flux while the interior
         // used the fresh phiInt_ -- inconsistent across the cut. Rebuild it from the maintained phiBnd_.
-        std::vector<DeviceBuffer<scalar>> phiFc;
+        std::vector<DeviceBuffer<scalar>> phiFc(nIface);   // ON-DEVICE cut-flux slices (Phase A: no phiBnd_.host())
+        for (int i = 0; i < nIface; ++i)
         {
-            const std::vector<scalar> pbH = phiBnd_.host();
-            label bi = 0;
-            for (std::size_t pi = 0; pi < lp.size(); ++pi)
-            {
-                if (lp[pi].type == "cyclic" || lp[pi].type == "cyclicAMI") continue;
-                if (lp[pi].type == "processor")
-                {
-                    DeviceBuffer<scalar> pf;
-                    pf.copyFrom(std::vector<scalar>(pbH.begin() + bi, pbH.begin() + bi + lp[pi].size));
-                    phiFc.push_back(std::move(pf));
-                }
-                bi += lp[pi].size;
-            }
+            const int n = static_cast<int>(halo_.size(i));
+            if (n <= 0) continue;
+            phiFc[i].resize(n);
+            cudaCheck(
+                cudaMemcpyAsync(phiFc[i].data(), phiBnd_.data() + procStart_[i], n * sizeof(scalar),
+                                cudaMemcpyDeviceToDevice, cudaStreamPerThread),
+                "phiFc cut slice");
         }
         if (turbModel_ == 1)   // k-omega SST (eps_ holds omega)
         {