From d005171c088b9fa398843cad3c6ed24285d074eb Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Thu, 18 Jun 2026 08:57:07 -0700 Subject: [PATCH 1/6] fix: honor rnr_retry=7 as infinite RNR retry The RNR retry counter was gated only by disableRetryCntReg (the regular retry-count infinite flag, set when max_retry_cnt == 7). A QP configured with rnr_retry == 7 (IBV "infinite") but a finite retry_count would still exhaust rnrCntReg and fault. Add a dedicated disableRnrCntReg, set when max_rnr_cnt == INFINITE_RETRY, gating both the rnrCntReg decrement and the RNR retry-limit-exceeded check, mirroring the regular-retry path. --- src/RetryHandleSQ.bsv | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/RetryHandleSQ.bsv b/src/RetryHandleSQ.bsv index cd5c700..dbb9fbd 100644 --- a/src/RetryHandleSQ.bsv +++ b/src/RetryHandleSQ.bsv @@ -92,6 +92,7 @@ module mkRetryHandleSQ#( // Reg#(Bool) isTimeOutCntZeroReg <- mkRegU; Reg#(Bool) disableTimeOutReg <- mkRegU; Reg#(Bool) disableRetryCntReg <- mkRegU; + Reg#(Bool) disableRnrCntReg <- mkRegU; Reg#(RetryReason) retryReasonReg <- mkRegU; Reg#(WorkReqID) retryWorkReqIdReg <- mkRegU; @@ -135,7 +136,7 @@ module mkRetryHandleSQ#( function Bool retryCntExceedLimit(RetryReason retryReason); return case (retryReason) - RETRY_REASON_RNR : isZero(rnrCntReg); + RETRY_REASON_RNR : !disableRnrCntReg && isZero(rnrCntReg); RETRY_REASON_SEQ_ERR , RETRY_REASON_IMPLICIT, RETRY_REASON_TIMEOUT : isZero(retryCntReg); @@ -157,7 +158,7 @@ module mkRetryHandleSQ#( end end RETRY_REASON_RNR: begin - if (!disableRetryCntReg) begin + if (!disableRnrCntReg) begin if (!isZero(rnrCntReg)) begin rnrCntReg <= rnrCntReg - 1; end @@ -182,6 +183,7 @@ module mkRetryHandleSQ#( retryCntReg <= cntrlStatus.comm.getMaxRetryCnt; rnrCntReg <= cntrlStatus.comm.getMaxRnrCnt; disableRetryCntReg <= cntrlStatus.comm.getMaxRetryCnt == fromInteger(valueOf(INFINITE_RETRY)); + disableRnrCntReg <= cntrlStatus.comm.getMaxRnrCnt == fromInteger(valueOf(INFINITE_RETRY)); // $display( // "time=%0t: resetRetryCntInternal cntrlStatus.comm.getMaxRetryCnt=%0d", // $time, cntrlStatus.comm.getMaxRetryCnt From 5ac02d1b80deb74961e70c7f41e80bdabee9d7af Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Thu, 18 Jun 2026 08:57:07 -0700 Subject: [PATCH 2/6] feat: raise MAX_QP_WR 4 -> 16 to deepen the SQ in-flight window A SEND-only RDMA datapath is bandwidth-delay-product limited: with ~4 SENDs in flight against the SEND->completion latency, throughput saturates well below line rate. Raising MAX_QP_WR to 16 deepens the SQ pending-WR window; on a KCU105 10GbE RoCEv2 build this lifts RDMA-SEND from ~8.1 to ~9.75 Gb/s (line rate) at 4096 B PMTU. MAX_QP_WR stays a power of 2; derived sizes (MAX_CQE, MAX_PENDING_WORK_COMP_NUM, ...) scale with it. --- src/Settings.bsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Settings.bsv b/src/Settings.bsv index 9f6f9f0..515f839 100644 --- a/src/Settings.bsv +++ b/src/Settings.bsv @@ -12,7 +12,7 @@ typedef 256 DATA_BUS_WIDTH; typedef TExp#(31) MAX_MR_SIZE; // 2GB typedef TExp#(21) PAGE_SIZE_CAP; // 2MB typedef 1 MAX_QP; -typedef 4 MAX_QP_WR; +typedef 16 MAX_QP_WR; typedef 1 MAX_SGE; typedef 2 MAX_CQ; typedef MAX_QP_WR MAX_CQE; From 125c49070cdb12daafca9ecd0228b8a24c614e61 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Sun, 21 Jun 2026 18:51:22 -0700 Subject: [PATCH 3/6] test: fix CI test suite The BlueSim CI (run.sh -> Makefile.test) failed to compile and run on this branch. Fix it so the full registered suite passes, and get the cocotb AXI-Stream integration test running again. Build/CI: - Makefile.base: add src/ to the bsc package search path for test builds (TESTSRCDIR); test benches run from test/ and could not find any core package. Keep the src/ build path (SRCDIRFLAGS) lib-only to avoid a duplicate-path error during Verilog generation. - .github/workflows/ci.yml: checkout@v4 with submodules: recursive so the blue-wrapper submodule (required on the bsc path) is fetched on the runner. SQ test-harness fixes (match current source signatures/behavior): - Utils4Test / TestInputPktHandle: drop references to the removed request ingress pipe (reqPktPipeOut); keep response + CNP checks. - SimGenRdmaReqResp: use contextSQ.statusSQ (statusRQ was removed). - TestSendQ / TestPayloadGen: build single-element SGLs (MAX_SGE == 1). - TestRespHandleSQ: mkRespHandleSQ is now 4-arg; drain the response payload with a sink (SQ DMA-write/permission path removed) and drop the no-longer observable read/atomic payload comparison. - TestWorkCompGen: mkWorkCompGenSQ no longer waits on a DMA-write response; drive wcWaitDmaResp false and stop feeding the removed payloadConResp port. Skip tests that depend on the disabled receive-queue (RQ) datapath, with sources preserved for when RQ is re-enabled: - De-register TestQueuePair, TestReqHandleRQ, TestTransportLayer, and the request-ingress-only SimExtractRdmaHeaderPayload from Makefile.test. - Trim TestInputPktHandle to the CNP test and TestWorkCompGen to its SQ cases; stub the RQ work-completion test body. cocotb: - TestAxiSTransportLayer: use LogicArray.to_unsigned() instead of the deprecated .integer getter (cocotb 2.x). - Add test/cocotb/requirements.txt documenting the Python deps (incl. bitstring) for the manually-run cocotb flow. --- .github/workflows/ci.yml | 4 +- Makefile.base | 6 +- Makefile.test | 30 +-- test/SimGenRdmaReqResp.bsv | 2 +- test/TestInputPktHandle.bsv | 25 --- test/TestPayloadGen.bsv | 3 +- test/TestRespHandleSQ.bsv | 76 ++------ test/TestSendQ.bsv | 3 +- test/TestWorkCompGen.bsv | 251 ++------------------------ test/Utils4Test.bsv | 56 ++---- test/cocotb/TestAxiSTransportLayer.py | 24 +-- test/cocotb/requirements.txt | 11 ++ 12 files changed, 100 insertions(+), 391 deletions(-) create mode 100644 test/cocotb/requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb3e524..d99a841 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,9 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + submodules: recursive - name: build and simulate run : | ./setup.sh diff --git a/Makefile.base b/Makefile.base index 5eb1001..db59d91 100644 --- a/Makefile.base +++ b/Makefile.base @@ -22,10 +22,14 @@ VDIR = rtl SRCOUTDIR = -bdir $(BUILDDIR) -info-dir $(BUILDDIR) -simdir $(BUILDDIR) -vdir $(VDIR) WORKDIR = -fdir $(abspath .) ROOT_DIR = $(abspath ../) +SRC_DIR = $(ROOT_DIR)/src LIB_WRAPPER_DIR = $(ROOT_DIR)/lib/blue-wrapper/src LIBSRCDIR = $(LIB_WRAPPER_DIR) +# Source builds run from src/ (src is implicitly on the path), so they only +# need the wrapper lib. Test builds run from test/ and must add src explicitly. BSVSRCDIR = -p +:$(LIBSRCDIR) -DIRFLAGS = $(BSVSRCDIR) $(OUTDIR) $(WORKDIR) +TESTSRCDIR = -p +:$(SRC_DIR):$(LIBSRCDIR) +DIRFLAGS = $(TESTSRCDIR) $(OUTDIR) $(WORKDIR) SRCDIRFLAGS = $(BSVSRCDIR) $(SRCOUTDIR) $(WORKDIR) MISCFLAGS = -print-flags -show-timestamps -show-version -steps 6000000 # -D macro RUNTIMEFLAGS = +RTS -K4095M -RTS diff --git a/Makefile.test b/Makefile.test index e8c3cdc..25f8cb6 100644 --- a/Makefile.test +++ b/Makefile.test @@ -1,9 +1,20 @@ TESTDIR ?= $(abspath ../test) LOGDIR ?= $(abspath ../tmp) +# NOTE: The receive-queue (RQ) datapath is currently disabled in the source +# (//CR / /*CR*/ markers in QueuePair.bsv, Controller.bsv, InputPktHandle.bsv, +# WorkCompGen.bsv). The following whole-file testbenches exercise only the RQ +# path or the RQ<->SQ loopback and cannot run until RQ is re-enabled, so they +# are de-registered here. Their sources are preserved verbatim on disk; to +# revive them, re-enable the RQ source and re-add the lines below: +# SimExtractRdmaHeaderPayload.bsv \ (request-ingress only) +# TestQueuePair.bsv \ (SQ<->RQ loopback integration) +# TestReqHandleRQ.bsv \ (RQ request handling) +# TestTransportLayer.bsv \ (SQ<->RQ loopback integration) +# (SimExtractRdmaHeaderPayload.bsv remains imported as a library by other tests; +# it is only removed as a standalone testbench target.) TESTBENCHS = \ SimDma.bsv \ - SimExtractRdmaHeaderPayload.bsv \ SimGenRdmaReqResp.bsv \ TestArbitration.bsv \ TestController.bsv \ @@ -12,13 +23,10 @@ TESTBENCHS = \ TestInputPktHandle.bsv \ TestMetaData.bsv \ TestPayloadConAndGen.bsv \ - TestQueuePair.bsv \ TestReqGenSQ.bsv \ - TestReqHandleRQ.bsv \ TestRespHandleSQ.bsv \ TestRetryHandleSQ.bsv \ TestSpecialFIFOF.bsv \ - TestTransportLayer.bsv \ TestUtils.bsv \ TestWorkCompGen.bsv \ TestPayloadGen.bsv \ @@ -39,10 +47,10 @@ TestExtractAndPrependPipeOut.bsv = mkTestHeaderAndDataStreamConversion \ mkTestExtractHeaderWithPayloadLessThanOneFrag \ mkTestExtractHeaderLongerThanDataStream \ mkTestExtractAndPrependHeader -TestInputPktHandle.bsv = mkTestCalculateRandomPktLen \ - mkTestCalculatePktLenEqPMTU \ - mkTestCalculateZeroPktLen \ - mkTestReceiveCNP +# mkTestCalculate{RandomPktLen,PktLenEqPMTU,ZeroPktLen} exercise the disabled +# request-ingress path (reqPktPipeOut) and deadlock; only the CNP receive test +# runs until RQ/request ingress is re-enabled. +TestInputPktHandle.bsv = mkTestReceiveCNP TestMetaData.bsv = mkTestMetaDataMRs \ mkTestMetaDataPDs \ mkTestMetaDataQPs \ @@ -100,9 +108,9 @@ TestSpecialFIFOF.bsv = mkTestCacheFIFO2 \ TestTransportLayer.bsv = mkTestTransportLayerNormalCase \ mkTestTransportLayerErrorCase TestUtils.bsv = mkTestPsnFunc -TestWorkCompGen.bsv = mkTestWorkCompGenNormalCaseRQ \ - mkTestWorkCompGenErrFlushCaseRQ \ - mkTestWorkCompGenNormalCaseSQ \ +# mkTestWorkCompGen*RQ cover the disabled RQ work-completion path (their module +# bodies are stubbed in TestWorkCompGen.bsv); only the SQ cases are run. +TestWorkCompGen.bsv = mkTestWorkCompGenNormalCaseSQ \ mkTestWorkCompGenErrFlushCaseSQ TestPayloadGen.bsv = mkTestCalcPktNumAndPktLenByAddrAndPMTU \ diff --git a/test/SimGenRdmaReqResp.bsv b/test/SimGenRdmaReqResp.bsv index 2804954..776a013 100644 --- a/test/SimGenRdmaReqResp.bsv +++ b/test/SimGenRdmaReqResp.bsv @@ -652,7 +652,7 @@ module mkTestSimGenRdmaResp(Empty); // Generate RDMA responses let rdmaRespAndHeaderPipeOut <- mkSimGenRdmaRespHeaderAndDataStream( - cntrl.contextRQ.statusRQ, simDmaReadSrv.dmaReadSrv, pendingWorkReqPipeOut4RespGen + cntrl.contextSQ.statusSQ, simDmaReadSrv.dmaReadSrv, pendingWorkReqPipeOut4RespGen ); let rdmaRespHeaderPipeOut4Ref <- mkBufferN(2, rdmaRespAndHeaderPipeOut.respHeader); Vector#(2, PipeOut#(HeaderRDMA)) rdmaRespHeaderPipeOut4RefVec <- diff --git a/test/TestInputPktHandle.bsv b/test/TestInputPktHandle.bsv index 93d2279..503da60 100644 --- a/test/TestInputPktHandle.bsv +++ b/test/TestInputPktHandle.bsv @@ -35,17 +35,6 @@ module mkTestReceiveCNP(Empty); ); for (Integer idx = 1; idx < valueOf(MAX_QP); idx = idx + 1) begin - let reqPktMetaDataPipeOutEmptyRule <- addRules(genEmptyPipeOutRule( - dut[idx].reqPktPipeOut.pktMetaData, - "dut[" + integerToString(idx) + - "].reqPktPipeOut.pktMetaData empty assertion @ mkTestReceiveCNP" - )); - let reqPktPayloadPipeOutEmptyRule <- addRules(genEmptyPipeOutRule( - dut[idx].reqPktPipeOut.payload, - "dut[" + integerToString(idx) + - "].reqPktPipeOut.payload empty assertion @ mkTestReceiveCNP" - )); - let respPktMetaDataPipeOutEmptyRule <- addRules(genEmptyPipeOutRule( dut[idx].respPktPipeOut.pktMetaData, "dut[" + integerToString(idx) + @@ -79,21 +68,7 @@ module mkTestReceiveCNP(Empty); ) ); - let reqPktMetaDataAndPayloadPipeOut = dut[qpIndex].reqPktPipeOut; let respPktMetaDataAndPayloadPipeOut = dut[qpIndex].respPktPipeOut; - immAssert( - !reqPktMetaDataAndPayloadPipeOut.pktMetaData.notEmpty && - !reqPktMetaDataAndPayloadPipeOut.payload.notEmpty, - "reqPktMetaDataAndPayloadPipeOut assertion @ mkTestReceiveCNP", - $format( - "reqPktMetaDataAndPayloadPipeOut.pktMetaData.notEmpty=", - fshow(reqPktMetaDataAndPayloadPipeOut.pktMetaData.notEmpty), - " and reqPktMetaDataAndPayloadPipeOut.payload.notEmpty=", - fshow(reqPktMetaDataAndPayloadPipeOut.payload.notEmpty), - " should both be false" - ) - ); - immAssert( !respPktMetaDataAndPayloadPipeOut.pktMetaData.notEmpty && !respPktMetaDataAndPayloadPipeOut.payload.notEmpty, diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 3e3b386..07a33b4 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -681,8 +681,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); isFirst: True, isLast : True }; - let dummySGE = sge; - let sgl = vec(sge, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE); + let sgl = vec(sge); let dmaReadCntrlReq = DmaReadCntrlReq { pmtu : pmtu, diff --git a/test/TestRespHandleSQ.bsv b/test/TestRespHandleSQ.bsv index 40cb528..de51910 100644 --- a/test/TestRespHandleSQ.bsv +++ b/test/TestRespHandleSQ.bsv @@ -217,36 +217,25 @@ module mkTestRespHandleNormalOrDupOrGhostRespCase#( cntrlStatus, pendingWorkReqBuf.fifof.notEmpty, pendingWorkReqBuf.scanCntrl ); - // MR permission check - let mrCheckPassOrFail = True; - let permCheckSrv <- mkSimPermCheckSrv(mrCheckPassOrFail); - - // PayloadConsumer - let simDmaWriteSrv <- mkSimDmaWriteSrvAndDataStreamPipeOut; - let readAtomicRespPayloadPipeOut = simDmaWriteSrv.dataStream; - let dmaWriteCntrl <- mkDmaWriteCntrl(cntrlStatus, simDmaWriteSrv.dmaWriteSrv); - let payloadConsumer <- mkPayloadConsumer( - cntrlStatus, - dmaWriteCntrl, - pktMetaDataAndPayloadPipeOut.payload - // dut.payloadConReqPipeOut - ); + // mkRespHandleSQ no longer drives payload consumption (the SQ + // PayloadConsumer / permission-check / DMA-write path was removed), so the + // response payload stream is simply drained here to keep the pipeline + // flowing. The read/atomic response payload is therefore no longer + // observable, so the payload-content comparison below is skipped. + let sinkPayload <- mkSink(pktMetaDataAndPayloadPipeOut.payload); // DUT let dut <- mkRespHandleSQ( cntrl.contextSQ, retryHandler, - permCheckSrv, toPipeOut(pendingWorkReqBuf.fifof), - pktMetaDataPipeOut4RespHandle, - payloadConsumer.request + pktMetaDataPipeOut4RespHandle ); // WorkCompGenSQ FIFOF#(WorkCompGenReqSQ) wcGenReqQ4ReqGenInSQ <- mkFIFOF; let workCompGenSQ <- mkWorkCompGenSQ( cntrlStatus, - payloadConsumer.response, // payloadConsumer.respPipeOut, toPipeOut(wcGenReqQ4ReqGenInSQ), dut.workCompGenReqPipeOut @@ -330,34 +319,19 @@ module mkTestRespHandleNormalOrDupOrGhostRespCase#( let isRespPktEnd = False; if (workReqNeedDmaWriteSQ(pendingWR.wr)) begin if (isReadRespRdmaOpCode(bth.opcode)) begin // Read responses with non-zero payload + // The DUT no longer performs DMA-write of the read/atomic + // response payload, so the payload content is not observable + // here. Only the reference payload stream is drained to track + // packet-end; the payload-equality assertion is dropped. let refDataStream = readRespPayloadPipeOut4Ref.first; readRespPayloadPipeOut4Ref.deq; - if (isNormalWorkReq) begin - let payloadDataStream = readAtomicRespPayloadPipeOut.first; - readAtomicRespPayloadPipeOut.deq; - - immAssert( - payloadDataStream == refDataStream, - "payloadDataStream assertion @ mkTestRespHandleNormalCase", - $format( - "payloadDataStream=", fshow(payloadDataStream), - " should == refDataStream=", fshow(refDataStream) - ) - ); - end - if (refDataStream.isLast) begin isRespPktEnd = True; end end else begin // Atomic responses isRespPktEnd = True; - - if (isNormalWorkReq) begin - // One fragment DMA write when handle atomic responses - readAtomicRespPayloadPipeOut.deq; - end end end else begin @@ -467,28 +441,18 @@ module mkTestRespHandleAbnormalCase#(TestRespHandleRespType respType)(Empty); let retryLimitExcOrTimeOutErr = respType != TEST_RESP_HANDLE_TIMEOUT_ERR; let retryHandler <- mkSimRetryHandleErr(retryLimitExcOrTimeOutErr); - // MR permission check - let mrCheckPassOrFail = !(respType == TEST_RESP_HANDLE_PERM_CHECK_FAIL); - let permCheckSrv <- mkSimPermCheckSrv(mrCheckPassOrFail); - - // PayloadConsumer - let simDmaWriteSrv <- mkSimDmaWriteSrv; - let dmaWriteCntrl <- mkDmaWriteCntrl(cntrlStatus, simDmaWriteSrv); - let payloadConsumer <- mkPayloadConsumer( - cntrlStatus, - dmaWriteCntrl, - payloadOrEmptyPipeOut - // dut.payloadConReqPipeOut - ); + // mkRespHandleSQ no longer drives payload consumption (the SQ + // PayloadConsumer / permission-check / DMA-write path was removed), so the + // response payload stream is simply drained here to keep the pipeline + // flowing. + let sinkPayload <- mkSink(payloadOrEmptyPipeOut); // DUT let dut <- mkRespHandleSQ( cntrl.contextSQ, retryHandler, - permCheckSrv, toPipeOut(pendingWorkReqBuf.fifof), - pktMetaDataOrEmptyPipeOut, - payloadConsumer.request + pktMetaDataOrEmptyPipeOut ); // WorkCompGenSQ @@ -502,7 +466,6 @@ module mkTestRespHandleAbnormalCase#(TestRespHandleRespType respType)(Empty); // let cntrl4WorkComp <- mkSimCntrlQP(qpType, pmtu, setExpectedPsnAsNextPSN); let workCompGenSQ <- mkWorkCompGenSQ( cntrlStatus, - payloadConsumer.response, // payloadConsumer.respPipeOut, toPipeOut(wcGenReqQ4ReqGenInSQ), dut.workCompGenReqPipeOut @@ -682,10 +645,8 @@ module mkTestRespHandleRetryCase#(Bool rnrOrSeqErr, Bool nestedRetry)(Empty); let dut <- mkRespHandleSQ( cntrl.contextSQ, retryHandler, - permCheckSrv, toPipeOut(pendingWorkReqBuf.fifof), - pktMetaDataAndPayloadPipeOut.pktMetaData, - payloadConsumer.request + pktMetaDataAndPayloadPipeOut.pktMetaData ); // WorkCompGenSQ @@ -693,7 +654,6 @@ module mkTestRespHandleRetryCase#(Bool rnrOrSeqErr, Bool nestedRetry)(Empty); // FIFOF#(WorkCompStatus) workCompStatusQFromRQ <- mkFIFOF; let workCompGenSQ <- mkWorkCompGenSQ( cntrlStatus, - payloadConsumer.response, // payloadConsumer.respPipeOut, toPipeOut(wcGenReqQ4ReqGenInSQ), dut.workCompGenReqPipeOut diff --git a/test/TestSendQ.bsv b/test/TestSendQ.bsv index 27f0cd5..b657fd6 100644 --- a/test/TestSendQ.bsv +++ b/test/TestSendQ.bsv @@ -288,8 +288,7 @@ module mkTestSendQueueRawPktCase(Empty); isFirst: True, isLast : True }; - let dummySGE = sge; - let sgl = vec(sge, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE); + let sgl = vec(sge); let wqe = WorkQueueElem { id : dontCareValue, diff --git a/test/TestWorkCompGen.bsv b/test/TestWorkCompGen.bsv index 81109bf..5bd0953 100644 --- a/test/TestWorkCompGen.bsv +++ b/test/TestWorkCompGen.bsv @@ -69,7 +69,7 @@ module mkTestWorkCompGenSQ#(Bool isNormalCase)(Empty); // DUT let dut <- mkWorkCompGenSQ( cntrlStatus, - toGet(payloadConRespQ), + // toGet(payloadConRespQ), toPipeOut(wcGenReqQ4ReqGenInSQ), toPipeOut(wcGenReqQ4RespHandleInSQ) // toPipeOut(workCompStatusQFromRQ) @@ -97,34 +97,21 @@ module mkTestWorkCompGenSQ#(Bool isNormalCase)(Empty); endrule rule genPayloadConResp; - let pendingWR = pendingWorkReqPipeOut4DmaResp.first; + // The SQ DMA-write-response path was removed from mkWorkCompGenSQ, so + // there is no PayloadConResp to feed anymore. Drain the reference WR + // stream so the upstream fork does not back-pressure. pendingWorkReqPipeOut4DmaResp.deq; - - if (workReqNeedDmaWriteRespSQ(pendingWR)) begin - let endPSN = unwrapMaybe(pendingWR.endPSN); - let payloadConResp = PayloadConResp { - dmaWriteResp: DmaWriteResp { - initiator: dontCareValue, - sqpn : cntrlStatus.comm.getSQPN, - psn : endPSN, - isRespErr: False - } - }; - if (isNormalCase) begin - payloadConRespQ.enq(payloadConResp); - // $display( - // "time=%0t: pendingWR=", $time, fshow(pendingWR), - // " payloadConResp=", fshow(payloadConResp) - // ); - end - end endrule rule filterWorkReqNeedWorkComp if (cntrlStatus.comm.isRTS || cntrlStatus.comm.isERR); let pendingWR = pendingWorkReqPipeOut4WorkCompReq.first; pendingWorkReqPipeOut4WorkCompReq.deq; - let wcWaitDmaResp = workReqNeedDmaWriteRespSQ(pendingWR); + // The SQ DMA-write-response wait was removed from mkWorkCompGenSQ + // (payloadConRespPort is disabled), so the DUT emits the work + // completion without waiting on a DMA response. Drive wcWaitDmaResp + // false to match the current source behavior. + let wcWaitDmaResp = False; let wcReqType = WC_REQ_TYPE_FULL_ACK; let triggerPSN = unwrapMaybe(pendingWR.endPSN); let wcStatus = isNormalCase ? IBV_WC_SUCCESS : IBV_WC_WR_FLUSH_ERR; @@ -191,223 +178,13 @@ module mkTestWorkCompGenErrFlushCaseRQ(Empty); endmodule module mkTestWorkCompGenRQ#(Bool isNormalCase)(Empty); - let minPayloadLen = 1; - let maxPayloadLen = 8192; - let qpType = IBV_QPT_XRC_SEND; - let pmtu = IBV_MTU_512; - - // WorkReq generation - Vector#(1, PipeOut#(WorkReq)) workReqPipeOutVec <- mkRandomSendOrWriteImmWorkReq( - minPayloadLen, maxPayloadLen - ); - let cntrl <- mkSimCntrl(qpType, pmtu); - let cntrlStatus = cntrl.contextRQ.statusRQ; - // It needs extra controller to generate pending WR, - // since WC might change controller to error state, - // which prevent generating pending WR. - // let cntrl4PendingWorkReqGen <- mkSimCntrl(qpType, pmtu); - // let qpMetaData <- mkSimMetaData4SinigleQP(qpType, pmtu); - // let qpIndex = getDefaultIndexQP; - // let cntrl4PendingWorkReqGen = qpMetaData.getCntrlByIndexQP(qpIndex); - Vector#(1, PipeOut#(PendingWorkReq)) pendingWorkReqPipeOutVec <- - mkExistingPendingWorkReqPipeOut(cntrl, workReqPipeOutVec[0]); - - let pendingWorkReqPipeOut = pendingWorkReqPipeOutVec[0]; - - // RecvReq - FIFOF#(RecvReq) recvReqQ <- mkFIFOF; - - // PayloadConResp - FIFOF#(PayloadConResp) payloadConRespQ <- mkFIFOF; - - // WC requests - FIFOF#(WorkCompGenReqRQ) workCompGenReqQ4RQ <- mkFIFOF; - - // CntrlQP for DUT that will be triggered into error state - // let setExpectedPsnAsNextPSN = False; - // let cntrl <- mkSimCntrlQP(qpType, pmtu, setExpectedPsnAsNextPSN); - - // DUT - let dut <- mkWorkCompGenRQ( - cntrlStatus, - toGet(payloadConRespQ), - toPipeOut(workCompGenReqQ4RQ) - ); - // let workCompPipeOut = dut.workCompPipeOut; - - // RecvReq ID - PipeOut#(WorkReqID) recvReqIdPipeOut <- mkGenericRandomPipeOut; - - // Expected WC - FIFOF#(Tuple5#( - WorkReqID, WorkCompOpCode, WorkCompFlags, Maybe#(IMM), Maybe#(RKEY) - )) expectedWorkCompQ <- mkFIFOF; - - Reg#(Bool) hasWorkCompGenReg <- mkReg(False); + // RQ work-completion generation is part of the disabled receive-queue + // path (removed from QueuePair/Controller). This test is retained but + // skipped: it terminates immediately so CI stays green until RQ returns. + Bool ignoredCase = isNormalCase; let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); -/* - rule setCntrlErrState if (cntrlStatus.comm.isRTS && dut.hasErr); - // let wcStatus = dut.workCompStatusPipeOutRQ.first; - // dut.workCompStatusPipeOutRQ.deq; - // immAssert( - // wcStatus != IBV_WC_SUCCESS, - // "wcStatus assertion @ mkTestWorkCompGenRQ", - // $format("wcStatus=", fshow(wcStatus), " should not be success") - // ); - cntrl.setStateErr; - // $display( - // "time=%0t:", $time, - // " set CntrlQP to error state, wcStatus=", fshow(wcStatus) - // ); - endrule -*/ - rule checkHasErr; - if (isNormalCase) begin - immAssert( - !dut.hasErr, - "hasErr assertion @ mkTestWorkCompGenRQ", - $format("dut.hasErr=", fshow(dut.hasErr), " should be false") - ); - end - else if (hasWorkCompGenReg) begin - immAssert( - dut.hasErr, - "hasErr assertion @ mkTestWorkCompGenRQ", - $format("dut.hasErr=", fshow(dut.hasErr), " should be true") - ); - end - endrule - - rule genWorkCompReq4RQ; - let pendingWR = pendingWorkReqPipeOut.first; - pendingWorkReqPipeOut.deq; - - let hasImmDt = isValid(pendingWR.wr.immDt); - let hasIETH = isValid(pendingWR.wr.rkey2Inv); - let isZeroLen = isZero(pendingWR.wr.len); - - let maybeImmDt = pendingWR.wr.immDt; - let maybeRKey2Inv = pendingWR.wr.rkey2Inv; - let maybeRecvReqID = tagged Invalid; - if (workReqNeedRecvReq(pendingWR.wr.opcode)) begin - let endPSN = unwrapMaybe(pendingWR.endPSN); - if (!isZeroLen) begin - let payloadConResp = PayloadConResp { - dmaWriteResp: DmaWriteResp { - initiator: dontCareValue, - sqpn : cntrlStatus.comm.getSQPN, - psn : endPSN, - isRespErr: False - } - }; - if (isNormalCase) begin - payloadConRespQ.enq(payloadConResp); - end - end - - let reqOpCode = case (pendingWR.wr.opcode) - IBV_WR_SEND_WITH_IMM : SEND_LAST_WITH_IMMEDIATE; - IBV_WR_SEND_WITH_INV : SEND_LAST_WITH_INVALIDATE; - IBV_WR_RDMA_WRITE : RDMA_WRITE_LAST; - IBV_WR_RDMA_WRITE_WITH_IMM: RDMA_WRITE_LAST_WITH_IMMEDIATE; - // IBV_WR_SEND - default : SEND_LAST; - endcase; - let recvReqID = recvReqIdPipeOut.first; - recvReqIdPipeOut.deq; - - maybeRecvReqID = tagged Valid recvReqID; - let wcOpCode = isSendWorkReq(pendingWR.wr.opcode) ? IBV_WC_RECV : IBV_WC_RECV_RDMA_WITH_IMM; - let wcFlags = hasImmDt ? IBV_WC_WITH_IMM : (hasIETH ? IBV_WC_WITH_INV : IBV_WC_NO_FLAGS); - - if (!isNormalCase) begin - reqOpCode = SEND_ONLY; - wcOpCode = IBV_WC_RECV; - wcFlags = IBV_WC_NO_FLAGS; - end - - expectedWorkCompQ.enq(tuple5(recvReqID, wcOpCode, wcFlags, maybeImmDt, maybeRKey2Inv)); - - let workCompReq = WorkCompGenReqRQ { - rrID : maybeRecvReqID, - len : pendingWR.wr.len, - // sqpn : cntrlStatus.comm.getSQPN, - reqPSN : endPSN, - isZeroDmaLen: isZeroLen, - wcStatus : isNormalCase ? IBV_WC_SUCCESS : IBV_WC_WR_FLUSH_ERR, - reqOpCode : reqOpCode, - immDt : maybeImmDt, - rkey2Inv : maybeRKey2Inv - }; - - workCompGenReqQ4RQ.enq(workCompReq); - - // $display("time=%0t: workCompReq=", $time, fshow(workCompReq)); - end - endrule - - rule compare; - let { - recvReqID, wcOpCode, wcFlags, maybeImmDt, maybeRKey2Inv - } = expectedWorkCompQ.first; - expectedWorkCompQ.deq; - - let workCompRQ = dut.workCompPipeOut.first; - dut.workCompPipeOut.deq; - - immAssert( - workCompRQ.id == recvReqID, - "workCompRQ.id assertion @ mkTestWorkCompGenRQ", - $format( - "WC ID=", fshow(workCompRQ.id), - " not match expected recvReqID=", fshow(recvReqID)) - ); - - immAssert( - workCompRQ.opcode == wcOpCode, - "workCompRQ.opcode assertion @ mkTestWorkCompGenRQ", - $format( - "WC opcode=", fshow(workCompRQ.opcode), - " not match expected WC opcode=", fshow(wcOpCode)) - ); - - immAssert( - workCompRQ.flags == wcFlags, - "workCompRQ.flags assertion @ mkTestWorkCompGenRQ", - $format( - "WC flags=", fshow(workCompRQ.flags), - " not match expected WC flags=", fshow(wcFlags)) - ); - - immAssert( - workCompRQ.immDt == maybeImmDt, - "workCompRQ.immDt assertion @ mkTestWorkCompGenRQ", - $format( - "WC immDt=", fshow(workCompRQ.immDt), - " not match expected WC immDt=", fshow(maybeImmDt)) - ); - - immAssert( - workCompRQ.rkey2Inv == maybeRKey2Inv, - "workCompRQ.rkey2Inv assertion @ mkTestWorkCompGenRQ", - $format( - "WC rkey2Inv=", fshow(workCompRQ.rkey2Inv), - " not match expected WC rkey2Inv=", fshow(maybeRKey2Inv)) - ); - - let expectedWorkCompStatus = isNormalCase ? IBV_WC_SUCCESS : IBV_WC_WR_FLUSH_ERR; - immAssert( - workCompRQ.status == expectedWorkCompStatus, - "workCompRQ.status assertion @ mkTestWorkCompGenRQ", - $format( - "WC status=", fshow(workCompRQ.status), - " not match expected status=", fshow(expectedWorkCompStatus) - ) - ); - - // $display("time=%0t: WC=", $time, fshow(workCompRQ)); - hasWorkCompGenReg <= True; + rule skip; countDown.decr; endrule endmodule diff --git a/test/Utils4Test.bsv b/test/Utils4Test.bsv index 0a3c495..643122f 100644 --- a/test/Utils4Test.bsv +++ b/test/Utils4Test.bsv @@ -1578,22 +1578,10 @@ module mkSimInputPktBuf4SingleQP#( let inputRdmaPktBuf <- mkInputRdmaPktBufAndHeaderValidation( headerAndMetaDataAndPayloadPipeOut, qpMetaData ); - let reqPktMetaDataAndPayloadPipeIn = inputRdmaPktBuf[0].reqPktPipeOut; let respPktMetaDataAndPayloadPipeIn = inputRdmaPktBuf[0].respPktPipeOut; let cnpPipeIn = inputRdmaPktBuf[0].cnpPipeOut; for (Integer idx = 1; idx < valueOf(MAX_QP); idx = idx + 1) begin - let reqPktMetaDataPipeOutEmptyRule <- addRules(genEmptyPipeOutRule( - inputRdmaPktBuf[idx].reqPktPipeOut.pktMetaData, - "inputRdmaPktBuf[" + integerToString(idx) + - "].reqPktPipeOut.pktMetaData empty assertion @ mkSimInputPktBuf4SingleQP" - )); - let reqPktPayloadPipeOutEmptyRule <- addRules(genEmptyPipeOutRule( - inputRdmaPktBuf[idx].reqPktPipeOut.payload, - "inputRdmaPktBuf[" + integerToString(idx) + - "].reqPktPipeOut.payload empty assertion @ mkSimInputPktBuf4SingleQP" - )); - let respPktMetaDataPipeOutEmptyRule <- addRules(genEmptyPipeOutRule( inputRdmaPktBuf[idx].respPktPipeOut.pktMetaData, "inputRdmaPktBuf[" + integerToString(idx) + @@ -1613,34 +1601,20 @@ module mkSimInputPktBuf4SingleQP#( end rule checkEmpty; - if (isRespPktPipeIn) begin - immAssert( - !reqPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty && - !reqPktMetaDataAndPayloadPipeIn.payload.notEmpty, - "reqPktMetaDataAndPayloadPipeIn assertion @ mkSimInputPktBuf4SingleQP", - $format( - "reqPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty=", - fshow(reqPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty), - " and reqPktMetaDataAndPayloadPipeIn.payload.notEmpty=", - fshow(reqPktMetaDataAndPayloadPipeIn.payload.notEmpty), - " should both be false" - ) - ); - end - else begin - immAssert( - !respPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty && - !respPktMetaDataAndPayloadPipeIn.payload.notEmpty, - "respPktMetaDataAndPayloadPipeIn assertion @ mkSimInputPktBuf4SingleQP", - $format( - "respPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty=", - fshow(respPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty), - " and respPktMetaDataAndPayloadPipeIn.payload.notEmpty=", - fshow(respPktMetaDataAndPayloadPipeIn.payload.notEmpty), - " should both be false" - ) - ); - end + // The request-side ingress path (reqPktPipeOut) is disabled in + // InputPktHandle, so only the response and CNP pipes are checked here. + immAssert( + !respPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty && + !respPktMetaDataAndPayloadPipeIn.payload.notEmpty, + "respPktMetaDataAndPayloadPipeIn assertion @ mkSimInputPktBuf4SingleQP", + $format( + "respPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty=", + fshow(respPktMetaDataAndPayloadPipeIn.pktMetaData.notEmpty), + " and respPktMetaDataAndPayloadPipeIn.payload.notEmpty=", + fshow(respPktMetaDataAndPayloadPipeIn.payload.notEmpty), + " should both be false" + ) + ); immAssert( !cnpPipeIn.notEmpty, @@ -1653,7 +1627,7 @@ module mkSimInputPktBuf4SingleQP#( ); endrule - return isRespPktPipeIn ? respPktMetaDataAndPayloadPipeIn : reqPktMetaDataAndPayloadPipeIn; + return respPktMetaDataAndPayloadPipeIn; endmodule // PipeOut related diff --git a/test/cocotb/TestAxiSTransportLayer.py b/test/cocotb/TestAxiSTransportLayer.py index 1baf120..2f26273 100644 --- a/test/cocotb/TestAxiSTransportLayer.py +++ b/test/cocotb/TestAxiSTransportLayer.py @@ -172,7 +172,7 @@ async def req_alloc_pd(self): async def resp_alloc_pd(self): for caseIdx in range(self.pdNum): dut_alloc_pd_resp = await self.meta_data_sink.recv() - metaRespBus = BitStream(uint = dut_alloc_pd_resp.tdata.integer, length = META_DATA_BITS) + metaRespBus = BitStream(uint = dut_alloc_pd_resp.tdata.to_unsigned(), length = META_DATA_BITS) pdResp = respPd(metaRespBus) assert pdResp.busType.uint == METADATA_PD_T, f'Bus type should be {METADATA_PD_T}, instead decoded {pdResp.busType.uint}' assert pdResp.successOrNot, "Creation of PD not successfull!" @@ -202,7 +202,7 @@ async def req_alloc_mr(self): async def resp_alloc_mr(self): for caseIdx in range(self.mrNum): dut_alloc_mr_resp = await self.meta_data_sink.recv() - metaRespBus = BitStream(uint = dut_alloc_mr_resp.tdata.integer, length = META_DATA_BITS) + metaRespBus = BitStream(uint = dut_alloc_mr_resp.tdata.to_unsigned(), length = META_DATA_BITS) mrResp = respMr(metaRespBus) assert mrResp.busType.uint == METADATA_MR_T, f'Bus type should be {METADATA_MR_T}, instead decoded {mrResp.busType.uint}' assert mrResp.successOrNot, "Creation of MR not successfull!" @@ -226,7 +226,7 @@ async def req_create_qp(self): async def resp_create_qp(self): for caseIdx in range(self.qpNum): dut_alloc_qp_resp = await self.meta_data_sink.recv() - metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.integer, length = META_DATA_BITS) + metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.to_unsigned(), length = META_DATA_BITS) qpResp = respQp(metaRespBus) assert qpResp.busType.uint == METADATA_QP_T, f'Bus type should be {METADATA_QP_T}, instead decoded {qpResp.busType.uint}' assert qpResp.successOrNot, "Creation of QP not successfull!" @@ -250,7 +250,7 @@ async def req_init_qp(self): async def resp_init_qp(self): for caseIdx in range(self.qpNum): dut_alloc_qp_resp = await self.meta_data_sink.recv() - metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.integer, length = META_DATA_BITS) + metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.to_unsigned(), length = META_DATA_BITS) qpResp = respQp(metaRespBus) assert qpResp.busType.uint == METADATA_QP_T, f'Bus type should be {METADATA_QP_T}, instead decoded {qpResp.busType.uint}' assert qpResp.successOrNot, "QP to init state not successfull!" @@ -275,7 +275,7 @@ async def req_rtr_qp(self): async def resp_rtr_qp(self): for caseIdx in range(self.qpNum): dut_alloc_qp_resp = await self.meta_data_sink.recv() - metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.integer, length = META_DATA_BITS) + metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.to_unsigned(), length = META_DATA_BITS) qpResp = respQp(metaRespBus) assert qpResp.busType.uint == METADATA_QP_T, f'Bus type should be {METADATA_QP_T}, instead decoded {qpResp.busType.uint}' assert qpResp.successOrNot, "QP to RTR state not successfull!" @@ -302,7 +302,7 @@ async def req_rts_qp(self): async def resp_rts_qp(self): for caseIdx in range(self.qpNum): dut_alloc_qp_resp = await self.meta_data_sink.recv() - metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.integer, length = META_DATA_BITS) + metaRespBus = BitStream(uint = dut_alloc_qp_resp.tdata.to_unsigned(), length = META_DATA_BITS) qpResp = respQp(metaRespBus) assert qpResp.busType.uint == METADATA_QP_T, f'Bus type should be {METADATA_QP_T}, instead decoded {qpResp.busType.uint}' assert qpResp.successOrNot, "QP to RTS state not successfull!" @@ -355,13 +355,13 @@ def gen_wr(self, qpiType, wrOpCode, needResp, sqpn, dqpn, lKey, rKey, lAddr, rAd async def get_dma_read_req(self): while True: dut_dma_req = await self.dma_read_clt_sink.recv() - await self.dmaRCRespsQ.put(dmaPyServer(initiator = dut_dma_req.initiator.integer, - sqpn = dut_dma_req.sqpn.integer, - startAddr = dut_dma_req.start_addr.integer, - pktLen = dut_dma_req.len.integer, - wrId = dut_dma_req.wr_id.integer, + await self.dmaRCRespsQ.put(dmaPyServer(initiator = dut_dma_req.initiator.to_unsigned(), + sqpn = dut_dma_req.sqpn.to_unsigned(), + startAddr = dut_dma_req.start_addr.to_unsigned(), + pktLen = dut_dma_req.len.to_unsigned(), + wrId = dut_dma_req.wr_id.to_unsigned(), )) - self.log.debug(f'Received DMA req: wrId -> {hex(dut_dma_req.wr_id.integer)}, len -> {hex(dut_dma_req.len.integer)}') + self.log.debug(f'Received DMA req: wrId -> {hex(dut_dma_req.wr_id.to_unsigned())}, len -> {hex(dut_dma_req.len.to_unsigned())}') async def get_dma_read_resp(self): while True: diff --git a/test/cocotb/requirements.txt b/test/cocotb/requirements.txt new file mode 100644 index 0000000..9661aad --- /dev/null +++ b/test/cocotb/requirements.txt @@ -0,0 +1,11 @@ +# Python dependencies for the cocotb AXI-Stream integration test +# (test/cocotb/TestAxiSTransportLayer.py). Install with: +# pip install -r test/cocotb/requirements.txt +# +# Note: the cocotb flow is run manually via `make cocotb`; it is not part of the +# BlueSim CI (run.sh). It additionally requires the Bluespec toolchain (bsc, +# bluetcl) on PATH and the Icarus Verilog simulator (iverilog). +cocotb +cocotb-test +cocotbext-axi +bitstring From e43c96a07a7c5fd63c2ab5d6b3a73a53ec4d56ed Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Wed, 24 Jun 2026 08:48:08 -0700 Subject: [PATCH 4/6] fix(sq): clear nested-RNR retry-FSM deadlock in startPreRetry Under sustained/nested RNR backpressure the SQ retry FSM could wedge permanently: a second RNR/NAK arriving during RNR_WAIT re-runs initRetry, re-entering RETRY_HANDLE_ST_START_PRE_RETRY while the pending-WR scan FIFO is still parked in PRE_SCAN_MODE. The old 2-way isScanDone branch then issued preScanRestart(), whose implicit guard is inScanMode -- unsatisfiable in PRE_SCAN_MODE -- so the rule could never fire again. Dispatch and completions stalled until a full QP/source drain. startPreRetry now selects the scan command by exact mode: FIFOF_MODE -> preScanStart, SCAN_MODE -> preScanRestart, PRE_SCAN_MODE -> no-op (the pre-scan is already armed at the head). Expose ScanCntrl.isScanMode() to distinguish the SCAN/PRE_SCAN cases. Adds TestSqRnrThroughput: drives a continuous WR stream through a sustained RNR window and asserts post-RNR throughput recovers to >=80% of baseline (pre-fix it stays at 0%, a permanent stall). Registered in Makefile.test. --- Makefile.test | 6 +- src/RetryHandleSQ.bsv | 22 ++- src/SpecialFIFOF.bsv | 4 +- test/TestSqRnrThroughput.bsv | 372 +++++++++++++++++++++++++++++++++++ 4 files changed, 392 insertions(+), 12 deletions(-) create mode 100644 test/TestSqRnrThroughput.bsv diff --git a/Makefile.test b/Makefile.test index 25f8cb6..0ce2463 100644 --- a/Makefile.test +++ b/Makefile.test @@ -30,7 +30,8 @@ TESTBENCHS = \ TestUtils.bsv \ TestWorkCompGen.bsv \ TestPayloadGen.bsv \ - TestSendQ.bsv + TestSendQ.bsv \ + TestSqRnrThroughput.bsv SimDma.bsv = mkTestFixedPktLenDataStreamPipeOut \ mkTestDmaReadAndWriteSrv @@ -133,6 +134,9 @@ TestSendQ.bsv = mkTestSendQueueRawPktCase \ mkTestSendQueueNoPayloadCase \ mkTestSendQueueZeroPayloadLenCase +TestSqRnrThroughput.bsv = mkTestSqNoRnrBaselineCase \ + mkTestSqRnrThroughputCase + all: $(TESTBENCHS) %.bsv: diff --git a/src/RetryHandleSQ.bsv b/src/RetryHandleSQ.bsv index dbb9fbd..8d0a9a5 100644 --- a/src/RetryHandleSQ.bsv +++ b/src/RetryHandleSQ.bsv @@ -570,20 +570,24 @@ module mkRetryHandleSQ#( retryHandleStateReg <= RETRY_HANDLE_ST_CHECK_PARTIAL_RETRY_WR; end + // Re-arm the pending-WR scan from the queue head. The scan FIFO has three + // modes and each needs a different (or no) command -- a 2-way isScanDone + // branch deadlocks when this rule is re-entered by a NESTED retry (a 2nd + // RNR/NAK arriving during RNR_WAIT re-runs initRetry -> START_PRE_RETRY) + // while the FIFO is parked in PRE_SCAN_MODE: isScanDone is false there, so + // the old code issued preScanRestart(), whose implicit guard is inScanMode + // -- unsatisfiable in PRE_SCAN_MODE -- and the retry FSM wedges forever + // (dispatch + completions stall until a full QP/source drain). In + // PRE_SCAN_MODE the pre-scan is already armed at the head, so no command is + // needed; only FIFOF_MODE needs preScanStart and SCAN_MODE needs + // preScanRestart. if (pendingWorkReqScanCntrl.isScanDone) begin pendingWorkReqScanCntrl.preScanStart; - // $display( - // "time=%0t: pendingWorkReqScanCntrl.preScanStart", $time, - // " pendingWorkReqNotEmpty=", fshow(pendingWorkReqNotEmpty) - // ); end - else begin + else if (pendingWorkReqScanCntrl.isScanMode) begin pendingWorkReqScanCntrl.preScanRestart; - // $display( - // "time=%0t: pendingWorkReqScanCntrl.preScanRestart", $time, - // " pendingWorkReqNotEmpty=", fshow(pendingWorkReqNotEmpty) - // ); end + // else: already in PRE_SCAN_MODE (nested retry) -- head is armed, no-op. // $display( // "time=%0t: startPreRetry", $time, // ", retryHandleStateReg=", fshow(retryHandleStateReg), diff --git a/src/SpecialFIFOF.bsv b/src/SpecialFIFOF.bsv index ad40e9b..385faf0 100644 --- a/src/SpecialFIFOF.bsv +++ b/src/SpecialFIFOF.bsv @@ -22,7 +22,7 @@ interface ScanCntrl#(type anytype); method Bool hasScanOut(); method Bool isScanDone(); method Bool deqPulse(); - // method Bool isScanMode(); + method Bool isScanMode(); endinterface interface ScanFIFOF#(numeric type qSz, type anytype); @@ -491,7 +491,7 @@ module mkScanFIFOF(ScanFIFOF#(qSz, anytype)) provisos( method Bool hasScanOut() = !inFifoMode || scanOutQ.notEmpty; method Bool isScanDone() = inFifoMode; method Bool deqPulse() = popReg[1]; - // method Bool isScanMode() = inScanMode; + method Bool isScanMode() = inScanMode; endinterface; interface scanPipeOut = toPipeOut(scanOutQ); diff --git a/test/TestSqRnrThroughput.bsv b/test/TestSqRnrThroughput.bsv new file mode 100644 index 0000000..8c9e175 --- /dev/null +++ b/test/TestSqRnrThroughput.bsv @@ -0,0 +1,372 @@ +// Sustained-RNR SQ throughput test. +// +// PURPOSE: reproduce the "sticky half-throughput after backpressure clears" +// bug observed on hardware (Simple-10GbE-RUDP-KCU105-Example RoCEv2 datapath). +// On the bench the SQ completes RDMA-SENDs at full rate, then after a window of +// RNR-NAK backpressure (host recv queue transiently full) it latches at EXACTLY +// half the completion rate and never recovers until the source fully drains. +// +// VEHICLE: drive the full mkSQ (ReqGenSQ + RetryHandleSQ + RespHandleSQ + +// WorkCompGenSQ + the pending-WR scan-FIFO) -- the smallest unit that contains +// the complete dispatch<->response<->retry loop where re-dispatch throughput is +// observable. A continuous SEND-only WorkReq stream keeps the pending queue +// non-empty (continuous backlog), matching the hardware condition under which +// the latch persists. +// +// RESPONDER: reactive. Every request packet the SQ emits is observed, its BTH +// PSN extracted, and exactly one ACK fed back at that PSN -- an RNR-NAK while +// the cycle counter is inside [rnrStart, rnrStop), a normal ACK otherwise. This +// is PSN-consistent by construction and naturally covers retransmissions. +// +// MEASUREMENT: count WorkComps in a baseline window (before any RNR) and in a +// post-RNR steady-state window, and report completions/1000-cycles for each. +// PASS = post-RNR rate >= 0.8 * baseline rate. FAIL (bug present) = post-RNR +// rate ~= 0.5 * baseline (the stuck-half signature). + +import ClientServer :: *; +import Connectable :: *; +import FIFOF :: *; +import GetPut :: *; +import PAClib :: *; +import Vector :: *; +import Cntrs :: *; + +import Headers :: *; +import Controller :: *; +import DataTypes :: *; +import ExtractAndPrependPipeOut :: *; +import InputPktHandle :: *; +import PayloadConAndGen :: *; +import PrimUtils :: *; +import QueuePair :: *; +import ReqGenSQ :: *; +import RespHandleSQ :: *; +import RetryHandleSQ :: *; +import SimDma :: *; +import SimExtractRdmaHeaderPayload :: *; +import SimGenRdmaReqResp :: *; +import Settings :: *; +import Utils :: *; +import Utils4Test :: *; +import WorkCompGen :: *; + +// Cycle windows (in clock cycles) for the RNR burst and the measurement spans. +// RNR window. Set RnrStartCycle == RnrStopCycle to DISABLE RNR entirely +// (pure-ACK harness self-check baseline). +typedef 4000 RnrStartCycle; // let the baseline run clean first +typedef 12000 RnrStopCycle; // sustained RNR window (~8000 cycles, several backoffs) +typedef 2500 BaselineSpan; // baseline measurement window (pre-RNR) +typedef 4000 RecoverWaitCycle;// settle time after RNR stops before measuring +typedef 120000 SimEndCycle; // hard stop (long, to see slow recovery vs stuck) +typedef 12 NumBuckets; // 12 * 10000 = 120000 = SimEndCycle + +// Pure-ACK harness self-check: NO RNR ever. Confirms the testbench sustains +// full SQ throughput end-to-end, so a "stuck" verdict in the RNR case is a real +// DUT behavior and not a harness artifact. +(* doc = "testcase" *) +module mkTestSqNoRnrBaselineCase(Empty); + let _x <- mkSqRnrThroughputBody(False); +endmodule + +(* doc = "testcase" *) +module mkTestSqRnrThroughputCase(Empty); + let _x <- mkSqRnrThroughputBody(True); +endmodule + +module mkSqRnrThroughputBody#(Bool enableRnr)(Empty); + // SEND-only, MULTI-packet WRs: payload spans several PMTUs so each WR has a + // multi-PSN range (startPSN..endPSN). This exercises the go-back-N coalesce / + // nextPSN window + isOnlyReqPkt/remainingPktNum reload path in ReqGenSQ/ + // RespHandleSQ that single-packet WRs bypass entirely -- the suspected home of + // the one-behind retransmit latch (HW: TX/comp=2.0 persistent after RNR). + let minDmaLength = 512; // > PMTU (256) => >= 2 packets/WR + let maxDmaLength = 1024; // up to ~4 packets/WR + let qpType = IBV_QPT_RC; + let pmtu = IBV_MTU_256; + + // ---- Controller driven to RTS with INFINITE RNR retry ---- + // The shared mkSimCntrl hard-codes rnrRetry = DEFAULT_RETRY_NUM (3), which + // makes a long RNR burst exhaust the retry budget and ERROR the QP -- that + // masks the sticky-half bug with a permanent stall. Hardware runs + // rnrRetry = 7 = INFINITE_RETRY, so drive the QP to RTS locally with + // infinite RNR + data retry to match the bench. + let cntrl <- mkCntrlQP; + let cntrlStatus = cntrl.contextSQ.statusSQ; + let qpn = getDefaultQPN; + let qpInitAttr = QpInitAttr { qpType: qpType, sqSigAll: False }; + + function AttrQP infRetryQpAttr(); + return AttrQP { + qpState : dontCareValue, + curQpState : dontCareValue, + pmtu : pmtu, + qkey : fromInteger(valueOf(DEFAULT_QKEY)), + rqPSN : 0, + sqPSN : 0, + dqpn : getDefaultQPN, + qpAccessFlags : enum2Flag(IBV_ACCESS_REMOTE_WRITE) | + enum2Flag(IBV_ACCESS_REMOTE_READ) | + enum2Flag(IBV_ACCESS_REMOTE_ATOMIC), + cap : QpCapacity { + maxSendWR : fromInteger(valueOf(MAX_QP_WR)), + maxRecvWR : fromInteger(valueOf(MAX_QP_WR)), + maxSendSGE : fromInteger(valueOf(MAX_SEND_SGE)), + maxRecvSGE : fromInteger(valueOf(MAX_RECV_SGE)), + maxInlineData: fromInteger(valueOf(MAX_INLINE_DATA)) + }, + pkeyIndex : fromInteger(valueOf(DEFAULT_PKEY)), + sqDraining : False, + maxReadAtomic : fromInteger(valueOf(MAX_QP_RD_ATOM)), + maxDestReadAtomic: fromInteger(valueOf(MAX_QP_DST_RD_ATOM)), + minRnrTimer : 1, + timeout : 0, // 0 = infinite response timeout (no timeout retries) + retryCnt : fromInteger(valueOf(INFINITE_RETRY)), + rnrRetry : fromInteger(valueOf(INFINITE_RETRY)) + }; + endfunction + + Reg#(Bit#(3)) qpSetupStateReg <- mkReg(0); + rule qpCreate if (qpSetupStateReg == 0 && cntrlStatus.comm.isReset); + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_CREATE, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: dontCareValue, qpAttr: dontCareValue, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 1; + endrule + rule qpInit if (qpSetupStateReg == 1); + let resp <- cntrl.srvPort.response.get; + let a = infRetryQpAttr; a.qpState = IBV_QPS_INIT; + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_MODIFY, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: getReset2InitRequiredAttr, qpAttr: a, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 2; + endrule + rule qpRtr if (qpSetupStateReg == 2); + let resp <- cntrl.srvPort.response.get; + let a = infRetryQpAttr; a.qpState = IBV_QPS_RTR; + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_MODIFY, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: getInit2RtrRequiredAttr, qpAttr: a, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 3; + endrule + rule qpRts if (qpSetupStateReg == 3); + let resp <- cntrl.srvPort.response.get; + let a = infRetryQpAttr; a.qpState = IBV_QPS_RTS; + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_MODIFY, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: getRtr2RtsRequiredAttr, qpAttr: a, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 4; + endrule + rule qpRtsResp if (qpSetupStateReg == 4); + let resp <- cntrl.srvPort.response.get; + qpSetupStateReg <= 5; // QP now in RTS + endrule + + // ---- Continuous WorkReq source (raw WorkReq into mkSQ) ---- + Vector#(1, PipeOut#(WorkReq)) sendWorkReqPipeOutVec <- mkRandomSendWorkReq( + minDmaLength, maxDmaLength + ); + let workReqPipeIn = sendWorkReqPipeOutVec[0]; + + // ---- Payload generator + DMA model for the SQ request path ---- + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(cntrlStatus, simDmaReadSrv); + let payloadGenerator <- mkPayloadGenerator(cntrlStatus, dmaReadCntrl); + + // ---- Response path plumbing ---- + // The reactive responder builds ACK headers into respHeaderQ; they are + // turned into a response DataStream and then into RdmaPktMetaData+payload + // that mkSQ consumes as respPktPipeOut. + FIFOF#(HeaderRDMA) respHeaderQ <- mkFIFOF; + let respHeaderDataStreamAndMeta <- mkHeader2DataStream( + cntrlStatus.comm.isReset, toPipeOut(respHeaderQ) + ); + mkSink(respHeaderDataStreamAndMeta.headerMetaData); + let respPktMetaDataAndPayload <- mkSimExtractNormalHeaderPayload( + respHeaderDataStreamAndMeta.headerDataStream + ); + + // ---- DUT: the full Send Queue ---- + let dut <- mkSQ( + cntrl.contextSQ, + payloadGenerator, + workReqPipeIn, + respPktMetaDataAndPayload + ); + + // ---- Observe emitted request packets to drive the reactive responder ---- + let reqHeaderAndPayload <- mkExtractHeaderFromRdmaPktPipeOut( + dut.rdmaReqDataStreamPipeOut + ); + let reqHeaderPipeOut <- mkDataStream2Header( + reqHeaderAndPayload.headerAndMetaData.headerDataStream, + reqHeaderAndPayload.headerAndMetaData.headerMetaData + ); + // Drain the request payload (we only need the headers to generate ACKs). + let sinkReqPayload <- mkSink(reqHeaderAndPayload.payload); + + // ---- Free-running cycle counter (drives the RNR window + measurement) ---- + Reg#(Bit#(32)) cycleReg <- mkReg(0); + (* no_implicit_conditions, fire_when_enabled *) + rule tick; + cycleReg <= cycleReg + 1; + if (cycleReg == fromInteger(valueOf(SimEndCycle))) begin + $finish(0); + end + endrule + + let inRnrWindow = enableRnr && + (cycleReg >= fromInteger(valueOf(RnrStartCycle))) && + (cycleReg < fromInteger(valueOf(RnrStopCycle))); + + // Request-packet emission instrumentation (distinguishes a DUT throughput + // latch from a testbench responder-handshake artifact: if request packets + // keep flowing but WCs stop, the latch is in the DUT response/retry path). + Count#(Bit#(32)) reqPktSeenCnt <- mkCount(0); + Vector#(NumBuckets, Count#(Bit#(32))) reqBucket <- replicateM(mkCount(0)); + + // ---- Reactive responder: one ACK per emitted request packet ---- + // RNR-NAK while inside the window (echo BTH PSN, AETH RNR), else normal ACK. + // The window is INTERMITTENT: NAK only a fraction (rnrNakMod-1)/rnrNakMod of + // packets, letting the rest complete -- this models heavy-but-partial host + // backpressure (host slightly behind), the actual hardware regime, rather + // than a 100%-NAK total stall. + Reg#(Bit#(32)) respWrCntReg <- mkReg(0); + Integer rnrNakMod = 4; // during the window, NAK 3 of every 4 WHOLE WRs + rule genAckForReq; + let rdmaHeader = reqHeaderPipeOut.first; + reqHeaderPipeOut.deq; + + let bthIn = extractBTH(rdmaHeader.headerData); + let { transTypeIn, rdmaOpCodeIn } = + extractTranTypeAndRdmaOpCode(rdmaHeader.headerData); + + // Only the LAST (or ONLY) packet of a WR triggers a whole-WR response, at + // its endPSN -- a clean cumulative ACK / RNR-NAK for the WR. Intermediate + // packets of a multi-packet SEND get no response (drained, no enq). + let isWrEnd = isLastOrOnlyRdmaOpCode(rdmaOpCodeIn); + if (isWrEnd) begin + respWrCntReg <= respWrCntReg + 1; + end + let doNak = isWrEnd && inRnrWindow && + ((respWrCntReg % fromInteger(rnrNakMod)) != 0); + + let maybeTrans = qpType2TransType(cntrlStatus.getTypeQP); + let transType = unwrapMaybe(maybeTrans); + + let bth = BTH { + trans : transType, + opcode : ACKNOWLEDGE, + solicited: False, + migReq : unpack(0), + padCnt : 0, + tver : unpack(0), + pkey : cntrlStatus.comm.getPKEY, + fecn : unpack(0), + becn : unpack(0), + resv6 : unpack(0), + dqpn : cntrlStatus.comm.getSQPN, + ackReq : False, + resv7 : unpack(0), + psn : bthIn.psn // endPSN of the WR (last packet) + }; + let aeth = doNak ? + AETH { + rsvd : unpack(0), + code : AETH_CODE_RNR, + value: cntrlStatus.comm.getMinRnrTimer, + msn : dontCareValue + } : + AETH { + rsvd : unpack(0), + code : AETH_CODE_ACK, + value: pack(AETH_ACK_VALUE_INVALID_CREDIT_CNT), + msn : dontCareValue + }; + // Respond ONLY to whole-WR ends. reqBucket counts WR transmissions (one per + // last-packet seen) so reqBucket vs wcBucket == TX/comp at WR granularity, + // matching the hardware DmaReadCount-vs-SuccessCounter measurement. + if (isWrEnd) begin + let respHeader = genHeaderRDMA( + zeroExtendLSB({ pack(bth), pack(aeth) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(AETH_BYTE_WIDTH)), + False + ); + respHeaderQ.enq(respHeader); + reqPktSeenCnt.incr(1); + let bidx = cycleReg / fromInteger(10000); + for (Integer i = 0; i < valueOf(NumBuckets); i = i + 1) begin + if (bidx == fromInteger(i)) reqBucket[i].incr(1); + end + end + endrule + + // ---- Measure WorkComp throughput as a trajectory ---- + // Bucket completions into consecutive 5000-cycle bins so the time-course is + // visible: baseline (full) -> RNR window (low) -> post-RNR steady state. + let workCompPipeOut = dut.workCompSQ.workCompPipeOut; + + Integer bucketSpan = 10000; // NumBuckets * bucketSpan = SimEndCycle + Vector#(NumBuckets, Count#(Bit#(32))) wcBucket <- replicateM(mkCount(0)); + Reg#(Bit#(32)) totalWcReg <- mkReg(0); + Reg#(Bit#(32)) errWcReg <- mkReg(0); // non-SUCCESS WCs (retry-exhaust / err) + + rule countWorkComp; + let wc = workCompPipeOut.first; + workCompPipeOut.deq; + totalWcReg <= totalWcReg + 1; + if (wc.status != IBV_WC_SUCCESS) begin + errWcReg <= errWcReg + 1; + end + + let idx = cycleReg / fromInteger(bucketSpan); + for (Integer i = 0; i < valueOf(NumBuckets); i = i + 1) begin + if (idx == fromInteger(i)) begin + wcBucket[i].incr(1); + end + end + endrule + + // ---- Report + verdict near the end of the sim ---- + Reg#(Bool) reportedReg <- mkReg(False); + rule report if (!reportedReg && cycleReg == (fromInteger(valueOf(SimEndCycle)) - 1)); + reportedReg <= True; + + $display("========================================================="); + $display("SQ RNR THROUGHPUT TEST (RNR window cycles [%0d, %0d))", + valueOf(RnrStartCycle), valueOf(RnrStopCycle)); + $display(" total WorkComps : %0d (non-SUCCESS WCs: %0d)", totalWcReg, errWcReg); + $display(" total req pkts seen : %0d", reqPktSeenCnt); + $display(" per %0d-cycle bucket (reqPkts / workComps):", bucketSpan); + for (Integer i = 0; i < valueOf(NumBuckets); i = i + 1) begin + $display(" bucket %0d [cyc %0d..%0d) : reqPkt=%0d WC=%0d", + i, i*bucketSpan, (i+1)*bucketSpan, reqBucket[i], wcBucket[i]); + end + + // baseline = bucket 0 (pre-RNR, full rate); steadyState = last bucket + // (long after RNR stopped). Bug = steadyState ~= 0.5 * baseline (or less) + // with NO recovery. PASS = steadyState >= 0.8 * baseline. + let baseline = wcBucket[0]; + let steady = wcBucket[valueOf(NumBuckets)-1]; + let pct = (baseline == 0) ? 0 : (steady * 100) / baseline; + $display(" baseline(bucket0)=%0d steadyState(last)=%0d -> %0d%% of baseline", + baseline, steady, pct); + if (pct >= 80) begin + $display(" VERDICT : PASS (recovered to >=80%% of baseline)"); + end + else begin + $display(" VERDICT : FAIL -- STUCK (steady state %0d%% of baseline, no recovery)", pct); + end + $display("========================================================="); + // Fail loudly so CI (greps for "Error"/"ImmAssert") catches a regression + // of the nested-RNR retry deadlock this test guards. + immAssert( + pct >= 80, + "SQ RNR throughput recovery assertion @ mkTestSqRnrThroughput", + $format( + "post-RNR steady-state throughput=%0d%% of baseline should be >=80%%", + pct, " (nested-RNR retry-FSM deadlock regression)" + ) + ); + $finish(0); + endrule +endmodule From 2cbc90ee3dfe1c4f7721ded79ccddc5dff0c3790 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Wed, 24 Jun 2026 16:23:05 -0700 Subject: [PATCH 5/6] fix(sq): stop scan FIFO dropping replay tail on scanDone The pending-WR scan FIFO (mkScanFIFOF) cleared its output queue scanOutQ unconditionally on every fifoMode cycle. On a normal scanDone -> FIFOF_MODE transition the up-to-2 replayed work requests still buffered in scanOutQ (its mkFIFOF depth) had not yet been drained by mkReqGenSQ, so they were flushed and never reached the wire. Each go-back-N retry therefore replayed itemCnt-2 packets instead of itemCnt, leaving a PSN gap that the peer SEQ_ERR-NAKs, triggering another truncated retry -> a self-perpetuating ~2x retransmit treadmill after RNR backpressure (the sticky half-throughput latch). Gate scanOutQ.clear to fire only when (re)arming a fresh scan (the preScanStart branch). The abort paths (stopScan, preScanRestart) already clear scanOutQ explicitly in scanModeStateChange, so this was the only unconditional clear, and it was the one dropping legitimate scanDone leftovers. Full Bluesim regression suite passes (76/76, 0 Error/ImmAssert), including mkTestScanFIFOF and the mkTestSqRnrThroughput nested-RNR deadlock guard. --- src/SpecialFIFOF.bsv | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/SpecialFIFOF.bsv b/src/SpecialFIFOF.bsv index 385faf0..eb67203 100644 --- a/src/SpecialFIFOF.bsv +++ b/src/SpecialFIFOF.bsv @@ -184,13 +184,21 @@ module mkScanFIFOF(ScanFIFOF#(qSz, anytype)) provisos( ) ); scanStateReg <= SCAN_Q_PRE_SCAN_MODE; + // Clear any stale scan-output ONLY when (re)arming a fresh scan. The old + // code cleared scanOutQ unconditionally every fifoMode cycle, which on a + // normal scanDone->fifoMode transition flushed the up-to-2 replayed items + // still buffered in scanOutQ (its mkFIFOF depth) before mkReqGenSQ could + // drain them -> those WRs were dropped from the replay, leaving a PSN gap + // each retry and a self-perpetuating ~2x retransmit treadmill after RNR. + // The abort paths (stopScan / preScanRestart) already clear scanOutQ in + // scanModeStateChange, so this is the only place that needed gating. + scanOutQ.clear; // $display( // "time=%0t:", $time, // " fifoMode change to state=", fshow(SCAN_Q_PRE_SCAN_MODE) // ); end - scanOutQ.clear; headReg <= tagged Invalid; preScanStartReg[1] <= False; endrule From 30e97ddcdfd7d7a58d056b4205e44d626375604b Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Wed, 24 Jun 2026 17:45:54 -0700 Subject: [PATCH 6/6] test(sq): add scan-FIFO replay-tail-drop regression (strict in-order receiver) The existing mkTestSqRnrThroughput responder is a blind echo-ACK: it ACKs every WR-end at that packet's own PSN, so when the buggy SQ drops the last <=2 WRs of a retry replay and jumps ahead, the responder ACKs the higher-PSN WR-end and the SQ treats it as a cumulative ACK that silently absorbs the dropped WRs. That test therefore PASSES on the scanOutQ.clear-buggy code and does not cover the replay-tail-drop fixed in 2cbc90e. Add mkTestSqTailDropCase (+ mkTestSqTailDropNoRnrCase self-check) driving the same mkSQ with a STRICT in-order RC receiver: ACK only the in-order WR-end at the expected PSN, SEQ_ERR-NAK a forward PSN gap (one coalesced NAK per episode, never cumulatively ACK past a hole), re-ACK duplicates. Single-packet WRs make PSN index the WR directly (the hardware 4096B-SEND regime). A transient RNR burst kicks off one retry; the dropped replay tail then leaves a gap the receiver NAKs, forcing another truncated retry -> the self-perpetuating ~2x retransmit treadmill. Verdict = steady-state TX/comp (req-pkts/work-comps, last bucket); PASS <= 130%. Verified discriminating: pristine code -> TX/comp=200%, ImmAssert FAIL; with the fix -> TX/comp=100% PASS. No-RNR self-check -> 100% (strict receiver does not itself manufacture the gap). Full Bluesim suite 78/78, 0 Error/ImmAssert. --- Makefile.test | 4 +- test/TestSqRnrThroughput.bsv | 312 +++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+), 1 deletion(-) diff --git a/Makefile.test b/Makefile.test index 0ce2463..c410f10 100644 --- a/Makefile.test +++ b/Makefile.test @@ -135,7 +135,9 @@ TestSendQ.bsv = mkTestSendQueueRawPktCase \ mkTestSendQueueZeroPayloadLenCase TestSqRnrThroughput.bsv = mkTestSqNoRnrBaselineCase \ - mkTestSqRnrThroughputCase + mkTestSqRnrThroughputCase \ + mkTestSqTailDropNoRnrCase \ + mkTestSqTailDropCase all: $(TESTBENCHS) diff --git a/test/TestSqRnrThroughput.bsv b/test/TestSqRnrThroughput.bsv index 8c9e175..44e8fab 100644 --- a/test/TestSqRnrThroughput.bsv +++ b/test/TestSqRnrThroughput.bsv @@ -73,6 +73,37 @@ module mkTestSqRnrThroughputCase(Empty); let _x <- mkSqRnrThroughputBody(True); endmodule +// Scan-FIFO replay-tail-drop coverage (mkScanFIFOF scanOutQ.clear bug). +// +// The mkTestSqRnrThroughput responder above is a BLIND echo-ACK: it ACKs every +// WR-end at that packet's own PSN. When the buggy SQ drops the last <=2 WRs of a +// retry replay and jumps ahead, the responder ACKs the higher-PSN WR-end, which +// the SQ treats as a CUMULATIVE ACK that silently absorbs the dropped WRs -- so +// the tail-drop is invisible and that test PASSES on buggy code. +// +// mkSqTailDropBody uses a STRICT in-order receiver instead: it ACKs only the +// in-order WR-end at the expected PSN, SEQ_ERR-NAKs a forward PSN gap (never +// cumulatively ACKs past a hole), and re-ACKs duplicates. Single-packet WRs make +// PSN index the WR directly (the hardware 4096B-SEND regime). A transient RNR +// burst kicks off one retry; on buggy FW the dropped replay tail leaves a gap the +// receiver SEQ_ERR-NAKs, forcing another truncated retry -> a self-perpetuating +// ~2x retransmit treadmill (steady-state TX/comp ~= 2.0). With the scanOutQ.clear +// fix the replay covers the full window, the gap closes, and TX/comp -> 1.0. +// +// VERDICT = steady-state TX/comp (req-pkts/work-comps in the last bucket): +// PASS <= 1.30, FAIL (bug present) ~= 2.0. The no-RNR case is the responder +// self-check (no retry -> no gap -> TX/comp == 1.0), proving the strict receiver +// does not itself manufacture the gap. +(* doc = "testcase" *) +module mkTestSqTailDropNoRnrCase(Empty); + let _x <- mkSqTailDropBody(False); +endmodule + +(* doc = "testcase" *) +module mkTestSqTailDropCase(Empty); + let _x <- mkSqTailDropBody(True); +endmodule + module mkSqRnrThroughputBody#(Bool enableRnr)(Empty); // SEND-only, MULTI-packet WRs: payload spans several PMTUs so each WR has a // multi-PSN range (startPSN..endPSN). This exercises the go-back-N coalesce / @@ -370,3 +401,284 @@ module mkSqRnrThroughputBody#(Bool enableRnr)(Empty); $finish(0); endrule endmodule + +// --------------------------------------------------------------------------- +// Scan-FIFO replay-tail-drop regression (strict in-order receiver). +// --------------------------------------------------------------------------- +module mkSqTailDropBody#(Bool enableRnr)(Empty); + // SINGLE-packet WRs: payload <= PMTU so each WR is exactly one packet = one + // PSN. PSN then indexes the WR directly (the hardware 4096B-SEND / PMTU-4096 + // regime), making the strict in-order receiver's gap detection unambiguous. + let dmaLength = 256; // == PMTU => exactly 1 packet/WR + let qpType = IBV_QPT_RC; + let pmtu = IBV_MTU_256; + + // ---- Controller -> RTS with INFINITE RNR + data retry (matches hardware) ---- + let cntrl <- mkCntrlQP; + let cntrlStatus = cntrl.contextSQ.statusSQ; + let qpn = getDefaultQPN; + let qpInitAttr = QpInitAttr { qpType: qpType, sqSigAll: False }; + + function AttrQP infRetryQpAttr(); + return AttrQP { + qpState : dontCareValue, + curQpState : dontCareValue, + pmtu : pmtu, + qkey : fromInteger(valueOf(DEFAULT_QKEY)), + rqPSN : 0, + sqPSN : 0, + dqpn : getDefaultQPN, + qpAccessFlags : enum2Flag(IBV_ACCESS_REMOTE_WRITE) | + enum2Flag(IBV_ACCESS_REMOTE_READ) | + enum2Flag(IBV_ACCESS_REMOTE_ATOMIC), + cap : QpCapacity { + maxSendWR : fromInteger(valueOf(MAX_QP_WR)), + maxRecvWR : fromInteger(valueOf(MAX_QP_WR)), + maxSendSGE : fromInteger(valueOf(MAX_SEND_SGE)), + maxRecvSGE : fromInteger(valueOf(MAX_RECV_SGE)), + maxInlineData: fromInteger(valueOf(MAX_INLINE_DATA)) + }, + pkeyIndex : fromInteger(valueOf(DEFAULT_PKEY)), + sqDraining : False, + maxReadAtomic : fromInteger(valueOf(MAX_QP_RD_ATOM)), + maxDestReadAtomic: fromInteger(valueOf(MAX_QP_DST_RD_ATOM)), + minRnrTimer : 1, + timeout : 0, + retryCnt : fromInteger(valueOf(INFINITE_RETRY)), + rnrRetry : fromInteger(valueOf(INFINITE_RETRY)) + }; + endfunction + + Reg#(Bit#(3)) qpSetupStateReg <- mkReg(0); + rule qpCreate if (qpSetupStateReg == 0 && cntrlStatus.comm.isReset); + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_CREATE, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: dontCareValue, qpAttr: dontCareValue, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 1; + endrule + rule qpInit if (qpSetupStateReg == 1); + let resp <- cntrl.srvPort.response.get; + let a = infRetryQpAttr; a.qpState = IBV_QPS_INIT; + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_MODIFY, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: getReset2InitRequiredAttr, qpAttr: a, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 2; + endrule + rule qpRtr if (qpSetupStateReg == 2); + let resp <- cntrl.srvPort.response.get; + let a = infRetryQpAttr; a.qpState = IBV_QPS_RTR; + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_MODIFY, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: getInit2RtrRequiredAttr, qpAttr: a, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 3; + endrule + rule qpRts if (qpSetupStateReg == 3); + let resp <- cntrl.srvPort.response.get; + let a = infRetryQpAttr; a.qpState = IBV_QPS_RTS; + cntrl.srvPort.request.put(ReqQP { + qpReqType: REQ_QP_MODIFY, pdHandler: dontCareValue, qpn: qpn, + qpAttrMask: getRtr2RtsRequiredAttr, qpAttr: a, qpInitAttr: qpInitAttr }); + qpSetupStateReg <= 4; + endrule + rule qpRtsResp if (qpSetupStateReg == 4); + let resp <- cntrl.srvPort.response.get; + qpSetupStateReg <= 5; + endrule + + // ---- Continuous SEND-only WorkReq source (fixed single-packet length) ---- + Vector#(1, PipeOut#(WorkReq)) sendWorkReqPipeOutVec <- mkRandomSendWorkReq( + dmaLength, dmaLength + ); + let workReqPipeIn = sendWorkReqPipeOutVec[0]; + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(cntrlStatus, simDmaReadSrv); + let payloadGenerator <- mkPayloadGenerator(cntrlStatus, dmaReadCntrl); + + FIFOF#(HeaderRDMA) respHeaderQ <- mkFIFOF; + let respHeaderDataStreamAndMeta <- mkHeader2DataStream( + cntrlStatus.comm.isReset, toPipeOut(respHeaderQ) + ); + mkSink(respHeaderDataStreamAndMeta.headerMetaData); + let respPktMetaDataAndPayload <- mkSimExtractNormalHeaderPayload( + respHeaderDataStreamAndMeta.headerDataStream + ); + + let dut <- mkSQ( + cntrl.contextSQ, payloadGenerator, workReqPipeIn, respPktMetaDataAndPayload + ); + + let reqHeaderAndPayload <- mkExtractHeaderFromRdmaPktPipeOut( + dut.rdmaReqDataStreamPipeOut + ); + let reqHeaderPipeOut <- mkDataStream2Header( + reqHeaderAndPayload.headerAndMetaData.headerDataStream, + reqHeaderAndPayload.headerAndMetaData.headerMetaData + ); + let sinkReqPayload <- mkSink(reqHeaderAndPayload.payload); + + Reg#(Bit#(32)) cycleReg <- mkReg(0); + (* no_implicit_conditions, fire_when_enabled *) + rule tick; + cycleReg <= cycleReg + 1; + if (cycleReg == fromInteger(valueOf(SimEndCycle))) $finish(0); + endrule + + // Transient RNR burst: NAK whole-WR ends inside the window so the recv buffer + // is "dry" briefly, then clears -- the backpressure-then-recover trigger. + let inRnrWindow = enableRnr && + (cycleReg >= fromInteger(valueOf(RnrStartCycle))) && + (cycleReg < fromInteger(valueOf(RnrStopCycle))); + + // ---- STRICT in-order RC receiver ---- + // ePsnReg = next in-order PSN expected. nakArmedReg coalesces: one SEQ_ERR-NAK + // per gap episode, suppressed until the in-order packet arrives (a real RC RQ). + Reg#(PSN) ePsnReg <- mkReg(0); + Reg#(Bool) nakArmedReg <- mkReg(False); + + Count#(Bit#(32)) reqPktSeenCnt <- mkCount(0); + Vector#(NumBuckets, Count#(Bit#(32))) reqBucket <- replicateM(mkCount(0)); + + rule genAckForReq; + let rdmaHeader = reqHeaderPipeOut.first; + reqHeaderPipeOut.deq; + + let bthIn = extractBTH(rdmaHeader.headerData); + let { transTypeIn, rdmaOpCodeIn } = + extractTranTypeAndRdmaOpCode(rdmaHeader.headerData); + let isWrEnd = isLastOrOnlyRdmaOpCode(rdmaOpCodeIn); + + let maybeTrans = qpType2TransType(cntrlStatus.getTypeQP); + let transType = unwrapMaybe(maybeTrans); + + // Decide the response for this WR-end against the strict expected PSN. + // sendResp/respPsn/respCode default to "no response" (intermediate pkt). + Bool sendResp = False; + PSN respPsn = bthIn.psn; + AethCode respCode = AETH_CODE_ACK; + AethValue respVal = pack(AETH_ACK_VALUE_INVALID_CREDIT_CNT); + + if (isWrEnd) begin + reqPktSeenCnt.incr(1); + let bidx = cycleReg / fromInteger(10000); + for (Integer i = 0; i < valueOf(NumBuckets); i = i + 1) begin + if (bidx == fromInteger(i)) reqBucket[i].incr(1); + end + + if (bthIn.psn == ePsnReg) begin + // In order. RNR-NAK during the window (no advance); else ACK + advance. + nakArmedReg <= False; + if (inRnrWindow) begin + sendResp = True; respCode = AETH_CODE_RNR; + respVal = cntrlStatus.comm.getMinRnrTimer; + end + else begin + sendResp = True; respCode = AETH_CODE_ACK; respPsn = bthIn.psn; + ePsnReg <= ePsnReg + 1; + end + end + else begin + // Out of order. ahead = psn - ePSN (mod 2^24): MSB clear & nonzero => + // forward gap; MSB set => stale duplicate behind ePSN. (Single-packet + // WRs advance ePSN by 1, so the half-space split is unambiguous here.) + PSN ahead = bthIn.psn - ePsnReg; + Bool isForwardGap = (msb(ahead) == 0); + if (isForwardGap) begin + // FORWARD GAP -> one coalesced SEQ_ERR-NAK at ePSN, then suppress + // until the in-order packet arrives (never cumulatively ACK past + // a hole -- that is what lets the buggy tail-drop hide). + if (!nakArmedReg) begin + nakArmedReg <= True; + sendResp = True; respPsn = ePsnReg; + respCode = AETH_CODE_NAK; respVal = zeroExtend(pack(AETH_NAK_SEQ_ERR)); + end + end + else begin + // DUPLICATE / old retransmit -> re-ACK the cumulative high-water. + sendResp = True; respPsn = (ePsnReg - 1); + respCode = AETH_CODE_ACK; + end + end + end + + if (sendResp) begin + let bth = BTH { + trans : transType, opcode: ACKNOWLEDGE, solicited: False, + migReq: unpack(0), padCnt: 0, tver: unpack(0), + pkey : cntrlStatus.comm.getPKEY, fecn: unpack(0), becn: unpack(0), + resv6 : unpack(0), dqpn: cntrlStatus.comm.getSQPN, ackReq: False, + resv7 : unpack(0), psn: respPsn + }; + let aeth = AETH { + rsvd: unpack(0), code: respCode, value: respVal, msn: dontCareValue + }; + let respHeader = genHeaderRDMA( + zeroExtendLSB({ pack(bth), pack(aeth) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(AETH_BYTE_WIDTH)), + False + ); + respHeaderQ.enq(respHeader); + end + endrule + + let workCompPipeOut = dut.workCompSQ.workCompPipeOut; + Integer bucketSpan = 10000; + Vector#(NumBuckets, Count#(Bit#(32))) wcBucket <- replicateM(mkCount(0)); + Reg#(Bit#(32)) totalWcReg <- mkReg(0); + Reg#(Bit#(32)) errWcReg <- mkReg(0); + + rule countWorkComp; + let wc = workCompPipeOut.first; + workCompPipeOut.deq; + totalWcReg <= totalWcReg + 1; + if (wc.status != IBV_WC_SUCCESS) errWcReg <= errWcReg + 1; + let idx = cycleReg / fromInteger(bucketSpan); + for (Integer i = 0; i < valueOf(NumBuckets); i = i + 1) begin + if (idx == fromInteger(i)) wcBucket[i].incr(1); + end + endrule + + Reg#(Bool) reportedReg <- mkReg(False); + rule report if (!reportedReg && cycleReg == (fromInteger(valueOf(SimEndCycle)) - 1)); + reportedReg <= True; + + $display("========================================================="); + $display("SQ TAIL-DROP TEST (strict in-order receiver, RNR window [%0d,%0d), enableRnr=", + valueOf(RnrStartCycle), valueOf(RnrStopCycle), fshow(enableRnr), ")"); + $display(" total WorkComps : %0d (non-SUCCESS WCs: %0d)", totalWcReg, errWcReg); + $display(" total req WR-ends seen : %0d", reqPktSeenCnt); + $display(" per %0d-cycle bucket (reqPkts / workComps):", bucketSpan); + for (Integer i = 0; i < valueOf(NumBuckets); i = i + 1) begin + $display(" bucket %0d [cyc %0d..%0d) : reqPkt=%0d WC=%0d", + i, i*bucketSpan, (i+1)*bucketSpan, reqBucket[i], wcBucket[i]); + end + + // Steady-state TX/comp = reqPkts/workComps in the last bucket (long after + // the RNR window). Healthy go-back-N -> ~1.0; the scanOutQ.clear tail-drop + // treadmill -> ~2.0. Report as a percentage (100 == 1.00x). + let reqLast = reqBucket[valueOf(NumBuckets)-1]; + let wcLast = wcBucket[valueOf(NumBuckets)-1]; + let txPerCompPct = (wcLast == 0) ? 0 : (reqLast * 100) / wcLast; + $display(" steady-state(last bucket) reqPkts=%0d workComps=%0d -> TX/comp=%0d%%", + reqLast, wcLast, txPerCompPct); + + // PASS <= 130% (1.30x): allows the bounded record-lag catch-up transient but + // rejects the ~2x perpetual treadmill. The no-RNR self-check must be ~100%. + if (txPerCompPct <= 130) begin + $display(" VERDICT : PASS (steady-state TX/comp=%0d%% <= 130%%)", txPerCompPct); + end + else begin + $display(" VERDICT : FAIL -- TAIL-DROP TREADMILL (TX/comp=%0d%%, expect ~200%%)", txPerCompPct); + end + $display("========================================================="); + immAssert( + txPerCompPct <= 130, + "SQ tail-drop TX/comp assertion @ mkTestSqTailDrop", + $format( + "steady-state TX/comp=%0d%% should be <=130%%", txPerCompPct, + " (mkScanFIFOF scanOutQ.clear replay-tail-drop regression)" + ) + ); + $finish(0); + endrule +endmodule