[LoopIdiomRecognize] Enable clmul optimization for CRC loops#203405
Merged
Conversation
llvmorg-github-actions
Bot
added
llvm:analysis
|
@llvm/pr-subscribers-backend-risc-v @llvm/pr-subscribers-llvm-analysis Author: Sean Clarke (xarkenz) ChangesThe current Hexagon currently bails out of This optimization fails to apply for some targets in some cases where the Assisted-by: Claude Opus 4.8 Patch is 20.39 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203405.diff 8 Files Affected:
diff --git a/llvm/include/llvm/Analysis/HashRecognize.h b/llvm/include/llvm/Analysis/HashRecognize.h
index 0f51227a6b010..b168278f4391b 100644
--- a/llvm/include/llvm/Analysis/HashRecognize.h
+++ b/llvm/include/llvm/Analysis/HashRecognize.h
@@ -35,6 +35,23 @@ struct CRCTable : public std::array<APInt, 256> {
#endif
};
+/// The constants used to perform polynomial division with a Barrett-style
+/// reduction into two polynomial multiplications instead.
+struct CRCBarrettConstants {
+ // An approximation of the polynomial quotient floor(x^2w / P(x)), where P(x)
+ // is the generating polynomial and w is the CRC width.
+ APInt Reciprocal;
+
+ // The generating polynomial P(x) in full, adjusted for endianness.
+ APInt Generator;
+
+ LLVM_ABI void print(raw_ostream &OS) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ LLVM_ABI LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
/// The structure that is returned when a polynomial algorithm was recognized by
/// the analysis. Currently, only the CRC algorithm is recognized.
struct PolynomialInfo {
@@ -87,6 +104,11 @@ class HashRecognize {
LLVM_ABI static CRCTable genSarwateTable(const APInt &GenPoly,
bool ByteOrderSwapped);
+ // Auxilary entry point after analysis to generate constants for a
+ // Barrett-style reduction.
+ LLVM_ABI static CRCBarrettConstants
+ genBarrettConstants(const APInt &GenPoly, bool ByteOrderSwapped);
+
LLVM_ABI void print(raw_ostream &OS) const;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 7d58473b81265..c7d8e46ac8433 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1181,6 +1181,10 @@ class TargetTransformInfo {
/// Return true if the hardware has a fast square-root instruction.
LLVM_ABI bool haveFastSqrt(Type *Ty) const;
+ /// Return true if the hardware has a fast carryless-multiplication
+ /// instruction.
+ LLVM_ABI bool haveFastClmul(Type *Ty) const;
+
/// Return true if the cost of the instruction is too high to speculatively
/// execute and should be kept behind a branch.
/// This normally just wraps around a getInstructionCost() call, but some
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 89fd4a1e7628e..d25e0ae81fb19 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -568,6 +568,8 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual bool haveFastSqrt(Type *Ty) const { return false; }
+ virtual bool haveFastClmul(Type *Ty) const { return false; }
+
virtual bool isExpensiveToSpeculativelyExecute(const Instruction *I) const {
return true;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e4b6bf51c7a4e..6f7bc0b21e030 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -669,6 +669,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
TLI->isOperationLegalOrCustom(ISD::FSQRT, VT);
}
+ bool haveFastClmul(Type *Ty) const override {
+ const TargetLoweringBase *TLI = getTLI();
+ EVT VT = TLI->getValueType(DL, Ty);
+ return TLI->isTypeLegal(VT) &&
+ TLI->isOperationLegalOrCustomOrPromote(ISD::CLMUL, VT);
+ }
+
bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const override { return true; }
InstructionCost getFPOpCost(Type *Ty) const override {
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 8974ce5734b13..9754c2863521d 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -376,6 +376,48 @@ CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
return Table;
}
+// Divide one GF(2) polynomial by another.
+static APInt calculateGF2Quotient(APInt Dividend, const APInt &Divisor) {
+ unsigned DivisorDeg = Divisor.getActiveBits() - 1;
+ APInt Quotient = APInt::getZero(Dividend.getBitWidth());
+ unsigned DividendDeg;
+ while (!Dividend.isZero() &&
+ (DividendDeg = Dividend.getActiveBits() - 1) >= DivisorDeg) {
+ unsigned Shift = DividendDeg - DivisorDeg;
+ Quotient.setBit(Shift);
+ Dividend ^= Divisor.shl(Shift);
+ }
+ return Quotient;
+}
+
+// Generate constants (mu/reciprocal, P/generator) for a Barrett-style
+// reduction. This reduction allows the Sarwate table entry to be computed on
+// the fly, rather than requiring a load from memory (on supporting hardware).
+CRCBarrettConstants HashRecognize::genBarrettConstants(const APInt &GenPoly,
+ bool ByteOrderSwapped) {
+ unsigned BW = GenPoly.getBitWidth();
+ unsigned ClmulBW = BW * 2;
+ unsigned DivBW = ClmulBW + 1;
+ APInt Dividend = APInt::getSignedMinValue(DivBW);
+ CRCBarrettConstants C;
+
+ if (ByteOrderSwapped) {
+ APInt G = GenPoly.zext(DivBW);
+ G.setBit(BW);
+ APInt Mu = calculateGF2Quotient(Dividend, G);
+ C.Reciprocal = Mu.trunc(ClmulBW);
+ C.Generator = G.trunc(ClmulBW);
+ return C;
+ }
+
+ APInt G = GenPoly.reverseBits().zext(DivBW);
+ G.setBit(BW);
+ APInt Mu = calculateGF2Quotient(Dividend, G);
+ C.Reciprocal = Mu.trunc(BW + 1).reverseBits().zext(ClmulBW).getLoBits(8);
+ C.Generator = GenPoly.zext(ClmulBW).shl(1);
+ return C;
+}
+
/// Checks that \p P1 and \p P2 are used together in an XOR in the use-def chain
/// of \p SI's condition, ignoring any casts. The purpose of this function is to
/// ensure that LHSAux from the SimpleRecurrence is used correctly in the CRC
@@ -532,6 +574,14 @@ void CRCTable::print(raw_ostream &OS) const {
void CRCTable::dump() const { print(dbgs()); }
#endif
+void CRCBarrettConstants::print(raw_ostream &OS) const {
+ OS << "Reciprocal = " << Reciprocal << ", Generator = " << Generator << '\n';
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void CRCBarrettConstants::dump() const { print(dbgs()); }
+#endif
+
void HashRecognize::print(raw_ostream &OS) const {
if (!L.isInnermost())
return;
@@ -566,6 +616,8 @@ void HashRecognize::print(raw_ostream &OS) const {
}
OS.indent(2) << "Computed CRC lookup table:\n";
genSarwateTable(Info.RHS, Info.ByteOrderSwapped).print(OS);
+ OS.indent(2) << "Computed CRC Barrett constants:\n";
+ genBarrettConstants(Info.RHS, Info.ByteOrderSwapped).print(OS);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 856950ccae595..b1604593febf8 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -739,6 +739,10 @@ bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
return TTIImpl->haveFastSqrt(Ty);
}
+bool TargetTransformInfo::haveFastClmul(Type *Ty) const {
+ return TTIImpl->haveFastClmul(Ty);
+}
+
bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(
const Instruction *I) const {
return TTIImpl->isExpensiveToSpeculativelyExecute(I);
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 7a08d99a9e505..be703f2377ae5 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -202,6 +202,8 @@ class LoopIdiomRecognize {
private:
using StoreList = SmallVector<StoreInst *, 8>;
using StoreListMap = MapVector<Value *, StoreList>;
+ using CRCTableFn =
+ function_ref<Value *(IRBuilderBase &Builder, Value *Indexer)>;
StoreListMap StoreRefsForMemset;
StoreListMap StoreRefsForMemsetPattern;
@@ -259,6 +261,9 @@ class LoopIdiomRecognize {
bool avoidLIRForMultiBlockLoop(bool IsMemset = false,
bool IsLoopMemset = false);
bool optimizeCRCLoop(const PolynomialInfo &Info);
+ bool optimizeCRCLoopWithFastClmul(const PolynomialInfo &Info);
+ void buildReducedCRCLoop(const PolynomialInfo &Info,
+ CRCTableFn GetTableEntry);
/// @}
/// \name Noncountable Loop Idiom Handling
@@ -1560,10 +1565,14 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
if (TT.getArch() == Triple::hexagon)
return false;
+ // If using llvm.clmul would be more efficient, use that instead of a lookup
+ // table.
+ if (optimizeCRCLoopWithFastClmul(Info))
+ return true;
+
// First, create a new GlobalVariable corresponding to the
// Sarwate-lookup-table.
Type *CRCTy = Info.LHS->getType();
- unsigned CRCBW = CRCTy->getIntegerBitWidth();
std::array<Constant *, 256> CRCConstants;
transform(HashRecognize::genSarwateTable(Info.RHS, Info.ByteOrderSwapped),
CRCConstants.begin(),
@@ -1574,10 +1583,94 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
new GlobalVariable(M, ConstArray->getType(), true,
GlobalValue::PrivateLinkage, ConstArray, ".crctable");
+ // Build a reduced per-byte loop which loads an entry from the Sarwate lookup
+ // table each iteration.
+ //
+ // Little-endian:
+ // crc = (crc >> 8) ^ tbl[(iv'th byte of data) ^ (bottom byte of crc)]
+ // Big-Endian:
+ // crc = (crc << 8) ^ tbl[(iv'th byte of data) ^ (top byte of crc)]
+ buildReducedCRCLoop(Info, [&](auto &Builder, Value *Indexer) {
+ // Always index into a GEP using the index type.
+ Indexer = Builder.CreateZExt(
+ Indexer, SE->getDataLayout().getIndexType(GV->getType()),
+ "indexer.ext");
+
+ // CRCTableLd = CRCTable[(iv'th byte of data) ^ (top|bottom) byte of CRC].
+ Value *CRCTableGEP =
+ Builder.CreateInBoundsGEP(CRCTy, GV, Indexer, "tbl.ptradd");
+ Value *CRCTableLd = Builder.CreateLoad(CRCTy, CRCTableGEP, "tbl.ld");
+
+ return CRCTableLd;
+ });
+
+ return true;
+}
+
+bool LoopIdiomRecognize::optimizeCRCLoopWithFastClmul(
+ const PolynomialInfo &Info) {
+ Type *CRCTy = Info.LHS->getType();
+ unsigned CRCBW = CRCTy->getIntegerBitWidth();
+ Type *ClmulTy = IntegerType::get(CRCTy->getContext(), 2 * CRCBW);
+
+ // Only apply this optimization strategy if llvm.clmul would lower into a
+ // native instruction.
+ if (!TTI->haveFastClmul(ClmulTy))
+ return false;
+
+ // First, generate the two constants required for a Barrett-style reduction.
+ CRCBarrettConstants Constants =
+ HashRecognize::genBarrettConstants(Info.RHS, Info.ByteOrderSwapped);
+ Value *Reciprocal = ConstantInt::get(ClmulTy, Constants.Reciprocal);
+ Value *Generator = ConstantInt::get(ClmulTy, Constants.Generator);
+
+ // Build a reduced per-byte loop which computes the lookup table entry on the
+ // fly each iteration using a Barrett-style reduction.
+ //
+ // Little-endian:
+ // q = ((iv'th byte of data) ^ (bottom byte of crc)) clmul (reciprocal term)
+ // entry = (bottom byte of q) clmul (generator term)
+ // crc = (crc >> 8) ^ (top byte of entry)
+ // Big-Endian:
+ // q = ((iv'th byte of data) ^ (top byte of crc)) clmul (reciprocal term)
+ // entry = (top byte of q) clmul (generator term)
+ // crc = (crc << 8) ^ (bottom byte of entry)
+ buildReducedCRCLoop(Info, [&](auto &Builder, Value *Indexer) {
+ // Approximate floor(Indexer / (generator term)) using Indexer * (reciprocal
+ // term).
+ Indexer = Builder.CreateZExt(Indexer, ClmulTy, "indexer.ext");
+ Value *Quotient = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, Indexer,
+ Reciprocal, {}, "quot");
+ Quotient = Info.ByteOrderSwapped
+ ? Builder.CreateLShr(Quotient, CRCBW, "quot.be.shift")
+ : Builder.CreateAnd(Quotient, 0xFF, "quot.le.mask");
+
+ // floor(Indexer / (generator term)) * (generator term) should give us what
+ // the Sarwate table entry would be.
+ Value *Entry = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, Quotient,
+ Generator, {}, "entry");
+ Entry =
+ Info.ByteOrderSwapped
+ ? Builder.CreateAnd(Entry, APInt::getLowBitsSet(2 * CRCBW, CRCBW),
+ "entry.be.mask")
+ : Builder.CreateLShr(Entry, 8, "entry.le.shift");
+ Entry = Builder.CreateTrunc(Entry, CRCTy, "entry.cast");
+
+ return Entry;
+ });
+
+ return true;
+}
+
+void LoopIdiomRecognize::buildReducedCRCLoop(
+ const PolynomialInfo &Info, LoopIdiomRecognize::CRCTableFn GetTableEntry) {
+ Type *CRCTy = Info.LHS->getType();
+ unsigned CRCBW = CRCTy->getIntegerBitWidth();
+
PHINode *IV = CurLoop->getCanonicalInductionVariable();
SmallVector<PHINode *, 2> Cleanup;
- // Next, mark all PHIs for removal except IV.
+ // First, mark all PHIs for removal except IV.
{
for (PHINode &PN : CurLoop->getHeader()->phis()) {
if (&PN == IV)
@@ -1606,11 +1699,6 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
// Finally, fill the loop with the Sarwate-table-lookup logic, and replace all
// uses of ComputedValue.
- //
- // Little-endian:
- // crc = (crc >> 8) ^ tbl[(iv'th byte of data) ^ (bottom byte of crc)]
- // Big-Endian:
- // crc = (crc << 8) ^ tbl[(iv'th byte of data) ^ (top byte of crc)]
{
auto LoByte = [](IRBuilderBase &Builder, Value *Op, const Twine &Name) {
return Builder.CreateZExtOrTrunc(
@@ -1665,24 +1753,18 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
Indexer = Info.ByteOrderSwapped ? HiIdx(Builder, Indexer, "indexer.hi")
: LoByte(Builder, Indexer, "indexer.lo");
- // Always index into a GEP using the index type.
- Indexer = Builder.CreateZExt(
- Indexer, SE->getDataLayout().getIndexType(GV->getType()),
- "indexer.ext");
-
- // CRCTableLd = CRCTable[(iv'th byte of data) ^ (top|bottom) byte of CRC].
- Value *CRCTableGEP =
- Builder.CreateInBoundsGEP(CRCTy, GV, Indexer, "tbl.ptradd");
- Value *CRCTableLd = Builder.CreateLoad(CRCTy, CRCTableGEP, "tbl.ld");
+ // Get the Sarwate lookup table entry corresponding to Indexer. This could
+ // either be a load or an on-the-fly computation, so let the caller decide.
+ Value *CRCTableEntry = GetTableEntry(Builder, Indexer);
// CRCNext = (CRC (<<|>>) 8) ^ CRCTableLd, or simply CRCTableLd in case of
// CRC-8.
- Value *CRCNext = CRCTableLd;
+ Value *CRCNext = CRCTableEntry;
if (CRCBW > 8) {
Value *CRCShift = Info.ByteOrderSwapped
? Builder.CreateShl(CRC, 8, "crc.be.shift")
: Builder.CreateLShr(CRC, 8, "crc.le.shift");
- CRCNext = Builder.CreateXor(CRCShift, CRCTableLd, "crc.next");
+ CRCNext = Builder.CreateXor(CRCShift, CRCTableEntry, "crc.next");
}
// Connect the back-edge for the loop, and RAUW the ComputedValue.
@@ -1697,7 +1779,6 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
RecursivelyDeleteDeadPHINode(PN);
SE->forgetLoop(CurLoop);
}
- return true;
}
bool LoopIdiomRecognize::runOnNoncountableLoop() {
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index 78b4139d21982..8ea699852fe48 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -25,6 +25,8 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) {
; CHECK-NEXT: 39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
; CHECK-NEXT: 34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
; CHECK-NEXT: 17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 255, Generator = 81922
;
entry:
br label %loop
@@ -73,6 +75,8 @@ define i16 @crc16.le.tc8.udiv(i8 %msg, i16 %checksum) {
; CHECK-NEXT: 39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
; CHECK-NEXT: 34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
; CHECK-NEXT: 17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 255, Generator = 81922
;
entry:
br label %loop
@@ -121,6 +125,8 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) {
; CHECK-NEXT: 39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
; CHECK-NEXT: 34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
; CHECK-NEXT: 17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 255, Generator = 81922
;
entry:
br label %loop
@@ -168,6 +174,8 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) {
; CHECK-NEXT: 4 13 22 31 27 18 9 0 1 8 19 26 30 23 12 5
; CHECK-NEXT: 26 19 8 1 5 12 23 30 31 22 13 4 0 9 18 27
; CHECK-NEXT: 16 25 2 11 15 6 29 20 21 28 7 14 10 3 24 17
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 103, Generator = 58
;
entry:
br label %loop
@@ -215,6 +223,8 @@ define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) {
; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 69936, Generator = 69665
;
entry:
%msg.ext = zext i8 %msg to i16
@@ -260,6 +270,8 @@ define i16 @crc16.be.tc8.crc.init.arg(i16 %crc.init) {
; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 69936, Generator = 69665
;
entry:
br label %loop
@@ -302,6 +314,8 @@ define i16 @crc16.be.tc8.crc.init.arg.flipped.sb.check(i16 %crc.init) {
; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 69936, Generator = 69665
;
entry:
br label %loop
@@ -344,6 +358,8 @@ define i8 @crc8.be.tc8.ptr.nested.loop(ptr %msg, i32 %loop.limit) {
; CHECK-NEXT: 248 229 194 223 140 145 182 171 16 13 42 55 100 121 94 67
; CHECK-NEXT: 178 175 136 149 198 219 252 225 90 71 96 125 46 51 20 9
; CHECK-NEXT: 127 98 69 88 11 22 49 44 151 138 173 176 227 254 217 196
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 284, Generator = 285
;
entry:
br label %outer.loop
@@ -404,6 +420,8 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) {
; CHECK-NEXT: 54925 50948 62879 58390 37033 33056 46011 41522 23237 19276 31191 26718 7393 3432 16371 11898
; CHECK-NEXT: 59150 63111 50204 54677 41258 45219 33336 37809 27462 31439 18516 23005 11618 15595 3696 8185
; CHECK-NEXT: 63375 58886 54429 50452 45483 40994 37561 33584 31687 27214 22741 18780 15843 11370 7921 3960
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 17, Generator = 67600
;
entry:
br label %loop
@@ -451,6 +469,8 @@ define i16 @crc16.be.tc8.zext.data(i8 %msg, i16 %checksum) {
; CHECK-NEXT: 53664 53410 54180 53926 54696 54442 55212 54958 55728 5547...
[truncated]
|
|
@llvm/pr-subscribers-llvm-transforms Author: Sean Clarke (xarkenz) ChangesThe current Hexagon currently bails out of This optimization fails to apply for some targets in some cases where the Assisted-by: Claude Opus 4.8 Patch is 20.39 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203405.diff 8 Files Affected:
diff --git a/llvm/include/llvm/Analysis/HashRecognize.h b/llvm/include/llvm/Analysis/HashRecognize.h
index 0f51227a6b010..b168278f4391b 100644
--- a/llvm/include/llvm/Analysis/HashRecognize.h
+++ b/llvm/include/llvm/Analysis/HashRecognize.h
@@ -35,6 +35,23 @@ struct CRCTable : public std::array<APInt, 256> {
#endif
};
+/// The constants used to perform polynomial division with a Barrett-style
+/// reduction into two polynomial multiplications instead.
+struct CRCBarrettConstants {
+ // An approximation of the polynomial quotient floor(x^2w / P(x)), where P(x)
+ // is the generating polynomial and w is the CRC width.
+ APInt Reciprocal;
+
+ // The generating polynomial P(x) in full, adjusted for endianness.
+ APInt Generator;
+
+ LLVM_ABI void print(raw_ostream &OS) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ LLVM_ABI LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
/// The structure that is returned when a polynomial algorithm was recognized by
/// the analysis. Currently, only the CRC algorithm is recognized.
struct PolynomialInfo {
@@ -87,6 +104,11 @@ class HashRecognize {
LLVM_ABI static CRCTable genSarwateTable(const APInt &GenPoly,
bool ByteOrderSwapped);
+ // Auxilary entry point after analysis to generate constants for a
+ // Barrett-style reduction.
+ LLVM_ABI static CRCBarrettConstants
+ genBarrettConstants(const APInt &GenPoly, bool ByteOrderSwapped);
+
LLVM_ABI void print(raw_ostream &OS) const;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 7d58473b81265..c7d8e46ac8433 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1181,6 +1181,10 @@ class TargetTransformInfo {
/// Return true if the hardware has a fast square-root instruction.
LLVM_ABI bool haveFastSqrt(Type *Ty) const;
+ /// Return true if the hardware has a fast carryless-multiplication
+ /// instruction.
+ LLVM_ABI bool haveFastClmul(Type *Ty) const;
+
/// Return true if the cost of the instruction is too high to speculatively
/// execute and should be kept behind a branch.
/// This normally just wraps around a getInstructionCost() call, but some
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 89fd4a1e7628e..d25e0ae81fb19 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -568,6 +568,8 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual bool haveFastSqrt(Type *Ty) const { return false; }
+ virtual bool haveFastClmul(Type *Ty) const { return false; }
+
virtual bool isExpensiveToSpeculativelyExecute(const Instruction *I) const {
return true;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e4b6bf51c7a4e..6f7bc0b21e030 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -669,6 +669,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
TLI->isOperationLegalOrCustom(ISD::FSQRT, VT);
}
+ bool haveFastClmul(Type *Ty) const override {
+ const TargetLoweringBase *TLI = getTLI();
+ EVT VT = TLI->getValueType(DL, Ty);
+ return TLI->isTypeLegal(VT) &&
+ TLI->isOperationLegalOrCustomOrPromote(ISD::CLMUL, VT);
+ }
+
bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const override { return true; }
InstructionCost getFPOpCost(Type *Ty) const override {
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 8974ce5734b13..9754c2863521d 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -376,6 +376,48 @@ CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
return Table;
}
+// Divide one GF(2) polynomial by another.
+static APInt calculateGF2Quotient(APInt Dividend, const APInt &Divisor) {
+ unsigned DivisorDeg = Divisor.getActiveBits() - 1;
+ APInt Quotient = APInt::getZero(Dividend.getBitWidth());
+ unsigned DividendDeg;
+ while (!Dividend.isZero() &&
+ (DividendDeg = Dividend.getActiveBits() - 1) >= DivisorDeg) {
+ unsigned Shift = DividendDeg - DivisorDeg;
+ Quotient.setBit(Shift);
+ Dividend ^= Divisor.shl(Shift);
+ }
+ return Quotient;
+}
+
+// Generate constants (mu/reciprocal, P/generator) for a Barrett-style
+// reduction. This reduction allows the Sarwate table entry to be computed on
+// the fly, rather than requiring a load from memory (on supporting hardware).
+CRCBarrettConstants HashRecognize::genBarrettConstants(const APInt &GenPoly,
+ bool ByteOrderSwapped) {
+ unsigned BW = GenPoly.getBitWidth();
+ unsigned ClmulBW = BW * 2;
+ unsigned DivBW = ClmulBW + 1;
+ APInt Dividend = APInt::getSignedMinValue(DivBW);
+ CRCBarrettConstants C;
+
+ if (ByteOrderSwapped) {
+ APInt G = GenPoly.zext(DivBW);
+ G.setBit(BW);
+ APInt Mu = calculateGF2Quotient(Dividend, G);
+ C.Reciprocal = Mu.trunc(ClmulBW);
+ C.Generator = G.trunc(ClmulBW);
+ return C;
+ }
+
+ APInt G = GenPoly.reverseBits().zext(DivBW);
+ G.setBit(BW);
+ APInt Mu = calculateGF2Quotient(Dividend, G);
+ C.Reciprocal = Mu.trunc(BW + 1).reverseBits().zext(ClmulBW).getLoBits(8);
+ C.Generator = GenPoly.zext(ClmulBW).shl(1);
+ return C;
+}
+
/// Checks that \p P1 and \p P2 are used together in an XOR in the use-def chain
/// of \p SI's condition, ignoring any casts. The purpose of this function is to
/// ensure that LHSAux from the SimpleRecurrence is used correctly in the CRC
@@ -532,6 +574,14 @@ void CRCTable::print(raw_ostream &OS) const {
void CRCTable::dump() const { print(dbgs()); }
#endif
+void CRCBarrettConstants::print(raw_ostream &OS) const {
+ OS << "Reciprocal = " << Reciprocal << ", Generator = " << Generator << '\n';
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void CRCBarrettConstants::dump() const { print(dbgs()); }
+#endif
+
void HashRecognize::print(raw_ostream &OS) const {
if (!L.isInnermost())
return;
@@ -566,6 +616,8 @@ void HashRecognize::print(raw_ostream &OS) const {
}
OS.indent(2) << "Computed CRC lookup table:\n";
genSarwateTable(Info.RHS, Info.ByteOrderSwapped).print(OS);
+ OS.indent(2) << "Computed CRC Barrett constants:\n";
+ genBarrettConstants(Info.RHS, Info.ByteOrderSwapped).print(OS);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 856950ccae595..b1604593febf8 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -739,6 +739,10 @@ bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
return TTIImpl->haveFastSqrt(Ty);
}
+bool TargetTransformInfo::haveFastClmul(Type *Ty) const {
+ return TTIImpl->haveFastClmul(Ty);
+}
+
bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(
const Instruction *I) const {
return TTIImpl->isExpensiveToSpeculativelyExecute(I);
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 7a08d99a9e505..be703f2377ae5 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -202,6 +202,8 @@ class LoopIdiomRecognize {
private:
using StoreList = SmallVector<StoreInst *, 8>;
using StoreListMap = MapVector<Value *, StoreList>;
+ using CRCTableFn =
+ function_ref<Value *(IRBuilderBase &Builder, Value *Indexer)>;
StoreListMap StoreRefsForMemset;
StoreListMap StoreRefsForMemsetPattern;
@@ -259,6 +261,9 @@ class LoopIdiomRecognize {
bool avoidLIRForMultiBlockLoop(bool IsMemset = false,
bool IsLoopMemset = false);
bool optimizeCRCLoop(const PolynomialInfo &Info);
+ bool optimizeCRCLoopWithFastClmul(const PolynomialInfo &Info);
+ void buildReducedCRCLoop(const PolynomialInfo &Info,
+ CRCTableFn GetTableEntry);
/// @}
/// \name Noncountable Loop Idiom Handling
@@ -1560,10 +1565,14 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
if (TT.getArch() == Triple::hexagon)
return false;
+ // If using llvm.clmul would be more efficient, use that instead of a lookup
+ // table.
+ if (optimizeCRCLoopWithFastClmul(Info))
+ return true;
+
// First, create a new GlobalVariable corresponding to the
// Sarwate-lookup-table.
Type *CRCTy = Info.LHS->getType();
- unsigned CRCBW = CRCTy->getIntegerBitWidth();
std::array<Constant *, 256> CRCConstants;
transform(HashRecognize::genSarwateTable(Info.RHS, Info.ByteOrderSwapped),
CRCConstants.begin(),
@@ -1574,10 +1583,94 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
new GlobalVariable(M, ConstArray->getType(), true,
GlobalValue::PrivateLinkage, ConstArray, ".crctable");
+ // Build a reduced per-byte loop which loads an entry from the Sarwate lookup
+ // table each iteration.
+ //
+ // Little-endian:
+ // crc = (crc >> 8) ^ tbl[(iv'th byte of data) ^ (bottom byte of crc)]
+ // Big-Endian:
+ // crc = (crc << 8) ^ tbl[(iv'th byte of data) ^ (top byte of crc)]
+ buildReducedCRCLoop(Info, [&](auto &Builder, Value *Indexer) {
+ // Always index into a GEP using the index type.
+ Indexer = Builder.CreateZExt(
+ Indexer, SE->getDataLayout().getIndexType(GV->getType()),
+ "indexer.ext");
+
+ // CRCTableLd = CRCTable[(iv'th byte of data) ^ (top|bottom) byte of CRC].
+ Value *CRCTableGEP =
+ Builder.CreateInBoundsGEP(CRCTy, GV, Indexer, "tbl.ptradd");
+ Value *CRCTableLd = Builder.CreateLoad(CRCTy, CRCTableGEP, "tbl.ld");
+
+ return CRCTableLd;
+ });
+
+ return true;
+}
+
+bool LoopIdiomRecognize::optimizeCRCLoopWithFastClmul(
+ const PolynomialInfo &Info) {
+ Type *CRCTy = Info.LHS->getType();
+ unsigned CRCBW = CRCTy->getIntegerBitWidth();
+ Type *ClmulTy = IntegerType::get(CRCTy->getContext(), 2 * CRCBW);
+
+ // Only apply this optimization strategy if llvm.clmul would lower into a
+ // native instruction.
+ if (!TTI->haveFastClmul(ClmulTy))
+ return false;
+
+ // First, generate the two constants required for a Barrett-style reduction.
+ CRCBarrettConstants Constants =
+ HashRecognize::genBarrettConstants(Info.RHS, Info.ByteOrderSwapped);
+ Value *Reciprocal = ConstantInt::get(ClmulTy, Constants.Reciprocal);
+ Value *Generator = ConstantInt::get(ClmulTy, Constants.Generator);
+
+ // Build a reduced per-byte loop which computes the lookup table entry on the
+ // fly each iteration using a Barrett-style reduction.
+ //
+ // Little-endian:
+ // q = ((iv'th byte of data) ^ (bottom byte of crc)) clmul (reciprocal term)
+ // entry = (bottom byte of q) clmul (generator term)
+ // crc = (crc >> 8) ^ (top byte of entry)
+ // Big-Endian:
+ // q = ((iv'th byte of data) ^ (top byte of crc)) clmul (reciprocal term)
+ // entry = (top byte of q) clmul (generator term)
+ // crc = (crc << 8) ^ (bottom byte of entry)
+ buildReducedCRCLoop(Info, [&](auto &Builder, Value *Indexer) {
+ // Approximate floor(Indexer / (generator term)) using Indexer * (reciprocal
+ // term).
+ Indexer = Builder.CreateZExt(Indexer, ClmulTy, "indexer.ext");
+ Value *Quotient = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, Indexer,
+ Reciprocal, {}, "quot");
+ Quotient = Info.ByteOrderSwapped
+ ? Builder.CreateLShr(Quotient, CRCBW, "quot.be.shift")
+ : Builder.CreateAnd(Quotient, 0xFF, "quot.le.mask");
+
+ // floor(Indexer / (generator term)) * (generator term) should give us what
+ // the Sarwate table entry would be.
+ Value *Entry = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, Quotient,
+ Generator, {}, "entry");
+ Entry =
+ Info.ByteOrderSwapped
+ ? Builder.CreateAnd(Entry, APInt::getLowBitsSet(2 * CRCBW, CRCBW),
+ "entry.be.mask")
+ : Builder.CreateLShr(Entry, 8, "entry.le.shift");
+ Entry = Builder.CreateTrunc(Entry, CRCTy, "entry.cast");
+
+ return Entry;
+ });
+
+ return true;
+}
+
+void LoopIdiomRecognize::buildReducedCRCLoop(
+ const PolynomialInfo &Info, LoopIdiomRecognize::CRCTableFn GetTableEntry) {
+ Type *CRCTy = Info.LHS->getType();
+ unsigned CRCBW = CRCTy->getIntegerBitWidth();
+
PHINode *IV = CurLoop->getCanonicalInductionVariable();
SmallVector<PHINode *, 2> Cleanup;
- // Next, mark all PHIs for removal except IV.
+ // First, mark all PHIs for removal except IV.
{
for (PHINode &PN : CurLoop->getHeader()->phis()) {
if (&PN == IV)
@@ -1606,11 +1699,6 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
// Finally, fill the loop with the Sarwate-table-lookup logic, and replace all
// uses of ComputedValue.
- //
- // Little-endian:
- // crc = (crc >> 8) ^ tbl[(iv'th byte of data) ^ (bottom byte of crc)]
- // Big-Endian:
- // crc = (crc << 8) ^ tbl[(iv'th byte of data) ^ (top byte of crc)]
{
auto LoByte = [](IRBuilderBase &Builder, Value *Op, const Twine &Name) {
return Builder.CreateZExtOrTrunc(
@@ -1665,24 +1753,18 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
Indexer = Info.ByteOrderSwapped ? HiIdx(Builder, Indexer, "indexer.hi")
: LoByte(Builder, Indexer, "indexer.lo");
- // Always index into a GEP using the index type.
- Indexer = Builder.CreateZExt(
- Indexer, SE->getDataLayout().getIndexType(GV->getType()),
- "indexer.ext");
-
- // CRCTableLd = CRCTable[(iv'th byte of data) ^ (top|bottom) byte of CRC].
- Value *CRCTableGEP =
- Builder.CreateInBoundsGEP(CRCTy, GV, Indexer, "tbl.ptradd");
- Value *CRCTableLd = Builder.CreateLoad(CRCTy, CRCTableGEP, "tbl.ld");
+ // Get the Sarwate lookup table entry corresponding to Indexer. This could
+ // either be a load or an on-the-fly computation, so let the caller decide.
+ Value *CRCTableEntry = GetTableEntry(Builder, Indexer);
// CRCNext = (CRC (<<|>>) 8) ^ CRCTableLd, or simply CRCTableLd in case of
// CRC-8.
- Value *CRCNext = CRCTableLd;
+ Value *CRCNext = CRCTableEntry;
if (CRCBW > 8) {
Value *CRCShift = Info.ByteOrderSwapped
? Builder.CreateShl(CRC, 8, "crc.be.shift")
: Builder.CreateLShr(CRC, 8, "crc.le.shift");
- CRCNext = Builder.CreateXor(CRCShift, CRCTableLd, "crc.next");
+ CRCNext = Builder.CreateXor(CRCShift, CRCTableEntry, "crc.next");
}
// Connect the back-edge for the loop, and RAUW the ComputedValue.
@@ -1697,7 +1779,6 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
RecursivelyDeleteDeadPHINode(PN);
SE->forgetLoop(CurLoop);
}
- return true;
}
bool LoopIdiomRecognize::runOnNoncountableLoop() {
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index 78b4139d21982..8ea699852fe48 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -25,6 +25,8 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) {
; CHECK-NEXT: 39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
; CHECK-NEXT: 34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
; CHECK-NEXT: 17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 255, Generator = 81922
;
entry:
br label %loop
@@ -73,6 +75,8 @@ define i16 @crc16.le.tc8.udiv(i8 %msg, i16 %checksum) {
; CHECK-NEXT: 39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
; CHECK-NEXT: 34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
; CHECK-NEXT: 17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 255, Generator = 81922
;
entry:
br label %loop
@@ -121,6 +125,8 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) {
; CHECK-NEXT: 39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
; CHECK-NEXT: 34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
; CHECK-NEXT: 17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 255, Generator = 81922
;
entry:
br label %loop
@@ -168,6 +174,8 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) {
; CHECK-NEXT: 4 13 22 31 27 18 9 0 1 8 19 26 30 23 12 5
; CHECK-NEXT: 26 19 8 1 5 12 23 30 31 22 13 4 0 9 18 27
; CHECK-NEXT: 16 25 2 11 15 6 29 20 21 28 7 14 10 3 24 17
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 103, Generator = 58
;
entry:
br label %loop
@@ -215,6 +223,8 @@ define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) {
; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 69936, Generator = 69665
;
entry:
%msg.ext = zext i8 %msg to i16
@@ -260,6 +270,8 @@ define i16 @crc16.be.tc8.crc.init.arg(i16 %crc.init) {
; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 69936, Generator = 69665
;
entry:
br label %loop
@@ -302,6 +314,8 @@ define i16 @crc16.be.tc8.crc.init.arg.flipped.sb.check(i16 %crc.init) {
; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 69936, Generator = 69665
;
entry:
br label %loop
@@ -344,6 +358,8 @@ define i8 @crc8.be.tc8.ptr.nested.loop(ptr %msg, i32 %loop.limit) {
; CHECK-NEXT: 248 229 194 223 140 145 182 171 16 13 42 55 100 121 94 67
; CHECK-NEXT: 178 175 136 149 198 219 252 225 90 71 96 125 46 51 20 9
; CHECK-NEXT: 127 98 69 88 11 22 49 44 151 138 173 176 227 254 217 196
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 284, Generator = 285
;
entry:
br label %outer.loop
@@ -404,6 +420,8 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) {
; CHECK-NEXT: 54925 50948 62879 58390 37033 33056 46011 41522 23237 19276 31191 26718 7393 3432 16371 11898
; CHECK-NEXT: 59150 63111 50204 54677 41258 45219 33336 37809 27462 31439 18516 23005 11618 15595 3696 8185
; CHECK-NEXT: 63375 58886 54429 50452 45483 40994 37561 33584 31687 27214 22741 18780 15843 11370 7921 3960
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Reciprocal = 17, Generator = 67600
;
entry:
br label %loop
@@ -451,6 +469,8 @@ define i16 @crc16.be.tc8.zext.data(i8 %msg, i16 %checksum) {
; CHECK-NEXT: 53664 53410 54180 53926 54696 54442 55212 54958 55728 5547...
[truncated]
|
artagnon
reviewed
Jun 11, 2026
artagnon
left a comment
artagnon
left a comment
Contributor
There was a problem hiding this comment.
Missing tests! Kindly duplicate cyclic-redundancy-check.ll in LoopIdiom, adding a target!
pfusik
requested changes
Jun 12, 2026
🐧 Linux x64 Test Results
✅ The build succeeded and all tests passed. |
artagnon
reviewed
Jun 13, 2026
artagnon
reviewed
Jun 13, 2026
artagnon
reviewed
Jun 13, 2026
artagnon
reviewed
Jun 13, 2026
Contributor
Author
|
The patch has been reworked for the most part. The whole CRC loop is now replaced with two clmul operations when possible. The algorithm has been tested in Python, but the actual implementation has yet to be properly tested. I will report back once I have done that. |
artagnon
reviewed
Jun 24, 2026
🪟 Windows x64 Test Results
✅ The build succeeded and all tests passed. |
|
✅ With the latest revision this PR passed the LLVM ABI annotation checker. |
artagnon
reviewed
Jun 25, 2026
arsenm
reviewed
Jun 25, 2026
pfusik
reviewed
Jun 26, 2026
artagnon
reviewed
Jun 26, 2026
xarkenz
force-pushed
the
crc-loop-clmul
branch
2 times, most recently
from
0b67189 to
318f815
Compare
Contributor
Author
This was referenced Jul 16, 2026
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
5 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
The current
optimizeCRCLoopfunction always optimizes CRC loops to use a 256-entry Sarwate lookup table to process a byte at a time (except on Hexagon and with optsize). However, some targets are able to process larger chunks of CRC data with carry-less/polynomial multiplication instructions. Implement this approach in IR for such targets using thellvm.clmulintrinsic. (Only with optsize for the moment, though.)Like the current CRC loop optimization, this does not apply to the Hexagon target, despite it having a
pmpywinstruction.Assisted-by: Claude Opus 4.8