[LoopIdiom] Use narrower bit widths where possible in optimizeCRCLoopUsingClmul#210139
Open
xarkenz wants to merge 1 commit into
Open
[LoopIdiom] Use narrower bit widths where possible in optimizeCRCLoopUsingClmul#210139xarkenz wants to merge 1 commit into
optimizeCRCLoopUsingClmul#210139xarkenz wants to merge 1 commit into
Conversation
|
@llvm/pr-subscribers-llvm-transforms @llvm/pr-subscribers-backend-risc-v Author: Sean Clarke (xarkenz) ChangesThe original implementation of Patch is 77.68 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/210139.diff 5 Files Affected:
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index e831a89a79113..ebdc079985126 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -266,7 +266,8 @@ class LoopIdiomRecognize {
bool IsLoopMemset = false);
bool optimizeCRCLoop(const PolynomialInfo &Info);
void optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
- IntegerType *ClmulTy);
+ IntegerType *ClmulMuTy,
+ IntegerType *ClmulGPTy);
void optimizeCRCLoopUsingTableLookup(const PolynomialInfo &Info);
/// @}
@@ -1590,17 +1591,16 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
if (TT.getArch() == Triple::hexagon)
return false;
- // In the clmul optimization, the first clmul uses 2*TC bits, and the second
- // clmul uses CRCBW+TC bits. For simplicity, have both clmuls operate on the
- // same bit width.
+ // Calculate the widths for the clmul operations that would be required.
+ LLVMContext &Ctx = Info.LHS->getContext();
unsigned CRCBW = Info.LHS->getType()->getIntegerBitWidth();
- unsigned ClmulBW = std::max(2 * Info.TripCount, CRCBW + Info.TripCount);
- auto *ClmulTy = IntegerType::get(Info.LHS->getContext(), ClmulBW);
+ auto *ClmulMuTy = IntegerType::get(Ctx, 2 * Info.TripCount);
+ auto *ClmulGPTy = IntegerType::get(Ctx, CRCBW + Info.TripCount);
// The force-crc-clmul flag should cause the clmul optimization to run
// unconditionally.
if (ForceCRCClmul) {
- optimizeCRCLoopUsingClmul(Info, ClmulTy);
+ optimizeCRCLoopUsingClmul(Info, ClmulMuTy, ClmulGPTy);
return true;
}
@@ -1617,8 +1617,8 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
// bit width is a fast operation on the target.
// TODO: If clmul exists on the target but not for the required width, it
// might be possible to split into multiple iterations of reduction.
- if (TTI->haveFastClmul(ClmulTy)) {
- optimizeCRCLoopUsingClmul(Info, ClmulTy);
+ if (TTI->haveFastClmul(ClmulMuTy) && TTI->haveFastClmul(ClmulGPTy)) {
+ optimizeCRCLoopUsingClmul(Info, ClmulMuTy, ClmulGPTy);
return true;
}
@@ -1629,7 +1629,8 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
// Reduction based on Intel's "Fast CRC Computation for Generic Polynomials
// Using PCLMULQDQ Instruction" white paper (December 2009).
void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
- IntegerType *ClmulTy) {
+ IntegerType *ClmulMuTy,
+ IntegerType *ClmulGPTy) {
Type *CRCTy = Info.LHS->getType();
LLVMContext &Ctx = CRCTy->getContext();
unsigned CRCBW = CRCTy->getIntegerBitWidth();
@@ -1637,23 +1638,28 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
// regardless of whether the actual data bit width matches (if auxiliary data
// is even used at all).
unsigned TC = Info.TripCount;
- unsigned ClmulBW = ClmulTy->getBitWidth();
// First, generate the constants required for GF(2) Barrett reduction.
auto [Mu, FullGenPoly] = HashRecognize::genBarrettConstants(Info);
- Value *MuConst = ConstantInt::get(Ctx, Mu.zext(ClmulBW));
- Value *GenPolyConst = ConstantInt::get(Ctx, FullGenPoly.zext(ClmulBW));
+ Value *MuConst = ConstantInt::get(Ctx, Mu.zext(ClmulMuTy->getBitWidth()));
+ Value *GenPolyConst =
+ ConstantInt::get(Ctx, FullGenPoly.zext(ClmulGPTy->getBitWidth()));
IRBuilder<> Builder(CurLoop->getLoopPreheader()->getTerminator());
auto LoTCBits = [TC, &Builder, &Ctx](Value *Op, const Twine &Name) {
unsigned OpBW = Op->getType()->getIntegerBitWidth();
- assert(OpBW >= TC && "Bit width should be at least TripCount");
+ if (OpBW <= TC)
+ return Op;
auto *Mask = ConstantInt::get(Ctx, APInt::getLowBitsSet(OpBW, TC));
return Builder.CreateAnd(Op, Mask, Name);
};
- Value *LHS = Builder.CreateZExt(Info.LHS, ClmulTy, "crc.cast");
+ // If a shift needs to occur in the setup for the first clmul with MuConst, it
+ // will be by abs(TC - CRCBW). To ensure that the shift can work without
+ // losing information or creating poison, give it CRCBW + TC bits.
+ bool SetupShiftNeeded = Info.IsBigEndian && TC != CRCBW;
+ auto *SetupTy = IntegerType::get(Ctx, SetupShiftNeeded ? CRCBW + TC : TC);
// Based on the Intel white paper, in our case, we have
// R(x) = (LHS*x^TC) xor (LHSAux ? getTCBits(LHSAux)*x^CRCBW : 0)
@@ -1668,19 +1674,20 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
// Thanks to restrictions imposed by HashRecognize for big-endian CRC loops,
// getTCBits(LHSAux) = LHSAux*x^(TC-CRCBW), so this can be further simplified
// to (LHS xor (LHSAux ? LHSAux : 0))*x^(TC-CRCBW).
- Value *ClmulMuInput = LHS;
+ Value *ClmulMuInput =
+ Builder.CreateZExtOrTrunc(Info.LHS, SetupTy, "crc.cast");
// If auxiliary data is present, XOR it in with the CRC.
if (Value *Data = Info.LHSAux) {
- // This is usually a zext, but DataBW may exceed ClmulBW if both CRCBW and
+ // This is usually a zext, but DataBW may exceed CRCBW+TC if both CRCBW and
// TC are small enough.
- Data = Builder.CreateZExtOrTrunc(Data, ClmulTy, "data.cast");
+ Data = Builder.CreateZExtOrTrunc(Data, SetupTy, "data.cast");
ClmulMuInput = Builder.CreateXor(ClmulMuInput, Data, "xor.crc.data");
}
// Align the current CRC with TripCount (multiply or divide by x^(TC-CRCBW)).
- if (Info.IsBigEndian && TC != CRCBW) {
+ if (SetupShiftNeeded) {
ClmulMuInput =
TC > CRCBW
? Builder.CreateShl(ClmulMuInput, TC - CRCBW, "crc.align.tc")
@@ -1693,6 +1700,8 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
// Step 1: T1(x) = floor(R(x)/x^CRCBW) * mu
// Input is TC bits and mu is TC+1 bits, so result will be 2*TC bits.
+ ClmulMuInput =
+ Builder.CreateZExtOrTrunc(ClmulMuInput, ClmulMuTy, "tcbits.cast");
Value *ClmulMu = Builder.CreateBinaryIntrinsic(
Intrinsic::clmul, ClmulMuInput, MuConst, /*FMFSource=*/{}, "clmul.mu");
@@ -1703,6 +1712,8 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
// Step 2: T2(x) = floor(T1(x)/x^TC) * P(x)
// Input is TC bits and P(x) is CRCBW+1 bits, so result will be CRCBW+TC bits.
+ ClmulGPInput =
+ Builder.CreateZExtOrTrunc(ClmulGPInput, ClmulGPTy, "quot.cast");
Value *ClmulGP = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, ClmulGPInput,
GenPolyConst,
/*FMFSource=*/{}, "clmul.gp");
@@ -1710,11 +1721,12 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
// Calculate the least significant part of R(x) for step 3 as specified above.
// R(x) mod x^CRCBW = LHS*x^TC mod x^CRCBW, though the (mod x^CRCBW) is
// handled later on when truncating back to CRCBW for ComputedValue.
- Value *CRCAlignClmul =
- Info.IsBigEndian ? Builder.CreateShl(LHS, TC, "crc.shl") : LHS;
+ Value *CRCNext = Builder.CreateZExt(Info.LHS, ClmulGPTy, "crc.recast");
+ if (Info.IsBigEndian)
+ CRCNext = Builder.CreateShl(CRCNext, TC, "crc.shl");
// Step 3: C(x) = (R(x) xor T2(x)) mod x^CRCBW
- Value *CRCNext = Builder.CreateXor(CRCAlignClmul, ClmulGP, "xor.crc.mult");
+ CRCNext = Builder.CreateXor(CRCNext, ClmulGP, "xor.crc.mult");
if (!Info.IsBigEndian)
CRCNext = Builder.CreateLShr(CRCNext, TC, "crc.lshr");
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll
index a381b89f651e8..e31fdd0164ddc 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll
@@ -29,14 +29,15 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) optsize {
; AES-LABEL: define i16 @crc16.le.tc8(
; AES-SAME: i8 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0:[0-9]+]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24
-; AES-NEXT: [[DATA_CAST:%.*]] = zext i8 [[MSG]] to i24
-; AES-NEXT: [[XOR_CRC_DATA:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]]
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[XOR_CRC_DATA]], 255
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 511)
-; AES-NEXT: [[QUOT_MASK:%.*]] = and i24 [[CLMUL_MU]], 255
-; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_MASK]], i24 81923)
-; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_CAST]], [[CLMUL_GP]]
+; AES-NEXT: [[CRC_CAST:%.*]] = trunc i16 [[CHECKSUM]] to i8
+; AES-NEXT: [[XOR_CRC_DATA:%.*]] = xor i8 [[CRC_CAST]], [[MSG]]
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA]] to i16
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 511)
+; AES-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255
+; AES-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i24
+; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 81923)
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24
+; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]]
; AES-NEXT: [[CRC_LSHR1:%.*]] = lshr i24 [[XOR_CRC_MULT]], 8
; AES-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[CRC_LSHR1]] to i16
; AES-NEXT: br label %[[LOOP:.*]]
@@ -95,14 +96,13 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) optsize {
; AES-LABEL: define i16 @crc16.le.tc16(
; AES-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32
-; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32
-; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]]
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 114687)
+; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]]
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 114687)
; AES-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535
; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 81923)
-; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]]
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32
+; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_RECAST]], [[CLMUL_GP]]
; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16
; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i16
; AES-NEXT: br label %[[LOOP:.*]]
@@ -161,16 +161,17 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) optsize {
; AES-LABEL: define i8 @crc8.le.tc16(
; AES-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32
-; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32
-; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]]
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 24423)
+; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16
+; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]]
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 24423)
; AES-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535
-; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 59)
-; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]]
-; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16
-; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i8
+; AES-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_MASK]] to i24
+; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 59)
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24
+; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]]
+; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i24 [[XOR_CRC_MULT]], 16
+; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[CRC_LSHR2]] to i8
; AES-NEXT: br label %[[LOOP:.*]]
; AES: [[LOOP]]:
; AES-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
@@ -226,14 +227,13 @@ define i16 @crc16.be.tc16(i16 %msg, i16 %checksum) optsize {
; AES-LABEL: define i16 @crc16.be.tc16(
; AES-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32
-; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32
-; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]]
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 69936)
+; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]]
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 69936)
; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16
; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 69665)
-; AES-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32
+; AES-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_RECAST]], 16
; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]]
; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i16
; AES-NEXT: br label %[[LOOP:.*]]
@@ -293,10 +293,13 @@ define i16 @crc16.be.tc8.misalign(i8 %msg, i16 %checksum) optsize {
; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24
; AES-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8
; AES-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273)
-; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8
-; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665)
-; AES-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8
+; AES-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273)
+; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8
+; AES-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24
+; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665)
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24
+; AES-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8
; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]]
; AES-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16
; AES-NEXT: br label %[[LOOP:.*]]
@@ -354,17 +357,20 @@ define i8 @crc8.be.tc16.misalign(i16 %msg, i8 %checksum) optsize {
; AES-LABEL: define i8 @crc8.be.tc16.misalign(
; AES-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32
-; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32
-; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]]
-; AES-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i32 [[XOR_CRC_DATA1]], 8
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[CRC_ALIGN_TC]], 65535
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 72779)
+; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i24
+; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i24
+; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]]
+; AES-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i24 [[XOR_CRC_DATA1]], 8
+; AES-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 65535
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i24 [[CRC_TCBITS]] to i32
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 72779)
; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16
-; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 285)
-; AES-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16
-; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]]
-; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i8
+; AES-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_LSHR]] to i24
+; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 285)
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24
+; AES-NEXT: [[CRC_SHL2:%.*]] = shl i24 [[CRC_RECAST]], 16
+; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL2]], [[CLMUL_GP]]
+; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i8
; AES-NEXT: br label %[[LOOP:.*]]
; AES: [[LOOP]]:
; AES-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
@@ -420,13 +426,14 @@ define i8 @crc8.be.tc8.data16.misalign(i16 %msg, i8 %checksum) optsize {
; AES-LABEL: define i8 @crc8.be.tc8.data16.misalign(
; AES-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16
-; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]]
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284)
+; AES-NEXT: [[DATA_CAST:%.*]] = trunc i16 [[MSG]] to i8
+; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]]
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 284)
; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8
; AES-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285)
-; AES-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_CAST]], 8
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16
+; AES-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_RECAST]], 8
; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL2]], [[CLMUL_GP]]
; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8
; AES-NEXT: br label %[[LOOP:.*]]
@@ -484,14 +491,16 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) optsize {
; AES-LABEL: define i32 @crc32.le.tc8.data32(
; AES-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] {
; AES-NEXT: [[ENTRY:.*:]]
-; AES-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40
-; AES-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40
-; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]]
-; AES-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255
-; AES-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273)
-; AES-NEXT: [[QUOT_MASK:%.*]] = and i40 [[CLMUL_MU]], 255
-; AES-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_MASK]], i40 67601)
-; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_CAST]], [[CLMUL_GP]]
+; AES-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i8
+; AES-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8
+; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CRC_CAST]], [[DATA_CAST]]
+; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16
+; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273)
+; AES-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255
+; AES-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i40
+; AES-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_CAST]], i40 67601)
+; AES-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i40
+; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_RECAST]], [[CLMUL_GP]]
; AES-NEXT: ...
[truncated]
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
The original implementation of
optimizeCRCLoopUsingClmul(#203405) uses a single conservative bit width for most operations, but this width is not always necessary. Use more restrictive bit widths for each clmul according to their inputs, and narrow the bit width for the CRC/data setup in most cases.