From e8b6e41b560834dca8ed0ecef92127a3f1d45a93 Mon Sep 17 00:00:00 2001 From: Sean Clarke Date: Thu, 16 Jul 2026 13:47:27 -0500 Subject: [PATCH 1/3] Narrow bit widths where possible --- .../Transforms/Scalar/LoopIdiomRecognize.cpp | 56 +++-- .../AArch64/cyclic-redundancy-check.ll | 144 ++++++------ .../RISCV/cyclic-redundancy-check.ll | 144 ++++++------ .../LoopIdiom/X86/cyclic-redundancy-check.ll | 144 ++++++------ .../LoopIdiom/cyclic-redundancy-check.ll | 220 ++++++++++-------- 5 files changed, 384 insertions(+), 324 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp index e674143099a53..3cb4c1fc6befe 100644 --- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp +++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp @@ -266,7 +266,8 @@ class LoopIdiomRecognize { bool IsLoopMemset = false); bool optimizeCRCLoop(const PolynomialInfo &Info); void optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, - IntegerType *ClmulTy); + IntegerType *ClmulMuTy, + IntegerType *ClmulGPTy); void optimizeCRCLoopUsingTableLookup(const PolynomialInfo &Info); /// @} @@ -1590,17 +1591,16 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) { if (TT.getArch() == Triple::hexagon) return false; - // In the clmul optimization, the first clmul uses 2*TC bits, and the second - // clmul uses CRCBW+TC bits. For simplicity, have both clmuls operate on the - // same bit width. + // Calculate the widths for the clmul operations that would be required. + LLVMContext &Ctx = Info.LHS->getContext(); unsigned CRCBW = Info.LHS->getType()->getIntegerBitWidth(); - unsigned ClmulBW = std::max(2 * Info.TripCount, CRCBW + Info.TripCount); - auto *ClmulTy = IntegerType::get(Info.LHS->getContext(), ClmulBW); + auto *ClmulMuTy = IntegerType::get(Ctx, 2 * Info.TripCount); + auto *ClmulGPTy = IntegerType::get(Ctx, CRCBW + Info.TripCount); // The force-crc-clmul flag should cause the clmul optimization to run // unconditionally. if (ForceCRCClmul) { - optimizeCRCLoopUsingClmul(Info, ClmulTy); + optimizeCRCLoopUsingClmul(Info, ClmulMuTy, ClmulGPTy); return true; } @@ -1617,8 +1617,8 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) { // bit width is a fast operation on the target. // TODO: If clmul exists on the target but not for the required width, it // might be possible to split into multiple iterations of reduction. - if (TTI->haveFastClmul(ClmulTy)) { - optimizeCRCLoopUsingClmul(Info, ClmulTy); + if (TTI->haveFastClmul(ClmulMuTy) && TTI->haveFastClmul(ClmulGPTy)) { + optimizeCRCLoopUsingClmul(Info, ClmulMuTy, ClmulGPTy); return true; } @@ -1629,7 +1629,8 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) { // Reduction based on Intel's "Fast CRC Computation for Generic Polynomials // Using PCLMULQDQ Instruction" white paper (December 2009). void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, - IntegerType *ClmulTy) { + IntegerType *ClmulMuTy, + IntegerType *ClmulGPTy) { Type *CRCTy = Info.LHS->getType(); LLVMContext &Ctx = CRCTy->getContext(); unsigned CRCBW = CRCTy->getIntegerBitWidth(); @@ -1637,23 +1638,28 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, // regardless of whether the actual data bit width matches (if auxiliary data // is even used at all). unsigned TC = Info.TripCount; - unsigned ClmulBW = ClmulTy->getBitWidth(); // First, generate the constants required for GF(2) Barrett reduction. auto [Mu, FullGenPoly] = HashRecognize::genBarrettConstants(Info); - Value *MuConst = ConstantInt::get(Ctx, Mu.zext(ClmulBW)); - Value *GenPolyConst = ConstantInt::get(Ctx, FullGenPoly.zext(ClmulBW)); + Value *MuConst = ConstantInt::get(Ctx, Mu.zext(ClmulMuTy->getBitWidth())); + Value *GenPolyConst = + ConstantInt::get(Ctx, FullGenPoly.zext(ClmulGPTy->getBitWidth())); IRBuilder<> Builder(CurLoop->getLoopPreheader()->getTerminator()); auto LoTCBits = [TC, &Builder, &Ctx](Value *Op, const Twine &Name) { unsigned OpBW = Op->getType()->getIntegerBitWidth(); - assert(OpBW >= TC && "Bit width should be at least TripCount"); + if (OpBW <= TC) + return Op; auto *Mask = ConstantInt::get(Ctx, APInt::getLowBitsSet(OpBW, TC)); return Builder.CreateAnd(Op, Mask, Name); }; - Value *LHS = Builder.CreateZExt(Info.LHS, ClmulTy, "crc.cast"); + // If a shift needs to occur in the setup for the first clmul with MuConst, it + // will be by abs(TC - CRCBW). To ensure that the shift can work without + // losing information or creating poison, give it CRCBW + TC bits. + bool SetupShiftNeeded = Info.IsBigEndian && TC != CRCBW; + auto *SetupTy = IntegerType::get(Ctx, SetupShiftNeeded ? CRCBW + TC : TC); // Based on the Intel white paper, in our case, we have // R(x) = (LHS*x^TC) xor (LHSAux ? getTCBits(LHSAux)*x^CRCBW : 0) @@ -1668,19 +1674,20 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, // Thanks to restrictions imposed by HashRecognize for big-endian CRC loops, // getTCBits(LHSAux) = LHSAux*x^(TC-CRCBW), so this can be further simplified // to (LHS xor (LHSAux ? LHSAux : 0))*x^(TC-CRCBW). - Value *ClmulMuInput = LHS; + Value *ClmulMuInput = + Builder.CreateZExtOrTrunc(Info.LHS, SetupTy, "crc.cast"); // If auxiliary data is present, XOR it in with the CRC. if (Value *Data = Info.LHSAux) { - // This is usually a zext, but DataBW may exceed ClmulBW if both CRCBW and + // This is usually a zext, but DataBW may exceed CRCBW+TC if both CRCBW and // TC are small enough. - Data = Builder.CreateZExtOrTrunc(Data, ClmulTy, "data.cast"); + Data = Builder.CreateZExtOrTrunc(Data, SetupTy, "data.cast"); ClmulMuInput = Builder.CreateXor(ClmulMuInput, Data, "xor.crc.data"); } // Align the current CRC with TripCount (multiply or divide by x^(TC-CRCBW)). - if (Info.IsBigEndian && TC != CRCBW) { + if (SetupShiftNeeded) { ClmulMuInput = TC > CRCBW ? Builder.CreateShl(ClmulMuInput, TC - CRCBW, "crc.align.tc") @@ -1693,6 +1700,8 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, // Step 1: T1(x) = floor(R(x)/x^CRCBW) * mu // Input is TC bits and mu is TC+1 bits, so result will be 2*TC bits. + ClmulMuInput = + Builder.CreateZExtOrTrunc(ClmulMuInput, ClmulMuTy, "tcbits.cast"); Value *ClmulMu = Builder.CreateBinaryIntrinsic( Intrinsic::clmul, ClmulMuInput, MuConst, /*FMFSource=*/{}, "clmul.mu"); @@ -1703,6 +1712,8 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, // Step 2: T2(x) = floor(T1(x)/x^TC) * P(x) // Input is TC bits and P(x) is CRCBW+1 bits, so result will be CRCBW+TC bits. + ClmulGPInput = + Builder.CreateZExtOrTrunc(ClmulGPInput, ClmulGPTy, "quot.cast"); Value *ClmulGP = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, ClmulGPInput, GenPolyConst, /*FMFSource=*/{}, "clmul.gp"); @@ -1710,11 +1721,12 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, // Calculate the least significant part of R(x) for step 3 as specified above. // R(x) mod x^CRCBW = LHS*x^TC mod x^CRCBW, though the (mod x^CRCBW) is // handled later on when truncating back to CRCBW for ComputedValue. - Value *CRCAlignClmul = - Info.IsBigEndian ? Builder.CreateShl(LHS, TC, "crc.shl") : LHS; + Value *CRCNext = Builder.CreateZExt(Info.LHS, ClmulGPTy, "crc.recast"); + if (Info.IsBigEndian) + CRCNext = Builder.CreateShl(CRCNext, TC, "crc.shl"); // Step 3: C(x) = (R(x) xor T2(x)) mod x^CRCBW - Value *CRCNext = Builder.CreateXor(CRCAlignClmul, ClmulGP, "xor.crc.mult"); + CRCNext = Builder.CreateXor(CRCNext, ClmulGP, "xor.crc.mult"); if (!Info.IsBigEndian) CRCNext = Builder.CreateLShr(CRCNext, TC, "crc.lshr"); diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll index a381b89f651e8..e31fdd0164ddc 100644 --- a/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll +++ b/llvm/test/Transforms/LoopIdiom/AArch64/cyclic-redundancy-check.ll @@ -29,14 +29,15 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) optsize { ; AES-LABEL: define i16 @crc16.le.tc8( ; AES-SAME: i8 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0:[0-9]+]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i8 [[MSG]] to i24 -; AES-NEXT: [[XOR_CRC_DATA:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[XOR_CRC_DATA]], 255 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 511) -; AES-NEXT: [[QUOT_MASK:%.*]] = and i24 [[CLMUL_MU]], 255 -; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_MASK]], i24 81923) -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_CAST]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_CAST:%.*]] = trunc i16 [[CHECKSUM]] to i8 +; AES-NEXT: [[XOR_CRC_DATA:%.*]] = xor i8 [[CRC_CAST]], [[MSG]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA]] to i16 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 511) +; AES-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; AES-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i24 +; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 81923) +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_LSHR1:%.*]] = lshr i24 [[XOR_CRC_MULT]], 8 ; AES-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[CRC_LSHR1]] to i16 ; AES-NEXT: br label %[[LOOP:.*]] @@ -95,14 +96,13 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) optsize { ; AES-LABEL: define i16 @crc16.le.tc16( ; AES-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 114687) +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 114687) ; AES-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 ; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 81923) -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_RECAST]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 ; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i16 ; AES-NEXT: br label %[[LOOP:.*]] @@ -161,16 +161,17 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) optsize { ; AES-LABEL: define i8 @crc8.le.tc16( ; AES-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 24423) +; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 24423) ; AES-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 -; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 59) -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] -; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 -; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i8 +; AES-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_MASK]] to i24 +; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 59) +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i24 [[XOR_CRC_MULT]], 16 +; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[CRC_LSHR2]] to i8 ; AES-NEXT: br label %[[LOOP:.*]] ; AES: [[LOOP]]: ; AES-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -226,14 +227,13 @@ define i16 @crc16.be.tc16(i16 %msg, i16 %checksum) optsize { ; AES-LABEL: define i16 @crc16.be.tc16( ; AES-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 69936) +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 69936) ; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 ; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 69665) -; AES-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; AES-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_RECAST]], 16 ; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i16 ; AES-NEXT: br label %[[LOOP:.*]] @@ -293,10 +293,13 @@ define i16 @crc16.be.tc8.misalign(i8 %msg, i16 %checksum) optsize { ; AES-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 ; AES-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; AES-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; AES-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; AES-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; AES-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; AES-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; AES-NEXT: br label %[[LOOP:.*]] @@ -354,17 +357,20 @@ define i8 @crc8.be.tc16.misalign(i16 %msg, i8 %checksum) optsize { ; AES-LABEL: define i8 @crc8.be.tc16.misalign( ; AES-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i32 [[XOR_CRC_DATA1]], 8 -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[CRC_ALIGN_TC]], 65535 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 72779) +; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; AES-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i24 +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] +; AES-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i24 [[XOR_CRC_DATA1]], 8 +; AES-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 65535 +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i24 [[CRC_TCBITS]] to i32 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 72779) ; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 -; AES-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 285) -; AES-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] -; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i8 +; AES-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_LSHR]] to i24 +; AES-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 285) +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; AES-NEXT: [[CRC_SHL2:%.*]] = shl i24 [[CRC_RECAST]], 16 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL2]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i8 ; AES-NEXT: br label %[[LOOP:.*]] ; AES: [[LOOP]]: ; AES-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -420,13 +426,14 @@ define i8 @crc8.be.tc8.data16.misalign(i16 %msg, i8 %checksum) optsize { ; AES-LABEL: define i8 @crc8.be.tc8.data16.misalign( ; AES-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284) +; AES-NEXT: [[DATA_CAST:%.*]] = trunc i16 [[MSG]] to i8 +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 284) ; AES-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 ; AES-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285) -; AES-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_CAST]], 8 +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; AES-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_RECAST]], 8 ; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL2]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8 ; AES-NEXT: br label %[[LOOP:.*]] @@ -484,14 +491,16 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) optsize { ; AES-LABEL: define i32 @crc32.le.tc8.data32( ; AES-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273) -; AES-NEXT: [[QUOT_MASK:%.*]] = and i40 [[CLMUL_MU]], 255 -; AES-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_MASK]], i40 67601) -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_CAST]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i8 +; AES-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CRC_CAST]], [[DATA_CAST]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; AES-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; AES-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i40 +; AES-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_CAST]], i40 67601) +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i40 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_RECAST]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i40 [[XOR_CRC_MULT]], 8 ; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i40 [[CRC_LSHR2]] to i32 ; AES-NEXT: br label %[[LOOP:.*]] @@ -550,14 +559,14 @@ define i8 @crc8.le.tc8.data32(i8 %checksum, i32 %msg) optsize { ; AES-LABEL: define i8 @crc8.le.tc8.data32( ; AES-SAME: i8 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; AES-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i16 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 107) +; AES-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 107) ; AES-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 ; AES-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_MASK]], i16 119) -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_CAST]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_RECAST]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i16 [[XOR_CRC_MULT]], 8 ; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[CRC_LSHR2]] to i8 ; AES-NEXT: br label %[[LOOP:.*]] @@ -616,14 +625,13 @@ define i32 @crc32.le.tc32(i32 %checksum, i32 %msg) optsize { ; AES-LABEL: define i32 @crc32.le.tc32( ; AES-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; AES-NEXT: [[ENTRY:.*:]] -; AES-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i64 -; AES-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i64 -; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i64 [[CRC_CAST]], [[DATA_CAST]] -; AES-NEXT: [[CRC_TCBITS:%.*]] = and i64 [[XOR_CRC_DATA1]], 4294967295 -; AES-NEXT: [[CLMUL_MU:%.*]] = call i64 @llvm.clmul.i64(i64 [[CRC_TCBITS]], i64 4770502929) +; AES-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CHECKSUM]], [[MSG]] +; AES-NEXT: [[TCBITS_CAST:%.*]] = zext i32 [[XOR_CRC_DATA1]] to i64 +; AES-NEXT: [[CLMUL_MU:%.*]] = call i64 @llvm.clmul.i64(i64 [[TCBITS_CAST]], i64 4770502929) ; AES-NEXT: [[QUOT_MASK:%.*]] = and i64 [[CLMUL_MU]], 4294967295 ; AES-NEXT: [[CLMUL_GP:%.*]] = call i64 @llvm.clmul.i64(i64 [[QUOT_MASK]], i64 67601) -; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i64 [[CRC_CAST]], [[CLMUL_GP]] +; AES-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i64 +; AES-NEXT: [[XOR_CRC_MULT:%.*]] = xor i64 [[CRC_RECAST]], [[CLMUL_GP]] ; AES-NEXT: [[CRC_LSHR2:%.*]] = lshr i64 [[XOR_CRC_MULT]], 32 ; AES-NEXT: [[CRC_NEXT3:%.*]] = trunc i64 [[CRC_LSHR2]] to i32 ; AES-NEXT: br label %[[LOOP:.*]] diff --git a/llvm/test/Transforms/LoopIdiom/RISCV/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/RISCV/cyclic-redundancy-check.ll index 9af6d463fafca..c7a861081515c 100644 --- a/llvm/test/Transforms/LoopIdiom/RISCV/cyclic-redundancy-check.ll +++ b/llvm/test/Transforms/LoopIdiom/RISCV/cyclic-redundancy-check.ll @@ -31,14 +31,15 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) optsize { ; ZBC-LABEL: define i16 @crc16.le.tc8( ; ZBC-SAME: i8 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0:[0-9]+]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 -; ZBC-NEXT: [[DATA_CAST:%.*]] = zext i8 [[MSG]] to i24 -; ZBC-NEXT: [[XOR_CRC_DATA:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[XOR_CRC_DATA]], 255 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 511) -; ZBC-NEXT: [[QUOT_MASK:%.*]] = and i24 [[CLMUL_MU]], 255 -; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_MASK]], i24 81923) -; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_CAST]], [[CLMUL_GP]] +; ZBC-NEXT: [[CRC_CAST:%.*]] = trunc i16 [[CHECKSUM]] to i8 +; ZBC-NEXT: [[XOR_CRC_DATA:%.*]] = xor i8 [[CRC_CAST]], [[MSG]] +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA]] to i16 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 511) +; ZBC-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; ZBC-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i24 +; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 81923) +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] ; ZBC-NEXT: [[CRC_LSHR1:%.*]] = lshr i24 [[XOR_CRC_MULT]], 8 ; ZBC-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[CRC_LSHR1]] to i16 ; ZBC-NEXT: br label %[[LOOP:.*]] @@ -97,14 +98,13 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) optsize { ; ZBC-LABEL: define i16 @crc16.le.tc16( ; ZBC-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; ZBC-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 114687) +; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 114687) ; ZBC-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 ; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 81923) -; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_RECAST]], [[CLMUL_GP]] ; ZBC-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 ; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i16 ; ZBC-NEXT: br label %[[LOOP:.*]] @@ -163,16 +163,17 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) optsize { ; ZBC-LABEL: define i8 @crc8.le.tc16( ; ZBC-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; ZBC-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 24423) +; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 24423) ; ZBC-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 -; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 59) -; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] -; ZBC-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 -; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i8 +; ZBC-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_MASK]] to i24 +; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 59) +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] +; ZBC-NEXT: [[CRC_LSHR2:%.*]] = lshr i24 [[XOR_CRC_MULT]], 16 +; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[CRC_LSHR2]] to i8 ; ZBC-NEXT: br label %[[LOOP:.*]] ; ZBC: [[LOOP]]: ; ZBC-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -228,14 +229,13 @@ define i16 @crc16.be.tc16(i16 %msg, i16 %checksum) optsize { ; ZBC-LABEL: define i16 @crc16.be.tc16( ; ZBC-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; ZBC-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 69936) +; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 69936) ; ZBC-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 ; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 69665) -; ZBC-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; ZBC-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_RECAST]], 16 ; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] ; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i16 ; ZBC-NEXT: br label %[[LOOP:.*]] @@ -295,10 +295,13 @@ define i16 @crc16.be.tc8.misalign(i8 %msg, i16 %checksum) optsize { ; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 ; ZBC-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; ZBC-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; ZBC-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; ZBC-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; ZBC-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; ZBC-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; ZBC-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; ZBC-NEXT: br label %[[LOOP:.*]] @@ -356,17 +359,20 @@ define i8 @crc8.be.tc16.misalign(i16 %msg, i8 %checksum) optsize { ; ZBC-LABEL: define i8 @crc8.be.tc16.misalign( ; ZBC-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; ZBC-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; ZBC-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i32 [[XOR_CRC_DATA1]], 8 -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[CRC_ALIGN_TC]], 65535 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 72779) +; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; ZBC-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i24 +; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] +; ZBC-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i24 [[XOR_CRC_DATA1]], 8 +; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 65535 +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i24 [[CRC_TCBITS]] to i32 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 72779) ; ZBC-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 -; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 285) -; ZBC-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 -; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] -; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i8 +; ZBC-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_LSHR]] to i24 +; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 285) +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; ZBC-NEXT: [[CRC_SHL2:%.*]] = shl i24 [[CRC_RECAST]], 16 +; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL2]], [[CLMUL_GP]] +; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i8 ; ZBC-NEXT: br label %[[LOOP:.*]] ; ZBC: [[LOOP]]: ; ZBC-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -422,13 +428,14 @@ define i8 @crc8.be.tc8.data16.misalign(i16 %msg, i8 %checksum) optsize { ; ZBC-LABEL: define i8 @crc8.be.tc8.data16.misalign( ; ZBC-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284) +; ZBC-NEXT: [[DATA_CAST:%.*]] = trunc i16 [[MSG]] to i8 +; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 284) ; ZBC-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 ; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285) -; ZBC-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_CAST]], 8 +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; ZBC-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_RECAST]], 8 ; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL2]], [[CLMUL_GP]] ; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8 ; ZBC-NEXT: br label %[[LOOP:.*]] @@ -508,14 +515,16 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) optsize { ; ZBC64-LABEL: define i32 @crc32.le.tc8.data32( ; ZBC64-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; ZBC64-NEXT: [[ENTRY:.*:]] -; ZBC64-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40 -; ZBC64-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40 -; ZBC64-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]] -; ZBC64-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255 -; ZBC64-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273) -; ZBC64-NEXT: [[QUOT_MASK:%.*]] = and i40 [[CLMUL_MU]], 255 -; ZBC64-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_MASK]], i40 67601) -; ZBC64-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_CAST]], [[CLMUL_GP]] +; ZBC64-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i8 +; ZBC64-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; ZBC64-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CRC_CAST]], [[DATA_CAST]] +; ZBC64-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; ZBC64-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; ZBC64-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; ZBC64-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i40 +; ZBC64-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_CAST]], i40 67601) +; ZBC64-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i40 +; ZBC64-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_RECAST]], [[CLMUL_GP]] ; ZBC64-NEXT: [[CRC_LSHR2:%.*]] = lshr i40 [[XOR_CRC_MULT]], 8 ; ZBC64-NEXT: [[CRC_NEXT3:%.*]] = trunc i40 [[CRC_LSHR2]] to i32 ; ZBC64-NEXT: br label %[[LOOP:.*]] @@ -574,14 +583,14 @@ define i8 @crc8.le.tc8.data32(i8 %checksum, i32 %msg) optsize { ; ZBC-LABEL: define i8 @crc8.le.tc8.data32( ; ZBC-SAME: i8 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; ZBC-NEXT: [[ENTRY:.*:]] -; ZBC-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; ZBC-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i16 -; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[DATA_CAST]] -; ZBC-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 107) +; ZBC-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; ZBC-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; ZBC-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; ZBC-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 107) ; ZBC-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 ; ZBC-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_MASK]], i16 119) -; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_CAST]], [[CLMUL_GP]] +; ZBC-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; ZBC-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_RECAST]], [[CLMUL_GP]] ; ZBC-NEXT: [[CRC_LSHR2:%.*]] = lshr i16 [[XOR_CRC_MULT]], 8 ; ZBC-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[CRC_LSHR2]] to i8 ; ZBC-NEXT: br label %[[LOOP:.*]] @@ -662,14 +671,13 @@ define i32 @crc32.le.tc32(i32 %checksum, i32 %msg) optsize { ; ZBC64-LABEL: define i32 @crc32.le.tc32( ; ZBC64-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; ZBC64-NEXT: [[ENTRY:.*:]] -; ZBC64-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i64 -; ZBC64-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i64 -; ZBC64-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i64 [[CRC_CAST]], [[DATA_CAST]] -; ZBC64-NEXT: [[CRC_TCBITS:%.*]] = and i64 [[XOR_CRC_DATA1]], 4294967295 -; ZBC64-NEXT: [[CLMUL_MU:%.*]] = call i64 @llvm.clmul.i64(i64 [[CRC_TCBITS]], i64 4770502929) +; ZBC64-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CHECKSUM]], [[MSG]] +; ZBC64-NEXT: [[TCBITS_CAST:%.*]] = zext i32 [[XOR_CRC_DATA1]] to i64 +; ZBC64-NEXT: [[CLMUL_MU:%.*]] = call i64 @llvm.clmul.i64(i64 [[TCBITS_CAST]], i64 4770502929) ; ZBC64-NEXT: [[QUOT_MASK:%.*]] = and i64 [[CLMUL_MU]], 4294967295 ; ZBC64-NEXT: [[CLMUL_GP:%.*]] = call i64 @llvm.clmul.i64(i64 [[QUOT_MASK]], i64 67601) -; ZBC64-NEXT: [[XOR_CRC_MULT:%.*]] = xor i64 [[CRC_CAST]], [[CLMUL_GP]] +; ZBC64-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i64 +; ZBC64-NEXT: [[XOR_CRC_MULT:%.*]] = xor i64 [[CRC_RECAST]], [[CLMUL_GP]] ; ZBC64-NEXT: [[CRC_LSHR2:%.*]] = lshr i64 [[XOR_CRC_MULT]], 32 ; ZBC64-NEXT: [[CRC_NEXT3:%.*]] = trunc i64 [[CRC_LSHR2]] to i32 ; ZBC64-NEXT: br label %[[LOOP:.*]] diff --git a/llvm/test/Transforms/LoopIdiom/X86/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/X86/cyclic-redundancy-check.ll index 5a4767841f8c7..1365e00c77921 100644 --- a/llvm/test/Transforms/LoopIdiom/X86/cyclic-redundancy-check.ll +++ b/llvm/test/Transforms/LoopIdiom/X86/cyclic-redundancy-check.ll @@ -29,14 +29,15 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) optsize { ; PCLMUL-LABEL: define i16 @crc16.le.tc8( ; PCLMUL-SAME: i8 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0:[0-9]+]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i8 [[MSG]] to i24 -; PCLMUL-NEXT: [[XOR_CRC_DATA:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[XOR_CRC_DATA]], 255 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 511) -; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i24 [[CLMUL_MU]], 255 -; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_MASK]], i24 81923) -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_CAST]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i16 [[CHECKSUM]] to i8 +; PCLMUL-NEXT: [[XOR_CRC_DATA:%.*]] = xor i8 [[CRC_CAST]], [[MSG]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA]] to i16 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 511) +; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; PCLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i24 +; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 81923) +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_LSHR1:%.*]] = lshr i24 [[XOR_CRC_MULT]], 8 ; PCLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[CRC_LSHR1]] to i16 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -95,14 +96,13 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) optsize { ; PCLMUL-LABEL: define i16 @crc16.le.tc16( ; PCLMUL-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 114687) +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 114687) ; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 ; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 81923) -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_RECAST]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 ; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i16 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -161,16 +161,17 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) optsize { ; PCLMUL-LABEL: define i8 @crc8.le.tc16( ; PCLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 24423) +; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 24423) ; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 -; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 59) -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] -; PCLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 -; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i8 +; PCLMUL-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_MASK]] to i24 +; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 59) +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i24 [[XOR_CRC_MULT]], 16 +; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[CRC_LSHR2]] to i8 ; PCLMUL-NEXT: br label %[[LOOP:.*]] ; PCLMUL: [[LOOP]]: ; PCLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -226,14 +227,13 @@ define i16 @crc16.be.tc16(i16 %msg, i16 %checksum) optsize { ; PCLMUL-LABEL: define i16 @crc16.be.tc16( ; PCLMUL-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 69936) +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 69936) ; PCLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 ; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 69665) -; PCLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; PCLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_RECAST]], 16 ; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i16 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -293,10 +293,13 @@ define i16 @crc16.be.tc8.misalign(i8 %msg, i16 %checksum) optsize { ; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 ; PCLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; PCLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; PCLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; PCLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; PCLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; PCLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -354,17 +357,20 @@ define i8 @crc8.be.tc16.misalign(i16 %msg, i8 %checksum) optsize { ; PCLMUL-LABEL: define i8 @crc8.be.tc16.misalign( ; PCLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i32 [[XOR_CRC_DATA1]], 8 -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[CRC_ALIGN_TC]], 65535 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 72779) +; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i24 +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] +; PCLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i24 [[XOR_CRC_DATA1]], 8 +; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 65535 +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i24 [[CRC_TCBITS]] to i32 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 72779) ; PCLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 -; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 285) -; PCLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] -; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i8 +; PCLMUL-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_LSHR]] to i24 +; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 285) +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; PCLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i24 [[CRC_RECAST]], 16 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL2]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i8 ; PCLMUL-NEXT: br label %[[LOOP:.*]] ; PCLMUL: [[LOOP]]: ; PCLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -420,13 +426,14 @@ define i8 @crc8.be.tc8.data16.misalign(i16 %msg, i8 %checksum) optsize { ; PCLMUL-LABEL: define i8 @crc8.be.tc8.data16.misalign( ; PCLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284) +; PCLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i16 [[MSG]] to i8 +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 284) ; PCLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 ; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285) -; PCLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_CAST]], 8 +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; PCLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_RECAST]], 8 ; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL2]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -484,14 +491,16 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) optsize { ; PCLMUL-LABEL: define i32 @crc32.le.tc8.data32( ; PCLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273) -; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i40 [[CLMUL_MU]], 255 -; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_MASK]], i40 67601) -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_CAST]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i8 +; PCLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CRC_CAST]], [[DATA_CAST]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; PCLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i40 +; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_CAST]], i40 67601) +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i40 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_RECAST]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i40 [[XOR_CRC_MULT]], 8 ; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i40 [[CRC_LSHR2]] to i32 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -550,14 +559,14 @@ define i8 @crc8.le.tc8.data32(i8 %checksum, i32 %msg) optsize { ; PCLMUL-LABEL: define i8 @crc8.le.tc8.data32( ; PCLMUL-SAME: i8 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i16 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 107) +; PCLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 107) ; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 ; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_MASK]], i16 119) -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_CAST]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_RECAST]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i16 [[XOR_CRC_MULT]], 8 ; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[CRC_LSHR2]] to i8 ; PCLMUL-NEXT: br label %[[LOOP:.*]] @@ -616,14 +625,13 @@ define i32 @crc32.le.tc32(i32 %checksum, i32 %msg) optsize { ; PCLMUL-LABEL: define i32 @crc32.le.tc32( ; PCLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; PCLMUL-NEXT: [[ENTRY:.*:]] -; PCLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i64 -; PCLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i64 -; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i64 [[CRC_CAST]], [[DATA_CAST]] -; PCLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i64 [[XOR_CRC_DATA1]], 4294967295 -; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i64 @llvm.clmul.i64(i64 [[CRC_TCBITS]], i64 4770502929) +; PCLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CHECKSUM]], [[MSG]] +; PCLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i32 [[XOR_CRC_DATA1]] to i64 +; PCLMUL-NEXT: [[CLMUL_MU:%.*]] = call i64 @llvm.clmul.i64(i64 [[TCBITS_CAST]], i64 4770502929) ; PCLMUL-NEXT: [[QUOT_MASK:%.*]] = and i64 [[CLMUL_MU]], 4294967295 ; PCLMUL-NEXT: [[CLMUL_GP:%.*]] = call i64 @llvm.clmul.i64(i64 [[QUOT_MASK]], i64 67601) -; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i64 [[CRC_CAST]], [[CLMUL_GP]] +; PCLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i64 +; PCLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i64 [[CRC_RECAST]], [[CLMUL_GP]] ; PCLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i64 [[XOR_CRC_MULT]], 32 ; PCLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i64 [[CRC_LSHR2]] to i32 ; PCLMUL-NEXT: br label %[[LOOP:.*]] diff --git a/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll index d4c2f59e9003a..ca90b34b5c4b5 100644 --- a/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll +++ b/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll @@ -46,14 +46,15 @@ define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) { ; CLMUL-LABEL: define i16 @crc16.le.tc8( ; CLMUL-SAME: i8 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i8 [[MSG]] to i24 -; CLMUL-NEXT: [[XOR_CRC_DATA:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[XOR_CRC_DATA]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 511) -; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i24 [[CLMUL_MU]], 255 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_MASK]], i24 81923) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i16 [[CHECKSUM]] to i8 +; CLMUL-NEXT: [[XOR_CRC_DATA:%.*]] = xor i8 [[CRC_CAST]], [[MSG]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 511) +; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 81923) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR1:%.*]] = lshr i24 [[XOR_CRC_MULT]], 8 ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[CRC_LSHR1]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -113,14 +114,15 @@ define i16 @crc16.le.tc8.udiv(i8 %msg, i16 %checksum) { ; CLMUL-LABEL: define i16 @crc16.le.tc8.udiv( ; CLMUL-SAME: i8 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i8 [[MSG]] to i24 -; CLMUL-NEXT: [[XOR_CRC_DATA:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[XOR_CRC_DATA]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 511) -; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i24 [[CLMUL_MU]], 255 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_MASK]], i24 81923) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i16 [[CHECKSUM]] to i8 +; CLMUL-NEXT: [[XOR_CRC_DATA:%.*]] = xor i8 [[CRC_CAST]], [[MSG]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 511) +; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 81923) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR1:%.*]] = lshr i24 [[XOR_CRC_MULT]], 8 ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[CRC_LSHR1]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -181,14 +183,13 @@ define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) { ; CLMUL-LABEL: define i16 @crc16.le.tc16( ; CLMUL-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 114687) +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 114687) ; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 ; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 81923) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -247,16 +248,17 @@ define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) { ; CLMUL-LABEL: define i8 @crc8.le.tc16( ; CLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 24423) +; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 24423) ; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i32 [[CLMUL_MU]], 65535 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_MASK]], i32 59) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_CAST]], [[CLMUL_GP]] -; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i32 [[XOR_CRC_MULT]], 16 -; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[CRC_LSHR2]] to i8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_MASK]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 59) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_RECAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i24 [[XOR_CRC_MULT]], 16 +; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[CRC_LSHR2]] to i8 ; CLMUL-NEXT: br label %[[LOOP:.*]] ; CLMUL: [[LOOP]]: ; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -321,10 +323,13 @@ define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) { ; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CRC_INIT]] to i24 ; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CRC_INIT]] to i24 +; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -383,10 +388,13 @@ define i16 @crc16.be.tc8.crc.init.arg(i16 %crc.init) { ; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CRC_INIT]] to i24 ; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CRC_INIT]] to i24 +; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -442,10 +450,13 @@ define i16 @crc16.be.tc8.crc.init.arg.flipped.sb.check(i16 %crc.init) { ; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CRC_INIT]] to i24 ; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CRC_INIT]] to i24 +; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -520,12 +531,12 @@ define i8 @crc8.be.tc8.ptr.nested.loop(ptr %msg, i32 %loop.limit) { ; CLMUL-NEXT: [[MSG_OUTER_IV:%.*]] = getelementptr inbounds i8, ptr [[MSG]], i64 [[OUTER_IV_EXT]] ; CLMUL-NEXT: [[MSG_LOAD:%.*]] = load i8, ptr [[MSG_OUTER_IV]], align 1 ; CLMUL-NEXT: [[CRC_INIT:%.*]] = xor i8 [[MSG_LOAD]], [[CRC_OUTER]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CRC_INIT]] to i16 -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[CRC_CAST]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284) +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[CRC_INIT]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 284) ; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 ; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285) -; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i16 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CRC_INIT]] to i16 +; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i16 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL1]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8 ; CLMUL-NEXT: br label %[[INNER_LOOP:.*]] @@ -602,10 +613,13 @@ define i16 @crc16.be.tc8.misalign(i8 %msg, i16 %checksum) { ; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 ; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 273) -; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 69665) -; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 69665) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -666,14 +680,13 @@ define i16 @crc16.be.tc16(i16 %msg, i16 %checksum) { ; CLMUL-LABEL: define i16 @crc16.be.tc16( ; CLMUL-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 69936) +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CHECKSUM]], [[MSG]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i16 [[XOR_CRC_DATA1]] to i32 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 69936) ; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 ; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 69665) -; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i32 +; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_RECAST]], 16 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -731,17 +744,20 @@ define i8 @crc8.be.tc16.misalign(i16 %msg, i8 %checksum) { ; CLMUL-LABEL: define i8 @crc8.be.tc16.misalign( ; CLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i32 [[XOR_CRC_DATA1]], 8 -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[CRC_ALIGN_TC]], 65535 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 72779) +; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i24 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i24 [[CRC_CAST]], [[DATA_CAST]] +; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i24 [[XOR_CRC_DATA1]], 8 +; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 65535 +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i24 [[CRC_TCBITS]] to i32 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[TCBITS_CAST]], i32 72779) ; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 285) -; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16 -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]] -; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = trunc i32 [[QUOT_LSHR]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 285) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i24 [[CRC_RECAST]], 16 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL2]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i8 ; CLMUL-NEXT: br label %[[LOOP:.*]] ; CLMUL: [[LOOP]]: ; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]] @@ -798,13 +814,14 @@ define i8 @crc8.be.tc8.data16.misalign(i16 %msg, i8 %checksum) { ; CLMUL-LABEL: define i8 @crc8.be.tc8.data16.misalign( ; CLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284) +; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i16 [[MSG]] to i8 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 284) ; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 ; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285) -; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL2]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -864,14 +881,16 @@ define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) { ; CLMUL-LABEL: define i32 @crc32.le.tc8.data32( ; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273) -; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i40 [[CLMUL_MU]], 255 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_MASK]], i40 67601) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i8 +; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CRC_CAST]], [[DATA_CAST]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i40 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_CAST]], i40 67601) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i40 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i40 [[XOR_CRC_MULT]], 8 ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i40 [[CRC_LSHR2]] to i32 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -930,14 +949,14 @@ define i8 @crc8.le.tc8.data32(i8 %checksum, i32 %msg) { ; CLMUL-LABEL: define i8 @crc8.le.tc8.data32( ; CLMUL-SAME: i8 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i16 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 107) +; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CHECKSUM]], [[DATA_CAST]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 107) ; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 ; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_MASK]], i16 119) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i8 [[CHECKSUM]] to i16 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i16 [[XOR_CRC_MULT]], 8 ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[CRC_LSHR2]] to i8 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -998,10 +1017,13 @@ define i16 @crc16.be.tc8.zext.data(i8 %msg, i16 %checksum) { ; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i24 ; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = lshr i24 [[CRC_CAST]], 8 ; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i24 [[CRC_ALIGN_TC]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i24 @llvm.clmul.i24(i24 [[CRC_TCBITS]], i24 257) -; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i24 [[CLMUL_MU]], 8 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_LSHR]], i24 65794) -; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_CAST]], 8 +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = trunc i24 [[CRC_TCBITS]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 257) +; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_LSHR]] to i24 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i24 @llvm.clmul.i24(i24 [[QUOT_CAST]], i24 65794) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i16 [[CHECKSUM]] to i24 +; CLMUL-NEXT: [[CRC_SHL1:%.*]] = shl i24 [[CRC_RECAST]], 8 ; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i24 [[CRC_SHL1]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_NEXT2:%.*]] = trunc i24 [[XOR_CRC_MULT]] to i16 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -1189,14 +1211,16 @@ define i32 @crc.disabled.optsize(i32 %checksum, i32 %msg) optsize { ; CLMUL-LABEL: define i32 @crc.disabled.optsize( ; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273) -; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i40 [[CLMUL_MU]], 255 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_MASK]], i40 67601) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i8 +; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i8 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i8 [[CRC_CAST]], [[DATA_CAST]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i8 [[XOR_CRC_DATA1]] to i16 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[TCBITS_CAST]], i16 273) +; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i16 [[CLMUL_MU]], 255 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i16 [[QUOT_MASK]] to i40 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i40 @llvm.clmul.i40(i40 [[QUOT_CAST]], i40 67601) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i40 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i40 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i40 [[XOR_CRC_MULT]], 8 ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i40 [[CRC_LSHR2]] to i32 ; CLMUL-NEXT: br label %[[LOOP:.*]] From 632434ce928684ecba87e2071f6f9aefed799779 Mon Sep 17 00:00:00 2001 From: Sean Clarke Date: Fri, 17 Jul 2026 09:37:39 -0500 Subject: [PATCH 2/3] Simplify clmul width check logic --- .../Transforms/Scalar/LoopIdiomRecognize.cpp | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp index 3cb4c1fc6befe..466a41f5604df 100644 --- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp +++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp @@ -265,9 +265,7 @@ class LoopIdiomRecognize { bool avoidLIRForMultiBlockLoop(bool IsMemset = false, bool IsLoopMemset = false); bool optimizeCRCLoop(const PolynomialInfo &Info); - void optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, - IntegerType *ClmulMuTy, - IntegerType *ClmulGPTy); + void optimizeCRCLoopUsingClmul(const PolynomialInfo &Info); void optimizeCRCLoopUsingTableLookup(const PolynomialInfo &Info); /// @} @@ -1591,16 +1589,10 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) { if (TT.getArch() == Triple::hexagon) return false; - // Calculate the widths for the clmul operations that would be required. - LLVMContext &Ctx = Info.LHS->getContext(); - unsigned CRCBW = Info.LHS->getType()->getIntegerBitWidth(); - auto *ClmulMuTy = IntegerType::get(Ctx, 2 * Info.TripCount); - auto *ClmulGPTy = IntegerType::get(Ctx, CRCBW + Info.TripCount); - // The force-crc-clmul flag should cause the clmul optimization to run // unconditionally. if (ForceCRCClmul) { - optimizeCRCLoopUsingClmul(Info, ClmulMuTy, ClmulGPTy); + optimizeCRCLoopUsingClmul(Info); return true; } @@ -1614,11 +1606,16 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) { } // The clmul optimization should only be applied if clmul with the required - // bit width is a fast operation on the target. + // bit width is a fast operation on the target. The first clmul needs 2*TC + // bits, and the second clmul needs CRCBW+TC bits, so test the widest clmul. // TODO: If clmul exists on the target but not for the required width, it // might be possible to split into multiple iterations of reduction. - if (TTI->haveFastClmul(ClmulMuTy) && TTI->haveFastClmul(ClmulGPTy)) { - optimizeCRCLoopUsingClmul(Info, ClmulMuTy, ClmulGPTy); + unsigned CRCBW = Info.LHS->getType()->getIntegerBitWidth(); + IntegerType *WidestClmulTy = + IntegerType::get(Info.LHS->getContext(), + std::max(2 * Info.TripCount, CRCBW + Info.TripCount)); + if (TTI->haveFastClmul(WidestClmulTy)) { + optimizeCRCLoopUsingClmul(Info); return true; } @@ -1628,9 +1625,7 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) { // The algorithm used in this optimization is a Polynomial (GF(2)) Barrett // Reduction based on Intel's "Fast CRC Computation for Generic Polynomials // Using PCLMULQDQ Instruction" white paper (December 2009). -void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, - IntegerType *ClmulMuTy, - IntegerType *ClmulGPTy) { +void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info) { Type *CRCTy = Info.LHS->getType(); LLVMContext &Ctx = CRCTy->getContext(); unsigned CRCBW = CRCTy->getIntegerBitWidth(); @@ -1638,6 +1633,10 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info, // regardless of whether the actual data bit width matches (if auxiliary data // is even used at all). unsigned TC = Info.TripCount; + // Based on the clmul inputs, the first clmul needs 2*TC bits, and the second + // needs CRCBW+TC bits. + IntegerType *ClmulMuTy = IntegerType::get(Ctx, 2 * TC); + IntegerType *ClmulGPTy = IntegerType::get(Ctx, CRCBW + TC); // First, generate the constants required for GF(2) Barrett reduction. auto [Mu, FullGenPoly] = HashRecognize::genBarrettConstants(Info); From a341e3deead70c62683e0d7712e3228d894c26d2 Mon Sep 17 00:00:00 2001 From: Sean Clarke Date: Fri, 17 Jul 2026 11:07:48 -0500 Subject: [PATCH 3/3] Update test after rebase --- .../LoopIdiom/cyclic-redundancy-check.ll | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll index ca90b34b5c4b5..b7cf735ef9a6d 100644 --- a/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll +++ b/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll @@ -1081,14 +1081,16 @@ define i32 @crc32.le.tc4.data32(i32 %checksum, i32 %msg) optsize { ; CLMUL-LABEL: define i32 @crc32.le.tc4.data32( ; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0:[0-9]+]] { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i36 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i36 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i36 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i36 [[XOR_CRC_DATA1]], 15 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i36 @llvm.clmul.i36(i36 [[CRC_TCBITS]], i36 17) -; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i36 [[CLMUL_MU]], 15 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i36 @llvm.clmul.i36(i36 [[QUOT_MASK]], i36 67601) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i36 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i4 +; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i4 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i4 [[CRC_CAST]], [[DATA_CAST]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i4 [[XOR_CRC_DATA1]] to i8 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i8 @llvm.clmul.i8(i8 [[TCBITS_CAST]], i8 17) +; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i8 [[CLMUL_MU]], 15 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i8 [[QUOT_MASK]] to i36 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i36 @llvm.clmul.i36(i36 [[QUOT_CAST]], i36 67601) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i36 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i36 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i36 [[XOR_CRC_MULT]], 4 ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i36 [[CRC_LSHR2]] to i32 ; CLMUL-NEXT: br label %[[LOOP:.*]] @@ -1146,14 +1148,16 @@ define i32 @crc32.le.tc30.data32(i32 %checksum, i32 %msg) optsize { ; CLMUL-LABEL: define i32 @crc32.le.tc30.data32( ; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] { ; CLMUL-NEXT: [[ENTRY:.*:]] -; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i62 -; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i62 -; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i62 [[CRC_CAST]], [[DATA_CAST]] -; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i62 [[XOR_CRC_DATA1]], 1073741823 -; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i62 @llvm.clmul.i62(i62 [[CRC_TCBITS]], i62 475535633) -; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i62 [[CLMUL_MU]], 1073741823 -; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i62 @llvm.clmul.i62(i62 [[QUOT_MASK]], i62 67601) -; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i62 [[CRC_CAST]], [[CLMUL_GP]] +; CLMUL-NEXT: [[CRC_CAST:%.*]] = trunc i32 [[CHECKSUM]] to i30 +; CLMUL-NEXT: [[DATA_CAST:%.*]] = trunc i32 [[MSG]] to i30 +; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i30 [[CRC_CAST]], [[DATA_CAST]] +; CLMUL-NEXT: [[TCBITS_CAST:%.*]] = zext i30 [[XOR_CRC_DATA1]] to i60 +; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i60 @llvm.clmul.i60(i60 [[TCBITS_CAST]], i60 475535633) +; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i60 [[CLMUL_MU]], 1073741823 +; CLMUL-NEXT: [[QUOT_CAST:%.*]] = zext i60 [[QUOT_MASK]] to i62 +; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i62 @llvm.clmul.i62(i62 [[QUOT_CAST]], i62 67601) +; CLMUL-NEXT: [[CRC_RECAST:%.*]] = zext i32 [[CHECKSUM]] to i62 +; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i62 [[CRC_RECAST]], [[CLMUL_GP]] ; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i62 [[XOR_CRC_MULT]], 30 ; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i62 [[CRC_LSHR2]] to i32 ; CLMUL-NEXT: br label %[[LOOP:.*]]