[HashRecognize] Remove byte-multiple trip count requirement#210086
Open
xarkenz wants to merge 3 commits into
Open
[HashRecognize] Remove byte-multiple trip count requirement#210086xarkenz wants to merge 3 commits into
xarkenz wants to merge 3 commits into
Conversation
|
@llvm/pr-subscribers-llvm-transforms @llvm/pr-subscribers-llvm-analysis Author: Sean Clarke (xarkenz) ChangesThe newly added carryless-multiply CRC loop optimization (#203405) does not require a byte-multiple trip count unlike the Sarwate method which was previously implemented. Loosen this requirement in HashRecognize and reapply it in Full diff: https://github.com/llvm/llvm-project/pull/210086.diff 4 Files Affected:
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 3846c822432e1..d29742ef571bd 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -511,8 +511,8 @@ std::variant<PolynomialInfo, StringRef> HashRecognize::recognizeCRC() const {
if (!Latch || !Exit || !IndVar || L.getNumBlocks() != 1)
return "Loop not in canonical form";
unsigned TC = SE.getSmallConstantTripCount(&L);
- if (!TC || TC % 8)
- return "Unable to find a small constant byte-multiple trip count";
+ if (!TC)
+ return "Unable to find a small constant trip count";
auto R = getRecurrences(Latch, IndVar, L);
if (!R)
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index e831a89a79113..e674143099a53 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1608,7 +1608,7 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
// be used to determine which optimization to use. Until then, only apply the
// clmul optimization when optimizing for size, since a lookup table is not
// viable in that case.
- if (!ApplyCodeSizeHeuristics) {
+ if (!ApplyCodeSizeHeuristics && Info.TripCount % 8 == 0) {
optimizeCRCLoopUsingTableLookup(Info);
return true;
}
@@ -1743,6 +1743,8 @@ void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
void LoopIdiomRecognize::optimizeCRCLoopUsingTableLookup(
const PolynomialInfo &Info) {
+ assert(Info.TripCount % 8 == 0 && "A byte-multiple trip count is required");
+
// First, create a new GlobalVariable corresponding to the
// Sarwate-lookup-table.
Type *CRCTy = Info.LHS->getType();
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index e3dc53a3e66ed..b7139a7367829 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -737,12 +737,31 @@ exit: ; preds = %loop
ret i16 %crc.next
}
-; Negative tests
-
-define i16 @not.crc.non.const.tc(i16 %crc.init, i32 %loop.limit) {
-; CHECK-LABEL: 'not.crc.non.const.tc'
-; CHECK-NEXT: Did not find a hash algorithm
-; CHECK-NEXT: Reason: Unable to find a small constant byte-multiple trip count
+define i16 @crc16.be.tc4.crc.init.arg(i16 %crc.init) {
+; CHECK-LABEL: 'crc16.be.tc4.crc.init.arg'
+; CHECK-NEXT: Found big-endian CRC-16 loop with trip count 4
+; CHECK-NEXT: Initial CRC: i16 %crc.init
+; CHECK-NEXT: Generating polynomial: 4129
+; CHECK-NEXT: Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+; CHECK-NEXT: Computed CRC lookup table:
+; CHECK-NEXT: 0 4129 8258 12387 16516 20645 24774 28903 33032 37161 41290 45419 49548 53677 57806 61935
+; CHECK-NEXT: 4657 528 12915 8786 21173 17044 29431 25302 37689 33560 45947 41818 54205 50076 62463 58334
+; CHECK-NEXT: 9314 13379 1056 5121 25830 29895 17572 21637 42346 46411 34088 38153 58862 62927 50604 54669
+; CHECK-NEXT: 13907 9842 5649 1584 30423 26358 22165 18100 46939 42874 38681 34616 63455 59390 55197 51132
+; CHECK-NEXT: 18628 22757 26758 30887 2112 6241 10242 14371 51660 55789 59790 63919 35144 39273 43274 47403
+; CHECK-NEXT: 23285 19156 31415 27286 6769 2640 14899 10770 56317 52188 64447 60318 39801 35672 47931 43802
+; CHECK-NEXT: 27814 31879 19684 23749 11298 15363 3168 7233 60846 64911 52716 56781 44330 48395 36200 40265
+; CHECK-NEXT: 32407 28342 24277 20212 15891 11826 7761 3696 65439 61374 57309 53244 48923 44858 40793 36728
+; CHECK-NEXT: 37256 33193 45514 41451 53516 49453 61774 57711 4224 161 12482 8419 20484 16421 28742 24679
+; CHECK-NEXT: 33721 37784 41979 46042 49981 54044 58239 62302 689 4752 8947 13010 16949 21012 25207 29270
+; CHECK-NEXT: 46570 42443 38312 34185 62830 58703 54572 50445 13538 9411 5280 1153 29798 25671 21540 17413
+; CHECK-NEXT: 42971 47098 34713 38840 59231 63358 50973 55100 9939 14066 1681 5808 26199 30326 17941 22068
+; CHECK-NEXT: 55628 51565 63758 59695 39368 35305 47498 43435 22596 18533 30726 26663 6336 2273 14466 10403
+; CHECK-NEXT: 52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
+; CHECK-NEXT: 64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
+; CHECK-NEXT: 61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
+; CHECK-NEXT: Computed CRC Barrett constants:
+; CHECK-NEXT: Mu = 17, FullGenPoly = 69665
;
entry:
br label %loop
@@ -752,20 +771,22 @@ loop: ; preds = %loop, %entry
%crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
%crc.shl = shl i16 %crc, 1
%crc.xor = xor i16 %crc.shl, 4129
- %check.sb = icmp sge i16 %crc, 0
- %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+ %check.sb = icmp slt i16 %crc, 0
+ %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
%iv.next = add nuw nsw i32 %iv, 1
- %exit.cond = icmp samesign ult i32 %iv, %loop.limit
- br i1 %exit.cond, label %loop, label %exit
+ %exit.cond = icmp samesign eq i32 %iv, 3
+ br i1 %exit.cond, label %exit, label %loop
exit: ; preds = %loop
ret i16 %crc.next
}
-define i16 @not.crc.non.canonical.not.multiple.8(i16 %crc.init) {
-; CHECK-LABEL: 'not.crc.non.canonical.not.multiple.8'
+; Negative tests
+
+define i16 @not.crc.non.const.tc(i16 %crc.init, i32 %loop.limit) {
+; CHECK-LABEL: 'not.crc.non.const.tc'
; CHECK-NEXT: Did not find a hash algorithm
-; CHECK-NEXT: Reason: Unable to find a small constant byte-multiple trip count
+; CHECK-NEXT: Reason: Unable to find a small constant trip count
;
entry:
br label %loop
@@ -775,11 +796,11 @@ loop: ; preds = %loop, %entry
%crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
%crc.shl = shl i16 %crc, 1
%crc.xor = xor i16 %crc.shl, 4129
- %check.sb = icmp slt i16 %crc, 0
- %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+ %check.sb = icmp sge i16 %crc, 0
+ %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
%iv.next = add nuw nsw i32 %iv, 1
- %exit.cond = icmp samesign eq i32 %iv, 3
- br i1 %exit.cond, label %exit, label %loop
+ %exit.cond = icmp samesign ult i32 %iv, %loop.limit
+ br i1 %exit.cond, label %loop, label %exit
exit: ; preds = %loop
ret i16 %crc.next
diff --git a/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll b/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll
index 299a84099cb54..d4c2f59e9003a 100644
--- a/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll
+++ b/llvm/test/Transforms/LoopIdiom/cyclic-redundancy-check.ll
@@ -1033,9 +1033,139 @@ exit: ; preds = %loop
ret i16 %crc.next
}
+define i32 @crc32.le.tc4.data32(i32 %checksum, i32 %msg) optsize {
+; TABLE-LABEL: define i32 @crc32.le.tc4.data32(
+; TABLE-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0:[0-9]+]] {
+; TABLE-NEXT: [[ENTRY:.*]]:
+; TABLE-NEXT: br label %[[LOOP:.*]]
+; TABLE: [[LOOP]]:
+; TABLE-NEXT: [[CRC:%.*]] = phi i32 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[CRC_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[DATA:%.*]] = phi i32 [ [[MSG]], %[[ENTRY]] ], [ [[DATA_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[XOR_CRC_DATA:%.*]] = xor i32 [[CRC]], [[DATA]]
+; TABLE-NEXT: [[SB_CRC_DATA:%.*]] = and i32 [[XOR_CRC_DATA]], 1
+; TABLE-NEXT: [[CHECK_SB:%.*]] = icmp eq i32 [[SB_CRC_DATA]], 0
+; TABLE-NEXT: [[CRC_LSHR:%.*]] = lshr i32 [[CRC]], 1
+; TABLE-NEXT: [[CRC_XOR:%.*]] = xor i32 [[CRC_LSHR]], 33800
+; TABLE-NEXT: [[CRC_NEXT]] = select i1 [[CHECK_SB]], i32 [[CRC_LSHR]], i32 [[CRC_XOR]]
+; TABLE-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; TABLE-NEXT: [[DATA_NEXT]] = lshr i32 [[DATA]], 1
+; TABLE-NEXT: [[EXIT_COND:%.*]] = icmp samesign ult i8 [[IV]], 3
+; TABLE-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; TABLE: [[EXIT]]:
+; TABLE-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i32 [ [[CRC_NEXT]], %[[LOOP]] ]
+; TABLE-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
+;
+; CLMUL-LABEL: define i32 @crc32.le.tc4.data32(
+; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CLMUL-NEXT: [[ENTRY:.*:]]
+; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i36
+; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i36
+; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i36 [[CRC_CAST]], [[DATA_CAST]]
+; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i36 [[XOR_CRC_DATA1]], 15
+; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i36 @llvm.clmul.i36(i36 [[CRC_TCBITS]], i36 17)
+; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i36 [[CLMUL_MU]], 15
+; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i36 @llvm.clmul.i36(i36 [[QUOT_MASK]], i36 67601)
+; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i36 [[CRC_CAST]], [[CLMUL_GP]]
+; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i36 [[XOR_CRC_MULT]], 4
+; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i36 [[CRC_LSHR2]] to i32
+; CLMUL-NEXT: br label %[[LOOP:.*]]
+; CLMUL: [[LOOP]]:
+; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CLMUL: [[EXIT]]:
+; CLMUL-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i32 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; CLMUL-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+ %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+ %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+ %xor.crc.data = xor i32 %crc, %data
+ %sb.crc.data = and i32 %xor.crc.data, 1
+ %check.sb = icmp eq i32 %sb.crc.data, 0
+ %crc.lshr = lshr i32 %crc, 1
+ %crc.xor = xor i32 %crc.lshr, 33800
+ %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+ %iv.next = add nuw nsw i8 %iv, 1
+ %data.next = lshr i32 %data, 1
+ %exit.cond = icmp samesign ult i8 %iv, 3
+ br i1 %exit.cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret i32 %crc.next
+}
+
+define i32 @crc32.le.tc30.data32(i32 %checksum, i32 %msg) optsize {
+; TABLE-LABEL: define i32 @crc32.le.tc30.data32(
+; TABLE-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] {
+; TABLE-NEXT: [[ENTRY:.*]]:
+; TABLE-NEXT: br label %[[LOOP:.*]]
+; TABLE: [[LOOP]]:
+; TABLE-NEXT: [[CRC:%.*]] = phi i32 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[CRC_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[DATA:%.*]] = phi i32 [ [[MSG]], %[[ENTRY]] ], [ [[DATA_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[XOR_CRC_DATA:%.*]] = xor i32 [[CRC]], [[DATA]]
+; TABLE-NEXT: [[SB_CRC_DATA:%.*]] = and i32 [[XOR_CRC_DATA]], 1
+; TABLE-NEXT: [[CHECK_SB:%.*]] = icmp eq i32 [[SB_CRC_DATA]], 0
+; TABLE-NEXT: [[CRC_LSHR:%.*]] = lshr i32 [[CRC]], 1
+; TABLE-NEXT: [[CRC_XOR:%.*]] = xor i32 [[CRC_LSHR]], 33800
+; TABLE-NEXT: [[CRC_NEXT]] = select i1 [[CHECK_SB]], i32 [[CRC_LSHR]], i32 [[CRC_XOR]]
+; TABLE-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; TABLE-NEXT: [[DATA_NEXT]] = lshr i32 [[DATA]], 1
+; TABLE-NEXT: [[EXIT_COND:%.*]] = icmp samesign ult i8 [[IV]], 29
+; TABLE-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; TABLE: [[EXIT]]:
+; TABLE-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i32 [ [[CRC_NEXT]], %[[LOOP]] ]
+; TABLE-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
+;
+; CLMUL-LABEL: define i32 @crc32.le.tc30.data32(
+; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] {
+; CLMUL-NEXT: [[ENTRY:.*:]]
+; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i62
+; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i62
+; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i62 [[CRC_CAST]], [[DATA_CAST]]
+; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i62 [[XOR_CRC_DATA1]], 1073741823
+; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i62 @llvm.clmul.i62(i62 [[CRC_TCBITS]], i62 475535633)
+; CLMUL-NEXT: [[QUOT_MASK:%.*]] = and i62 [[CLMUL_MU]], 1073741823
+; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i62 @llvm.clmul.i62(i62 [[QUOT_MASK]], i62 67601)
+; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i62 [[CRC_CAST]], [[CLMUL_GP]]
+; CLMUL-NEXT: [[CRC_LSHR2:%.*]] = lshr i62 [[XOR_CRC_MULT]], 30
+; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i62 [[CRC_LSHR2]] to i32
+; CLMUL-NEXT: br label %[[LOOP:.*]]
+; CLMUL: [[LOOP]]:
+; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CLMUL: [[EXIT]]:
+; CLMUL-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i32 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; CLMUL-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+ %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+ %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+ %xor.crc.data = xor i32 %crc, %data
+ %sb.crc.data = and i32 %xor.crc.data, 1
+ %check.sb = icmp eq i32 %sb.crc.data, 0
+ %crc.lshr = lshr i32 %crc, 1
+ %crc.xor = xor i32 %crc.lshr, 33800
+ %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+ %iv.next = add nuw nsw i8 %iv, 1
+ %data.next = lshr i32 %data, 1
+ %exit.cond = icmp samesign ult i8 %iv, 29
+ br i1 %exit.cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret i32 %crc.next
+}
+
define i32 @crc.disabled.optsize(i32 %checksum, i32 %msg) optsize {
; TABLE-LABEL: define i32 @crc.disabled.optsize(
-; TABLE-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0:[0-9]+]] {
+; TABLE-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] {
; TABLE-NEXT: [[ENTRY:.*]]:
; TABLE-NEXT: br label %[[LOOP:.*]]
; TABLE: [[LOOP]]:
@@ -1057,7 +1187,7 @@ define i32 @crc.disabled.optsize(i32 %checksum, i32 %msg) optsize {
; TABLE-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
;
; CLMUL-LABEL: define i32 @crc.disabled.optsize(
-; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) #[[ATTR0]] {
; CLMUL-NEXT: [[ENTRY:.*:]]
; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40
; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
The newly added carryless-multiply CRC loop optimization (#203405) does not require a byte-multiple trip count unlike the Sarwate method which was previously implemented. Loosen this requirement in HashRecognize and reapply it in
optimizeCRCLoop.