[SLP] NFC. Use InstructionsState::getOpcode only when necessary. #120210
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-llvm-transforms Author: Han-Kuan Chen (HanKuanChen) ChangesFull diff: https://github.com/llvm/llvm-project/pull/120210.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d967813075bb9f..e97e414b3a95af 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1845,7 +1845,7 @@ class BoUpSLP {
InstructionsState S = getSameOpcode(Ops, TLI);
// Note: Only consider instructions with <= 2 operands to avoid
// complexity explosion.
- if (S.getOpcode() &&
+ if (S.getMainOp() &&
(S.getMainOp()->getNumOperands() <= 2 || !MainAltOps.empty() ||
!S.isAltShuffle()) &&
all_of(Ops, [&S](Value *V) {
@@ -2696,7 +2696,7 @@ class BoUpSLP {
OperandData &AltOp = getData(OpIdx, Lane);
InstructionsState OpS =
getSameOpcode({MainAltOps[OpIdx].front(), AltOp.V}, TLI);
- if (OpS.getOpcode() && OpS.isAltShuffle())
+ if (OpS.getMainOp() && OpS.isAltShuffle())
MainAltOps[OpIdx].push_back(AltOp.V);
}
}
@@ -8067,7 +8067,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
// Check if this is a duplicate of another entry.
- if (S.getOpcode()) {
+ if (S.getMainOp()) {
if (TreeEntry *E = getTreeEntry(S.getMainOp())) {
LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp()
<< ".\n");
@@ -8144,15 +8144,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
// Don't handle scalable vectors
- if (S.getOpcode() == Instruction::ExtractElement &&
- isa<ScalableVectorType>(
- cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
- LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
- return;
- }
+ if (auto *EE = dyn_cast_if_present<ExtractElementInst>(S.getMainOp()))
+ if (isa<ScalableVectorType>(EE->getVectorOperandType())) {
+ LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
+ if (TryToFindDuplicates(S))
+ newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
+ return;
+ }
// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
@@ -8168,7 +8167,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// vectorize.
auto &&NotProfitableForVectorization = [&S, this,
Depth](ArrayRef<Value *> VL) {
- if (!S.getOpcode() || !S.isAltShuffle() || VL.size() > 2)
+ if (!S.getMainOp() || !S.isAltShuffle() || VL.size() > 2)
return false;
if (VectorizableTree.size() < MinTreeSize)
return false;
@@ -8223,7 +8222,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
bool IsScatterVectorizeUserTE =
UserTreeIdx.UserTE &&
UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
- bool AreAllSameBlock = S.getOpcode() && allSameBlock(VL);
+ bool AreAllSameBlock = S.getMainOp() && allSameBlock(VL);
bool AreScatterAllGEPSameBlock =
(IsScatterVectorizeUserTE && VL.front()->getType()->isPointerTy() &&
VL.size() > 2 &&
@@ -8240,7 +8239,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE,
SortedIndices));
bool AreAllSameInsts = AreAllSameBlock || AreScatterAllGEPSameBlock;
- if (!AreAllSameInsts || (!S.getOpcode() && allConstant(VL)) || isSplat(VL) ||
+ if (!AreAllSameInsts || (!S.getMainOp() && allConstant(VL)) || isSplat(VL) ||
(isa_and_present<InsertElementInst, ExtractValueInst, ExtractElementInst>(
S.getMainOp()) &&
!all_of(VL, isVectorLikeInstWithConstOps)) ||
@@ -8253,7 +8252,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
// Don't vectorize ephemeral values.
- if (S.getOpcode() && !EphValues.empty()) {
+ if (S.getMainOp() && !EphValues.empty()) {
for (Value *V : VL) {
if (EphValues.count(V)) {
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
@@ -9730,7 +9729,7 @@ void BoUpSLP::transformNodes() {
if (IsSplat)
continue;
InstructionsState S = getSameOpcode(Slice, *TLI);
- if (!S.getOpcode() || S.isAltShuffle() || !allSameBlock(Slice) ||
+ if (!S.getMainOp() || S.isAltShuffle() || !allSameBlock(Slice) ||
(S.getOpcode() == Instruction::Load &&
areKnownNonVectorizableLoads(Slice)) ||
(S.getOpcode() != Instruction::Load && !has_single_bit(VF)))
@@ -14394,12 +14393,12 @@ BoUpSLP::TreeEntry *BoUpSLP::getMatchedVectorizedOperand(const TreeEntry *E,
ArrayRef<Value *> VL = E->getOperand(NodeIdx);
InstructionsState S = getSameOpcode(VL, *TLI);
// Special processing for GEPs bundle, which may include non-gep values.
- if (!S.getOpcode() && VL.front()->getType()->isPointerTy()) {
+ if (!S.getMainOp() && VL.front()->getType()->isPointerTy()) {
const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
if (It != VL.end())
S = getSameOpcode(*It, *TLI);
}
- if (!S.getOpcode())
+ if (!S.getMainOp())
return nullptr;
auto CheckSameVE = [&](const TreeEntry *VE) {
return VE->isSame(VL) &&
@@ -18376,7 +18375,7 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
hasFullVectorsOrPowerOf2(*TTI, ValOps.front()->getType(),
ValOps.size()) ||
(VectorizeNonPowerOf2 && has_single_bit(ValOps.size() + 1));
- if ((!IsAllowedSize && S.getOpcode() &&
+ if ((!IsAllowedSize && S.getMainOp() &&
S.getOpcode() != Instruction::Load &&
(!S.getMainOp()->isSafeToRemove() ||
any_of(ValOps.getArrayRef(),
@@ -18387,8 +18386,8 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
return !Stores.contains(U);
}));
}))) ||
- (ValOps.size() > Chain.size() / 2 && !S.getOpcode())) {
- Size = (!IsAllowedSize && S.getOpcode()) ? 1 : 2;
+ (ValOps.size() > Chain.size() / 2 && !S.getMainOp())) {
+ Size = (!IsAllowedSize && S.getMainOp()) ? 1 : 2;
return false;
}
}
@@ -18912,7 +18911,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
// Check that all of the parts are instructions of the same type,
// we permit an alternate opcode via InstructionsState.
InstructionsState S = getSameOpcode(VL, *TLI);
- if (!S.getOpcode())
+ if (!S.getMainOp())
return false;
Instruction *I0 = S.getMainOp();
@@ -19724,8 +19723,8 @@ class HorizontalReduction {
// Also check if the instruction was folded to constant/other value.
auto *Inst = dyn_cast<Instruction>(RdxVal);
if ((Inst && isVectorLikeInstWithConstOps(Inst) &&
- (!S.getOpcode() || !S.isOpcodeOrAlt(Inst))) ||
- (S.getOpcode() && !Inst))
+ (!S.getMainOp() || !S.isOpcodeOrAlt(Inst))) ||
+ (S.getMainOp() && !Inst))
continue;
Candidates.push_back(RdxVal);
TrackedToOrig.try_emplace(RdxVal, OrigReducedVals[Cnt]);
@@ -21128,7 +21127,7 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI,
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
}
InstructionsState S = getSameOpcode({I1, I2}, TLI);
- if (S.getOpcode() && (IsCompatibility || !S.isAltShuffle()))
+ if (S.getMainOp() && (IsCompatibility || !S.isAltShuffle()))
continue;
if (IsCompatibility)
return false;
@@ -21283,7 +21282,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
if (NodeI1 != NodeI2)
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
InstructionsState S = getSameOpcode({I1, I2}, *TLI);
- if (S.getOpcode() && !S.isAltShuffle())
+ if (S.getMainOp() && !S.isAltShuffle())
continue;
return I1->getOpcode() < I2->getOpcode();
}
|
|
@llvm/pr-subscribers-vectorizers Author: Han-Kuan Chen (HanKuanChen) ChangesFull diff: https://github.com/llvm/llvm-project/pull/120210.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d967813075bb9f..e97e414b3a95af 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1845,7 +1845,7 @@ class BoUpSLP {
InstructionsState S = getSameOpcode(Ops, TLI);
// Note: Only consider instructions with <= 2 operands to avoid
// complexity explosion.
- if (S.getOpcode() &&
+ if (S.getMainOp() &&
(S.getMainOp()->getNumOperands() <= 2 || !MainAltOps.empty() ||
!S.isAltShuffle()) &&
all_of(Ops, [&S](Value *V) {
@@ -2696,7 +2696,7 @@ class BoUpSLP {
OperandData &AltOp = getData(OpIdx, Lane);
InstructionsState OpS =
getSameOpcode({MainAltOps[OpIdx].front(), AltOp.V}, TLI);
- if (OpS.getOpcode() && OpS.isAltShuffle())
+ if (OpS.getMainOp() && OpS.isAltShuffle())
MainAltOps[OpIdx].push_back(AltOp.V);
}
}
@@ -8067,7 +8067,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
// Check if this is a duplicate of another entry.
- if (S.getOpcode()) {
+ if (S.getMainOp()) {
if (TreeEntry *E = getTreeEntry(S.getMainOp())) {
LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp()
<< ".\n");
@@ -8144,15 +8144,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
// Don't handle scalable vectors
- if (S.getOpcode() == Instruction::ExtractElement &&
- isa<ScalableVectorType>(
- cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
- LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
- return;
- }
+ if (auto *EE = dyn_cast_if_present<ExtractElementInst>(S.getMainOp()))
+ if (isa<ScalableVectorType>(EE->getVectorOperandType())) {
+ LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
+ if (TryToFindDuplicates(S))
+ newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
+ return;
+ }
// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
@@ -8168,7 +8167,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// vectorize.
auto &&NotProfitableForVectorization = [&S, this,
Depth](ArrayRef<Value *> VL) {
- if (!S.getOpcode() || !S.isAltShuffle() || VL.size() > 2)
+ if (!S.getMainOp() || !S.isAltShuffle() || VL.size() > 2)
return false;
if (VectorizableTree.size() < MinTreeSize)
return false;
@@ -8223,7 +8222,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
bool IsScatterVectorizeUserTE =
UserTreeIdx.UserTE &&
UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
- bool AreAllSameBlock = S.getOpcode() && allSameBlock(VL);
+ bool AreAllSameBlock = S.getMainOp() && allSameBlock(VL);
bool AreScatterAllGEPSameBlock =
(IsScatterVectorizeUserTE && VL.front()->getType()->isPointerTy() &&
VL.size() > 2 &&
@@ -8240,7 +8239,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE,
SortedIndices));
bool AreAllSameInsts = AreAllSameBlock || AreScatterAllGEPSameBlock;
- if (!AreAllSameInsts || (!S.getOpcode() && allConstant(VL)) || isSplat(VL) ||
+ if (!AreAllSameInsts || (!S.getMainOp() && allConstant(VL)) || isSplat(VL) ||
(isa_and_present<InsertElementInst, ExtractValueInst, ExtractElementInst>(
S.getMainOp()) &&
!all_of(VL, isVectorLikeInstWithConstOps)) ||
@@ -8253,7 +8252,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
// Don't vectorize ephemeral values.
- if (S.getOpcode() && !EphValues.empty()) {
+ if (S.getMainOp() && !EphValues.empty()) {
for (Value *V : VL) {
if (EphValues.count(V)) {
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
@@ -9730,7 +9729,7 @@ void BoUpSLP::transformNodes() {
if (IsSplat)
continue;
InstructionsState S = getSameOpcode(Slice, *TLI);
- if (!S.getOpcode() || S.isAltShuffle() || !allSameBlock(Slice) ||
+ if (!S.getMainOp() || S.isAltShuffle() || !allSameBlock(Slice) ||
(S.getOpcode() == Instruction::Load &&
areKnownNonVectorizableLoads(Slice)) ||
(S.getOpcode() != Instruction::Load && !has_single_bit(VF)))
@@ -14394,12 +14393,12 @@ BoUpSLP::TreeEntry *BoUpSLP::getMatchedVectorizedOperand(const TreeEntry *E,
ArrayRef<Value *> VL = E->getOperand(NodeIdx);
InstructionsState S = getSameOpcode(VL, *TLI);
// Special processing for GEPs bundle, which may include non-gep values.
- if (!S.getOpcode() && VL.front()->getType()->isPointerTy()) {
+ if (!S.getMainOp() && VL.front()->getType()->isPointerTy()) {
const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
if (It != VL.end())
S = getSameOpcode(*It, *TLI);
}
- if (!S.getOpcode())
+ if (!S.getMainOp())
return nullptr;
auto CheckSameVE = [&](const TreeEntry *VE) {
return VE->isSame(VL) &&
@@ -18376,7 +18375,7 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
hasFullVectorsOrPowerOf2(*TTI, ValOps.front()->getType(),
ValOps.size()) ||
(VectorizeNonPowerOf2 && has_single_bit(ValOps.size() + 1));
- if ((!IsAllowedSize && S.getOpcode() &&
+ if ((!IsAllowedSize && S.getMainOp() &&
S.getOpcode() != Instruction::Load &&
(!S.getMainOp()->isSafeToRemove() ||
any_of(ValOps.getArrayRef(),
@@ -18387,8 +18386,8 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
return !Stores.contains(U);
}));
}))) ||
- (ValOps.size() > Chain.size() / 2 && !S.getOpcode())) {
- Size = (!IsAllowedSize && S.getOpcode()) ? 1 : 2;
+ (ValOps.size() > Chain.size() / 2 && !S.getMainOp())) {
+ Size = (!IsAllowedSize && S.getMainOp()) ? 1 : 2;
return false;
}
}
@@ -18912,7 +18911,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
// Check that all of the parts are instructions of the same type,
// we permit an alternate opcode via InstructionsState.
InstructionsState S = getSameOpcode(VL, *TLI);
- if (!S.getOpcode())
+ if (!S.getMainOp())
return false;
Instruction *I0 = S.getMainOp();
@@ -19724,8 +19723,8 @@ class HorizontalReduction {
// Also check if the instruction was folded to constant/other value.
auto *Inst = dyn_cast<Instruction>(RdxVal);
if ((Inst && isVectorLikeInstWithConstOps(Inst) &&
- (!S.getOpcode() || !S.isOpcodeOrAlt(Inst))) ||
- (S.getOpcode() && !Inst))
+ (!S.getMainOp() || !S.isOpcodeOrAlt(Inst))) ||
+ (S.getMainOp() && !Inst))
continue;
Candidates.push_back(RdxVal);
TrackedToOrig.try_emplace(RdxVal, OrigReducedVals[Cnt]);
@@ -21128,7 +21127,7 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI,
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
}
InstructionsState S = getSameOpcode({I1, I2}, TLI);
- if (S.getOpcode() && (IsCompatibility || !S.isAltShuffle()))
+ if (S.getMainOp() && (IsCompatibility || !S.isAltShuffle()))
continue;
if (IsCompatibility)
return false;
@@ -21283,7 +21282,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
if (NodeI1 != NodeI2)
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
InstructionsState S = getSameOpcode({I1, I2}, *TLI);
- if (S.getOpcode() && !S.isAltShuffle())
+ if (S.getMainOp() && !S.isAltShuffle())
continue;
return I1->getOpcode() < I2->getOpcode();
}
|
HanKuanChen
force-pushed
the
slp-nfc-InstructionsState-use-getMainOp
branch
from
28c4948 to
75ec467
Compare
Use isa, isa_and_present and dyn_cast_if_present instead of getOpcode.
HanKuanChen
force-pushed
the
slp-nfc-InstructionsState-use-getMainOp
branch
from
75ec467 to
eae78c0
Compare
|
Use isa_and_present: valid check: |
Better to have last ( |
|
Move to #120217. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
No description provided.