diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h index 535b623d31ac..023998d41141 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -495,10 +495,29 @@ namespace llvm { /// The typedef for ExprValueMap. /// - typedef DenseMap> ExprValueMapType; + typedef std::pair ValueOffsetPair; + typedef DenseMap> ExprValueMapType; /// ExprValueMap -- This map records the original values from which /// the SCEV expr is generated from. + /// + /// We want to represent the mapping as SCEV -> ValueOffsetPair instead + /// of SCEV -> Value: + /// Suppose we know S1 expands to V1, and + /// S1 = S2 + C_a + /// S3 = S2 + C_b + /// where C_a and C_b are different SCEVConstants. Then we'd like to + /// expand S3 as V1 - C_a + C_b instead of expanding S2 literally. + /// It is helpful when S2 is a complex SCEV expr. + /// + /// In order to do that, we represent ExprValueMap as a mapping from + /// SCEV to ValueOffsetPair. We will save both S1->{V1, 0} and + /// S2->{V1, C_a} into the map when we create SCEV for V1. When S3 + /// is expanded, it will first expand S2 to V1 - C_a because of + /// S2->{V1, C_a} in the map, then expand S3 to V1 - C_a + C_b. + /// + /// Note: S->{V, Offset} in the ExprValueMap means S can be expanded + /// to V - Offset. ExprValueMapType ExprValueMap; /// The typedef for ValueExprMap. @@ -1181,7 +1200,7 @@ namespace llvm { bool containsAddRecurrence(const SCEV *S); /// Return the Value set from which the SCEV expr is generated. - SetVector *getSCEVValues(const SCEV *S); + SetVector *getSCEVValues(const SCEV *S); /// Erase Value from ValueExprMap and ExprValueMap. void eraseValueFromMap(Value *V); diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h index 1acf952ab70c..337a11b6dd00 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -14,6 +14,7 @@ #ifndef LLVM_ANALYSIS_SCALAREVOLUTIONEXPANDER_H #define LLVM_ANALYSIS_SCALAREVOLUTIONEXPANDER_H +#include "llvm/ADT/Optional.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ScalarEvolutionNormalization.h" #include "llvm/Analysis/TargetFolder.h" @@ -284,7 +285,15 @@ namespace llvm { void setChainedPhi(PHINode *PN) { ChainedPhis.insert(PN); } - /// \brief Try to find LLVM IR value for S available at the point At. + /// Try to find existing LLVM IR value for S available at the point At. + Value *getExactExistingExpansion(const SCEV *S, const Instruction *At, + Loop *L); + + /// Try to find the ValueOffsetPair for S. The function is mainly + /// used to check whether S can be expanded cheaply. + /// If this returns a non-None value, we know we can codegen the + /// `ValueOffsetPair` into a suitable expansion identical with S + /// so that S can be expanded cheaply. /// /// L is a hint which tells in which loop to look for the suitable value. /// On success return value which is equivalent to the expanded S at point @@ -292,7 +301,9 @@ namespace llvm { /// /// Note that this function does not perform an exhaustive search. I.e if it /// didn't find any value it does not mean that there is no such value. - Value *findExistingExpansion(const SCEV *S, const Instruction *At, Loop *L); + /// + Optional + getRelatedExistingExpansion(const SCEV *S, const Instruction *At, Loop *L); private: LLVMContext &getContext() const { return SE.getContext(); } @@ -325,7 +336,8 @@ namespace llvm { PointerType *PTy, Type *Ty, Value *V); /// \brief Find a previous Value in ExprValueMap for expand. - Value *FindValueInExprValueMap(const SCEV *S, const Instruction *InsertPt); + ScalarEvolution::ValueOffsetPair + FindValueInExprValueMap(const SCEV *S, const Instruction *InsertPt); Value *expand(const SCEV *S); diff --git a/contrib/llvm/lib/Analysis/ScalarEvolution.cpp b/contrib/llvm/lib/Analysis/ScalarEvolution.cpp index e42a4b574d90..8fefada6fdbc 100644 --- a/contrib/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/contrib/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3378,8 +3378,28 @@ bool ScalarEvolution::containsAddRecurrence(const SCEV *S) { return F.FoundOne; } -/// Return the Value set from S. -SetVector *ScalarEvolution::getSCEVValues(const SCEV *S) { +/// Try to split a SCEVAddExpr into a pair of {SCEV, ConstantInt}. +/// If \p S is a SCEVAddExpr and is composed of a sub SCEV S' and an +/// offset I, then return {S', I}, else return {\p S, nullptr}. +static std::pair splitAddExpr(const SCEV *S) { + const auto *Add = dyn_cast(S); + if (!Add) + return {S, nullptr}; + + if (Add->getNumOperands() != 2) + return {S, nullptr}; + + auto *ConstOp = dyn_cast(Add->getOperand(0)); + if (!ConstOp) + return {S, nullptr}; + + return {Add->getOperand(1), ConstOp->getValue()}; +} + +/// Return the ValueOffsetPair set for \p S. \p S can be represented +/// by the value and offset from any ValueOffsetPair in the set. +SetVector * +ScalarEvolution::getSCEVValues(const SCEV *S) { ExprValueMapType::iterator SI = ExprValueMap.find_as(S); if (SI == ExprValueMap.end()) return nullptr; @@ -3387,24 +3407,31 @@ SetVector *ScalarEvolution::getSCEVValues(const SCEV *S) { if (VerifySCEVMap) { // Check there is no dangling Value in the set returned. for (const auto &VE : SI->second) - assert(ValueExprMap.count(VE)); + assert(ValueExprMap.count(VE.first)); } #endif return &SI->second; } -/// Erase Value from ValueExprMap and ExprValueMap. If ValueExprMap.erase(V) is -/// not used together with forgetMemoizedResults(S), eraseValueFromMap should be -/// used instead to ensure whenever V->S is removed from ValueExprMap, V is also -/// removed from the set of ExprValueMap[S]. +/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V) +/// cannot be used separately. eraseValueFromMap should be used to remove +/// V from ValueExprMap and ExprValueMap at the same time. void ScalarEvolution::eraseValueFromMap(Value *V) { ValueExprMapType::iterator I = ValueExprMap.find_as(V); if (I != ValueExprMap.end()) { const SCEV *S = I->second; - SetVector *SV = getSCEVValues(S); - // Remove V from the set of ExprValueMap[S] - if (SV) - SV->remove(V); + // Remove {V, 0} from the set of ExprValueMap[S] + if (SetVector *SV = getSCEVValues(S)) + SV->remove({V, nullptr}); + + // Remove {V, Offset} from the set of ExprValueMap[Stripped] + const SCEV *Stripped; + ConstantInt *Offset; + std::tie(Stripped, Offset) = splitAddExpr(S); + if (Offset != nullptr) { + if (SetVector *SV = getSCEVValues(Stripped)) + SV->remove({V, Offset}); + } ValueExprMap.erase(V); } } @@ -3419,11 +3446,26 @@ const SCEV *ScalarEvolution::getSCEV(Value *V) { S = createSCEV(V); // During PHI resolution, it is possible to create two SCEVs for the same // V, so it is needed to double check whether V->S is inserted into - // ValueExprMap before insert S->V into ExprValueMap. + // ValueExprMap before insert S->{V, 0} into ExprValueMap. std::pair Pair = ValueExprMap.insert({SCEVCallbackVH(V, this), S}); - if (Pair.second) - ExprValueMap[S].insert(V); + if (Pair.second) { + ExprValueMap[S].insert({V, nullptr}); + + // If S == Stripped + Offset, add Stripped -> {V, Offset} into + // ExprValueMap. + const SCEV *Stripped = S; + ConstantInt *Offset = nullptr; + std::tie(Stripped, Offset) = splitAddExpr(S); + // If stripped is SCEVUnknown, don't bother to save + // Stripped -> {V, offset}. It doesn't simplify and sometimes even + // increase the complexity of the expansion code. + // If V is GetElementPtrInst, don't save Stripped -> {V, offset} + // because it may generate add/sub instead of GEP in SCEV expansion. + if (Offset != nullptr && !isa(Stripped) && + !isa(V)) + ExprValueMap[Stripped].insert({V, Offset}); + } } return S; } @@ -3436,8 +3478,8 @@ const SCEV *ScalarEvolution::getExistingSCEV(Value *V) { const SCEV *S = I->second; if (checkValidity(S)) return S; + eraseValueFromMap(V); forgetMemoizedResults(S); - ValueExprMap.erase(I); } return nullptr; } @@ -3675,8 +3717,8 @@ void ScalarEvolution::forgetSymbolicName(Instruction *PN, const SCEV *SymName) { if (!isa(I) || !isa(Old) || (I != PN && Old == SymName)) { + eraseValueFromMap(It->first); forgetMemoizedResults(Old); - ValueExprMap.erase(It); } } @@ -4055,7 +4097,7 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) { // to create an AddRecExpr for this PHI node. We can not keep this temporary // as it will prevent later (possibly simpler) SCEV expressions to be added // to the ValueExprMap. - ValueExprMap.erase(PN); + eraseValueFromMap(PN); } return nullptr; @@ -5435,8 +5477,8 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { // case, createNodeForPHI will perform the necessary updates on its // own when it gets to that point. if (!isa(I) || !isa(Old)) { + eraseValueFromMap(It->first); forgetMemoizedResults(Old); - ValueExprMap.erase(It); } if (PHINode *PN = dyn_cast(I)) ConstantEvolutionLoopExitValue.erase(PN); @@ -5481,8 +5523,8 @@ void ScalarEvolution::forgetLoop(const Loop *L) { ValueExprMapType::iterator It = ValueExprMap.find_as(static_cast(I)); if (It != ValueExprMap.end()) { + eraseValueFromMap(It->first); forgetMemoizedResults(It->second); - ValueExprMap.erase(It); if (PHINode *PN = dyn_cast(I)) ConstantEvolutionLoopExitValue.erase(PN); } @@ -5515,8 +5557,8 @@ void ScalarEvolution::forgetValue(Value *V) { ValueExprMapType::iterator It = ValueExprMap.find_as(static_cast(I)); if (It != ValueExprMap.end()) { + eraseValueFromMap(It->first); forgetMemoizedResults(It->second); - ValueExprMap.erase(It); if (PHINode *PN = dyn_cast(I)) ConstantEvolutionLoopExitValue.erase(PN); } diff --git a/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp b/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp index 2e45bb840946..6036dccbf513 100644 --- a/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp @@ -1625,9 +1625,10 @@ Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) { return V; } -Value *SCEVExpander::FindValueInExprValueMap(const SCEV *S, - const Instruction *InsertPt) { - SetVector *Set = SE.getSCEVValues(S); +ScalarEvolution::ValueOffsetPair +SCEVExpander::FindValueInExprValueMap(const SCEV *S, + const Instruction *InsertPt) { + SetVector *Set = SE.getSCEVValues(S); // If the expansion is not in CanonicalMode, and the SCEV contains any // sub scAddRecExpr type SCEV, it is required to expand the SCEV literally. if (CanonicalMode || !SE.containsAddRecurrence(S)) { @@ -1636,21 +1637,21 @@ Value *SCEVExpander::FindValueInExprValueMap(const SCEV *S, // Choose a Value from the set which dominates the insertPt. // insertPt should be inside the Value's parent loop so as not to break // the LCSSA form. - for (auto const &Ent : *Set) { + for (auto const &VOPair : *Set) { + Value *V = VOPair.first; + ConstantInt *Offset = VOPair.second; Instruction *EntInst = nullptr; - if (Ent && isa(Ent) && - (EntInst = cast(Ent)) && - S->getType() == Ent->getType() && + if (V && isa(V) && (EntInst = cast(V)) && + S->getType() == V->getType() && EntInst->getFunction() == InsertPt->getFunction() && SE.DT.dominates(EntInst, InsertPt) && (SE.LI.getLoopFor(EntInst->getParent()) == nullptr || - SE.LI.getLoopFor(EntInst->getParent())->contains(InsertPt))) { - return Ent; - } + SE.LI.getLoopFor(EntInst->getParent())->contains(InsertPt))) + return {V, Offset}; } } } - return nullptr; + return {nullptr, nullptr}; } // The expansion of SCEV will either reuse a previous Value in ExprValueMap, @@ -1698,11 +1699,33 @@ Value *SCEVExpander::expand(const SCEV *S) { Builder.SetInsertPoint(InsertPt); // Expand the expression into instructions. - Value *V = FindValueInExprValueMap(S, InsertPt); + ScalarEvolution::ValueOffsetPair VO = FindValueInExprValueMap(S, InsertPt); + Value *V = VO.first; if (!V) V = visit(S); - + else if (VO.second) { + if (PointerType *Vty = dyn_cast(V->getType())) { + Type *Ety = Vty->getPointerElementType(); + int64_t Offset = VO.second->getSExtValue(); + int64_t ESize = SE.getTypeSizeInBits(Ety); + if ((Offset * 8) % ESize == 0) { + ConstantInt *Idx = + ConstantInt::getSigned(VO.second->getType(), -(Offset * 8) / ESize); + V = Builder.CreateGEP(Ety, V, Idx, "scevgep"); + } else { + ConstantInt *Idx = + ConstantInt::getSigned(VO.second->getType(), -Offset); + unsigned AS = Vty->getAddressSpace(); + V = Builder.CreateBitCast(V, Type::getInt8PtrTy(SE.getContext(), AS)); + V = Builder.CreateGEP(Type::getInt8Ty(SE.getContext()), V, Idx, + "uglygep"); + V = Builder.CreateBitCast(V, Vty); + } + } else { + V = Builder.CreateSub(V, VO.second); + } + } // Remember the expanded value for this SCEV at this location. // // This is independent of PostIncLoops. The mapped value simply materializes @@ -1887,8 +1910,18 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT, return NumElim; } -Value *SCEVExpander::findExistingExpansion(const SCEV *S, - const Instruction *At, Loop *L) { +Value *SCEVExpander::getExactExistingExpansion(const SCEV *S, + const Instruction *At, Loop *L) { + Optional VO = + getRelatedExistingExpansion(S, At, L); + if (VO && VO.getValue().second == nullptr) + return VO.getValue().first; + return nullptr; +} + +Optional +SCEVExpander::getRelatedExistingExpansion(const SCEV *S, const Instruction *At, + Loop *L) { using namespace llvm::PatternMatch; SmallVector ExitingBlocks; @@ -1906,22 +1939,23 @@ Value *SCEVExpander::findExistingExpansion(const SCEV *S, continue; if (SE.getSCEV(LHS) == S && SE.DT.dominates(LHS, At)) - return LHS; + return ScalarEvolution::ValueOffsetPair(LHS, nullptr); if (SE.getSCEV(RHS) == S && SE.DT.dominates(RHS, At)) - return RHS; + return ScalarEvolution::ValueOffsetPair(RHS, nullptr); } // Use expand's logic which is used for reusing a previous Value in // ExprValueMap. - if (Value *Val = FindValueInExprValueMap(S, At)) - return Val; + ScalarEvolution::ValueOffsetPair VO = FindValueInExprValueMap(S, At); + if (VO.first) + return VO; // There is potential to make this significantly smarter, but this simple // heuristic already gets some interesting cases. // Can not find suitable value. - return nullptr; + return None; } bool SCEVExpander::isHighCostExpansionHelper( @@ -1930,7 +1964,7 @@ bool SCEVExpander::isHighCostExpansionHelper( // If we can find an existing value for this scev avaliable at the point "At" // then consider the expression cheap. - if (At && findExistingExpansion(S, At, L) != nullptr) + if (At && getRelatedExistingExpansion(S, At, L)) return false; // Zero/One operand expressions @@ -1978,7 +2012,7 @@ bool SCEVExpander::isHighCostExpansionHelper( // involving division. This is just a simple search heuristic. if (!At) At = &ExitingBB->back(); - if (!findExistingExpansion( + if (!getRelatedExistingExpansion( SE.getAddExpr(S, SE.getConstant(S->getType(), 1)), At, L)) return true; } diff --git a/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp index e958563e2d10..cf3e7c568158 100644 --- a/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp @@ -481,7 +481,7 @@ Value *IndVarSimplify::expandSCEVIfNeeded(SCEVExpander &Rewriter, const SCEV *S, Type *ResultTy) { // Before expanding S into an expensive LLVM expression, see if we can use an // already existing value as the expansion for S. - if (Value *ExistingValue = Rewriter.findExistingExpansion(S, InsertPt, L)) + if (Value *ExistingValue = Rewriter.getExactExistingExpansion(S, InsertPt, L)) if (ExistingValue->getType() == ResultTy) return ExistingValue;