Lines Matching refs:SCEV
136 typedef DenseMap<const SCEV *, RegSortData> RegUsesTy;
139 SmallVector<const SCEV *, 16> RegSequence;
142 void CountRegister(const SCEV *Reg, size_t LUIdx);
143 void DropRegister(const SCEV *Reg, size_t LUIdx);
146 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
148 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
152 typedef SmallVectorImpl<const SCEV *>::iterator iterator;
153 typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator;
163 RegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { in CountRegister()
174 RegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { in DropRegister()
199 RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { in isRegUsedByUsesOtherThan()
210 const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { in getUsedByIndices()
249 SmallVector<const SCEV *, 4> BaseRegs;
253 const SCEV *ScaledReg;
264 void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
275 void DeleteBaseReg(const SCEV *&S);
277 bool referencesReg(const SCEV *S) const;
288 static void DoInitialMatch(const SCEV *S, Loop *L, in DoInitialMatch()
289 SmallVectorImpl<const SCEV *> &Good, in DoInitialMatch()
290 SmallVectorImpl<const SCEV *> &Bad, in DoInitialMatch()
313 AR->getLoop(), SCEV::FlagAnyWrap), in DoInitialMatch()
321 SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); in DoInitialMatch()
322 const SCEV *NewMul = SE.getMulExpr(Ops); in DoInitialMatch()
324 SmallVector<const SCEV *, 4> MyGood; in DoInitialMatch()
325 SmallVector<const SCEV *, 4> MyBad; in DoInitialMatch()
327 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( in DoInitialMatch()
329 for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(), in DoInitialMatch()
332 for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(), in DoInitialMatch()
346 void Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { in InitialMatch()
347 SmallVector<const SCEV *, 4> Good; in InitialMatch()
348 SmallVector<const SCEV *, 4> Bad; in InitialMatch()
351 const SCEV *Sum = SE.getAddExpr(Good); in InitialMatch()
357 const SCEV *Sum = SE.getAddExpr(Bad); in InitialMatch()
427 void Formula::DeleteBaseReg(const SCEV *&S) { in DeleteBaseReg()
434 bool Formula::referencesReg(const SCEV *S) const { in referencesReg()
446 for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), in hasRegsUsedByUsesOtherThan()
463 for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), in print()
527 static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, in getExactSDiv()
561 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, in getExactSDiv()
564 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, in getExactSDiv()
570 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); in getExactSDiv()
578 SmallVector<const SCEV *, 8> Ops; in getExactSDiv()
581 const SCEV *Op = getExactSDiv(*I, RHS, SE, in getExactSDiv()
594 SmallVector<const SCEV *, 4> Ops; in getExactSDiv()
598 const SCEV *S = *I; in getExactSDiv()
600 if (const SCEV *Q = getExactSDiv(S, RHS, SE, in getExactSDiv()
619 static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { in ExtractImmediate()
626 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); in ExtractImmediate()
632 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); in ExtractImmediate()
637 SCEV::FlagAnyWrap); in ExtractImmediate()
646 static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { in ExtractSymbol()
653 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); in ExtractSymbol()
659 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); in ExtractSymbol()
664 SCEV::FlagAnyWrap); in ExtractSymbol()
745 static bool isHighCostExpansion(const SCEV *S, in isHighCostExpansion()
746 SmallPtrSetImpl<const SCEV*> &Processed, in isHighCostExpansion()
894 SmallPtrSetImpl<const SCEV *> &Regs,
895 const DenseSet<const SCEV *> &VisitedRegs,
900 SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr);
906 void RateRegister(const SCEV *Reg,
907 SmallPtrSetImpl<const SCEV *> &Regs,
910 void RatePrimaryRegister(const SCEV *Reg,
911 SmallPtrSetImpl<const SCEV *> &Regs,
914 SmallPtrSetImpl<const SCEV *> *LoserRegs);
920 void Cost::RateRegister(const SCEV *Reg, in RateRegister()
921 SmallPtrSetImpl<const SCEV *> &Regs, in RateRegister()
968 void Cost::RatePrimaryRegister(const SCEV *Reg, in RatePrimaryRegister()
969 SmallPtrSetImpl<const SCEV *> &Regs, in RatePrimaryRegister()
972 SmallPtrSetImpl<const SCEV *> *LoserRegs) { in RatePrimaryRegister()
986 SmallPtrSetImpl<const SCEV *> &Regs, in RateFormula()
987 const DenseSet<const SCEV *> &VisitedRegs, in RateFormula()
992 SmallPtrSetImpl<const SCEV *> *LoserRegs) { in RateFormula()
995 if (const SCEV *ScaledReg = F.ScaledReg) { in RateFormula()
1004 for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), in RateFormula()
1006 const SCEV *BaseReg = *I; in RateFormula()
1178 static SmallVector<const SCEV *, 4> getEmptyKey() { in getEmptyKey()
1179 SmallVector<const SCEV *, 4> V; in getEmptyKey()
1180 V.push_back(reinterpret_cast<const SCEV *>(-1)); in getEmptyKey()
1184 static SmallVector<const SCEV *, 4> getTombstoneKey() { in getTombstoneKey()
1185 SmallVector<const SCEV *, 4> V; in getTombstoneKey()
1186 V.push_back(reinterpret_cast<const SCEV *>(-2)); in getTombstoneKey()
1190 static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) { in getHashValue()
1194 static bool isEqual(const SmallVector<const SCEV *, 4> &LHS, in isEqual()
1195 const SmallVector<const SCEV *, 4> &RHS) { in isEqual()
1206 DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier;
1219 typedef PointerIntPair<const SCEV *, 2, KindType> SCEVUseKindPair;
1252 SmallPtrSet<const SCEV *, 4> Regs;
1275 SmallVector<const SCEV *, 4> Key = F.BaseRegs; in HasFormulaWithSameRegs()
1291 SmallVector<const SCEV *, 4> Key = F.BaseRegs; in InsertFormula()
1303 for (SmallVectorImpl<const SCEV *>::const_iterator I = in InsertFormula()
1329 SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs); in RecomputeRegs()
1337 for (const SCEV *S : OldRegs) in RecomputeRegs()
1561 Type *AccessTy, const SCEV *S, bool HasBaseReg) { in isAlwaysFoldable()
1598 const SCEV *IncExpr;
1600 IVInc(Instruction *U, Value *O, const SCEV *E): in IVInc()
1608 const SCEV *ExprBase;
1612 IVChain(const IVInc &Head, const SCEV *Base) in IVChain()
1637 bool isProfitableIncrement(const SCEV *OperExpr,
1638 const SCEV *IncExpr,
1720 std::pair<size_t, int64_t> getUse(const SCEV *&Expr,
1728 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1729 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1770 const SmallPtrSet<const SCEV *, 16> &CurRegs,
1771 DenseSet<const SCEV *> &VisitedRegs) const;
1818 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeShadowIV()
1998 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeMax()
2001 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); in OptimizeMax()
2004 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); in OptimizeMax()
2031 const SCEV *MaxLHS = Max->getOperand(0); in OptimizeMax()
2032 const SCEV *MaxRHS = Max->getOperand(1); in OptimizeMax()
2042 const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); in OptimizeMax()
2153 const SCEV *A = IU.getStride(*CondUse, L); in OptimizeLoopTermCond()
2154 const SCEV *B = IU.getStride(*UI, L); in OptimizeLoopTermCond()
2284 LSRInstance::getUse(const SCEV *&Expr, in getUse()
2286 const SCEV *Copy = Expr; in getUse()
2378 SmallSetVector<const SCEV *, 4> Strides; in CollectInterestingTypesAndFactors()
2381 SmallVector<const SCEV *, 4> Worklist; in CollectInterestingTypesAndFactors()
2383 const SCEV *Expr = IU.getExpr(*UI); in CollectInterestingTypesAndFactors()
2391 const SCEV *S = Worklist.pop_back_val(); in CollectInterestingTypesAndFactors()
2403 for (SmallSetVector<const SCEV *, 4>::const_iterator in CollectInterestingTypesAndFactors()
2405 for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = in CollectInterestingTypesAndFactors()
2407 const SCEV *OldStride = *I; in CollectInterestingTypesAndFactors()
2408 const SCEV *NewStride = *NewStrideIter; in CollectInterestingTypesAndFactors()
2487 static const SCEV *getExprBase(const SCEV *S) { in getExprBase()
2506 const SCEV *SubExpr = *I; in getExprBase()
2525 bool IVChain::isProfitableIncrement(const SCEV *OperExpr, in isProfitableIncrement()
2526 const SCEV *IncExpr, in isProfitableIncrement()
2535 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand)); in isProfitableIncrement()
2540 SmallPtrSet<const SCEV*, 8> Processed; in isProfitableIncrement()
2582 const SCEV *LastIncExpr = nullptr; in isProfitableChain()
2635 const SCEV *const OperExpr = SE.getSCEV(NextIV); in ChainInstruction()
2636 const SCEV *const OperExprBase = getExprBase(OperExpr); in ChainInstruction()
2641 const SCEV *LastIncExpr = nullptr; in ChainInstruction()
2661 const SCEV *PrevExpr = SE.getSCEV(PrevIV); in ChainInstruction()
2662 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); in ChainInstruction()
2850 static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, in canFoldIVIncExpr()
2906 const SCEV *LeftOverExpr = nullptr; in GenerateIVChain()
2920 const SCEV *IncExpr = SE.getNoopOrSignExtend(IncI->IncExpr, IntTy); in GenerateIVChain()
2928 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), in GenerateIVChain()
2998 const SCEV *S = IU.getExpr(*UI); in CollectFixupsAndInitialFormulae()
3019 const SCEV *N = SE.getSCEV(NV); in CollectFixupsAndInitialFormulae()
3062 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { in InsertInitialFormula()
3076 LSRInstance::InsertSupplementalFormula(const SCEV *S, in InsertSupplementalFormula()
3090 for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), in CountRegisters()
3115 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); in CollectLoopInvariantFixupsAndFormulae()
3116 SmallPtrSet<const SCEV *, 32> Visited; in CollectLoopInvariantFixupsAndFormulae()
3119 const SCEV *S = Worklist.pop_back_val(); in CollectLoopInvariantFixupsAndFormulae()
3159 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); in CollectLoopInvariantFixupsAndFormulae()
3202 static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, in CollectSubexprs()
3203 SmallVectorImpl<const SCEV *> &Ops, in CollectSubexprs()
3215 const SCEV *Remainder = CollectSubexprs(*I, C, Ops, L, SE, Depth+1); in CollectSubexprs()
3225 const SCEV *Remainder = CollectSubexprs(AR->getStart(), in CollectSubexprs()
3240 SCEV::FlagAnyWrap); in CollectSubexprs()
3249 const SCEV *Remainder = in CollectSubexprs()
3264 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateReassociationsImpl()
3265 SmallVector<const SCEV *, 8> AddOps; in GenerateReassociationsImpl()
3266 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE); in GenerateReassociationsImpl()
3273 for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), in GenerateReassociationsImpl()
3289 SmallVector<const SCEV *, 8> InnerAddOps( in GenerateReassociationsImpl()
3290 ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); in GenerateReassociationsImpl()
3292 ((const SmallVector<const SCEV *, 8> &)AddOps).end()); in GenerateReassociationsImpl()
3301 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); in GenerateReassociationsImpl()
3372 SmallVector<const SCEV *, 4> Ops; in GenerateCombinations()
3373 for (SmallVectorImpl<const SCEV *>::const_iterator in GenerateCombinations()
3375 const SCEV *BaseReg = *I; in GenerateCombinations()
3383 const SCEV *Sum = SE.getAddExpr(Ops); in GenerateCombinations()
3399 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateSymbolicOffsetsImpl()
3431 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateConstantOffsetsImpl()
3440 const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G); in GenerateConstantOffsetsImpl()
3543 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateICmpZeroScales()
3621 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateScales()
3626 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { in GenerateScales()
3659 for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(), in GenerateTruncates()
3681 const SCEV *OrigReg;
3683 WorkItem(size_t LI, int64_t I, const SCEV *R) in WorkItem()
3707 typedef std::map<int64_t, const SCEV *> ImmMapTy; in GenerateCrossUseConstantOffsets()
3708 typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy; in GenerateCrossUseConstantOffsets()
3710 DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; in GenerateCrossUseConstantOffsets()
3711 SmallVector<const SCEV *, 8> Sequence; in GenerateCrossUseConstantOffsets()
3714 const SCEV *Reg = *I; in GenerateCrossUseConstantOffsets()
3729 for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(), in GenerateCrossUseConstantOffsets()
3731 const SCEV *Reg = *I; in GenerateCrossUseConstantOffsets()
3747 const SCEV *OrigReg = J->second; in GenerateCrossUseConstantOffsets()
3792 const SCEV *OrigReg = WI.OrigReg; in GenerateCrossUseConstantOffsets()
3795 const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); in GenerateCrossUseConstantOffsets()
3836 const SCEV *BaseReg = F.BaseRegs[N]; in GenerateCrossUseConstantOffsets()
3853 for (SmallVectorImpl<const SCEV *>::const_iterator in GenerateCrossUseConstantOffsets()
3914 DenseSet<const SCEV *> VisitedRegs; in FilterOutUndesirableDedicatedRegisters()
3915 SmallPtrSet<const SCEV *, 16> Regs; in FilterOutUndesirableDedicatedRegisters()
3916 SmallPtrSet<const SCEV *, 16> LoserRegs; in FilterOutUndesirableDedicatedRegisters()
3923 typedef DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo> in FilterOutUndesirableDedicatedRegisters()
3958 SmallVector<const SCEV *, 4> Key; in FilterOutUndesirableDedicatedRegisters()
3959 for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(), in FilterOutUndesirableDedicatedRegisters()
3961 const SCEV *Reg = *J; in FilterOutUndesirableDedicatedRegisters()
4057 for (SmallVectorImpl<const SCEV *>::const_iterator in NarrowSearchSpaceByDetectingSupersets()
4208 SmallPtrSet<const SCEV *, 4> Taken; in NarrowSearchSpaceByPickingWinnerRegs()
4216 const SCEV *Best = nullptr; in NarrowSearchSpaceByPickingWinnerRegs()
4220 const SCEV *Reg = *I; in NarrowSearchSpaceByPickingWinnerRegs()
4283 const SmallPtrSet<const SCEV *, 16> &CurRegs, in SolveRecurse() argument
4284 DenseSet<const SCEV *> &VisitedRegs) const { in SolveRecurse()
4301 SmallSetVector<const SCEV *, 4> ReqRegs; in SolveRecurse()
4302 for (const SCEV *S : CurRegs) in SolveRecurse()
4306 SmallPtrSet<const SCEV *, 16> NewRegs; in SolveRecurse()
4316 for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(), in SolveRecurse()
4318 const SCEV *Reg = *J; in SolveRecurse()
4349 for (const SCEV *S : NewRegs) in SolveRecurse()
4368 SmallPtrSet<const SCEV *, 16> CurRegs; in Solve()
4369 DenseSet<const SCEV *> VisitedRegs; in Solve()
4549 SmallVector<const SCEV *, 8> Ops; in Expand()
4552 for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), in Expand()
4554 const SCEV *Reg = *I; in Expand()
4569 const SCEV *ScaledS = F.ScaledReg; in Expand()
4657 const SCEV *FullS = Ops.empty() ? in Expand()