xref: /aosp_15_r20/external/llvm/lib/CodeGen/CodeGenPrepare.cpp (revision 9880d6810fe72a1726cb53787c6711e909410d58)
1*9880d681SAndroid Build Coastguard Worker //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
2*9880d681SAndroid Build Coastguard Worker //
3*9880d681SAndroid Build Coastguard Worker //                     The LLVM Compiler Infrastructure
4*9880d681SAndroid Build Coastguard Worker //
5*9880d681SAndroid Build Coastguard Worker // This file is distributed under the University of Illinois Open Source
6*9880d681SAndroid Build Coastguard Worker // License. See LICENSE.TXT for details.
7*9880d681SAndroid Build Coastguard Worker //
8*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
9*9880d681SAndroid Build Coastguard Worker //
10*9880d681SAndroid Build Coastguard Worker // This pass munges the code in the input function to better prepare it for
11*9880d681SAndroid Build Coastguard Worker // SelectionDAG-based code generation. This works around limitations in it's
12*9880d681SAndroid Build Coastguard Worker // basic-block-at-a-time approach. It should eventually be removed.
13*9880d681SAndroid Build Coastguard Worker //
14*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
15*9880d681SAndroid Build Coastguard Worker 
16*9880d681SAndroid Build Coastguard Worker #include "llvm/CodeGen/Passes.h"
17*9880d681SAndroid Build Coastguard Worker #include "llvm/ADT/DenseMap.h"
18*9880d681SAndroid Build Coastguard Worker #include "llvm/ADT/SmallSet.h"
19*9880d681SAndroid Build Coastguard Worker #include "llvm/ADT/Statistic.h"
20*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/InstructionSimplify.h"
21*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/LoopInfo.h"
22*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/TargetLibraryInfo.h"
23*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/TargetTransformInfo.h"
24*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/ValueTracking.h"
25*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/MemoryBuiltins.h"
26*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/CallSite.h"
27*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Constants.h"
28*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/DataLayout.h"
29*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/DerivedTypes.h"
30*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Dominators.h"
31*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Function.h"
32*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/GetElementPtrTypeIterator.h"
33*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/IRBuilder.h"
34*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/InlineAsm.h"
35*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Instructions.h"
36*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/IntrinsicInst.h"
37*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/MDBuilder.h"
38*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/PatternMatch.h"
39*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Statepoint.h"
40*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/ValueHandle.h"
41*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/ValueMap.h"
42*9880d681SAndroid Build Coastguard Worker #include "llvm/Pass.h"
43*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/BranchProbability.h"
44*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/CommandLine.h"
45*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/Debug.h"
46*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/raw_ostream.h"
47*9880d681SAndroid Build Coastguard Worker #include "llvm/Target/TargetLowering.h"
48*9880d681SAndroid Build Coastguard Worker #include "llvm/Target/TargetSubtargetInfo.h"
49*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/BasicBlockUtils.h"
50*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/BuildLibCalls.h"
51*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/BypassSlowDivision.h"
52*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/Local.h"
53*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
54*9880d681SAndroid Build Coastguard Worker using namespace llvm;
55*9880d681SAndroid Build Coastguard Worker using namespace llvm::PatternMatch;
56*9880d681SAndroid Build Coastguard Worker 
57*9880d681SAndroid Build Coastguard Worker #define DEBUG_TYPE "codegenprepare"
58*9880d681SAndroid Build Coastguard Worker 
59*9880d681SAndroid Build Coastguard Worker STATISTIC(NumBlocksElim, "Number of blocks eliminated");
60*9880d681SAndroid Build Coastguard Worker STATISTIC(NumPHIsElim,   "Number of trivial PHIs eliminated");
61*9880d681SAndroid Build Coastguard Worker STATISTIC(NumGEPsElim,   "Number of GEPs converted to casts");
62*9880d681SAndroid Build Coastguard Worker STATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of "
63*9880d681SAndroid Build Coastguard Worker                       "sunken Cmps");
64*9880d681SAndroid Build Coastguard Worker STATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses "
65*9880d681SAndroid Build Coastguard Worker                        "of sunken Casts");
66*9880d681SAndroid Build Coastguard Worker STATISTIC(NumMemoryInsts, "Number of memory instructions whose address "
67*9880d681SAndroid Build Coastguard Worker                           "computations were sunk");
68*9880d681SAndroid Build Coastguard Worker STATISTIC(NumExtsMoved,  "Number of [s|z]ext instructions combined with loads");
69*9880d681SAndroid Build Coastguard Worker STATISTIC(NumExtUses,    "Number of uses of [s|z]ext instructions optimized");
70*9880d681SAndroid Build Coastguard Worker STATISTIC(NumAndsAdded,
71*9880d681SAndroid Build Coastguard Worker           "Number of and mask instructions added to form ext loads");
72*9880d681SAndroid Build Coastguard Worker STATISTIC(NumAndUses, "Number of uses of and mask instructions optimized");
73*9880d681SAndroid Build Coastguard Worker STATISTIC(NumRetsDup,    "Number of return instructions duplicated");
74*9880d681SAndroid Build Coastguard Worker STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
75*9880d681SAndroid Build Coastguard Worker STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");
76*9880d681SAndroid Build Coastguard Worker STATISTIC(NumAndCmpsMoved, "Number of and/cmp's pushed into branches");
77*9880d681SAndroid Build Coastguard Worker STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");
78*9880d681SAndroid Build Coastguard Worker 
79*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableBranchOpts(
80*9880d681SAndroid Build Coastguard Worker   "disable-cgp-branch-opts", cl::Hidden, cl::init(false),
81*9880d681SAndroid Build Coastguard Worker   cl::desc("Disable branch optimizations in CodeGenPrepare"));
82*9880d681SAndroid Build Coastguard Worker 
83*9880d681SAndroid Build Coastguard Worker static cl::opt<bool>
84*9880d681SAndroid Build Coastguard Worker     DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false),
85*9880d681SAndroid Build Coastguard Worker                   cl::desc("Disable GC optimizations in CodeGenPrepare"));
86*9880d681SAndroid Build Coastguard Worker 
87*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableSelectToBranch(
88*9880d681SAndroid Build Coastguard Worker   "disable-cgp-select2branch", cl::Hidden, cl::init(false),
89*9880d681SAndroid Build Coastguard Worker   cl::desc("Disable select to branch conversion."));
90*9880d681SAndroid Build Coastguard Worker 
91*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> AddrSinkUsingGEPs(
92*9880d681SAndroid Build Coastguard Worker   "addr-sink-using-gep", cl::Hidden, cl::init(false),
93*9880d681SAndroid Build Coastguard Worker   cl::desc("Address sinking in CGP using GEPs."));
94*9880d681SAndroid Build Coastguard Worker 
95*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> EnableAndCmpSinking(
96*9880d681SAndroid Build Coastguard Worker    "enable-andcmp-sinking", cl::Hidden, cl::init(true),
97*9880d681SAndroid Build Coastguard Worker    cl::desc("Enable sinkinig and/cmp into branches."));
98*9880d681SAndroid Build Coastguard Worker 
99*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableStoreExtract(
100*9880d681SAndroid Build Coastguard Worker     "disable-cgp-store-extract", cl::Hidden, cl::init(false),
101*9880d681SAndroid Build Coastguard Worker     cl::desc("Disable store(extract) optimizations in CodeGenPrepare"));
102*9880d681SAndroid Build Coastguard Worker 
103*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> StressStoreExtract(
104*9880d681SAndroid Build Coastguard Worker     "stress-cgp-store-extract", cl::Hidden, cl::init(false),
105*9880d681SAndroid Build Coastguard Worker     cl::desc("Stress test store(extract) optimizations in CodeGenPrepare"));
106*9880d681SAndroid Build Coastguard Worker 
107*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableExtLdPromotion(
108*9880d681SAndroid Build Coastguard Worker     "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
109*9880d681SAndroid Build Coastguard Worker     cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
110*9880d681SAndroid Build Coastguard Worker              "CodeGenPrepare"));
111*9880d681SAndroid Build Coastguard Worker 
112*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> StressExtLdPromotion(
113*9880d681SAndroid Build Coastguard Worker     "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
114*9880d681SAndroid Build Coastguard Worker     cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
115*9880d681SAndroid Build Coastguard Worker              "optimization in CodeGenPrepare"));
116*9880d681SAndroid Build Coastguard Worker 
117*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisablePreheaderProtect(
118*9880d681SAndroid Build Coastguard Worker     "disable-preheader-prot", cl::Hidden, cl::init(false),
119*9880d681SAndroid Build Coastguard Worker     cl::desc("Disable protection against removing loop preheaders"));
120*9880d681SAndroid Build Coastguard Worker 
121*9880d681SAndroid Build Coastguard Worker namespace {
122*9880d681SAndroid Build Coastguard Worker typedef SmallPtrSet<Instruction *, 16> SetOfInstrs;
123*9880d681SAndroid Build Coastguard Worker typedef PointerIntPair<Type *, 1, bool> TypeIsSExt;
124*9880d681SAndroid Build Coastguard Worker typedef DenseMap<Instruction *, TypeIsSExt> InstrToOrigTy;
125*9880d681SAndroid Build Coastguard Worker class TypePromotionTransaction;
126*9880d681SAndroid Build Coastguard Worker 
127*9880d681SAndroid Build Coastguard Worker   class CodeGenPrepare : public FunctionPass {
128*9880d681SAndroid Build Coastguard Worker     const TargetMachine *TM;
129*9880d681SAndroid Build Coastguard Worker     const TargetLowering *TLI;
130*9880d681SAndroid Build Coastguard Worker     const TargetTransformInfo *TTI;
131*9880d681SAndroid Build Coastguard Worker     const TargetLibraryInfo *TLInfo;
132*9880d681SAndroid Build Coastguard Worker     const LoopInfo *LI;
133*9880d681SAndroid Build Coastguard Worker 
134*9880d681SAndroid Build Coastguard Worker     /// As we scan instructions optimizing them, this is the next instruction
135*9880d681SAndroid Build Coastguard Worker     /// to optimize. Transforms that can invalidate this should update it.
136*9880d681SAndroid Build Coastguard Worker     BasicBlock::iterator CurInstIterator;
137*9880d681SAndroid Build Coastguard Worker 
138*9880d681SAndroid Build Coastguard Worker     /// Keeps track of non-local addresses that have been sunk into a block.
139*9880d681SAndroid Build Coastguard Worker     /// This allows us to avoid inserting duplicate code for blocks with
140*9880d681SAndroid Build Coastguard Worker     /// multiple load/stores of the same address.
141*9880d681SAndroid Build Coastguard Worker     ValueMap<Value*, Value*> SunkAddrs;
142*9880d681SAndroid Build Coastguard Worker 
143*9880d681SAndroid Build Coastguard Worker     /// Keeps track of all instructions inserted for the current function.
144*9880d681SAndroid Build Coastguard Worker     SetOfInstrs InsertedInsts;
145*9880d681SAndroid Build Coastguard Worker     /// Keeps track of the type of the related instruction before their
146*9880d681SAndroid Build Coastguard Worker     /// promotion for the current function.
147*9880d681SAndroid Build Coastguard Worker     InstrToOrigTy PromotedInsts;
148*9880d681SAndroid Build Coastguard Worker 
149*9880d681SAndroid Build Coastguard Worker     /// True if CFG is modified in any way.
150*9880d681SAndroid Build Coastguard Worker     bool ModifiedDT;
151*9880d681SAndroid Build Coastguard Worker 
152*9880d681SAndroid Build Coastguard Worker     /// True if optimizing for size.
153*9880d681SAndroid Build Coastguard Worker     bool OptSize;
154*9880d681SAndroid Build Coastguard Worker 
155*9880d681SAndroid Build Coastguard Worker     /// DataLayout for the Function being processed.
156*9880d681SAndroid Build Coastguard Worker     const DataLayout *DL;
157*9880d681SAndroid Build Coastguard Worker 
158*9880d681SAndroid Build Coastguard Worker   public:
159*9880d681SAndroid Build Coastguard Worker     static char ID; // Pass identification, replacement for typeid
CodeGenPrepare(const TargetMachine * TM=nullptr)160*9880d681SAndroid Build Coastguard Worker     explicit CodeGenPrepare(const TargetMachine *TM = nullptr)
161*9880d681SAndroid Build Coastguard Worker         : FunctionPass(ID), TM(TM), TLI(nullptr), TTI(nullptr), DL(nullptr) {
162*9880d681SAndroid Build Coastguard Worker         initializeCodeGenPreparePass(*PassRegistry::getPassRegistry());
163*9880d681SAndroid Build Coastguard Worker       }
164*9880d681SAndroid Build Coastguard Worker     bool runOnFunction(Function &F) override;
165*9880d681SAndroid Build Coastguard Worker 
getPassName() const166*9880d681SAndroid Build Coastguard Worker     const char *getPassName() const override { return "CodeGen Prepare"; }
167*9880d681SAndroid Build Coastguard Worker 
getAnalysisUsage(AnalysisUsage & AU) const168*9880d681SAndroid Build Coastguard Worker     void getAnalysisUsage(AnalysisUsage &AU) const override {
169*9880d681SAndroid Build Coastguard Worker       // FIXME: When we can selectively preserve passes, preserve the domtree.
170*9880d681SAndroid Build Coastguard Worker       AU.addRequired<TargetLibraryInfoWrapperPass>();
171*9880d681SAndroid Build Coastguard Worker       AU.addRequired<TargetTransformInfoWrapperPass>();
172*9880d681SAndroid Build Coastguard Worker       AU.addRequired<LoopInfoWrapperPass>();
173*9880d681SAndroid Build Coastguard Worker     }
174*9880d681SAndroid Build Coastguard Worker 
175*9880d681SAndroid Build Coastguard Worker   private:
176*9880d681SAndroid Build Coastguard Worker     bool eliminateFallThrough(Function &F);
177*9880d681SAndroid Build Coastguard Worker     bool eliminateMostlyEmptyBlocks(Function &F);
178*9880d681SAndroid Build Coastguard Worker     bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const;
179*9880d681SAndroid Build Coastguard Worker     void eliminateMostlyEmptyBlock(BasicBlock *BB);
180*9880d681SAndroid Build Coastguard Worker     bool optimizeBlock(BasicBlock &BB, bool& ModifiedDT);
181*9880d681SAndroid Build Coastguard Worker     bool optimizeInst(Instruction *I, bool& ModifiedDT);
182*9880d681SAndroid Build Coastguard Worker     bool optimizeMemoryInst(Instruction *I, Value *Addr,
183*9880d681SAndroid Build Coastguard Worker                             Type *AccessTy, unsigned AS);
184*9880d681SAndroid Build Coastguard Worker     bool optimizeInlineAsmInst(CallInst *CS);
185*9880d681SAndroid Build Coastguard Worker     bool optimizeCallInst(CallInst *CI, bool& ModifiedDT);
186*9880d681SAndroid Build Coastguard Worker     bool moveExtToFormExtLoad(Instruction *&I);
187*9880d681SAndroid Build Coastguard Worker     bool optimizeExtUses(Instruction *I);
188*9880d681SAndroid Build Coastguard Worker     bool optimizeLoadExt(LoadInst *I);
189*9880d681SAndroid Build Coastguard Worker     bool optimizeSelectInst(SelectInst *SI);
190*9880d681SAndroid Build Coastguard Worker     bool optimizeShuffleVectorInst(ShuffleVectorInst *SI);
191*9880d681SAndroid Build Coastguard Worker     bool optimizeSwitchInst(SwitchInst *CI);
192*9880d681SAndroid Build Coastguard Worker     bool optimizeExtractElementInst(Instruction *Inst);
193*9880d681SAndroid Build Coastguard Worker     bool dupRetToEnableTailCallOpts(BasicBlock *BB);
194*9880d681SAndroid Build Coastguard Worker     bool placeDbgValues(Function &F);
195*9880d681SAndroid Build Coastguard Worker     bool sinkAndCmp(Function &F);
196*9880d681SAndroid Build Coastguard Worker     bool extLdPromotion(TypePromotionTransaction &TPT, LoadInst *&LI,
197*9880d681SAndroid Build Coastguard Worker                         Instruction *&Inst,
198*9880d681SAndroid Build Coastguard Worker                         const SmallVectorImpl<Instruction *> &Exts,
199*9880d681SAndroid Build Coastguard Worker                         unsigned CreatedInstCost);
200*9880d681SAndroid Build Coastguard Worker     bool splitBranchCondition(Function &F);
201*9880d681SAndroid Build Coastguard Worker     bool simplifyOffsetableRelocate(Instruction &I);
202*9880d681SAndroid Build Coastguard Worker     void stripInvariantGroupMetadata(Instruction &I);
203*9880d681SAndroid Build Coastguard Worker   };
204*9880d681SAndroid Build Coastguard Worker }
205*9880d681SAndroid Build Coastguard Worker 
206*9880d681SAndroid Build Coastguard Worker char CodeGenPrepare::ID = 0;
207*9880d681SAndroid Build Coastguard Worker INITIALIZE_TM_PASS(CodeGenPrepare, "codegenprepare",
208*9880d681SAndroid Build Coastguard Worker                    "Optimize for code generation", false, false)
209*9880d681SAndroid Build Coastguard Worker 
createCodeGenPreparePass(const TargetMachine * TM)210*9880d681SAndroid Build Coastguard Worker FunctionPass *llvm::createCodeGenPreparePass(const TargetMachine *TM) {
211*9880d681SAndroid Build Coastguard Worker   return new CodeGenPrepare(TM);
212*9880d681SAndroid Build Coastguard Worker }
213*9880d681SAndroid Build Coastguard Worker 
runOnFunction(Function & F)214*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::runOnFunction(Function &F) {
215*9880d681SAndroid Build Coastguard Worker   if (skipFunction(F))
216*9880d681SAndroid Build Coastguard Worker     return false;
217*9880d681SAndroid Build Coastguard Worker 
218*9880d681SAndroid Build Coastguard Worker   DL = &F.getParent()->getDataLayout();
219*9880d681SAndroid Build Coastguard Worker 
220*9880d681SAndroid Build Coastguard Worker   bool EverMadeChange = false;
221*9880d681SAndroid Build Coastguard Worker   // Clear per function information.
222*9880d681SAndroid Build Coastguard Worker   InsertedInsts.clear();
223*9880d681SAndroid Build Coastguard Worker   PromotedInsts.clear();
224*9880d681SAndroid Build Coastguard Worker 
225*9880d681SAndroid Build Coastguard Worker   ModifiedDT = false;
226*9880d681SAndroid Build Coastguard Worker   if (TM)
227*9880d681SAndroid Build Coastguard Worker     TLI = TM->getSubtargetImpl(F)->getTargetLowering();
228*9880d681SAndroid Build Coastguard Worker   TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
229*9880d681SAndroid Build Coastguard Worker   TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
230*9880d681SAndroid Build Coastguard Worker   LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
231*9880d681SAndroid Build Coastguard Worker   OptSize = F.optForSize();
232*9880d681SAndroid Build Coastguard Worker 
233*9880d681SAndroid Build Coastguard Worker   /// This optimization identifies DIV instructions that can be
234*9880d681SAndroid Build Coastguard Worker   /// profitably bypassed and carried out with a shorter, faster divide.
235*9880d681SAndroid Build Coastguard Worker   if (!OptSize && TLI && TLI->isSlowDivBypassed()) {
236*9880d681SAndroid Build Coastguard Worker     const DenseMap<unsigned int, unsigned int> &BypassWidths =
237*9880d681SAndroid Build Coastguard Worker        TLI->getBypassSlowDivWidths();
238*9880d681SAndroid Build Coastguard Worker     BasicBlock* BB = &*F.begin();
239*9880d681SAndroid Build Coastguard Worker     while (BB != nullptr) {
240*9880d681SAndroid Build Coastguard Worker       // bypassSlowDivision may create new BBs, but we don't want to reapply the
241*9880d681SAndroid Build Coastguard Worker       // optimization to those blocks.
242*9880d681SAndroid Build Coastguard Worker       BasicBlock* Next = BB->getNextNode();
243*9880d681SAndroid Build Coastguard Worker       EverMadeChange |= bypassSlowDivision(BB, BypassWidths);
244*9880d681SAndroid Build Coastguard Worker       BB = Next;
245*9880d681SAndroid Build Coastguard Worker     }
246*9880d681SAndroid Build Coastguard Worker   }
247*9880d681SAndroid Build Coastguard Worker 
248*9880d681SAndroid Build Coastguard Worker   // Eliminate blocks that contain only PHI nodes and an
249*9880d681SAndroid Build Coastguard Worker   // unconditional branch.
250*9880d681SAndroid Build Coastguard Worker   EverMadeChange |= eliminateMostlyEmptyBlocks(F);
251*9880d681SAndroid Build Coastguard Worker 
252*9880d681SAndroid Build Coastguard Worker   // llvm.dbg.value is far away from the value then iSel may not be able
253*9880d681SAndroid Build Coastguard Worker   // handle it properly. iSel will drop llvm.dbg.value if it can not
254*9880d681SAndroid Build Coastguard Worker   // find a node corresponding to the value.
255*9880d681SAndroid Build Coastguard Worker   EverMadeChange |= placeDbgValues(F);
256*9880d681SAndroid Build Coastguard Worker 
257*9880d681SAndroid Build Coastguard Worker   // If there is a mask, compare against zero, and branch that can be combined
258*9880d681SAndroid Build Coastguard Worker   // into a single target instruction, push the mask and compare into branch
259*9880d681SAndroid Build Coastguard Worker   // users. Do this before OptimizeBlock -> OptimizeInst ->
260*9880d681SAndroid Build Coastguard Worker   // OptimizeCmpExpression, which perturbs the pattern being searched for.
261*9880d681SAndroid Build Coastguard Worker   if (!DisableBranchOpts) {
262*9880d681SAndroid Build Coastguard Worker     EverMadeChange |= sinkAndCmp(F);
263*9880d681SAndroid Build Coastguard Worker     EverMadeChange |= splitBranchCondition(F);
264*9880d681SAndroid Build Coastguard Worker   }
265*9880d681SAndroid Build Coastguard Worker 
266*9880d681SAndroid Build Coastguard Worker   bool MadeChange = true;
267*9880d681SAndroid Build Coastguard Worker   while (MadeChange) {
268*9880d681SAndroid Build Coastguard Worker     MadeChange = false;
269*9880d681SAndroid Build Coastguard Worker     for (Function::iterator I = F.begin(); I != F.end(); ) {
270*9880d681SAndroid Build Coastguard Worker       BasicBlock *BB = &*I++;
271*9880d681SAndroid Build Coastguard Worker       bool ModifiedDTOnIteration = false;
272*9880d681SAndroid Build Coastguard Worker       MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration);
273*9880d681SAndroid Build Coastguard Worker 
274*9880d681SAndroid Build Coastguard Worker       // Restart BB iteration if the dominator tree of the Function was changed
275*9880d681SAndroid Build Coastguard Worker       if (ModifiedDTOnIteration)
276*9880d681SAndroid Build Coastguard Worker         break;
277*9880d681SAndroid Build Coastguard Worker     }
278*9880d681SAndroid Build Coastguard Worker     EverMadeChange |= MadeChange;
279*9880d681SAndroid Build Coastguard Worker   }
280*9880d681SAndroid Build Coastguard Worker 
281*9880d681SAndroid Build Coastguard Worker   SunkAddrs.clear();
282*9880d681SAndroid Build Coastguard Worker 
283*9880d681SAndroid Build Coastguard Worker   if (!DisableBranchOpts) {
284*9880d681SAndroid Build Coastguard Worker     MadeChange = false;
285*9880d681SAndroid Build Coastguard Worker     SmallPtrSet<BasicBlock*, 8> WorkList;
286*9880d681SAndroid Build Coastguard Worker     for (BasicBlock &BB : F) {
287*9880d681SAndroid Build Coastguard Worker       SmallVector<BasicBlock *, 2> Successors(succ_begin(&BB), succ_end(&BB));
288*9880d681SAndroid Build Coastguard Worker       MadeChange |= ConstantFoldTerminator(&BB, true);
289*9880d681SAndroid Build Coastguard Worker       if (!MadeChange) continue;
290*9880d681SAndroid Build Coastguard Worker 
291*9880d681SAndroid Build Coastguard Worker       for (SmallVectorImpl<BasicBlock*>::iterator
292*9880d681SAndroid Build Coastguard Worker              II = Successors.begin(), IE = Successors.end(); II != IE; ++II)
293*9880d681SAndroid Build Coastguard Worker         if (pred_begin(*II) == pred_end(*II))
294*9880d681SAndroid Build Coastguard Worker           WorkList.insert(*II);
295*9880d681SAndroid Build Coastguard Worker     }
296*9880d681SAndroid Build Coastguard Worker 
297*9880d681SAndroid Build Coastguard Worker     // Delete the dead blocks and any of their dead successors.
298*9880d681SAndroid Build Coastguard Worker     MadeChange |= !WorkList.empty();
299*9880d681SAndroid Build Coastguard Worker     while (!WorkList.empty()) {
300*9880d681SAndroid Build Coastguard Worker       BasicBlock *BB = *WorkList.begin();
301*9880d681SAndroid Build Coastguard Worker       WorkList.erase(BB);
302*9880d681SAndroid Build Coastguard Worker       SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB));
303*9880d681SAndroid Build Coastguard Worker 
304*9880d681SAndroid Build Coastguard Worker       DeleteDeadBlock(BB);
305*9880d681SAndroid Build Coastguard Worker 
306*9880d681SAndroid Build Coastguard Worker       for (SmallVectorImpl<BasicBlock*>::iterator
307*9880d681SAndroid Build Coastguard Worker              II = Successors.begin(), IE = Successors.end(); II != IE; ++II)
308*9880d681SAndroid Build Coastguard Worker         if (pred_begin(*II) == pred_end(*II))
309*9880d681SAndroid Build Coastguard Worker           WorkList.insert(*II);
310*9880d681SAndroid Build Coastguard Worker     }
311*9880d681SAndroid Build Coastguard Worker 
312*9880d681SAndroid Build Coastguard Worker     // Merge pairs of basic blocks with unconditional branches, connected by
313*9880d681SAndroid Build Coastguard Worker     // a single edge.
314*9880d681SAndroid Build Coastguard Worker     if (EverMadeChange || MadeChange)
315*9880d681SAndroid Build Coastguard Worker       MadeChange |= eliminateFallThrough(F);
316*9880d681SAndroid Build Coastguard Worker 
317*9880d681SAndroid Build Coastguard Worker     EverMadeChange |= MadeChange;
318*9880d681SAndroid Build Coastguard Worker   }
319*9880d681SAndroid Build Coastguard Worker 
320*9880d681SAndroid Build Coastguard Worker   if (!DisableGCOpts) {
321*9880d681SAndroid Build Coastguard Worker     SmallVector<Instruction *, 2> Statepoints;
322*9880d681SAndroid Build Coastguard Worker     for (BasicBlock &BB : F)
323*9880d681SAndroid Build Coastguard Worker       for (Instruction &I : BB)
324*9880d681SAndroid Build Coastguard Worker         if (isStatepoint(I))
325*9880d681SAndroid Build Coastguard Worker           Statepoints.push_back(&I);
326*9880d681SAndroid Build Coastguard Worker     for (auto &I : Statepoints)
327*9880d681SAndroid Build Coastguard Worker       EverMadeChange |= simplifyOffsetableRelocate(*I);
328*9880d681SAndroid Build Coastguard Worker   }
329*9880d681SAndroid Build Coastguard Worker 
330*9880d681SAndroid Build Coastguard Worker   return EverMadeChange;
331*9880d681SAndroid Build Coastguard Worker }
332*9880d681SAndroid Build Coastguard Worker 
333*9880d681SAndroid Build Coastguard Worker /// Merge basic blocks which are connected by a single edge, where one of the
334*9880d681SAndroid Build Coastguard Worker /// basic blocks has a single successor pointing to the other basic block,
335*9880d681SAndroid Build Coastguard Worker /// which has a single predecessor.
eliminateFallThrough(Function & F)336*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::eliminateFallThrough(Function &F) {
337*9880d681SAndroid Build Coastguard Worker   bool Changed = false;
338*9880d681SAndroid Build Coastguard Worker   // Scan all of the blocks in the function, except for the entry block.
339*9880d681SAndroid Build Coastguard Worker   for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) {
340*9880d681SAndroid Build Coastguard Worker     BasicBlock *BB = &*I++;
341*9880d681SAndroid Build Coastguard Worker     // If the destination block has a single pred, then this is a trivial
342*9880d681SAndroid Build Coastguard Worker     // edge, just collapse it.
343*9880d681SAndroid Build Coastguard Worker     BasicBlock *SinglePred = BB->getSinglePredecessor();
344*9880d681SAndroid Build Coastguard Worker 
345*9880d681SAndroid Build Coastguard Worker     // Don't merge if BB's address is taken.
346*9880d681SAndroid Build Coastguard Worker     if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue;
347*9880d681SAndroid Build Coastguard Worker 
348*9880d681SAndroid Build Coastguard Worker     BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator());
349*9880d681SAndroid Build Coastguard Worker     if (Term && !Term->isConditional()) {
350*9880d681SAndroid Build Coastguard Worker       Changed = true;
351*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "To merge:\n"<< *SinglePred << "\n\n\n");
352*9880d681SAndroid Build Coastguard Worker       // Remember if SinglePred was the entry block of the function.
353*9880d681SAndroid Build Coastguard Worker       // If so, we will need to move BB back to the entry position.
354*9880d681SAndroid Build Coastguard Worker       bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
355*9880d681SAndroid Build Coastguard Worker       MergeBasicBlockIntoOnlyPred(BB, nullptr);
356*9880d681SAndroid Build Coastguard Worker 
357*9880d681SAndroid Build Coastguard Worker       if (isEntry && BB != &BB->getParent()->getEntryBlock())
358*9880d681SAndroid Build Coastguard Worker         BB->moveBefore(&BB->getParent()->getEntryBlock());
359*9880d681SAndroid Build Coastguard Worker 
360*9880d681SAndroid Build Coastguard Worker       // We have erased a block. Update the iterator.
361*9880d681SAndroid Build Coastguard Worker       I = BB->getIterator();
362*9880d681SAndroid Build Coastguard Worker     }
363*9880d681SAndroid Build Coastguard Worker   }
364*9880d681SAndroid Build Coastguard Worker   return Changed;
365*9880d681SAndroid Build Coastguard Worker }
366*9880d681SAndroid Build Coastguard Worker 
367*9880d681SAndroid Build Coastguard Worker /// Eliminate blocks that contain only PHI nodes, debug info directives, and an
368*9880d681SAndroid Build Coastguard Worker /// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split
369*9880d681SAndroid Build Coastguard Worker /// edges in ways that are non-optimal for isel. Start by eliminating these
370*9880d681SAndroid Build Coastguard Worker /// blocks so we can split them the way we want them.
eliminateMostlyEmptyBlocks(Function & F)371*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F) {
372*9880d681SAndroid Build Coastguard Worker   SmallPtrSet<BasicBlock *, 16> Preheaders;
373*9880d681SAndroid Build Coastguard Worker   SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end());
374*9880d681SAndroid Build Coastguard Worker   while (!LoopList.empty()) {
375*9880d681SAndroid Build Coastguard Worker     Loop *L = LoopList.pop_back_val();
376*9880d681SAndroid Build Coastguard Worker     LoopList.insert(LoopList.end(), L->begin(), L->end());
377*9880d681SAndroid Build Coastguard Worker     if (BasicBlock *Preheader = L->getLoopPreheader())
378*9880d681SAndroid Build Coastguard Worker       Preheaders.insert(Preheader);
379*9880d681SAndroid Build Coastguard Worker   }
380*9880d681SAndroid Build Coastguard Worker 
381*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
382*9880d681SAndroid Build Coastguard Worker   // Note that this intentionally skips the entry block.
383*9880d681SAndroid Build Coastguard Worker   for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) {
384*9880d681SAndroid Build Coastguard Worker     BasicBlock *BB = &*I++;
385*9880d681SAndroid Build Coastguard Worker 
386*9880d681SAndroid Build Coastguard Worker     // If this block doesn't end with an uncond branch, ignore it.
387*9880d681SAndroid Build Coastguard Worker     BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());
388*9880d681SAndroid Build Coastguard Worker     if (!BI || !BI->isUnconditional())
389*9880d681SAndroid Build Coastguard Worker       continue;
390*9880d681SAndroid Build Coastguard Worker 
391*9880d681SAndroid Build Coastguard Worker     // If the instruction before the branch (skipping debug info) isn't a phi
392*9880d681SAndroid Build Coastguard Worker     // node, then other stuff is happening here.
393*9880d681SAndroid Build Coastguard Worker     BasicBlock::iterator BBI = BI->getIterator();
394*9880d681SAndroid Build Coastguard Worker     if (BBI != BB->begin()) {
395*9880d681SAndroid Build Coastguard Worker       --BBI;
396*9880d681SAndroid Build Coastguard Worker       while (isa<DbgInfoIntrinsic>(BBI)) {
397*9880d681SAndroid Build Coastguard Worker         if (BBI == BB->begin())
398*9880d681SAndroid Build Coastguard Worker           break;
399*9880d681SAndroid Build Coastguard Worker         --BBI;
400*9880d681SAndroid Build Coastguard Worker       }
401*9880d681SAndroid Build Coastguard Worker       if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI))
402*9880d681SAndroid Build Coastguard Worker         continue;
403*9880d681SAndroid Build Coastguard Worker     }
404*9880d681SAndroid Build Coastguard Worker 
405*9880d681SAndroid Build Coastguard Worker     // Do not break infinite loops.
406*9880d681SAndroid Build Coastguard Worker     BasicBlock *DestBB = BI->getSuccessor(0);
407*9880d681SAndroid Build Coastguard Worker     if (DestBB == BB)
408*9880d681SAndroid Build Coastguard Worker       continue;
409*9880d681SAndroid Build Coastguard Worker 
410*9880d681SAndroid Build Coastguard Worker     if (!canMergeBlocks(BB, DestBB))
411*9880d681SAndroid Build Coastguard Worker       continue;
412*9880d681SAndroid Build Coastguard Worker 
413*9880d681SAndroid Build Coastguard Worker     // Do not delete loop preheaders if doing so would create a critical edge.
414*9880d681SAndroid Build Coastguard Worker     // Loop preheaders can be good locations to spill registers. If the
415*9880d681SAndroid Build Coastguard Worker     // preheader is deleted and we create a critical edge, registers may be
416*9880d681SAndroid Build Coastguard Worker     // spilled in the loop body instead.
417*9880d681SAndroid Build Coastguard Worker     if (!DisablePreheaderProtect && Preheaders.count(BB) &&
418*9880d681SAndroid Build Coastguard Worker         !(BB->getSinglePredecessor() && BB->getSinglePredecessor()->getSingleSuccessor()))
419*9880d681SAndroid Build Coastguard Worker      continue;
420*9880d681SAndroid Build Coastguard Worker 
421*9880d681SAndroid Build Coastguard Worker     eliminateMostlyEmptyBlock(BB);
422*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
423*9880d681SAndroid Build Coastguard Worker   }
424*9880d681SAndroid Build Coastguard Worker   return MadeChange;
425*9880d681SAndroid Build Coastguard Worker }
426*9880d681SAndroid Build Coastguard Worker 
427*9880d681SAndroid Build Coastguard Worker /// Return true if we can merge BB into DestBB if there is a single
428*9880d681SAndroid Build Coastguard Worker /// unconditional branch between them, and BB contains no other non-phi
429*9880d681SAndroid Build Coastguard Worker /// instructions.
canMergeBlocks(const BasicBlock * BB,const BasicBlock * DestBB) const430*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB,
431*9880d681SAndroid Build Coastguard Worker                                     const BasicBlock *DestBB) const {
432*9880d681SAndroid Build Coastguard Worker   // We only want to eliminate blocks whose phi nodes are used by phi nodes in
433*9880d681SAndroid Build Coastguard Worker   // the successor.  If there are more complex condition (e.g. preheaders),
434*9880d681SAndroid Build Coastguard Worker   // don't mess around with them.
435*9880d681SAndroid Build Coastguard Worker   BasicBlock::const_iterator BBI = BB->begin();
436*9880d681SAndroid Build Coastguard Worker   while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
437*9880d681SAndroid Build Coastguard Worker     for (const User *U : PN->users()) {
438*9880d681SAndroid Build Coastguard Worker       const Instruction *UI = cast<Instruction>(U);
439*9880d681SAndroid Build Coastguard Worker       if (UI->getParent() != DestBB || !isa<PHINode>(UI))
440*9880d681SAndroid Build Coastguard Worker         return false;
441*9880d681SAndroid Build Coastguard Worker       // If User is inside DestBB block and it is a PHINode then check
442*9880d681SAndroid Build Coastguard Worker       // incoming value. If incoming value is not from BB then this is
443*9880d681SAndroid Build Coastguard Worker       // a complex condition (e.g. preheaders) we want to avoid here.
444*9880d681SAndroid Build Coastguard Worker       if (UI->getParent() == DestBB) {
445*9880d681SAndroid Build Coastguard Worker         if (const PHINode *UPN = dyn_cast<PHINode>(UI))
446*9880d681SAndroid Build Coastguard Worker           for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) {
447*9880d681SAndroid Build Coastguard Worker             Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I));
448*9880d681SAndroid Build Coastguard Worker             if (Insn && Insn->getParent() == BB &&
449*9880d681SAndroid Build Coastguard Worker                 Insn->getParent() != UPN->getIncomingBlock(I))
450*9880d681SAndroid Build Coastguard Worker               return false;
451*9880d681SAndroid Build Coastguard Worker           }
452*9880d681SAndroid Build Coastguard Worker       }
453*9880d681SAndroid Build Coastguard Worker     }
454*9880d681SAndroid Build Coastguard Worker   }
455*9880d681SAndroid Build Coastguard Worker 
456*9880d681SAndroid Build Coastguard Worker   // If BB and DestBB contain any common predecessors, then the phi nodes in BB
457*9880d681SAndroid Build Coastguard Worker   // and DestBB may have conflicting incoming values for the block.  If so, we
458*9880d681SAndroid Build Coastguard Worker   // can't merge the block.
459*9880d681SAndroid Build Coastguard Worker   const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin());
460*9880d681SAndroid Build Coastguard Worker   if (!DestBBPN) return true;  // no conflict.
461*9880d681SAndroid Build Coastguard Worker 
462*9880d681SAndroid Build Coastguard Worker   // Collect the preds of BB.
463*9880d681SAndroid Build Coastguard Worker   SmallPtrSet<const BasicBlock*, 16> BBPreds;
464*9880d681SAndroid Build Coastguard Worker   if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
465*9880d681SAndroid Build Coastguard Worker     // It is faster to get preds from a PHI than with pred_iterator.
466*9880d681SAndroid Build Coastguard Worker     for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
467*9880d681SAndroid Build Coastguard Worker       BBPreds.insert(BBPN->getIncomingBlock(i));
468*9880d681SAndroid Build Coastguard Worker   } else {
469*9880d681SAndroid Build Coastguard Worker     BBPreds.insert(pred_begin(BB), pred_end(BB));
470*9880d681SAndroid Build Coastguard Worker   }
471*9880d681SAndroid Build Coastguard Worker 
472*9880d681SAndroid Build Coastguard Worker   // Walk the preds of DestBB.
473*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) {
474*9880d681SAndroid Build Coastguard Worker     BasicBlock *Pred = DestBBPN->getIncomingBlock(i);
475*9880d681SAndroid Build Coastguard Worker     if (BBPreds.count(Pred)) {   // Common predecessor?
476*9880d681SAndroid Build Coastguard Worker       BBI = DestBB->begin();
477*9880d681SAndroid Build Coastguard Worker       while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
478*9880d681SAndroid Build Coastguard Worker         const Value *V1 = PN->getIncomingValueForBlock(Pred);
479*9880d681SAndroid Build Coastguard Worker         const Value *V2 = PN->getIncomingValueForBlock(BB);
480*9880d681SAndroid Build Coastguard Worker 
481*9880d681SAndroid Build Coastguard Worker         // If V2 is a phi node in BB, look up what the mapped value will be.
482*9880d681SAndroid Build Coastguard Worker         if (const PHINode *V2PN = dyn_cast<PHINode>(V2))
483*9880d681SAndroid Build Coastguard Worker           if (V2PN->getParent() == BB)
484*9880d681SAndroid Build Coastguard Worker             V2 = V2PN->getIncomingValueForBlock(Pred);
485*9880d681SAndroid Build Coastguard Worker 
486*9880d681SAndroid Build Coastguard Worker         // If there is a conflict, bail out.
487*9880d681SAndroid Build Coastguard Worker         if (V1 != V2) return false;
488*9880d681SAndroid Build Coastguard Worker       }
489*9880d681SAndroid Build Coastguard Worker     }
490*9880d681SAndroid Build Coastguard Worker   }
491*9880d681SAndroid Build Coastguard Worker 
492*9880d681SAndroid Build Coastguard Worker   return true;
493*9880d681SAndroid Build Coastguard Worker }
494*9880d681SAndroid Build Coastguard Worker 
495*9880d681SAndroid Build Coastguard Worker 
496*9880d681SAndroid Build Coastguard Worker /// Eliminate a basic block that has only phi's and an unconditional branch in
497*9880d681SAndroid Build Coastguard Worker /// it.
eliminateMostlyEmptyBlock(BasicBlock * BB)498*9880d681SAndroid Build Coastguard Worker void CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
499*9880d681SAndroid Build Coastguard Worker   BranchInst *BI = cast<BranchInst>(BB->getTerminator());
500*9880d681SAndroid Build Coastguard Worker   BasicBlock *DestBB = BI->getSuccessor(0);
501*9880d681SAndroid Build Coastguard Worker 
502*9880d681SAndroid Build Coastguard Worker   DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB);
503*9880d681SAndroid Build Coastguard Worker 
504*9880d681SAndroid Build Coastguard Worker   // If the destination block has a single pred, then this is a trivial edge,
505*9880d681SAndroid Build Coastguard Worker   // just collapse it.
506*9880d681SAndroid Build Coastguard Worker   if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) {
507*9880d681SAndroid Build Coastguard Worker     if (SinglePred != DestBB) {
508*9880d681SAndroid Build Coastguard Worker       // Remember if SinglePred was the entry block of the function.  If so, we
509*9880d681SAndroid Build Coastguard Worker       // will need to move BB back to the entry position.
510*9880d681SAndroid Build Coastguard Worker       bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
511*9880d681SAndroid Build Coastguard Worker       MergeBasicBlockIntoOnlyPred(DestBB, nullptr);
512*9880d681SAndroid Build Coastguard Worker 
513*9880d681SAndroid Build Coastguard Worker       if (isEntry && BB != &BB->getParent()->getEntryBlock())
514*9880d681SAndroid Build Coastguard Worker         BB->moveBefore(&BB->getParent()->getEntryBlock());
515*9880d681SAndroid Build Coastguard Worker 
516*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
517*9880d681SAndroid Build Coastguard Worker       return;
518*9880d681SAndroid Build Coastguard Worker     }
519*9880d681SAndroid Build Coastguard Worker   }
520*9880d681SAndroid Build Coastguard Worker 
521*9880d681SAndroid Build Coastguard Worker   // Otherwise, we have multiple predecessors of BB.  Update the PHIs in DestBB
522*9880d681SAndroid Build Coastguard Worker   // to handle the new incoming edges it is about to have.
523*9880d681SAndroid Build Coastguard Worker   PHINode *PN;
524*9880d681SAndroid Build Coastguard Worker   for (BasicBlock::iterator BBI = DestBB->begin();
525*9880d681SAndroid Build Coastguard Worker        (PN = dyn_cast<PHINode>(BBI)); ++BBI) {
526*9880d681SAndroid Build Coastguard Worker     // Remove the incoming value for BB, and remember it.
527*9880d681SAndroid Build Coastguard Worker     Value *InVal = PN->removeIncomingValue(BB, false);
528*9880d681SAndroid Build Coastguard Worker 
529*9880d681SAndroid Build Coastguard Worker     // Two options: either the InVal is a phi node defined in BB or it is some
530*9880d681SAndroid Build Coastguard Worker     // value that dominates BB.
531*9880d681SAndroid Build Coastguard Worker     PHINode *InValPhi = dyn_cast<PHINode>(InVal);
532*9880d681SAndroid Build Coastguard Worker     if (InValPhi && InValPhi->getParent() == BB) {
533*9880d681SAndroid Build Coastguard Worker       // Add all of the input values of the input PHI as inputs of this phi.
534*9880d681SAndroid Build Coastguard Worker       for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i)
535*9880d681SAndroid Build Coastguard Worker         PN->addIncoming(InValPhi->getIncomingValue(i),
536*9880d681SAndroid Build Coastguard Worker                         InValPhi->getIncomingBlock(i));
537*9880d681SAndroid Build Coastguard Worker     } else {
538*9880d681SAndroid Build Coastguard Worker       // Otherwise, add one instance of the dominating value for each edge that
539*9880d681SAndroid Build Coastguard Worker       // we will be adding.
540*9880d681SAndroid Build Coastguard Worker       if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
541*9880d681SAndroid Build Coastguard Worker         for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
542*9880d681SAndroid Build Coastguard Worker           PN->addIncoming(InVal, BBPN->getIncomingBlock(i));
543*9880d681SAndroid Build Coastguard Worker       } else {
544*9880d681SAndroid Build Coastguard Worker         for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
545*9880d681SAndroid Build Coastguard Worker           PN->addIncoming(InVal, *PI);
546*9880d681SAndroid Build Coastguard Worker       }
547*9880d681SAndroid Build Coastguard Worker     }
548*9880d681SAndroid Build Coastguard Worker   }
549*9880d681SAndroid Build Coastguard Worker 
550*9880d681SAndroid Build Coastguard Worker   // The PHIs are now updated, change everything that refers to BB to use
551*9880d681SAndroid Build Coastguard Worker   // DestBB and remove BB.
552*9880d681SAndroid Build Coastguard Worker   BB->replaceAllUsesWith(DestBB);
553*9880d681SAndroid Build Coastguard Worker   BB->eraseFromParent();
554*9880d681SAndroid Build Coastguard Worker   ++NumBlocksElim;
555*9880d681SAndroid Build Coastguard Worker 
556*9880d681SAndroid Build Coastguard Worker   DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
557*9880d681SAndroid Build Coastguard Worker }
558*9880d681SAndroid Build Coastguard Worker 
559*9880d681SAndroid Build Coastguard Worker // Computes a map of base pointer relocation instructions to corresponding
560*9880d681SAndroid Build Coastguard Worker // derived pointer relocation instructions given a vector of all relocate calls
computeBaseDerivedRelocateMap(const SmallVectorImpl<GCRelocateInst * > & AllRelocateCalls,DenseMap<GCRelocateInst *,SmallVector<GCRelocateInst *,2>> & RelocateInstMap)561*9880d681SAndroid Build Coastguard Worker static void computeBaseDerivedRelocateMap(
562*9880d681SAndroid Build Coastguard Worker     const SmallVectorImpl<GCRelocateInst *> &AllRelocateCalls,
563*9880d681SAndroid Build Coastguard Worker     DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>>
564*9880d681SAndroid Build Coastguard Worker         &RelocateInstMap) {
565*9880d681SAndroid Build Coastguard Worker   // Collect information in two maps: one primarily for locating the base object
566*9880d681SAndroid Build Coastguard Worker   // while filling the second map; the second map is the final structure holding
567*9880d681SAndroid Build Coastguard Worker   // a mapping between Base and corresponding Derived relocate calls
568*9880d681SAndroid Build Coastguard Worker   DenseMap<std::pair<unsigned, unsigned>, GCRelocateInst *> RelocateIdxMap;
569*9880d681SAndroid Build Coastguard Worker   for (auto *ThisRelocate : AllRelocateCalls) {
570*9880d681SAndroid Build Coastguard Worker     auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
571*9880d681SAndroid Build Coastguard Worker                             ThisRelocate->getDerivedPtrIndex());
572*9880d681SAndroid Build Coastguard Worker     RelocateIdxMap.insert(std::make_pair(K, ThisRelocate));
573*9880d681SAndroid Build Coastguard Worker   }
574*9880d681SAndroid Build Coastguard Worker   for (auto &Item : RelocateIdxMap) {
575*9880d681SAndroid Build Coastguard Worker     std::pair<unsigned, unsigned> Key = Item.first;
576*9880d681SAndroid Build Coastguard Worker     if (Key.first == Key.second)
577*9880d681SAndroid Build Coastguard Worker       // Base relocation: nothing to insert
578*9880d681SAndroid Build Coastguard Worker       continue;
579*9880d681SAndroid Build Coastguard Worker 
580*9880d681SAndroid Build Coastguard Worker     GCRelocateInst *I = Item.second;
581*9880d681SAndroid Build Coastguard Worker     auto BaseKey = std::make_pair(Key.first, Key.first);
582*9880d681SAndroid Build Coastguard Worker 
583*9880d681SAndroid Build Coastguard Worker     // We're iterating over RelocateIdxMap so we cannot modify it.
584*9880d681SAndroid Build Coastguard Worker     auto MaybeBase = RelocateIdxMap.find(BaseKey);
585*9880d681SAndroid Build Coastguard Worker     if (MaybeBase == RelocateIdxMap.end())
586*9880d681SAndroid Build Coastguard Worker       // TODO: We might want to insert a new base object relocate and gep off
587*9880d681SAndroid Build Coastguard Worker       // that, if there are enough derived object relocates.
588*9880d681SAndroid Build Coastguard Worker       continue;
589*9880d681SAndroid Build Coastguard Worker 
590*9880d681SAndroid Build Coastguard Worker     RelocateInstMap[MaybeBase->second].push_back(I);
591*9880d681SAndroid Build Coastguard Worker   }
592*9880d681SAndroid Build Coastguard Worker }
593*9880d681SAndroid Build Coastguard Worker 
594*9880d681SAndroid Build Coastguard Worker // Accepts a GEP and extracts the operands into a vector provided they're all
595*9880d681SAndroid Build Coastguard Worker // small integer constants
getGEPSmallConstantIntOffsetV(GetElementPtrInst * GEP,SmallVectorImpl<Value * > & OffsetV)596*9880d681SAndroid Build Coastguard Worker static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP,
597*9880d681SAndroid Build Coastguard Worker                                           SmallVectorImpl<Value *> &OffsetV) {
598*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 1; i < GEP->getNumOperands(); i++) {
599*9880d681SAndroid Build Coastguard Worker     // Only accept small constant integer operands
600*9880d681SAndroid Build Coastguard Worker     auto Op = dyn_cast<ConstantInt>(GEP->getOperand(i));
601*9880d681SAndroid Build Coastguard Worker     if (!Op || Op->getZExtValue() > 20)
602*9880d681SAndroid Build Coastguard Worker       return false;
603*9880d681SAndroid Build Coastguard Worker   }
604*9880d681SAndroid Build Coastguard Worker 
605*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 1; i < GEP->getNumOperands(); i++)
606*9880d681SAndroid Build Coastguard Worker     OffsetV.push_back(GEP->getOperand(i));
607*9880d681SAndroid Build Coastguard Worker   return true;
608*9880d681SAndroid Build Coastguard Worker }
609*9880d681SAndroid Build Coastguard Worker 
610*9880d681SAndroid Build Coastguard Worker // Takes a RelocatedBase (base pointer relocation instruction) and Targets to
611*9880d681SAndroid Build Coastguard Worker // replace, computes a replacement, and affects it.
612*9880d681SAndroid Build Coastguard Worker static bool
simplifyRelocatesOffABase(GCRelocateInst * RelocatedBase,const SmallVectorImpl<GCRelocateInst * > & Targets)613*9880d681SAndroid Build Coastguard Worker simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase,
614*9880d681SAndroid Build Coastguard Worker                           const SmallVectorImpl<GCRelocateInst *> &Targets) {
615*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
616*9880d681SAndroid Build Coastguard Worker   for (GCRelocateInst *ToReplace : Targets) {
617*9880d681SAndroid Build Coastguard Worker     assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() &&
618*9880d681SAndroid Build Coastguard Worker            "Not relocating a derived object of the original base object");
619*9880d681SAndroid Build Coastguard Worker     if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
620*9880d681SAndroid Build Coastguard Worker       // A duplicate relocate call. TODO: coalesce duplicates.
621*9880d681SAndroid Build Coastguard Worker       continue;
622*9880d681SAndroid Build Coastguard Worker     }
623*9880d681SAndroid Build Coastguard Worker 
624*9880d681SAndroid Build Coastguard Worker     if (RelocatedBase->getParent() != ToReplace->getParent()) {
625*9880d681SAndroid Build Coastguard Worker       // Base and derived relocates are in different basic blocks.
626*9880d681SAndroid Build Coastguard Worker       // In this case transform is only valid when base dominates derived
627*9880d681SAndroid Build Coastguard Worker       // relocate. However it would be too expensive to check dominance
628*9880d681SAndroid Build Coastguard Worker       // for each such relocate, so we skip the whole transformation.
629*9880d681SAndroid Build Coastguard Worker       continue;
630*9880d681SAndroid Build Coastguard Worker     }
631*9880d681SAndroid Build Coastguard Worker 
632*9880d681SAndroid Build Coastguard Worker     Value *Base = ToReplace->getBasePtr();
633*9880d681SAndroid Build Coastguard Worker     auto Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr());
634*9880d681SAndroid Build Coastguard Worker     if (!Derived || Derived->getPointerOperand() != Base)
635*9880d681SAndroid Build Coastguard Worker       continue;
636*9880d681SAndroid Build Coastguard Worker 
637*9880d681SAndroid Build Coastguard Worker     SmallVector<Value *, 2> OffsetV;
638*9880d681SAndroid Build Coastguard Worker     if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV))
639*9880d681SAndroid Build Coastguard Worker       continue;
640*9880d681SAndroid Build Coastguard Worker 
641*9880d681SAndroid Build Coastguard Worker     // Create a Builder and replace the target callsite with a gep
642*9880d681SAndroid Build Coastguard Worker     assert(RelocatedBase->getNextNode() &&
643*9880d681SAndroid Build Coastguard Worker            "Should always have one since it's not a terminator");
644*9880d681SAndroid Build Coastguard Worker 
645*9880d681SAndroid Build Coastguard Worker     // Insert after RelocatedBase
646*9880d681SAndroid Build Coastguard Worker     IRBuilder<> Builder(RelocatedBase->getNextNode());
647*9880d681SAndroid Build Coastguard Worker     Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
648*9880d681SAndroid Build Coastguard Worker 
649*9880d681SAndroid Build Coastguard Worker     // If gc_relocate does not match the actual type, cast it to the right type.
650*9880d681SAndroid Build Coastguard Worker     // In theory, there must be a bitcast after gc_relocate if the type does not
651*9880d681SAndroid Build Coastguard Worker     // match, and we should reuse it to get the derived pointer. But it could be
652*9880d681SAndroid Build Coastguard Worker     // cases like this:
653*9880d681SAndroid Build Coastguard Worker     // bb1:
654*9880d681SAndroid Build Coastguard Worker     //  ...
655*9880d681SAndroid Build Coastguard Worker     //  %g1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
656*9880d681SAndroid Build Coastguard Worker     //  br label %merge
657*9880d681SAndroid Build Coastguard Worker     //
658*9880d681SAndroid Build Coastguard Worker     // bb2:
659*9880d681SAndroid Build Coastguard Worker     //  ...
660*9880d681SAndroid Build Coastguard Worker     //  %g2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
661*9880d681SAndroid Build Coastguard Worker     //  br label %merge
662*9880d681SAndroid Build Coastguard Worker     //
663*9880d681SAndroid Build Coastguard Worker     // merge:
664*9880d681SAndroid Build Coastguard Worker     //  %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ]
665*9880d681SAndroid Build Coastguard Worker     //  %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)*
666*9880d681SAndroid Build Coastguard Worker     //
667*9880d681SAndroid Build Coastguard Worker     // In this case, we can not find the bitcast any more. So we insert a new bitcast
668*9880d681SAndroid Build Coastguard Worker     // no matter there is already one or not. In this way, we can handle all cases, and
669*9880d681SAndroid Build Coastguard Worker     // the extra bitcast should be optimized away in later passes.
670*9880d681SAndroid Build Coastguard Worker     Value *ActualRelocatedBase = RelocatedBase;
671*9880d681SAndroid Build Coastguard Worker     if (RelocatedBase->getType() != Base->getType()) {
672*9880d681SAndroid Build Coastguard Worker       ActualRelocatedBase =
673*9880d681SAndroid Build Coastguard Worker           Builder.CreateBitCast(RelocatedBase, Base->getType());
674*9880d681SAndroid Build Coastguard Worker     }
675*9880d681SAndroid Build Coastguard Worker     Value *Replacement = Builder.CreateGEP(
676*9880d681SAndroid Build Coastguard Worker         Derived->getSourceElementType(), ActualRelocatedBase, makeArrayRef(OffsetV));
677*9880d681SAndroid Build Coastguard Worker     Replacement->takeName(ToReplace);
678*9880d681SAndroid Build Coastguard Worker     // If the newly generated derived pointer's type does not match the original derived
679*9880d681SAndroid Build Coastguard Worker     // pointer's type, cast the new derived pointer to match it. Same reasoning as above.
680*9880d681SAndroid Build Coastguard Worker     Value *ActualReplacement = Replacement;
681*9880d681SAndroid Build Coastguard Worker     if (Replacement->getType() != ToReplace->getType()) {
682*9880d681SAndroid Build Coastguard Worker       ActualReplacement =
683*9880d681SAndroid Build Coastguard Worker           Builder.CreateBitCast(Replacement, ToReplace->getType());
684*9880d681SAndroid Build Coastguard Worker     }
685*9880d681SAndroid Build Coastguard Worker     ToReplace->replaceAllUsesWith(ActualReplacement);
686*9880d681SAndroid Build Coastguard Worker     ToReplace->eraseFromParent();
687*9880d681SAndroid Build Coastguard Worker 
688*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
689*9880d681SAndroid Build Coastguard Worker   }
690*9880d681SAndroid Build Coastguard Worker   return MadeChange;
691*9880d681SAndroid Build Coastguard Worker }
692*9880d681SAndroid Build Coastguard Worker 
693*9880d681SAndroid Build Coastguard Worker // Turns this:
694*9880d681SAndroid Build Coastguard Worker //
695*9880d681SAndroid Build Coastguard Worker // %base = ...
696*9880d681SAndroid Build Coastguard Worker // %ptr = gep %base + 15
697*9880d681SAndroid Build Coastguard Worker // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
698*9880d681SAndroid Build Coastguard Worker // %base' = relocate(%tok, i32 4, i32 4)
699*9880d681SAndroid Build Coastguard Worker // %ptr' = relocate(%tok, i32 4, i32 5)
700*9880d681SAndroid Build Coastguard Worker // %val = load %ptr'
701*9880d681SAndroid Build Coastguard Worker //
702*9880d681SAndroid Build Coastguard Worker // into this:
703*9880d681SAndroid Build Coastguard Worker //
704*9880d681SAndroid Build Coastguard Worker // %base = ...
705*9880d681SAndroid Build Coastguard Worker // %ptr = gep %base + 15
706*9880d681SAndroid Build Coastguard Worker // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
707*9880d681SAndroid Build Coastguard Worker // %base' = gc.relocate(%tok, i32 4, i32 4)
708*9880d681SAndroid Build Coastguard Worker // %ptr' = gep %base' + 15
709*9880d681SAndroid Build Coastguard Worker // %val = load %ptr'
simplifyOffsetableRelocate(Instruction & I)710*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::simplifyOffsetableRelocate(Instruction &I) {
711*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
712*9880d681SAndroid Build Coastguard Worker   SmallVector<GCRelocateInst *, 2> AllRelocateCalls;
713*9880d681SAndroid Build Coastguard Worker 
714*9880d681SAndroid Build Coastguard Worker   for (auto *U : I.users())
715*9880d681SAndroid Build Coastguard Worker     if (GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U))
716*9880d681SAndroid Build Coastguard Worker       // Collect all the relocate calls associated with a statepoint
717*9880d681SAndroid Build Coastguard Worker       AllRelocateCalls.push_back(Relocate);
718*9880d681SAndroid Build Coastguard Worker 
719*9880d681SAndroid Build Coastguard Worker   // We need atleast one base pointer relocation + one derived pointer
720*9880d681SAndroid Build Coastguard Worker   // relocation to mangle
721*9880d681SAndroid Build Coastguard Worker   if (AllRelocateCalls.size() < 2)
722*9880d681SAndroid Build Coastguard Worker     return false;
723*9880d681SAndroid Build Coastguard Worker 
724*9880d681SAndroid Build Coastguard Worker   // RelocateInstMap is a mapping from the base relocate instruction to the
725*9880d681SAndroid Build Coastguard Worker   // corresponding derived relocate instructions
726*9880d681SAndroid Build Coastguard Worker   DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>> RelocateInstMap;
727*9880d681SAndroid Build Coastguard Worker   computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap);
728*9880d681SAndroid Build Coastguard Worker   if (RelocateInstMap.empty())
729*9880d681SAndroid Build Coastguard Worker     return false;
730*9880d681SAndroid Build Coastguard Worker 
731*9880d681SAndroid Build Coastguard Worker   for (auto &Item : RelocateInstMap)
732*9880d681SAndroid Build Coastguard Worker     // Item.first is the RelocatedBase to offset against
733*9880d681SAndroid Build Coastguard Worker     // Item.second is the vector of Targets to replace
734*9880d681SAndroid Build Coastguard Worker     MadeChange = simplifyRelocatesOffABase(Item.first, Item.second);
735*9880d681SAndroid Build Coastguard Worker   return MadeChange;
736*9880d681SAndroid Build Coastguard Worker }
737*9880d681SAndroid Build Coastguard Worker 
738*9880d681SAndroid Build Coastguard Worker /// SinkCast - Sink the specified cast instruction into its user blocks
SinkCast(CastInst * CI)739*9880d681SAndroid Build Coastguard Worker static bool SinkCast(CastInst *CI) {
740*9880d681SAndroid Build Coastguard Worker   BasicBlock *DefBB = CI->getParent();
741*9880d681SAndroid Build Coastguard Worker 
742*9880d681SAndroid Build Coastguard Worker   /// InsertedCasts - Only insert a cast in each block once.
743*9880d681SAndroid Build Coastguard Worker   DenseMap<BasicBlock*, CastInst*> InsertedCasts;
744*9880d681SAndroid Build Coastguard Worker 
745*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
746*9880d681SAndroid Build Coastguard Worker   for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
747*9880d681SAndroid Build Coastguard Worker        UI != E; ) {
748*9880d681SAndroid Build Coastguard Worker     Use &TheUse = UI.getUse();
749*9880d681SAndroid Build Coastguard Worker     Instruction *User = cast<Instruction>(*UI);
750*9880d681SAndroid Build Coastguard Worker 
751*9880d681SAndroid Build Coastguard Worker     // Figure out which BB this cast is used in.  For PHI's this is the
752*9880d681SAndroid Build Coastguard Worker     // appropriate predecessor block.
753*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = User->getParent();
754*9880d681SAndroid Build Coastguard Worker     if (PHINode *PN = dyn_cast<PHINode>(User)) {
755*9880d681SAndroid Build Coastguard Worker       UserBB = PN->getIncomingBlock(TheUse);
756*9880d681SAndroid Build Coastguard Worker     }
757*9880d681SAndroid Build Coastguard Worker 
758*9880d681SAndroid Build Coastguard Worker     // Preincrement use iterator so we don't invalidate it.
759*9880d681SAndroid Build Coastguard Worker     ++UI;
760*9880d681SAndroid Build Coastguard Worker 
761*9880d681SAndroid Build Coastguard Worker     // The first insertion point of a block containing an EH pad is after the
762*9880d681SAndroid Build Coastguard Worker     // pad.  If the pad is the user, we cannot sink the cast past the pad.
763*9880d681SAndroid Build Coastguard Worker     if (User->isEHPad())
764*9880d681SAndroid Build Coastguard Worker       continue;
765*9880d681SAndroid Build Coastguard Worker 
766*9880d681SAndroid Build Coastguard Worker     // If the block selected to receive the cast is an EH pad that does not
767*9880d681SAndroid Build Coastguard Worker     // allow non-PHI instructions before the terminator, we can't sink the
768*9880d681SAndroid Build Coastguard Worker     // cast.
769*9880d681SAndroid Build Coastguard Worker     if (UserBB->getTerminator()->isEHPad())
770*9880d681SAndroid Build Coastguard Worker       continue;
771*9880d681SAndroid Build Coastguard Worker 
772*9880d681SAndroid Build Coastguard Worker     // If this user is in the same block as the cast, don't change the cast.
773*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) continue;
774*9880d681SAndroid Build Coastguard Worker 
775*9880d681SAndroid Build Coastguard Worker     // If we have already inserted a cast into this block, use it.
776*9880d681SAndroid Build Coastguard Worker     CastInst *&InsertedCast = InsertedCasts[UserBB];
777*9880d681SAndroid Build Coastguard Worker 
778*9880d681SAndroid Build Coastguard Worker     if (!InsertedCast) {
779*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
780*9880d681SAndroid Build Coastguard Worker       assert(InsertPt != UserBB->end());
781*9880d681SAndroid Build Coastguard Worker       InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0),
782*9880d681SAndroid Build Coastguard Worker                                       CI->getType(), "", &*InsertPt);
783*9880d681SAndroid Build Coastguard Worker     }
784*9880d681SAndroid Build Coastguard Worker 
785*9880d681SAndroid Build Coastguard Worker     // Replace a use of the cast with a use of the new cast.
786*9880d681SAndroid Build Coastguard Worker     TheUse = InsertedCast;
787*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
788*9880d681SAndroid Build Coastguard Worker     ++NumCastUses;
789*9880d681SAndroid Build Coastguard Worker   }
790*9880d681SAndroid Build Coastguard Worker 
791*9880d681SAndroid Build Coastguard Worker   // If we removed all uses, nuke the cast.
792*9880d681SAndroid Build Coastguard Worker   if (CI->use_empty()) {
793*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
794*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
795*9880d681SAndroid Build Coastguard Worker   }
796*9880d681SAndroid Build Coastguard Worker 
797*9880d681SAndroid Build Coastguard Worker   return MadeChange;
798*9880d681SAndroid Build Coastguard Worker }
799*9880d681SAndroid Build Coastguard Worker 
800*9880d681SAndroid Build Coastguard Worker /// If the specified cast instruction is a noop copy (e.g. it's casting from
801*9880d681SAndroid Build Coastguard Worker /// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
802*9880d681SAndroid Build Coastguard Worker /// reduce the number of virtual registers that must be created and coalesced.
803*9880d681SAndroid Build Coastguard Worker ///
804*9880d681SAndroid Build Coastguard Worker /// Return true if any changes are made.
805*9880d681SAndroid Build Coastguard Worker ///
OptimizeNoopCopyExpression(CastInst * CI,const TargetLowering & TLI,const DataLayout & DL)806*9880d681SAndroid Build Coastguard Worker static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI,
807*9880d681SAndroid Build Coastguard Worker                                        const DataLayout &DL) {
808*9880d681SAndroid Build Coastguard Worker   // If this is a noop copy,
809*9880d681SAndroid Build Coastguard Worker   EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType());
810*9880d681SAndroid Build Coastguard Worker   EVT DstVT = TLI.getValueType(DL, CI->getType());
811*9880d681SAndroid Build Coastguard Worker 
812*9880d681SAndroid Build Coastguard Worker   // This is an fp<->int conversion?
813*9880d681SAndroid Build Coastguard Worker   if (SrcVT.isInteger() != DstVT.isInteger())
814*9880d681SAndroid Build Coastguard Worker     return false;
815*9880d681SAndroid Build Coastguard Worker 
816*9880d681SAndroid Build Coastguard Worker   // If this is an extension, it will be a zero or sign extension, which
817*9880d681SAndroid Build Coastguard Worker   // isn't a noop.
818*9880d681SAndroid Build Coastguard Worker   if (SrcVT.bitsLT(DstVT)) return false;
819*9880d681SAndroid Build Coastguard Worker 
820*9880d681SAndroid Build Coastguard Worker   // If these values will be promoted, find out what they will be promoted
821*9880d681SAndroid Build Coastguard Worker   // to.  This helps us consider truncates on PPC as noop copies when they
822*9880d681SAndroid Build Coastguard Worker   // are.
823*9880d681SAndroid Build Coastguard Worker   if (TLI.getTypeAction(CI->getContext(), SrcVT) ==
824*9880d681SAndroid Build Coastguard Worker       TargetLowering::TypePromoteInteger)
825*9880d681SAndroid Build Coastguard Worker     SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT);
826*9880d681SAndroid Build Coastguard Worker   if (TLI.getTypeAction(CI->getContext(), DstVT) ==
827*9880d681SAndroid Build Coastguard Worker       TargetLowering::TypePromoteInteger)
828*9880d681SAndroid Build Coastguard Worker     DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT);
829*9880d681SAndroid Build Coastguard Worker 
830*9880d681SAndroid Build Coastguard Worker   // If, after promotion, these are the same types, this is a noop copy.
831*9880d681SAndroid Build Coastguard Worker   if (SrcVT != DstVT)
832*9880d681SAndroid Build Coastguard Worker     return false;
833*9880d681SAndroid Build Coastguard Worker 
834*9880d681SAndroid Build Coastguard Worker   return SinkCast(CI);
835*9880d681SAndroid Build Coastguard Worker }
836*9880d681SAndroid Build Coastguard Worker 
837*9880d681SAndroid Build Coastguard Worker /// Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if
838*9880d681SAndroid Build Coastguard Worker /// possible.
839*9880d681SAndroid Build Coastguard Worker ///
840*9880d681SAndroid Build Coastguard Worker /// Return true if any changes were made.
CombineUAddWithOverflow(CmpInst * CI)841*9880d681SAndroid Build Coastguard Worker static bool CombineUAddWithOverflow(CmpInst *CI) {
842*9880d681SAndroid Build Coastguard Worker   Value *A, *B;
843*9880d681SAndroid Build Coastguard Worker   Instruction *AddI;
844*9880d681SAndroid Build Coastguard Worker   if (!match(CI,
845*9880d681SAndroid Build Coastguard Worker              m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI))))
846*9880d681SAndroid Build Coastguard Worker     return false;
847*9880d681SAndroid Build Coastguard Worker 
848*9880d681SAndroid Build Coastguard Worker   Type *Ty = AddI->getType();
849*9880d681SAndroid Build Coastguard Worker   if (!isa<IntegerType>(Ty))
850*9880d681SAndroid Build Coastguard Worker     return false;
851*9880d681SAndroid Build Coastguard Worker 
852*9880d681SAndroid Build Coastguard Worker   // We don't want to move around uses of condition values this late, so we we
853*9880d681SAndroid Build Coastguard Worker   // check if it is legal to create the call to the intrinsic in the basic
854*9880d681SAndroid Build Coastguard Worker   // block containing the icmp:
855*9880d681SAndroid Build Coastguard Worker 
856*9880d681SAndroid Build Coastguard Worker   if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse())
857*9880d681SAndroid Build Coastguard Worker     return false;
858*9880d681SAndroid Build Coastguard Worker 
859*9880d681SAndroid Build Coastguard Worker #ifndef NDEBUG
860*9880d681SAndroid Build Coastguard Worker   // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption
861*9880d681SAndroid Build Coastguard Worker   // for now:
862*9880d681SAndroid Build Coastguard Worker   if (AddI->hasOneUse())
863*9880d681SAndroid Build Coastguard Worker     assert(*AddI->user_begin() == CI && "expected!");
864*9880d681SAndroid Build Coastguard Worker #endif
865*9880d681SAndroid Build Coastguard Worker 
866*9880d681SAndroid Build Coastguard Worker   Module *M = CI->getModule();
867*9880d681SAndroid Build Coastguard Worker   Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty);
868*9880d681SAndroid Build Coastguard Worker 
869*9880d681SAndroid Build Coastguard Worker   auto *InsertPt = AddI->hasOneUse() ? CI : AddI;
870*9880d681SAndroid Build Coastguard Worker 
871*9880d681SAndroid Build Coastguard Worker   auto *UAddWithOverflow =
872*9880d681SAndroid Build Coastguard Worker       CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt);
873*9880d681SAndroid Build Coastguard Worker   auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt);
874*9880d681SAndroid Build Coastguard Worker   auto *Overflow =
875*9880d681SAndroid Build Coastguard Worker       ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt);
876*9880d681SAndroid Build Coastguard Worker 
877*9880d681SAndroid Build Coastguard Worker   CI->replaceAllUsesWith(Overflow);
878*9880d681SAndroid Build Coastguard Worker   AddI->replaceAllUsesWith(UAdd);
879*9880d681SAndroid Build Coastguard Worker   CI->eraseFromParent();
880*9880d681SAndroid Build Coastguard Worker   AddI->eraseFromParent();
881*9880d681SAndroid Build Coastguard Worker   return true;
882*9880d681SAndroid Build Coastguard Worker }
883*9880d681SAndroid Build Coastguard Worker 
884*9880d681SAndroid Build Coastguard Worker /// Sink the given CmpInst into user blocks to reduce the number of virtual
885*9880d681SAndroid Build Coastguard Worker /// registers that must be created and coalesced. This is a clear win except on
886*9880d681SAndroid Build Coastguard Worker /// targets with multiple condition code registers (PowerPC), where it might
887*9880d681SAndroid Build Coastguard Worker /// lose; some adjustment may be wanted there.
888*9880d681SAndroid Build Coastguard Worker ///
889*9880d681SAndroid Build Coastguard Worker /// Return true if any changes are made.
SinkCmpExpression(CmpInst * CI,const TargetLowering * TLI)890*9880d681SAndroid Build Coastguard Worker static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
891*9880d681SAndroid Build Coastguard Worker   BasicBlock *DefBB = CI->getParent();
892*9880d681SAndroid Build Coastguard Worker 
893*9880d681SAndroid Build Coastguard Worker   // Avoid sinking soft-FP comparisons, since this can move them into a loop.
894*9880d681SAndroid Build Coastguard Worker   if (TLI && TLI->useSoftFloat() && isa<FCmpInst>(CI))
895*9880d681SAndroid Build Coastguard Worker     return false;
896*9880d681SAndroid Build Coastguard Worker 
897*9880d681SAndroid Build Coastguard Worker   // Only insert a cmp in each block once.
898*9880d681SAndroid Build Coastguard Worker   DenseMap<BasicBlock*, CmpInst*> InsertedCmps;
899*9880d681SAndroid Build Coastguard Worker 
900*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
901*9880d681SAndroid Build Coastguard Worker   for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
902*9880d681SAndroid Build Coastguard Worker        UI != E; ) {
903*9880d681SAndroid Build Coastguard Worker     Use &TheUse = UI.getUse();
904*9880d681SAndroid Build Coastguard Worker     Instruction *User = cast<Instruction>(*UI);
905*9880d681SAndroid Build Coastguard Worker 
906*9880d681SAndroid Build Coastguard Worker     // Preincrement use iterator so we don't invalidate it.
907*9880d681SAndroid Build Coastguard Worker     ++UI;
908*9880d681SAndroid Build Coastguard Worker 
909*9880d681SAndroid Build Coastguard Worker     // Don't bother for PHI nodes.
910*9880d681SAndroid Build Coastguard Worker     if (isa<PHINode>(User))
911*9880d681SAndroid Build Coastguard Worker       continue;
912*9880d681SAndroid Build Coastguard Worker 
913*9880d681SAndroid Build Coastguard Worker     // Figure out which BB this cmp is used in.
914*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = User->getParent();
915*9880d681SAndroid Build Coastguard Worker 
916*9880d681SAndroid Build Coastguard Worker     // If this user is in the same block as the cmp, don't change the cmp.
917*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) continue;
918*9880d681SAndroid Build Coastguard Worker 
919*9880d681SAndroid Build Coastguard Worker     // If we have already inserted a cmp into this block, use it.
920*9880d681SAndroid Build Coastguard Worker     CmpInst *&InsertedCmp = InsertedCmps[UserBB];
921*9880d681SAndroid Build Coastguard Worker 
922*9880d681SAndroid Build Coastguard Worker     if (!InsertedCmp) {
923*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
924*9880d681SAndroid Build Coastguard Worker       assert(InsertPt != UserBB->end());
925*9880d681SAndroid Build Coastguard Worker       InsertedCmp =
926*9880d681SAndroid Build Coastguard Worker           CmpInst::Create(CI->getOpcode(), CI->getPredicate(),
927*9880d681SAndroid Build Coastguard Worker                           CI->getOperand(0), CI->getOperand(1), "", &*InsertPt);
928*9880d681SAndroid Build Coastguard Worker     }
929*9880d681SAndroid Build Coastguard Worker 
930*9880d681SAndroid Build Coastguard Worker     // Replace a use of the cmp with a use of the new cmp.
931*9880d681SAndroid Build Coastguard Worker     TheUse = InsertedCmp;
932*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
933*9880d681SAndroid Build Coastguard Worker     ++NumCmpUses;
934*9880d681SAndroid Build Coastguard Worker   }
935*9880d681SAndroid Build Coastguard Worker 
936*9880d681SAndroid Build Coastguard Worker   // If we removed all uses, nuke the cmp.
937*9880d681SAndroid Build Coastguard Worker   if (CI->use_empty()) {
938*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
939*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
940*9880d681SAndroid Build Coastguard Worker   }
941*9880d681SAndroid Build Coastguard Worker 
942*9880d681SAndroid Build Coastguard Worker   return MadeChange;
943*9880d681SAndroid Build Coastguard Worker }
944*9880d681SAndroid Build Coastguard Worker 
OptimizeCmpExpression(CmpInst * CI,const TargetLowering * TLI)945*9880d681SAndroid Build Coastguard Worker static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
946*9880d681SAndroid Build Coastguard Worker   if (SinkCmpExpression(CI, TLI))
947*9880d681SAndroid Build Coastguard Worker     return true;
948*9880d681SAndroid Build Coastguard Worker 
949*9880d681SAndroid Build Coastguard Worker   if (CombineUAddWithOverflow(CI))
950*9880d681SAndroid Build Coastguard Worker     return true;
951*9880d681SAndroid Build Coastguard Worker 
952*9880d681SAndroid Build Coastguard Worker   return false;
953*9880d681SAndroid Build Coastguard Worker }
954*9880d681SAndroid Build Coastguard Worker 
955*9880d681SAndroid Build Coastguard Worker /// Check if the candidates could be combined with a shift instruction, which
956*9880d681SAndroid Build Coastguard Worker /// includes:
957*9880d681SAndroid Build Coastguard Worker /// 1. Truncate instruction
958*9880d681SAndroid Build Coastguard Worker /// 2. And instruction and the imm is a mask of the low bits:
959*9880d681SAndroid Build Coastguard Worker /// imm & (imm+1) == 0
isExtractBitsCandidateUse(Instruction * User)960*9880d681SAndroid Build Coastguard Worker static bool isExtractBitsCandidateUse(Instruction *User) {
961*9880d681SAndroid Build Coastguard Worker   if (!isa<TruncInst>(User)) {
962*9880d681SAndroid Build Coastguard Worker     if (User->getOpcode() != Instruction::And ||
963*9880d681SAndroid Build Coastguard Worker         !isa<ConstantInt>(User->getOperand(1)))
964*9880d681SAndroid Build Coastguard Worker       return false;
965*9880d681SAndroid Build Coastguard Worker 
966*9880d681SAndroid Build Coastguard Worker     const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue();
967*9880d681SAndroid Build Coastguard Worker 
968*9880d681SAndroid Build Coastguard Worker     if ((Cimm & (Cimm + 1)).getBoolValue())
969*9880d681SAndroid Build Coastguard Worker       return false;
970*9880d681SAndroid Build Coastguard Worker   }
971*9880d681SAndroid Build Coastguard Worker   return true;
972*9880d681SAndroid Build Coastguard Worker }
973*9880d681SAndroid Build Coastguard Worker 
974*9880d681SAndroid Build Coastguard Worker /// Sink both shift and truncate instruction to the use of truncate's BB.
975*9880d681SAndroid Build Coastguard Worker static bool
SinkShiftAndTruncate(BinaryOperator * ShiftI,Instruction * User,ConstantInt * CI,DenseMap<BasicBlock *,BinaryOperator * > & InsertedShifts,const TargetLowering & TLI,const DataLayout & DL)976*9880d681SAndroid Build Coastguard Worker SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
977*9880d681SAndroid Build Coastguard Worker                      DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts,
978*9880d681SAndroid Build Coastguard Worker                      const TargetLowering &TLI, const DataLayout &DL) {
979*9880d681SAndroid Build Coastguard Worker   BasicBlock *UserBB = User->getParent();
980*9880d681SAndroid Build Coastguard Worker   DenseMap<BasicBlock *, CastInst *> InsertedTruncs;
981*9880d681SAndroid Build Coastguard Worker   TruncInst *TruncI = dyn_cast<TruncInst>(User);
982*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
983*9880d681SAndroid Build Coastguard Worker 
984*9880d681SAndroid Build Coastguard Worker   for (Value::user_iterator TruncUI = TruncI->user_begin(),
985*9880d681SAndroid Build Coastguard Worker                             TruncE = TruncI->user_end();
986*9880d681SAndroid Build Coastguard Worker        TruncUI != TruncE;) {
987*9880d681SAndroid Build Coastguard Worker 
988*9880d681SAndroid Build Coastguard Worker     Use &TruncTheUse = TruncUI.getUse();
989*9880d681SAndroid Build Coastguard Worker     Instruction *TruncUser = cast<Instruction>(*TruncUI);
990*9880d681SAndroid Build Coastguard Worker     // Preincrement use iterator so we don't invalidate it.
991*9880d681SAndroid Build Coastguard Worker 
992*9880d681SAndroid Build Coastguard Worker     ++TruncUI;
993*9880d681SAndroid Build Coastguard Worker 
994*9880d681SAndroid Build Coastguard Worker     int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode());
995*9880d681SAndroid Build Coastguard Worker     if (!ISDOpcode)
996*9880d681SAndroid Build Coastguard Worker       continue;
997*9880d681SAndroid Build Coastguard Worker 
998*9880d681SAndroid Build Coastguard Worker     // If the use is actually a legal node, there will not be an
999*9880d681SAndroid Build Coastguard Worker     // implicit truncate.
1000*9880d681SAndroid Build Coastguard Worker     // FIXME: always querying the result type is just an
1001*9880d681SAndroid Build Coastguard Worker     // approximation; some nodes' legality is determined by the
1002*9880d681SAndroid Build Coastguard Worker     // operand or other means. There's no good way to find out though.
1003*9880d681SAndroid Build Coastguard Worker     if (TLI.isOperationLegalOrCustom(
1004*9880d681SAndroid Build Coastguard Worker             ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true)))
1005*9880d681SAndroid Build Coastguard Worker       continue;
1006*9880d681SAndroid Build Coastguard Worker 
1007*9880d681SAndroid Build Coastguard Worker     // Don't bother for PHI nodes.
1008*9880d681SAndroid Build Coastguard Worker     if (isa<PHINode>(TruncUser))
1009*9880d681SAndroid Build Coastguard Worker       continue;
1010*9880d681SAndroid Build Coastguard Worker 
1011*9880d681SAndroid Build Coastguard Worker     BasicBlock *TruncUserBB = TruncUser->getParent();
1012*9880d681SAndroid Build Coastguard Worker 
1013*9880d681SAndroid Build Coastguard Worker     if (UserBB == TruncUserBB)
1014*9880d681SAndroid Build Coastguard Worker       continue;
1015*9880d681SAndroid Build Coastguard Worker 
1016*9880d681SAndroid Build Coastguard Worker     BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB];
1017*9880d681SAndroid Build Coastguard Worker     CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
1018*9880d681SAndroid Build Coastguard Worker 
1019*9880d681SAndroid Build Coastguard Worker     if (!InsertedShift && !InsertedTrunc) {
1020*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt();
1021*9880d681SAndroid Build Coastguard Worker       assert(InsertPt != TruncUserBB->end());
1022*9880d681SAndroid Build Coastguard Worker       // Sink the shift
1023*9880d681SAndroid Build Coastguard Worker       if (ShiftI->getOpcode() == Instruction::AShr)
1024*9880d681SAndroid Build Coastguard Worker         InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
1025*9880d681SAndroid Build Coastguard Worker                                                    "", &*InsertPt);
1026*9880d681SAndroid Build Coastguard Worker       else
1027*9880d681SAndroid Build Coastguard Worker         InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
1028*9880d681SAndroid Build Coastguard Worker                                                    "", &*InsertPt);
1029*9880d681SAndroid Build Coastguard Worker 
1030*9880d681SAndroid Build Coastguard Worker       // Sink the trunc
1031*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt();
1032*9880d681SAndroid Build Coastguard Worker       TruncInsertPt++;
1033*9880d681SAndroid Build Coastguard Worker       assert(TruncInsertPt != TruncUserBB->end());
1034*9880d681SAndroid Build Coastguard Worker 
1035*9880d681SAndroid Build Coastguard Worker       InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift,
1036*9880d681SAndroid Build Coastguard Worker                                        TruncI->getType(), "", &*TruncInsertPt);
1037*9880d681SAndroid Build Coastguard Worker 
1038*9880d681SAndroid Build Coastguard Worker       MadeChange = true;
1039*9880d681SAndroid Build Coastguard Worker 
1040*9880d681SAndroid Build Coastguard Worker       TruncTheUse = InsertedTrunc;
1041*9880d681SAndroid Build Coastguard Worker     }
1042*9880d681SAndroid Build Coastguard Worker   }
1043*9880d681SAndroid Build Coastguard Worker   return MadeChange;
1044*9880d681SAndroid Build Coastguard Worker }
1045*9880d681SAndroid Build Coastguard Worker 
1046*9880d681SAndroid Build Coastguard Worker /// Sink the shift *right* instruction into user blocks if the uses could
1047*9880d681SAndroid Build Coastguard Worker /// potentially be combined with this shift instruction and generate BitExtract
1048*9880d681SAndroid Build Coastguard Worker /// instruction. It will only be applied if the architecture supports BitExtract
1049*9880d681SAndroid Build Coastguard Worker /// instruction. Here is an example:
1050*9880d681SAndroid Build Coastguard Worker /// BB1:
1051*9880d681SAndroid Build Coastguard Worker ///   %x.extract.shift = lshr i64 %arg1, 32
1052*9880d681SAndroid Build Coastguard Worker /// BB2:
1053*9880d681SAndroid Build Coastguard Worker ///   %x.extract.trunc = trunc i64 %x.extract.shift to i16
1054*9880d681SAndroid Build Coastguard Worker /// ==>
1055*9880d681SAndroid Build Coastguard Worker ///
1056*9880d681SAndroid Build Coastguard Worker /// BB2:
1057*9880d681SAndroid Build Coastguard Worker ///   %x.extract.shift.1 = lshr i64 %arg1, 32
1058*9880d681SAndroid Build Coastguard Worker ///   %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16
1059*9880d681SAndroid Build Coastguard Worker ///
1060*9880d681SAndroid Build Coastguard Worker /// CodeGen will recoginze the pattern in BB2 and generate BitExtract
1061*9880d681SAndroid Build Coastguard Worker /// instruction.
1062*9880d681SAndroid Build Coastguard Worker /// Return true if any changes are made.
OptimizeExtractBits(BinaryOperator * ShiftI,ConstantInt * CI,const TargetLowering & TLI,const DataLayout & DL)1063*9880d681SAndroid Build Coastguard Worker static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
1064*9880d681SAndroid Build Coastguard Worker                                 const TargetLowering &TLI,
1065*9880d681SAndroid Build Coastguard Worker                                 const DataLayout &DL) {
1066*9880d681SAndroid Build Coastguard Worker   BasicBlock *DefBB = ShiftI->getParent();
1067*9880d681SAndroid Build Coastguard Worker 
1068*9880d681SAndroid Build Coastguard Worker   /// Only insert instructions in each block once.
1069*9880d681SAndroid Build Coastguard Worker   DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts;
1070*9880d681SAndroid Build Coastguard Worker 
1071*9880d681SAndroid Build Coastguard Worker   bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType()));
1072*9880d681SAndroid Build Coastguard Worker 
1073*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
1074*9880d681SAndroid Build Coastguard Worker   for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end();
1075*9880d681SAndroid Build Coastguard Worker        UI != E;) {
1076*9880d681SAndroid Build Coastguard Worker     Use &TheUse = UI.getUse();
1077*9880d681SAndroid Build Coastguard Worker     Instruction *User = cast<Instruction>(*UI);
1078*9880d681SAndroid Build Coastguard Worker     // Preincrement use iterator so we don't invalidate it.
1079*9880d681SAndroid Build Coastguard Worker     ++UI;
1080*9880d681SAndroid Build Coastguard Worker 
1081*9880d681SAndroid Build Coastguard Worker     // Don't bother for PHI nodes.
1082*9880d681SAndroid Build Coastguard Worker     if (isa<PHINode>(User))
1083*9880d681SAndroid Build Coastguard Worker       continue;
1084*9880d681SAndroid Build Coastguard Worker 
1085*9880d681SAndroid Build Coastguard Worker     if (!isExtractBitsCandidateUse(User))
1086*9880d681SAndroid Build Coastguard Worker       continue;
1087*9880d681SAndroid Build Coastguard Worker 
1088*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = User->getParent();
1089*9880d681SAndroid Build Coastguard Worker 
1090*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) {
1091*9880d681SAndroid Build Coastguard Worker       // If the shift and truncate instruction are in the same BB. The use of
1092*9880d681SAndroid Build Coastguard Worker       // the truncate(TruncUse) may still introduce another truncate if not
1093*9880d681SAndroid Build Coastguard Worker       // legal. In this case, we would like to sink both shift and truncate
1094*9880d681SAndroid Build Coastguard Worker       // instruction to the BB of TruncUse.
1095*9880d681SAndroid Build Coastguard Worker       // for example:
1096*9880d681SAndroid Build Coastguard Worker       // BB1:
1097*9880d681SAndroid Build Coastguard Worker       // i64 shift.result = lshr i64 opnd, imm
1098*9880d681SAndroid Build Coastguard Worker       // trunc.result = trunc shift.result to i16
1099*9880d681SAndroid Build Coastguard Worker       //
1100*9880d681SAndroid Build Coastguard Worker       // BB2:
1101*9880d681SAndroid Build Coastguard Worker       //   ----> We will have an implicit truncate here if the architecture does
1102*9880d681SAndroid Build Coastguard Worker       //   not have i16 compare.
1103*9880d681SAndroid Build Coastguard Worker       // cmp i16 trunc.result, opnd2
1104*9880d681SAndroid Build Coastguard Worker       //
1105*9880d681SAndroid Build Coastguard Worker       if (isa<TruncInst>(User) && shiftIsLegal
1106*9880d681SAndroid Build Coastguard Worker           // If the type of the truncate is legal, no trucate will be
1107*9880d681SAndroid Build Coastguard Worker           // introduced in other basic blocks.
1108*9880d681SAndroid Build Coastguard Worker           &&
1109*9880d681SAndroid Build Coastguard Worker           (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType()))))
1110*9880d681SAndroid Build Coastguard Worker         MadeChange =
1111*9880d681SAndroid Build Coastguard Worker             SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL);
1112*9880d681SAndroid Build Coastguard Worker 
1113*9880d681SAndroid Build Coastguard Worker       continue;
1114*9880d681SAndroid Build Coastguard Worker     }
1115*9880d681SAndroid Build Coastguard Worker     // If we have already inserted a shift into this block, use it.
1116*9880d681SAndroid Build Coastguard Worker     BinaryOperator *&InsertedShift = InsertedShifts[UserBB];
1117*9880d681SAndroid Build Coastguard Worker 
1118*9880d681SAndroid Build Coastguard Worker     if (!InsertedShift) {
1119*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
1120*9880d681SAndroid Build Coastguard Worker       assert(InsertPt != UserBB->end());
1121*9880d681SAndroid Build Coastguard Worker 
1122*9880d681SAndroid Build Coastguard Worker       if (ShiftI->getOpcode() == Instruction::AShr)
1123*9880d681SAndroid Build Coastguard Worker         InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
1124*9880d681SAndroid Build Coastguard Worker                                                    "", &*InsertPt);
1125*9880d681SAndroid Build Coastguard Worker       else
1126*9880d681SAndroid Build Coastguard Worker         InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
1127*9880d681SAndroid Build Coastguard Worker                                                    "", &*InsertPt);
1128*9880d681SAndroid Build Coastguard Worker 
1129*9880d681SAndroid Build Coastguard Worker       MadeChange = true;
1130*9880d681SAndroid Build Coastguard Worker     }
1131*9880d681SAndroid Build Coastguard Worker 
1132*9880d681SAndroid Build Coastguard Worker     // Replace a use of the shift with a use of the new shift.
1133*9880d681SAndroid Build Coastguard Worker     TheUse = InsertedShift;
1134*9880d681SAndroid Build Coastguard Worker   }
1135*9880d681SAndroid Build Coastguard Worker 
1136*9880d681SAndroid Build Coastguard Worker   // If we removed all uses, nuke the shift.
1137*9880d681SAndroid Build Coastguard Worker   if (ShiftI->use_empty())
1138*9880d681SAndroid Build Coastguard Worker     ShiftI->eraseFromParent();
1139*9880d681SAndroid Build Coastguard Worker 
1140*9880d681SAndroid Build Coastguard Worker   return MadeChange;
1141*9880d681SAndroid Build Coastguard Worker }
1142*9880d681SAndroid Build Coastguard Worker 
1143*9880d681SAndroid Build Coastguard Worker // Translate a masked load intrinsic like
1144*9880d681SAndroid Build Coastguard Worker // <16 x i32 > @llvm.masked.load( <16 x i32>* %addr, i32 align,
1145*9880d681SAndroid Build Coastguard Worker //                               <16 x i1> %mask, <16 x i32> %passthru)
1146*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, with loading element one-by-one if
1147*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set
1148*9880d681SAndroid Build Coastguard Worker //
1149*9880d681SAndroid Build Coastguard Worker //  %1 = bitcast i8* %addr to i32*
1150*9880d681SAndroid Build Coastguard Worker //  %2 = extractelement <16 x i1> %mask, i32 0
1151*9880d681SAndroid Build Coastguard Worker //  %3 = icmp eq i1 %2, true
1152*9880d681SAndroid Build Coastguard Worker //  br i1 %3, label %cond.load, label %else
1153*9880d681SAndroid Build Coastguard Worker //
1154*9880d681SAndroid Build Coastguard Worker //cond.load:                                        ; preds = %0
1155*9880d681SAndroid Build Coastguard Worker //  %4 = getelementptr i32* %1, i32 0
1156*9880d681SAndroid Build Coastguard Worker //  %5 = load i32* %4
1157*9880d681SAndroid Build Coastguard Worker //  %6 = insertelement <16 x i32> undef, i32 %5, i32 0
1158*9880d681SAndroid Build Coastguard Worker //  br label %else
1159*9880d681SAndroid Build Coastguard Worker //
1160*9880d681SAndroid Build Coastguard Worker //else:                                             ; preds = %0, %cond.load
1161*9880d681SAndroid Build Coastguard Worker //  %res.phi.else = phi <16 x i32> [ %6, %cond.load ], [ undef, %0 ]
1162*9880d681SAndroid Build Coastguard Worker //  %7 = extractelement <16 x i1> %mask, i32 1
1163*9880d681SAndroid Build Coastguard Worker //  %8 = icmp eq i1 %7, true
1164*9880d681SAndroid Build Coastguard Worker //  br i1 %8, label %cond.load1, label %else2
1165*9880d681SAndroid Build Coastguard Worker //
1166*9880d681SAndroid Build Coastguard Worker //cond.load1:                                       ; preds = %else
1167*9880d681SAndroid Build Coastguard Worker //  %9 = getelementptr i32* %1, i32 1
1168*9880d681SAndroid Build Coastguard Worker //  %10 = load i32* %9
1169*9880d681SAndroid Build Coastguard Worker //  %11 = insertelement <16 x i32> %res.phi.else, i32 %10, i32 1
1170*9880d681SAndroid Build Coastguard Worker //  br label %else2
1171*9880d681SAndroid Build Coastguard Worker //
1172*9880d681SAndroid Build Coastguard Worker //else2:                                            ; preds = %else, %cond.load1
1173*9880d681SAndroid Build Coastguard Worker //  %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
1174*9880d681SAndroid Build Coastguard Worker //  %12 = extractelement <16 x i1> %mask, i32 2
1175*9880d681SAndroid Build Coastguard Worker //  %13 = icmp eq i1 %12, true
1176*9880d681SAndroid Build Coastguard Worker //  br i1 %13, label %cond.load4, label %else5
1177*9880d681SAndroid Build Coastguard Worker //
scalarizeMaskedLoad(CallInst * CI)1178*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedLoad(CallInst *CI) {
1179*9880d681SAndroid Build Coastguard Worker   Value *Ptr  = CI->getArgOperand(0);
1180*9880d681SAndroid Build Coastguard Worker   Value *Alignment = CI->getArgOperand(1);
1181*9880d681SAndroid Build Coastguard Worker   Value *Mask = CI->getArgOperand(2);
1182*9880d681SAndroid Build Coastguard Worker   Value *Src0 = CI->getArgOperand(3);
1183*9880d681SAndroid Build Coastguard Worker 
1184*9880d681SAndroid Build Coastguard Worker   unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1185*9880d681SAndroid Build Coastguard Worker   VectorType *VecType = dyn_cast<VectorType>(CI->getType());
1186*9880d681SAndroid Build Coastguard Worker   assert(VecType && "Unexpected return type of masked load intrinsic");
1187*9880d681SAndroid Build Coastguard Worker 
1188*9880d681SAndroid Build Coastguard Worker   Type *EltTy = CI->getType()->getVectorElementType();
1189*9880d681SAndroid Build Coastguard Worker 
1190*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(CI->getContext());
1191*9880d681SAndroid Build Coastguard Worker   Instruction *InsertPt = CI;
1192*9880d681SAndroid Build Coastguard Worker   BasicBlock *IfBlock = CI->getParent();
1193*9880d681SAndroid Build Coastguard Worker   BasicBlock *CondBlock = nullptr;
1194*9880d681SAndroid Build Coastguard Worker   BasicBlock *PrevIfBlock = CI->getParent();
1195*9880d681SAndroid Build Coastguard Worker 
1196*9880d681SAndroid Build Coastguard Worker   Builder.SetInsertPoint(InsertPt);
1197*9880d681SAndroid Build Coastguard Worker   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1198*9880d681SAndroid Build Coastguard Worker 
1199*9880d681SAndroid Build Coastguard Worker   // Short-cut if the mask is all-true.
1200*9880d681SAndroid Build Coastguard Worker   bool IsAllOnesMask = isa<Constant>(Mask) &&
1201*9880d681SAndroid Build Coastguard Worker     cast<Constant>(Mask)->isAllOnesValue();
1202*9880d681SAndroid Build Coastguard Worker 
1203*9880d681SAndroid Build Coastguard Worker   if (IsAllOnesMask) {
1204*9880d681SAndroid Build Coastguard Worker     Value *NewI = Builder.CreateAlignedLoad(Ptr, AlignVal);
1205*9880d681SAndroid Build Coastguard Worker     CI->replaceAllUsesWith(NewI);
1206*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1207*9880d681SAndroid Build Coastguard Worker     return;
1208*9880d681SAndroid Build Coastguard Worker   }
1209*9880d681SAndroid Build Coastguard Worker 
1210*9880d681SAndroid Build Coastguard Worker   // Adjust alignment for the scalar instruction.
1211*9880d681SAndroid Build Coastguard Worker   AlignVal = std::min(AlignVal, VecType->getScalarSizeInBits()/8);
1212*9880d681SAndroid Build Coastguard Worker   // Bitcast %addr fron i8* to EltTy*
1213*9880d681SAndroid Build Coastguard Worker   Type *NewPtrType =
1214*9880d681SAndroid Build Coastguard Worker     EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace());
1215*9880d681SAndroid Build Coastguard Worker   Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
1216*9880d681SAndroid Build Coastguard Worker   unsigned VectorWidth = VecType->getNumElements();
1217*9880d681SAndroid Build Coastguard Worker 
1218*9880d681SAndroid Build Coastguard Worker   Value *UndefVal = UndefValue::get(VecType);
1219*9880d681SAndroid Build Coastguard Worker 
1220*9880d681SAndroid Build Coastguard Worker   // The result vector
1221*9880d681SAndroid Build Coastguard Worker   Value *VResult = UndefVal;
1222*9880d681SAndroid Build Coastguard Worker 
1223*9880d681SAndroid Build Coastguard Worker   if (isa<ConstantVector>(Mask)) {
1224*9880d681SAndroid Build Coastguard Worker     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1225*9880d681SAndroid Build Coastguard Worker       if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1226*9880d681SAndroid Build Coastguard Worker           continue;
1227*9880d681SAndroid Build Coastguard Worker       Value *Gep =
1228*9880d681SAndroid Build Coastguard Worker           Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1229*9880d681SAndroid Build Coastguard Worker       LoadInst* Load = Builder.CreateAlignedLoad(Gep, AlignVal);
1230*9880d681SAndroid Build Coastguard Worker       VResult = Builder.CreateInsertElement(VResult, Load,
1231*9880d681SAndroid Build Coastguard Worker                                             Builder.getInt32(Idx));
1232*9880d681SAndroid Build Coastguard Worker     }
1233*9880d681SAndroid Build Coastguard Worker     Value *NewI = Builder.CreateSelect(Mask, VResult, Src0);
1234*9880d681SAndroid Build Coastguard Worker     CI->replaceAllUsesWith(NewI);
1235*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1236*9880d681SAndroid Build Coastguard Worker     return;
1237*9880d681SAndroid Build Coastguard Worker   }
1238*9880d681SAndroid Build Coastguard Worker 
1239*9880d681SAndroid Build Coastguard Worker   PHINode *Phi = nullptr;
1240*9880d681SAndroid Build Coastguard Worker   Value *PrevPhi = UndefVal;
1241*9880d681SAndroid Build Coastguard Worker 
1242*9880d681SAndroid Build Coastguard Worker   for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1243*9880d681SAndroid Build Coastguard Worker 
1244*9880d681SAndroid Build Coastguard Worker     // Fill the "else" block, created in the previous iteration
1245*9880d681SAndroid Build Coastguard Worker     //
1246*9880d681SAndroid Build Coastguard Worker     //  %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
1247*9880d681SAndroid Build Coastguard Worker     //  %mask_1 = extractelement <16 x i1> %mask, i32 Idx
1248*9880d681SAndroid Build Coastguard Worker     //  %to_load = icmp eq i1 %mask_1, true
1249*9880d681SAndroid Build Coastguard Worker     //  br i1 %to_load, label %cond.load, label %else
1250*9880d681SAndroid Build Coastguard Worker     //
1251*9880d681SAndroid Build Coastguard Worker     if (Idx > 0) {
1252*9880d681SAndroid Build Coastguard Worker       Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
1253*9880d681SAndroid Build Coastguard Worker       Phi->addIncoming(VResult, CondBlock);
1254*9880d681SAndroid Build Coastguard Worker       Phi->addIncoming(PrevPhi, PrevIfBlock);
1255*9880d681SAndroid Build Coastguard Worker       PrevPhi = Phi;
1256*9880d681SAndroid Build Coastguard Worker       VResult = Phi;
1257*9880d681SAndroid Build Coastguard Worker     }
1258*9880d681SAndroid Build Coastguard Worker 
1259*9880d681SAndroid Build Coastguard Worker     Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx));
1260*9880d681SAndroid Build Coastguard Worker     Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1261*9880d681SAndroid Build Coastguard Worker                                     ConstantInt::get(Predicate->getType(), 1));
1262*9880d681SAndroid Build Coastguard Worker 
1263*9880d681SAndroid Build Coastguard Worker     // Create "cond" block
1264*9880d681SAndroid Build Coastguard Worker     //
1265*9880d681SAndroid Build Coastguard Worker     //  %EltAddr = getelementptr i32* %1, i32 0
1266*9880d681SAndroid Build Coastguard Worker     //  %Elt = load i32* %EltAddr
1267*9880d681SAndroid Build Coastguard Worker     //  VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
1268*9880d681SAndroid Build Coastguard Worker     //
1269*9880d681SAndroid Build Coastguard Worker     CondBlock = IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.load");
1270*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1271*9880d681SAndroid Build Coastguard Worker 
1272*9880d681SAndroid Build Coastguard Worker     Value *Gep =
1273*9880d681SAndroid Build Coastguard Worker         Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1274*9880d681SAndroid Build Coastguard Worker     LoadInst *Load = Builder.CreateAlignedLoad(Gep, AlignVal);
1275*9880d681SAndroid Build Coastguard Worker     VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx));
1276*9880d681SAndroid Build Coastguard Worker 
1277*9880d681SAndroid Build Coastguard Worker     // Create "else" block, fill it in the next iteration
1278*9880d681SAndroid Build Coastguard Worker     BasicBlock *NewIfBlock =
1279*9880d681SAndroid Build Coastguard Worker         CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
1280*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1281*9880d681SAndroid Build Coastguard Worker     Instruction *OldBr = IfBlock->getTerminator();
1282*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1283*9880d681SAndroid Build Coastguard Worker     OldBr->eraseFromParent();
1284*9880d681SAndroid Build Coastguard Worker     PrevIfBlock = IfBlock;
1285*9880d681SAndroid Build Coastguard Worker     IfBlock = NewIfBlock;
1286*9880d681SAndroid Build Coastguard Worker   }
1287*9880d681SAndroid Build Coastguard Worker 
1288*9880d681SAndroid Build Coastguard Worker   Phi = Builder.CreatePHI(VecType, 2, "res.phi.select");
1289*9880d681SAndroid Build Coastguard Worker   Phi->addIncoming(VResult, CondBlock);
1290*9880d681SAndroid Build Coastguard Worker   Phi->addIncoming(PrevPhi, PrevIfBlock);
1291*9880d681SAndroid Build Coastguard Worker   Value *NewI = Builder.CreateSelect(Mask, Phi, Src0);
1292*9880d681SAndroid Build Coastguard Worker   CI->replaceAllUsesWith(NewI);
1293*9880d681SAndroid Build Coastguard Worker   CI->eraseFromParent();
1294*9880d681SAndroid Build Coastguard Worker }
1295*9880d681SAndroid Build Coastguard Worker 
1296*9880d681SAndroid Build Coastguard Worker // Translate a masked store intrinsic, like
1297*9880d681SAndroid Build Coastguard Worker // void @llvm.masked.store(<16 x i32> %src, <16 x i32>* %addr, i32 align,
1298*9880d681SAndroid Build Coastguard Worker //                               <16 x i1> %mask)
1299*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, that stores element one-by-one if
1300*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set
1301*9880d681SAndroid Build Coastguard Worker //
1302*9880d681SAndroid Build Coastguard Worker //   %1 = bitcast i8* %addr to i32*
1303*9880d681SAndroid Build Coastguard Worker //   %2 = extractelement <16 x i1> %mask, i32 0
1304*9880d681SAndroid Build Coastguard Worker //   %3 = icmp eq i1 %2, true
1305*9880d681SAndroid Build Coastguard Worker //   br i1 %3, label %cond.store, label %else
1306*9880d681SAndroid Build Coastguard Worker //
1307*9880d681SAndroid Build Coastguard Worker // cond.store:                                       ; preds = %0
1308*9880d681SAndroid Build Coastguard Worker //   %4 = extractelement <16 x i32> %val, i32 0
1309*9880d681SAndroid Build Coastguard Worker //   %5 = getelementptr i32* %1, i32 0
1310*9880d681SAndroid Build Coastguard Worker //   store i32 %4, i32* %5
1311*9880d681SAndroid Build Coastguard Worker //   br label %else
1312*9880d681SAndroid Build Coastguard Worker //
1313*9880d681SAndroid Build Coastguard Worker // else:                                             ; preds = %0, %cond.store
1314*9880d681SAndroid Build Coastguard Worker //   %6 = extractelement <16 x i1> %mask, i32 1
1315*9880d681SAndroid Build Coastguard Worker //   %7 = icmp eq i1 %6, true
1316*9880d681SAndroid Build Coastguard Worker //   br i1 %7, label %cond.store1, label %else2
1317*9880d681SAndroid Build Coastguard Worker //
1318*9880d681SAndroid Build Coastguard Worker // cond.store1:                                      ; preds = %else
1319*9880d681SAndroid Build Coastguard Worker //   %8 = extractelement <16 x i32> %val, i32 1
1320*9880d681SAndroid Build Coastguard Worker //   %9 = getelementptr i32* %1, i32 1
1321*9880d681SAndroid Build Coastguard Worker //   store i32 %8, i32* %9
1322*9880d681SAndroid Build Coastguard Worker //   br label %else2
1323*9880d681SAndroid Build Coastguard Worker //   . . .
scalarizeMaskedStore(CallInst * CI)1324*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedStore(CallInst *CI) {
1325*9880d681SAndroid Build Coastguard Worker   Value *Src = CI->getArgOperand(0);
1326*9880d681SAndroid Build Coastguard Worker   Value *Ptr  = CI->getArgOperand(1);
1327*9880d681SAndroid Build Coastguard Worker   Value *Alignment = CI->getArgOperand(2);
1328*9880d681SAndroid Build Coastguard Worker   Value *Mask = CI->getArgOperand(3);
1329*9880d681SAndroid Build Coastguard Worker 
1330*9880d681SAndroid Build Coastguard Worker   unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1331*9880d681SAndroid Build Coastguard Worker   VectorType *VecType = dyn_cast<VectorType>(Src->getType());
1332*9880d681SAndroid Build Coastguard Worker   assert(VecType && "Unexpected data type in masked store intrinsic");
1333*9880d681SAndroid Build Coastguard Worker 
1334*9880d681SAndroid Build Coastguard Worker   Type *EltTy = VecType->getElementType();
1335*9880d681SAndroid Build Coastguard Worker 
1336*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(CI->getContext());
1337*9880d681SAndroid Build Coastguard Worker   Instruction *InsertPt = CI;
1338*9880d681SAndroid Build Coastguard Worker   BasicBlock *IfBlock = CI->getParent();
1339*9880d681SAndroid Build Coastguard Worker   Builder.SetInsertPoint(InsertPt);
1340*9880d681SAndroid Build Coastguard Worker   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1341*9880d681SAndroid Build Coastguard Worker 
1342*9880d681SAndroid Build Coastguard Worker   // Short-cut if the mask is all-true.
1343*9880d681SAndroid Build Coastguard Worker   bool IsAllOnesMask = isa<Constant>(Mask) &&
1344*9880d681SAndroid Build Coastguard Worker     cast<Constant>(Mask)->isAllOnesValue();
1345*9880d681SAndroid Build Coastguard Worker 
1346*9880d681SAndroid Build Coastguard Worker   if (IsAllOnesMask) {
1347*9880d681SAndroid Build Coastguard Worker     Builder.CreateAlignedStore(Src, Ptr, AlignVal);
1348*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1349*9880d681SAndroid Build Coastguard Worker     return;
1350*9880d681SAndroid Build Coastguard Worker   }
1351*9880d681SAndroid Build Coastguard Worker 
1352*9880d681SAndroid Build Coastguard Worker   // Adjust alignment for the scalar instruction.
1353*9880d681SAndroid Build Coastguard Worker   AlignVal = std::max(AlignVal, VecType->getScalarSizeInBits()/8);
1354*9880d681SAndroid Build Coastguard Worker   // Bitcast %addr fron i8* to EltTy*
1355*9880d681SAndroid Build Coastguard Worker   Type *NewPtrType =
1356*9880d681SAndroid Build Coastguard Worker     EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace());
1357*9880d681SAndroid Build Coastguard Worker   Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
1358*9880d681SAndroid Build Coastguard Worker   unsigned VectorWidth = VecType->getNumElements();
1359*9880d681SAndroid Build Coastguard Worker 
1360*9880d681SAndroid Build Coastguard Worker   if (isa<ConstantVector>(Mask)) {
1361*9880d681SAndroid Build Coastguard Worker     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1362*9880d681SAndroid Build Coastguard Worker       if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1363*9880d681SAndroid Build Coastguard Worker           continue;
1364*9880d681SAndroid Build Coastguard Worker       Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx));
1365*9880d681SAndroid Build Coastguard Worker       Value *Gep =
1366*9880d681SAndroid Build Coastguard Worker           Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1367*9880d681SAndroid Build Coastguard Worker       Builder.CreateAlignedStore(OneElt, Gep, AlignVal);
1368*9880d681SAndroid Build Coastguard Worker     }
1369*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1370*9880d681SAndroid Build Coastguard Worker     return;
1371*9880d681SAndroid Build Coastguard Worker   }
1372*9880d681SAndroid Build Coastguard Worker 
1373*9880d681SAndroid Build Coastguard Worker   for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1374*9880d681SAndroid Build Coastguard Worker 
1375*9880d681SAndroid Build Coastguard Worker     // Fill the "else" block, created in the previous iteration
1376*9880d681SAndroid Build Coastguard Worker     //
1377*9880d681SAndroid Build Coastguard Worker     //  %mask_1 = extractelement <16 x i1> %mask, i32 Idx
1378*9880d681SAndroid Build Coastguard Worker     //  %to_store = icmp eq i1 %mask_1, true
1379*9880d681SAndroid Build Coastguard Worker     //  br i1 %to_store, label %cond.store, label %else
1380*9880d681SAndroid Build Coastguard Worker     //
1381*9880d681SAndroid Build Coastguard Worker     Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx));
1382*9880d681SAndroid Build Coastguard Worker     Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1383*9880d681SAndroid Build Coastguard Worker                                     ConstantInt::get(Predicate->getType(), 1));
1384*9880d681SAndroid Build Coastguard Worker 
1385*9880d681SAndroid Build Coastguard Worker     // Create "cond" block
1386*9880d681SAndroid Build Coastguard Worker     //
1387*9880d681SAndroid Build Coastguard Worker     //  %OneElt = extractelement <16 x i32> %Src, i32 Idx
1388*9880d681SAndroid Build Coastguard Worker     //  %EltAddr = getelementptr i32* %1, i32 0
1389*9880d681SAndroid Build Coastguard Worker     //  %store i32 %OneElt, i32* %EltAddr
1390*9880d681SAndroid Build Coastguard Worker     //
1391*9880d681SAndroid Build Coastguard Worker     BasicBlock *CondBlock =
1392*9880d681SAndroid Build Coastguard Worker         IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store");
1393*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1394*9880d681SAndroid Build Coastguard Worker 
1395*9880d681SAndroid Build Coastguard Worker     Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx));
1396*9880d681SAndroid Build Coastguard Worker     Value *Gep =
1397*9880d681SAndroid Build Coastguard Worker         Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1398*9880d681SAndroid Build Coastguard Worker     Builder.CreateAlignedStore(OneElt, Gep, AlignVal);
1399*9880d681SAndroid Build Coastguard Worker 
1400*9880d681SAndroid Build Coastguard Worker     // Create "else" block, fill it in the next iteration
1401*9880d681SAndroid Build Coastguard Worker     BasicBlock *NewIfBlock =
1402*9880d681SAndroid Build Coastguard Worker         CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
1403*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1404*9880d681SAndroid Build Coastguard Worker     Instruction *OldBr = IfBlock->getTerminator();
1405*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1406*9880d681SAndroid Build Coastguard Worker     OldBr->eraseFromParent();
1407*9880d681SAndroid Build Coastguard Worker     IfBlock = NewIfBlock;
1408*9880d681SAndroid Build Coastguard Worker   }
1409*9880d681SAndroid Build Coastguard Worker   CI->eraseFromParent();
1410*9880d681SAndroid Build Coastguard Worker }
1411*9880d681SAndroid Build Coastguard Worker 
1412*9880d681SAndroid Build Coastguard Worker // Translate a masked gather intrinsic like
1413*9880d681SAndroid Build Coastguard Worker // <16 x i32 > @llvm.masked.gather.v16i32( <16 x i32*> %Ptrs, i32 4,
1414*9880d681SAndroid Build Coastguard Worker //                               <16 x i1> %Mask, <16 x i32> %Src)
1415*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, with loading element one-by-one if
1416*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set
1417*9880d681SAndroid Build Coastguard Worker //
1418*9880d681SAndroid Build Coastguard Worker // % Ptrs = getelementptr i32, i32* %base, <16 x i64> %ind
1419*9880d681SAndroid Build Coastguard Worker // % Mask0 = extractelement <16 x i1> %Mask, i32 0
1420*9880d681SAndroid Build Coastguard Worker // % ToLoad0 = icmp eq i1 % Mask0, true
1421*9880d681SAndroid Build Coastguard Worker // br i1 % ToLoad0, label %cond.load, label %else
1422*9880d681SAndroid Build Coastguard Worker //
1423*9880d681SAndroid Build Coastguard Worker // cond.load:
1424*9880d681SAndroid Build Coastguard Worker // % Ptr0 = extractelement <16 x i32*> %Ptrs, i32 0
1425*9880d681SAndroid Build Coastguard Worker // % Load0 = load i32, i32* % Ptr0, align 4
1426*9880d681SAndroid Build Coastguard Worker // % Res0 = insertelement <16 x i32> undef, i32 % Load0, i32 0
1427*9880d681SAndroid Build Coastguard Worker // br label %else
1428*9880d681SAndroid Build Coastguard Worker //
1429*9880d681SAndroid Build Coastguard Worker // else:
1430*9880d681SAndroid Build Coastguard Worker // %res.phi.else = phi <16 x i32>[% Res0, %cond.load], [undef, % 0]
1431*9880d681SAndroid Build Coastguard Worker // % Mask1 = extractelement <16 x i1> %Mask, i32 1
1432*9880d681SAndroid Build Coastguard Worker // % ToLoad1 = icmp eq i1 % Mask1, true
1433*9880d681SAndroid Build Coastguard Worker // br i1 % ToLoad1, label %cond.load1, label %else2
1434*9880d681SAndroid Build Coastguard Worker //
1435*9880d681SAndroid Build Coastguard Worker // cond.load1:
1436*9880d681SAndroid Build Coastguard Worker // % Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
1437*9880d681SAndroid Build Coastguard Worker // % Load1 = load i32, i32* % Ptr1, align 4
1438*9880d681SAndroid Build Coastguard Worker // % Res1 = insertelement <16 x i32> %res.phi.else, i32 % Load1, i32 1
1439*9880d681SAndroid Build Coastguard Worker // br label %else2
1440*9880d681SAndroid Build Coastguard Worker // . . .
1441*9880d681SAndroid Build Coastguard Worker // % Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
1442*9880d681SAndroid Build Coastguard Worker // ret <16 x i32> %Result
scalarizeMaskedGather(CallInst * CI)1443*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedGather(CallInst *CI) {
1444*9880d681SAndroid Build Coastguard Worker   Value *Ptrs = CI->getArgOperand(0);
1445*9880d681SAndroid Build Coastguard Worker   Value *Alignment = CI->getArgOperand(1);
1446*9880d681SAndroid Build Coastguard Worker   Value *Mask = CI->getArgOperand(2);
1447*9880d681SAndroid Build Coastguard Worker   Value *Src0 = CI->getArgOperand(3);
1448*9880d681SAndroid Build Coastguard Worker 
1449*9880d681SAndroid Build Coastguard Worker   VectorType *VecType = dyn_cast<VectorType>(CI->getType());
1450*9880d681SAndroid Build Coastguard Worker 
1451*9880d681SAndroid Build Coastguard Worker   assert(VecType && "Unexpected return type of masked load intrinsic");
1452*9880d681SAndroid Build Coastguard Worker 
1453*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(CI->getContext());
1454*9880d681SAndroid Build Coastguard Worker   Instruction *InsertPt = CI;
1455*9880d681SAndroid Build Coastguard Worker   BasicBlock *IfBlock = CI->getParent();
1456*9880d681SAndroid Build Coastguard Worker   BasicBlock *CondBlock = nullptr;
1457*9880d681SAndroid Build Coastguard Worker   BasicBlock *PrevIfBlock = CI->getParent();
1458*9880d681SAndroid Build Coastguard Worker   Builder.SetInsertPoint(InsertPt);
1459*9880d681SAndroid Build Coastguard Worker   unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1460*9880d681SAndroid Build Coastguard Worker 
1461*9880d681SAndroid Build Coastguard Worker   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1462*9880d681SAndroid Build Coastguard Worker 
1463*9880d681SAndroid Build Coastguard Worker   Value *UndefVal = UndefValue::get(VecType);
1464*9880d681SAndroid Build Coastguard Worker 
1465*9880d681SAndroid Build Coastguard Worker   // The result vector
1466*9880d681SAndroid Build Coastguard Worker   Value *VResult = UndefVal;
1467*9880d681SAndroid Build Coastguard Worker   unsigned VectorWidth = VecType->getNumElements();
1468*9880d681SAndroid Build Coastguard Worker 
1469*9880d681SAndroid Build Coastguard Worker   // Shorten the way if the mask is a vector of constants.
1470*9880d681SAndroid Build Coastguard Worker   bool IsConstMask = isa<ConstantVector>(Mask);
1471*9880d681SAndroid Build Coastguard Worker 
1472*9880d681SAndroid Build Coastguard Worker   if (IsConstMask) {
1473*9880d681SAndroid Build Coastguard Worker     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1474*9880d681SAndroid Build Coastguard Worker       if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1475*9880d681SAndroid Build Coastguard Worker         continue;
1476*9880d681SAndroid Build Coastguard Worker       Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1477*9880d681SAndroid Build Coastguard Worker                                                 "Ptr" + Twine(Idx));
1478*9880d681SAndroid Build Coastguard Worker       LoadInst *Load = Builder.CreateAlignedLoad(Ptr, AlignVal,
1479*9880d681SAndroid Build Coastguard Worker                                                  "Load" + Twine(Idx));
1480*9880d681SAndroid Build Coastguard Worker       VResult = Builder.CreateInsertElement(VResult, Load,
1481*9880d681SAndroid Build Coastguard Worker                                             Builder.getInt32(Idx),
1482*9880d681SAndroid Build Coastguard Worker                                             "Res" + Twine(Idx));
1483*9880d681SAndroid Build Coastguard Worker     }
1484*9880d681SAndroid Build Coastguard Worker     Value *NewI = Builder.CreateSelect(Mask, VResult, Src0);
1485*9880d681SAndroid Build Coastguard Worker     CI->replaceAllUsesWith(NewI);
1486*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1487*9880d681SAndroid Build Coastguard Worker     return;
1488*9880d681SAndroid Build Coastguard Worker   }
1489*9880d681SAndroid Build Coastguard Worker 
1490*9880d681SAndroid Build Coastguard Worker   PHINode *Phi = nullptr;
1491*9880d681SAndroid Build Coastguard Worker   Value *PrevPhi = UndefVal;
1492*9880d681SAndroid Build Coastguard Worker 
1493*9880d681SAndroid Build Coastguard Worker   for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1494*9880d681SAndroid Build Coastguard Worker 
1495*9880d681SAndroid Build Coastguard Worker     // Fill the "else" block, created in the previous iteration
1496*9880d681SAndroid Build Coastguard Worker     //
1497*9880d681SAndroid Build Coastguard Worker     //  %Mask1 = extractelement <16 x i1> %Mask, i32 1
1498*9880d681SAndroid Build Coastguard Worker     //  %ToLoad1 = icmp eq i1 %Mask1, true
1499*9880d681SAndroid Build Coastguard Worker     //  br i1 %ToLoad1, label %cond.load, label %else
1500*9880d681SAndroid Build Coastguard Worker     //
1501*9880d681SAndroid Build Coastguard Worker     if (Idx > 0) {
1502*9880d681SAndroid Build Coastguard Worker       Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
1503*9880d681SAndroid Build Coastguard Worker       Phi->addIncoming(VResult, CondBlock);
1504*9880d681SAndroid Build Coastguard Worker       Phi->addIncoming(PrevPhi, PrevIfBlock);
1505*9880d681SAndroid Build Coastguard Worker       PrevPhi = Phi;
1506*9880d681SAndroid Build Coastguard Worker       VResult = Phi;
1507*9880d681SAndroid Build Coastguard Worker     }
1508*9880d681SAndroid Build Coastguard Worker 
1509*9880d681SAndroid Build Coastguard Worker     Value *Predicate = Builder.CreateExtractElement(Mask,
1510*9880d681SAndroid Build Coastguard Worker                                                     Builder.getInt32(Idx),
1511*9880d681SAndroid Build Coastguard Worker                                                     "Mask" + Twine(Idx));
1512*9880d681SAndroid Build Coastguard Worker     Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1513*9880d681SAndroid Build Coastguard Worker                                     ConstantInt::get(Predicate->getType(), 1),
1514*9880d681SAndroid Build Coastguard Worker                                     "ToLoad" + Twine(Idx));
1515*9880d681SAndroid Build Coastguard Worker 
1516*9880d681SAndroid Build Coastguard Worker     // Create "cond" block
1517*9880d681SAndroid Build Coastguard Worker     //
1518*9880d681SAndroid Build Coastguard Worker     //  %EltAddr = getelementptr i32* %1, i32 0
1519*9880d681SAndroid Build Coastguard Worker     //  %Elt = load i32* %EltAddr
1520*9880d681SAndroid Build Coastguard Worker     //  VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
1521*9880d681SAndroid Build Coastguard Worker     //
1522*9880d681SAndroid Build Coastguard Worker     CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load");
1523*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1524*9880d681SAndroid Build Coastguard Worker 
1525*9880d681SAndroid Build Coastguard Worker     Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1526*9880d681SAndroid Build Coastguard Worker                                               "Ptr" + Twine(Idx));
1527*9880d681SAndroid Build Coastguard Worker     LoadInst *Load = Builder.CreateAlignedLoad(Ptr, AlignVal,
1528*9880d681SAndroid Build Coastguard Worker                                                "Load" + Twine(Idx));
1529*9880d681SAndroid Build Coastguard Worker     VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx),
1530*9880d681SAndroid Build Coastguard Worker                                           "Res" + Twine(Idx));
1531*9880d681SAndroid Build Coastguard Worker 
1532*9880d681SAndroid Build Coastguard Worker     // Create "else" block, fill it in the next iteration
1533*9880d681SAndroid Build Coastguard Worker     BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else");
1534*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1535*9880d681SAndroid Build Coastguard Worker     Instruction *OldBr = IfBlock->getTerminator();
1536*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1537*9880d681SAndroid Build Coastguard Worker     OldBr->eraseFromParent();
1538*9880d681SAndroid Build Coastguard Worker     PrevIfBlock = IfBlock;
1539*9880d681SAndroid Build Coastguard Worker     IfBlock = NewIfBlock;
1540*9880d681SAndroid Build Coastguard Worker   }
1541*9880d681SAndroid Build Coastguard Worker 
1542*9880d681SAndroid Build Coastguard Worker   Phi = Builder.CreatePHI(VecType, 2, "res.phi.select");
1543*9880d681SAndroid Build Coastguard Worker   Phi->addIncoming(VResult, CondBlock);
1544*9880d681SAndroid Build Coastguard Worker   Phi->addIncoming(PrevPhi, PrevIfBlock);
1545*9880d681SAndroid Build Coastguard Worker   Value *NewI = Builder.CreateSelect(Mask, Phi, Src0);
1546*9880d681SAndroid Build Coastguard Worker   CI->replaceAllUsesWith(NewI);
1547*9880d681SAndroid Build Coastguard Worker   CI->eraseFromParent();
1548*9880d681SAndroid Build Coastguard Worker }
1549*9880d681SAndroid Build Coastguard Worker 
1550*9880d681SAndroid Build Coastguard Worker // Translate a masked scatter intrinsic, like
1551*9880d681SAndroid Build Coastguard Worker // void @llvm.masked.scatter.v16i32(<16 x i32> %Src, <16 x i32*>* %Ptrs, i32 4,
1552*9880d681SAndroid Build Coastguard Worker //                                  <16 x i1> %Mask)
1553*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, that stores element one-by-one if
1554*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set.
1555*9880d681SAndroid Build Coastguard Worker //
1556*9880d681SAndroid Build Coastguard Worker // % Ptrs = getelementptr i32, i32* %ptr, <16 x i64> %ind
1557*9880d681SAndroid Build Coastguard Worker // % Mask0 = extractelement <16 x i1> % Mask, i32 0
1558*9880d681SAndroid Build Coastguard Worker // % ToStore0 = icmp eq i1 % Mask0, true
1559*9880d681SAndroid Build Coastguard Worker // br i1 %ToStore0, label %cond.store, label %else
1560*9880d681SAndroid Build Coastguard Worker //
1561*9880d681SAndroid Build Coastguard Worker // cond.store:
1562*9880d681SAndroid Build Coastguard Worker // % Elt0 = extractelement <16 x i32> %Src, i32 0
1563*9880d681SAndroid Build Coastguard Worker // % Ptr0 = extractelement <16 x i32*> %Ptrs, i32 0
1564*9880d681SAndroid Build Coastguard Worker // store i32 %Elt0, i32* % Ptr0, align 4
1565*9880d681SAndroid Build Coastguard Worker // br label %else
1566*9880d681SAndroid Build Coastguard Worker //
1567*9880d681SAndroid Build Coastguard Worker // else:
1568*9880d681SAndroid Build Coastguard Worker // % Mask1 = extractelement <16 x i1> % Mask, i32 1
1569*9880d681SAndroid Build Coastguard Worker // % ToStore1 = icmp eq i1 % Mask1, true
1570*9880d681SAndroid Build Coastguard Worker // br i1 % ToStore1, label %cond.store1, label %else2
1571*9880d681SAndroid Build Coastguard Worker //
1572*9880d681SAndroid Build Coastguard Worker // cond.store1:
1573*9880d681SAndroid Build Coastguard Worker // % Elt1 = extractelement <16 x i32> %Src, i32 1
1574*9880d681SAndroid Build Coastguard Worker // % Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
1575*9880d681SAndroid Build Coastguard Worker // store i32 % Elt1, i32* % Ptr1, align 4
1576*9880d681SAndroid Build Coastguard Worker // br label %else2
1577*9880d681SAndroid Build Coastguard Worker //   . . .
scalarizeMaskedScatter(CallInst * CI)1578*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedScatter(CallInst *CI) {
1579*9880d681SAndroid Build Coastguard Worker   Value *Src = CI->getArgOperand(0);
1580*9880d681SAndroid Build Coastguard Worker   Value *Ptrs = CI->getArgOperand(1);
1581*9880d681SAndroid Build Coastguard Worker   Value *Alignment = CI->getArgOperand(2);
1582*9880d681SAndroid Build Coastguard Worker   Value *Mask = CI->getArgOperand(3);
1583*9880d681SAndroid Build Coastguard Worker 
1584*9880d681SAndroid Build Coastguard Worker   assert(isa<VectorType>(Src->getType()) &&
1585*9880d681SAndroid Build Coastguard Worker          "Unexpected data type in masked scatter intrinsic");
1586*9880d681SAndroid Build Coastguard Worker   assert(isa<VectorType>(Ptrs->getType()) &&
1587*9880d681SAndroid Build Coastguard Worker          isa<PointerType>(Ptrs->getType()->getVectorElementType()) &&
1588*9880d681SAndroid Build Coastguard Worker          "Vector of pointers is expected in masked scatter intrinsic");
1589*9880d681SAndroid Build Coastguard Worker 
1590*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(CI->getContext());
1591*9880d681SAndroid Build Coastguard Worker   Instruction *InsertPt = CI;
1592*9880d681SAndroid Build Coastguard Worker   BasicBlock *IfBlock = CI->getParent();
1593*9880d681SAndroid Build Coastguard Worker   Builder.SetInsertPoint(InsertPt);
1594*9880d681SAndroid Build Coastguard Worker   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1595*9880d681SAndroid Build Coastguard Worker 
1596*9880d681SAndroid Build Coastguard Worker   unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1597*9880d681SAndroid Build Coastguard Worker   unsigned VectorWidth = Src->getType()->getVectorNumElements();
1598*9880d681SAndroid Build Coastguard Worker 
1599*9880d681SAndroid Build Coastguard Worker   // Shorten the way if the mask is a vector of constants.
1600*9880d681SAndroid Build Coastguard Worker   bool IsConstMask = isa<ConstantVector>(Mask);
1601*9880d681SAndroid Build Coastguard Worker 
1602*9880d681SAndroid Build Coastguard Worker   if (IsConstMask) {
1603*9880d681SAndroid Build Coastguard Worker     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1604*9880d681SAndroid Build Coastguard Worker       if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1605*9880d681SAndroid Build Coastguard Worker         continue;
1606*9880d681SAndroid Build Coastguard Worker       Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx),
1607*9880d681SAndroid Build Coastguard Worker                                                    "Elt" + Twine(Idx));
1608*9880d681SAndroid Build Coastguard Worker       Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1609*9880d681SAndroid Build Coastguard Worker                                                 "Ptr" + Twine(Idx));
1610*9880d681SAndroid Build Coastguard Worker       Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
1611*9880d681SAndroid Build Coastguard Worker     }
1612*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1613*9880d681SAndroid Build Coastguard Worker     return;
1614*9880d681SAndroid Build Coastguard Worker   }
1615*9880d681SAndroid Build Coastguard Worker   for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1616*9880d681SAndroid Build Coastguard Worker     // Fill the "else" block, created in the previous iteration
1617*9880d681SAndroid Build Coastguard Worker     //
1618*9880d681SAndroid Build Coastguard Worker     //  % Mask1 = extractelement <16 x i1> % Mask, i32 Idx
1619*9880d681SAndroid Build Coastguard Worker     //  % ToStore = icmp eq i1 % Mask1, true
1620*9880d681SAndroid Build Coastguard Worker     //  br i1 % ToStore, label %cond.store, label %else
1621*9880d681SAndroid Build Coastguard Worker     //
1622*9880d681SAndroid Build Coastguard Worker     Value *Predicate = Builder.CreateExtractElement(Mask,
1623*9880d681SAndroid Build Coastguard Worker                                                     Builder.getInt32(Idx),
1624*9880d681SAndroid Build Coastguard Worker                                                     "Mask" + Twine(Idx));
1625*9880d681SAndroid Build Coastguard Worker     Value *Cmp =
1626*9880d681SAndroid Build Coastguard Worker        Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1627*9880d681SAndroid Build Coastguard Worker                           ConstantInt::get(Predicate->getType(), 1),
1628*9880d681SAndroid Build Coastguard Worker                           "ToStore" + Twine(Idx));
1629*9880d681SAndroid Build Coastguard Worker 
1630*9880d681SAndroid Build Coastguard Worker     // Create "cond" block
1631*9880d681SAndroid Build Coastguard Worker     //
1632*9880d681SAndroid Build Coastguard Worker     //  % Elt1 = extractelement <16 x i32> %Src, i32 1
1633*9880d681SAndroid Build Coastguard Worker     //  % Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
1634*9880d681SAndroid Build Coastguard Worker     //  %store i32 % Elt1, i32* % Ptr1
1635*9880d681SAndroid Build Coastguard Worker     //
1636*9880d681SAndroid Build Coastguard Worker     BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.store");
1637*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1638*9880d681SAndroid Build Coastguard Worker 
1639*9880d681SAndroid Build Coastguard Worker     Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx),
1640*9880d681SAndroid Build Coastguard Worker                                                  "Elt" + Twine(Idx));
1641*9880d681SAndroid Build Coastguard Worker     Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1642*9880d681SAndroid Build Coastguard Worker                                               "Ptr" + Twine(Idx));
1643*9880d681SAndroid Build Coastguard Worker     Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
1644*9880d681SAndroid Build Coastguard Worker 
1645*9880d681SAndroid Build Coastguard Worker     // Create "else" block, fill it in the next iteration
1646*9880d681SAndroid Build Coastguard Worker     BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else");
1647*9880d681SAndroid Build Coastguard Worker     Builder.SetInsertPoint(InsertPt);
1648*9880d681SAndroid Build Coastguard Worker     Instruction *OldBr = IfBlock->getTerminator();
1649*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1650*9880d681SAndroid Build Coastguard Worker     OldBr->eraseFromParent();
1651*9880d681SAndroid Build Coastguard Worker     IfBlock = NewIfBlock;
1652*9880d681SAndroid Build Coastguard Worker   }
1653*9880d681SAndroid Build Coastguard Worker   CI->eraseFromParent();
1654*9880d681SAndroid Build Coastguard Worker }
1655*9880d681SAndroid Build Coastguard Worker 
1656*9880d681SAndroid Build Coastguard Worker /// If counting leading or trailing zeros is an expensive operation and a zero
1657*9880d681SAndroid Build Coastguard Worker /// input is defined, add a check for zero to avoid calling the intrinsic.
1658*9880d681SAndroid Build Coastguard Worker ///
1659*9880d681SAndroid Build Coastguard Worker /// We want to transform:
1660*9880d681SAndroid Build Coastguard Worker ///     %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
1661*9880d681SAndroid Build Coastguard Worker ///
1662*9880d681SAndroid Build Coastguard Worker /// into:
1663*9880d681SAndroid Build Coastguard Worker ///   entry:
1664*9880d681SAndroid Build Coastguard Worker ///     %cmpz = icmp eq i64 %A, 0
1665*9880d681SAndroid Build Coastguard Worker ///     br i1 %cmpz, label %cond.end, label %cond.false
1666*9880d681SAndroid Build Coastguard Worker ///   cond.false:
1667*9880d681SAndroid Build Coastguard Worker ///     %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)
1668*9880d681SAndroid Build Coastguard Worker ///     br label %cond.end
1669*9880d681SAndroid Build Coastguard Worker ///   cond.end:
1670*9880d681SAndroid Build Coastguard Worker ///     %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
1671*9880d681SAndroid Build Coastguard Worker ///
1672*9880d681SAndroid Build Coastguard Worker /// If the transform is performed, return true and set ModifiedDT to true.
despeculateCountZeros(IntrinsicInst * CountZeros,const TargetLowering * TLI,const DataLayout * DL,bool & ModifiedDT)1673*9880d681SAndroid Build Coastguard Worker static bool despeculateCountZeros(IntrinsicInst *CountZeros,
1674*9880d681SAndroid Build Coastguard Worker                                   const TargetLowering *TLI,
1675*9880d681SAndroid Build Coastguard Worker                                   const DataLayout *DL,
1676*9880d681SAndroid Build Coastguard Worker                                   bool &ModifiedDT) {
1677*9880d681SAndroid Build Coastguard Worker   if (!TLI || !DL)
1678*9880d681SAndroid Build Coastguard Worker     return false;
1679*9880d681SAndroid Build Coastguard Worker 
1680*9880d681SAndroid Build Coastguard Worker   // If a zero input is undefined, it doesn't make sense to despeculate that.
1681*9880d681SAndroid Build Coastguard Worker   if (match(CountZeros->getOperand(1), m_One()))
1682*9880d681SAndroid Build Coastguard Worker     return false;
1683*9880d681SAndroid Build Coastguard Worker 
1684*9880d681SAndroid Build Coastguard Worker   // If it's cheap to speculate, there's nothing to do.
1685*9880d681SAndroid Build Coastguard Worker   auto IntrinsicID = CountZeros->getIntrinsicID();
1686*9880d681SAndroid Build Coastguard Worker   if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz()) ||
1687*9880d681SAndroid Build Coastguard Worker       (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz()))
1688*9880d681SAndroid Build Coastguard Worker     return false;
1689*9880d681SAndroid Build Coastguard Worker 
1690*9880d681SAndroid Build Coastguard Worker   // Only handle legal scalar cases. Anything else requires too much work.
1691*9880d681SAndroid Build Coastguard Worker   Type *Ty = CountZeros->getType();
1692*9880d681SAndroid Build Coastguard Worker   unsigned SizeInBits = Ty->getPrimitiveSizeInBits();
1693*9880d681SAndroid Build Coastguard Worker   if (Ty->isVectorTy() || SizeInBits > DL->getLargestLegalIntTypeSizeInBits())
1694*9880d681SAndroid Build Coastguard Worker     return false;
1695*9880d681SAndroid Build Coastguard Worker 
1696*9880d681SAndroid Build Coastguard Worker   // The intrinsic will be sunk behind a compare against zero and branch.
1697*9880d681SAndroid Build Coastguard Worker   BasicBlock *StartBlock = CountZeros->getParent();
1698*9880d681SAndroid Build Coastguard Worker   BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false");
1699*9880d681SAndroid Build Coastguard Worker 
1700*9880d681SAndroid Build Coastguard Worker   // Create another block after the count zero intrinsic. A PHI will be added
1701*9880d681SAndroid Build Coastguard Worker   // in this block to select the result of the intrinsic or the bit-width
1702*9880d681SAndroid Build Coastguard Worker   // constant if the input to the intrinsic is zero.
1703*9880d681SAndroid Build Coastguard Worker   BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(CountZeros));
1704*9880d681SAndroid Build Coastguard Worker   BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end");
1705*9880d681SAndroid Build Coastguard Worker 
1706*9880d681SAndroid Build Coastguard Worker   // Set up a builder to create a compare, conditional branch, and PHI.
1707*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(CountZeros->getContext());
1708*9880d681SAndroid Build Coastguard Worker   Builder.SetInsertPoint(StartBlock->getTerminator());
1709*9880d681SAndroid Build Coastguard Worker   Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc());
1710*9880d681SAndroid Build Coastguard Worker 
1711*9880d681SAndroid Build Coastguard Worker   // Replace the unconditional branch that was created by the first split with
1712*9880d681SAndroid Build Coastguard Worker   // a compare against zero and a conditional branch.
1713*9880d681SAndroid Build Coastguard Worker   Value *Zero = Constant::getNullValue(Ty);
1714*9880d681SAndroid Build Coastguard Worker   Value *Cmp = Builder.CreateICmpEQ(CountZeros->getOperand(0), Zero, "cmpz");
1715*9880d681SAndroid Build Coastguard Worker   Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
1716*9880d681SAndroid Build Coastguard Worker   StartBlock->getTerminator()->eraseFromParent();
1717*9880d681SAndroid Build Coastguard Worker 
1718*9880d681SAndroid Build Coastguard Worker   // Create a PHI in the end block to select either the output of the intrinsic
1719*9880d681SAndroid Build Coastguard Worker   // or the bit width of the operand.
1720*9880d681SAndroid Build Coastguard Worker   Builder.SetInsertPoint(&EndBlock->front());
1721*9880d681SAndroid Build Coastguard Worker   PHINode *PN = Builder.CreatePHI(Ty, 2, "ctz");
1722*9880d681SAndroid Build Coastguard Worker   CountZeros->replaceAllUsesWith(PN);
1723*9880d681SAndroid Build Coastguard Worker   Value *BitWidth = Builder.getInt(APInt(SizeInBits, SizeInBits));
1724*9880d681SAndroid Build Coastguard Worker   PN->addIncoming(BitWidth, StartBlock);
1725*9880d681SAndroid Build Coastguard Worker   PN->addIncoming(CountZeros, CallBlock);
1726*9880d681SAndroid Build Coastguard Worker 
1727*9880d681SAndroid Build Coastguard Worker   // We are explicitly handling the zero case, so we can set the intrinsic's
1728*9880d681SAndroid Build Coastguard Worker   // undefined zero argument to 'true'. This will also prevent reprocessing the
1729*9880d681SAndroid Build Coastguard Worker   // intrinsic; we only despeculate when a zero input is defined.
1730*9880d681SAndroid Build Coastguard Worker   CountZeros->setArgOperand(1, Builder.getTrue());
1731*9880d681SAndroid Build Coastguard Worker   ModifiedDT = true;
1732*9880d681SAndroid Build Coastguard Worker   return true;
1733*9880d681SAndroid Build Coastguard Worker }
1734*9880d681SAndroid Build Coastguard Worker 
optimizeCallInst(CallInst * CI,bool & ModifiedDT)1735*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool& ModifiedDT) {
1736*9880d681SAndroid Build Coastguard Worker   BasicBlock *BB = CI->getParent();
1737*9880d681SAndroid Build Coastguard Worker 
1738*9880d681SAndroid Build Coastguard Worker   // Lower inline assembly if we can.
1739*9880d681SAndroid Build Coastguard Worker   // If we found an inline asm expession, and if the target knows how to
1740*9880d681SAndroid Build Coastguard Worker   // lower it to normal LLVM code, do so now.
1741*9880d681SAndroid Build Coastguard Worker   if (TLI && isa<InlineAsm>(CI->getCalledValue())) {
1742*9880d681SAndroid Build Coastguard Worker     if (TLI->ExpandInlineAsm(CI)) {
1743*9880d681SAndroid Build Coastguard Worker       // Avoid invalidating the iterator.
1744*9880d681SAndroid Build Coastguard Worker       CurInstIterator = BB->begin();
1745*9880d681SAndroid Build Coastguard Worker       // Avoid processing instructions out of order, which could cause
1746*9880d681SAndroid Build Coastguard Worker       // reuse before a value is defined.
1747*9880d681SAndroid Build Coastguard Worker       SunkAddrs.clear();
1748*9880d681SAndroid Build Coastguard Worker       return true;
1749*9880d681SAndroid Build Coastguard Worker     }
1750*9880d681SAndroid Build Coastguard Worker     // Sink address computing for memory operands into the block.
1751*9880d681SAndroid Build Coastguard Worker     if (optimizeInlineAsmInst(CI))
1752*9880d681SAndroid Build Coastguard Worker       return true;
1753*9880d681SAndroid Build Coastguard Worker   }
1754*9880d681SAndroid Build Coastguard Worker 
1755*9880d681SAndroid Build Coastguard Worker   // Align the pointer arguments to this call if the target thinks it's a good
1756*9880d681SAndroid Build Coastguard Worker   // idea
1757*9880d681SAndroid Build Coastguard Worker   unsigned MinSize, PrefAlign;
1758*9880d681SAndroid Build Coastguard Worker   if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
1759*9880d681SAndroid Build Coastguard Worker     for (auto &Arg : CI->arg_operands()) {
1760*9880d681SAndroid Build Coastguard Worker       // We want to align both objects whose address is used directly and
1761*9880d681SAndroid Build Coastguard Worker       // objects whose address is used in casts and GEPs, though it only makes
1762*9880d681SAndroid Build Coastguard Worker       // sense for GEPs if the offset is a multiple of the desired alignment and
1763*9880d681SAndroid Build Coastguard Worker       // if size - offset meets the size threshold.
1764*9880d681SAndroid Build Coastguard Worker       if (!Arg->getType()->isPointerTy())
1765*9880d681SAndroid Build Coastguard Worker         continue;
1766*9880d681SAndroid Build Coastguard Worker       APInt Offset(DL->getPointerSizeInBits(
1767*9880d681SAndroid Build Coastguard Worker                        cast<PointerType>(Arg->getType())->getAddressSpace()),
1768*9880d681SAndroid Build Coastguard Worker                    0);
1769*9880d681SAndroid Build Coastguard Worker       Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset);
1770*9880d681SAndroid Build Coastguard Worker       uint64_t Offset2 = Offset.getLimitedValue();
1771*9880d681SAndroid Build Coastguard Worker       if ((Offset2 & (PrefAlign-1)) != 0)
1772*9880d681SAndroid Build Coastguard Worker         continue;
1773*9880d681SAndroid Build Coastguard Worker       AllocaInst *AI;
1774*9880d681SAndroid Build Coastguard Worker       if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign &&
1775*9880d681SAndroid Build Coastguard Worker           DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)
1776*9880d681SAndroid Build Coastguard Worker         AI->setAlignment(PrefAlign);
1777*9880d681SAndroid Build Coastguard Worker       // Global variables can only be aligned if they are defined in this
1778*9880d681SAndroid Build Coastguard Worker       // object (i.e. they are uniquely initialized in this object), and
1779*9880d681SAndroid Build Coastguard Worker       // over-aligning global variables that have an explicit section is
1780*9880d681SAndroid Build Coastguard Worker       // forbidden.
1781*9880d681SAndroid Build Coastguard Worker       GlobalVariable *GV;
1782*9880d681SAndroid Build Coastguard Worker       if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() &&
1783*9880d681SAndroid Build Coastguard Worker           GV->getAlignment() < PrefAlign &&
1784*9880d681SAndroid Build Coastguard Worker           DL->getTypeAllocSize(GV->getValueType()) >=
1785*9880d681SAndroid Build Coastguard Worker               MinSize + Offset2)
1786*9880d681SAndroid Build Coastguard Worker         GV->setAlignment(PrefAlign);
1787*9880d681SAndroid Build Coastguard Worker     }
1788*9880d681SAndroid Build Coastguard Worker     // If this is a memcpy (or similar) then we may be able to improve the
1789*9880d681SAndroid Build Coastguard Worker     // alignment
1790*9880d681SAndroid Build Coastguard Worker     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) {
1791*9880d681SAndroid Build Coastguard Worker       unsigned Align = getKnownAlignment(MI->getDest(), *DL);
1792*9880d681SAndroid Build Coastguard Worker       if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
1793*9880d681SAndroid Build Coastguard Worker         Align = std::min(Align, getKnownAlignment(MTI->getSource(), *DL));
1794*9880d681SAndroid Build Coastguard Worker       if (Align > MI->getAlignment())
1795*9880d681SAndroid Build Coastguard Worker         MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align));
1796*9880d681SAndroid Build Coastguard Worker     }
1797*9880d681SAndroid Build Coastguard Worker   }
1798*9880d681SAndroid Build Coastguard Worker 
1799*9880d681SAndroid Build Coastguard Worker   // If we have a cold call site, try to sink addressing computation into the
1800*9880d681SAndroid Build Coastguard Worker   // cold block.  This interacts with our handling for loads and stores to
1801*9880d681SAndroid Build Coastguard Worker   // ensure that we can fold all uses of a potential addressing computation
1802*9880d681SAndroid Build Coastguard Worker   // into their uses.  TODO: generalize this to work over profiling data
1803*9880d681SAndroid Build Coastguard Worker   if (!OptSize && CI->hasFnAttr(Attribute::Cold))
1804*9880d681SAndroid Build Coastguard Worker     for (auto &Arg : CI->arg_operands()) {
1805*9880d681SAndroid Build Coastguard Worker       if (!Arg->getType()->isPointerTy())
1806*9880d681SAndroid Build Coastguard Worker         continue;
1807*9880d681SAndroid Build Coastguard Worker       unsigned AS = Arg->getType()->getPointerAddressSpace();
1808*9880d681SAndroid Build Coastguard Worker       return optimizeMemoryInst(CI, Arg, Arg->getType(), AS);
1809*9880d681SAndroid Build Coastguard Worker     }
1810*9880d681SAndroid Build Coastguard Worker 
1811*9880d681SAndroid Build Coastguard Worker   IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
1812*9880d681SAndroid Build Coastguard Worker   if (II) {
1813*9880d681SAndroid Build Coastguard Worker     switch (II->getIntrinsicID()) {
1814*9880d681SAndroid Build Coastguard Worker     default: break;
1815*9880d681SAndroid Build Coastguard Worker     case Intrinsic::objectsize: {
1816*9880d681SAndroid Build Coastguard Worker       // Lower all uses of llvm.objectsize.*
1817*9880d681SAndroid Build Coastguard Worker       uint64_t Size;
1818*9880d681SAndroid Build Coastguard Worker       Type *ReturnTy = CI->getType();
1819*9880d681SAndroid Build Coastguard Worker       Constant *RetVal = nullptr;
1820*9880d681SAndroid Build Coastguard Worker       ConstantInt *Op1 = cast<ConstantInt>(II->getArgOperand(1));
1821*9880d681SAndroid Build Coastguard Worker       ObjSizeMode Mode = Op1->isZero() ? ObjSizeMode::Max : ObjSizeMode::Min;
1822*9880d681SAndroid Build Coastguard Worker       if (getObjectSize(II->getArgOperand(0),
1823*9880d681SAndroid Build Coastguard Worker                         Size, *DL, TLInfo, false, Mode)) {
1824*9880d681SAndroid Build Coastguard Worker         RetVal = ConstantInt::get(ReturnTy, Size);
1825*9880d681SAndroid Build Coastguard Worker       } else {
1826*9880d681SAndroid Build Coastguard Worker         RetVal = ConstantInt::get(ReturnTy,
1827*9880d681SAndroid Build Coastguard Worker                                   Mode == ObjSizeMode::Min ? 0 : -1ULL);
1828*9880d681SAndroid Build Coastguard Worker       }
1829*9880d681SAndroid Build Coastguard Worker       // Substituting this can cause recursive simplifications, which can
1830*9880d681SAndroid Build Coastguard Worker       // invalidate our iterator.  Use a WeakVH to hold onto it in case this
1831*9880d681SAndroid Build Coastguard Worker       // happens.
1832*9880d681SAndroid Build Coastguard Worker       Value *CurValue = &*CurInstIterator;
1833*9880d681SAndroid Build Coastguard Worker       WeakVH IterHandle(CurValue);
1834*9880d681SAndroid Build Coastguard Worker 
1835*9880d681SAndroid Build Coastguard Worker       replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr);
1836*9880d681SAndroid Build Coastguard Worker 
1837*9880d681SAndroid Build Coastguard Worker       // If the iterator instruction was recursively deleted, start over at the
1838*9880d681SAndroid Build Coastguard Worker       // start of the block.
1839*9880d681SAndroid Build Coastguard Worker       if (IterHandle != CurValue) {
1840*9880d681SAndroid Build Coastguard Worker         CurInstIterator = BB->begin();
1841*9880d681SAndroid Build Coastguard Worker         SunkAddrs.clear();
1842*9880d681SAndroid Build Coastguard Worker       }
1843*9880d681SAndroid Build Coastguard Worker       return true;
1844*9880d681SAndroid Build Coastguard Worker     }
1845*9880d681SAndroid Build Coastguard Worker     case Intrinsic::masked_load: {
1846*9880d681SAndroid Build Coastguard Worker       // Scalarize unsupported vector masked load
1847*9880d681SAndroid Build Coastguard Worker       if (!TTI->isLegalMaskedLoad(CI->getType())) {
1848*9880d681SAndroid Build Coastguard Worker         scalarizeMaskedLoad(CI);
1849*9880d681SAndroid Build Coastguard Worker         ModifiedDT = true;
1850*9880d681SAndroid Build Coastguard Worker         return true;
1851*9880d681SAndroid Build Coastguard Worker       }
1852*9880d681SAndroid Build Coastguard Worker       return false;
1853*9880d681SAndroid Build Coastguard Worker     }
1854*9880d681SAndroid Build Coastguard Worker     case Intrinsic::masked_store: {
1855*9880d681SAndroid Build Coastguard Worker       if (!TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType())) {
1856*9880d681SAndroid Build Coastguard Worker         scalarizeMaskedStore(CI);
1857*9880d681SAndroid Build Coastguard Worker         ModifiedDT = true;
1858*9880d681SAndroid Build Coastguard Worker         return true;
1859*9880d681SAndroid Build Coastguard Worker       }
1860*9880d681SAndroid Build Coastguard Worker       return false;
1861*9880d681SAndroid Build Coastguard Worker     }
1862*9880d681SAndroid Build Coastguard Worker     case Intrinsic::masked_gather: {
1863*9880d681SAndroid Build Coastguard Worker       if (!TTI->isLegalMaskedGather(CI->getType())) {
1864*9880d681SAndroid Build Coastguard Worker         scalarizeMaskedGather(CI);
1865*9880d681SAndroid Build Coastguard Worker         ModifiedDT = true;
1866*9880d681SAndroid Build Coastguard Worker         return true;
1867*9880d681SAndroid Build Coastguard Worker       }
1868*9880d681SAndroid Build Coastguard Worker       return false;
1869*9880d681SAndroid Build Coastguard Worker     }
1870*9880d681SAndroid Build Coastguard Worker     case Intrinsic::masked_scatter: {
1871*9880d681SAndroid Build Coastguard Worker       if (!TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType())) {
1872*9880d681SAndroid Build Coastguard Worker         scalarizeMaskedScatter(CI);
1873*9880d681SAndroid Build Coastguard Worker         ModifiedDT = true;
1874*9880d681SAndroid Build Coastguard Worker         return true;
1875*9880d681SAndroid Build Coastguard Worker       }
1876*9880d681SAndroid Build Coastguard Worker       return false;
1877*9880d681SAndroid Build Coastguard Worker     }
1878*9880d681SAndroid Build Coastguard Worker     case Intrinsic::aarch64_stlxr:
1879*9880d681SAndroid Build Coastguard Worker     case Intrinsic::aarch64_stxr: {
1880*9880d681SAndroid Build Coastguard Worker       ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0));
1881*9880d681SAndroid Build Coastguard Worker       if (!ExtVal || !ExtVal->hasOneUse() ||
1882*9880d681SAndroid Build Coastguard Worker           ExtVal->getParent() == CI->getParent())
1883*9880d681SAndroid Build Coastguard Worker         return false;
1884*9880d681SAndroid Build Coastguard Worker       // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.
1885*9880d681SAndroid Build Coastguard Worker       ExtVal->moveBefore(CI);
1886*9880d681SAndroid Build Coastguard Worker       // Mark this instruction as "inserted by CGP", so that other
1887*9880d681SAndroid Build Coastguard Worker       // optimizations don't touch it.
1888*9880d681SAndroid Build Coastguard Worker       InsertedInsts.insert(ExtVal);
1889*9880d681SAndroid Build Coastguard Worker       return true;
1890*9880d681SAndroid Build Coastguard Worker     }
1891*9880d681SAndroid Build Coastguard Worker     case Intrinsic::invariant_group_barrier:
1892*9880d681SAndroid Build Coastguard Worker       II->replaceAllUsesWith(II->getArgOperand(0));
1893*9880d681SAndroid Build Coastguard Worker       II->eraseFromParent();
1894*9880d681SAndroid Build Coastguard Worker       return true;
1895*9880d681SAndroid Build Coastguard Worker 
1896*9880d681SAndroid Build Coastguard Worker     case Intrinsic::cttz:
1897*9880d681SAndroid Build Coastguard Worker     case Intrinsic::ctlz:
1898*9880d681SAndroid Build Coastguard Worker       // If counting zeros is expensive, try to avoid it.
1899*9880d681SAndroid Build Coastguard Worker       return despeculateCountZeros(II, TLI, DL, ModifiedDT);
1900*9880d681SAndroid Build Coastguard Worker     }
1901*9880d681SAndroid Build Coastguard Worker 
1902*9880d681SAndroid Build Coastguard Worker     if (TLI) {
1903*9880d681SAndroid Build Coastguard Worker       // Unknown address space.
1904*9880d681SAndroid Build Coastguard Worker       // TODO: Target hook to pick which address space the intrinsic cares
1905*9880d681SAndroid Build Coastguard Worker       // about?
1906*9880d681SAndroid Build Coastguard Worker       unsigned AddrSpace = ~0u;
1907*9880d681SAndroid Build Coastguard Worker       SmallVector<Value*, 2> PtrOps;
1908*9880d681SAndroid Build Coastguard Worker       Type *AccessTy;
1909*9880d681SAndroid Build Coastguard Worker       if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy, AddrSpace))
1910*9880d681SAndroid Build Coastguard Worker         while (!PtrOps.empty())
1911*9880d681SAndroid Build Coastguard Worker           if (optimizeMemoryInst(II, PtrOps.pop_back_val(), AccessTy, AddrSpace))
1912*9880d681SAndroid Build Coastguard Worker             return true;
1913*9880d681SAndroid Build Coastguard Worker     }
1914*9880d681SAndroid Build Coastguard Worker   }
1915*9880d681SAndroid Build Coastguard Worker 
1916*9880d681SAndroid Build Coastguard Worker   // From here on out we're working with named functions.
1917*9880d681SAndroid Build Coastguard Worker   if (!CI->getCalledFunction()) return false;
1918*9880d681SAndroid Build Coastguard Worker 
1919*9880d681SAndroid Build Coastguard Worker   // Lower all default uses of _chk calls.  This is very similar
1920*9880d681SAndroid Build Coastguard Worker   // to what InstCombineCalls does, but here we are only lowering calls
1921*9880d681SAndroid Build Coastguard Worker   // to fortified library functions (e.g. __memcpy_chk) that have the default
1922*9880d681SAndroid Build Coastguard Worker   // "don't know" as the objectsize.  Anything else should be left alone.
1923*9880d681SAndroid Build Coastguard Worker   FortifiedLibCallSimplifier Simplifier(TLInfo, true);
1924*9880d681SAndroid Build Coastguard Worker   if (Value *V = Simplifier.optimizeCall(CI)) {
1925*9880d681SAndroid Build Coastguard Worker     CI->replaceAllUsesWith(V);
1926*9880d681SAndroid Build Coastguard Worker     CI->eraseFromParent();
1927*9880d681SAndroid Build Coastguard Worker     return true;
1928*9880d681SAndroid Build Coastguard Worker   }
1929*9880d681SAndroid Build Coastguard Worker   return false;
1930*9880d681SAndroid Build Coastguard Worker }
1931*9880d681SAndroid Build Coastguard Worker 
1932*9880d681SAndroid Build Coastguard Worker /// Look for opportunities to duplicate return instructions to the predecessor
1933*9880d681SAndroid Build Coastguard Worker /// to enable tail call optimizations. The case it is currently looking for is:
1934*9880d681SAndroid Build Coastguard Worker /// @code
1935*9880d681SAndroid Build Coastguard Worker /// bb0:
1936*9880d681SAndroid Build Coastguard Worker ///   %tmp0 = tail call i32 @f0()
1937*9880d681SAndroid Build Coastguard Worker ///   br label %return
1938*9880d681SAndroid Build Coastguard Worker /// bb1:
1939*9880d681SAndroid Build Coastguard Worker ///   %tmp1 = tail call i32 @f1()
1940*9880d681SAndroid Build Coastguard Worker ///   br label %return
1941*9880d681SAndroid Build Coastguard Worker /// bb2:
1942*9880d681SAndroid Build Coastguard Worker ///   %tmp2 = tail call i32 @f2()
1943*9880d681SAndroid Build Coastguard Worker ///   br label %return
1944*9880d681SAndroid Build Coastguard Worker /// return:
1945*9880d681SAndroid Build Coastguard Worker ///   %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
1946*9880d681SAndroid Build Coastguard Worker ///   ret i32 %retval
1947*9880d681SAndroid Build Coastguard Worker /// @endcode
1948*9880d681SAndroid Build Coastguard Worker ///
1949*9880d681SAndroid Build Coastguard Worker /// =>
1950*9880d681SAndroid Build Coastguard Worker ///
1951*9880d681SAndroid Build Coastguard Worker /// @code
1952*9880d681SAndroid Build Coastguard Worker /// bb0:
1953*9880d681SAndroid Build Coastguard Worker ///   %tmp0 = tail call i32 @f0()
1954*9880d681SAndroid Build Coastguard Worker ///   ret i32 %tmp0
1955*9880d681SAndroid Build Coastguard Worker /// bb1:
1956*9880d681SAndroid Build Coastguard Worker ///   %tmp1 = tail call i32 @f1()
1957*9880d681SAndroid Build Coastguard Worker ///   ret i32 %tmp1
1958*9880d681SAndroid Build Coastguard Worker /// bb2:
1959*9880d681SAndroid Build Coastguard Worker ///   %tmp2 = tail call i32 @f2()
1960*9880d681SAndroid Build Coastguard Worker ///   ret i32 %tmp2
1961*9880d681SAndroid Build Coastguard Worker /// @endcode
dupRetToEnableTailCallOpts(BasicBlock * BB)1962*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) {
1963*9880d681SAndroid Build Coastguard Worker   if (!TLI)
1964*9880d681SAndroid Build Coastguard Worker     return false;
1965*9880d681SAndroid Build Coastguard Worker 
1966*9880d681SAndroid Build Coastguard Worker   ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator());
1967*9880d681SAndroid Build Coastguard Worker   if (!RI)
1968*9880d681SAndroid Build Coastguard Worker     return false;
1969*9880d681SAndroid Build Coastguard Worker 
1970*9880d681SAndroid Build Coastguard Worker   PHINode *PN = nullptr;
1971*9880d681SAndroid Build Coastguard Worker   BitCastInst *BCI = nullptr;
1972*9880d681SAndroid Build Coastguard Worker   Value *V = RI->getReturnValue();
1973*9880d681SAndroid Build Coastguard Worker   if (V) {
1974*9880d681SAndroid Build Coastguard Worker     BCI = dyn_cast<BitCastInst>(V);
1975*9880d681SAndroid Build Coastguard Worker     if (BCI)
1976*9880d681SAndroid Build Coastguard Worker       V = BCI->getOperand(0);
1977*9880d681SAndroid Build Coastguard Worker 
1978*9880d681SAndroid Build Coastguard Worker     PN = dyn_cast<PHINode>(V);
1979*9880d681SAndroid Build Coastguard Worker     if (!PN)
1980*9880d681SAndroid Build Coastguard Worker       return false;
1981*9880d681SAndroid Build Coastguard Worker   }
1982*9880d681SAndroid Build Coastguard Worker 
1983*9880d681SAndroid Build Coastguard Worker   if (PN && PN->getParent() != BB)
1984*9880d681SAndroid Build Coastguard Worker     return false;
1985*9880d681SAndroid Build Coastguard Worker 
1986*9880d681SAndroid Build Coastguard Worker   // It's not safe to eliminate the sign / zero extension of the return value.
1987*9880d681SAndroid Build Coastguard Worker   // See llvm::isInTailCallPosition().
1988*9880d681SAndroid Build Coastguard Worker   const Function *F = BB->getParent();
1989*9880d681SAndroid Build Coastguard Worker   AttributeSet CallerAttrs = F->getAttributes();
1990*9880d681SAndroid Build Coastguard Worker   if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) ||
1991*9880d681SAndroid Build Coastguard Worker       CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt))
1992*9880d681SAndroid Build Coastguard Worker     return false;
1993*9880d681SAndroid Build Coastguard Worker 
1994*9880d681SAndroid Build Coastguard Worker   // Make sure there are no instructions between the PHI and return, or that the
1995*9880d681SAndroid Build Coastguard Worker   // return is the first instruction in the block.
1996*9880d681SAndroid Build Coastguard Worker   if (PN) {
1997*9880d681SAndroid Build Coastguard Worker     BasicBlock::iterator BI = BB->begin();
1998*9880d681SAndroid Build Coastguard Worker     do { ++BI; } while (isa<DbgInfoIntrinsic>(BI));
1999*9880d681SAndroid Build Coastguard Worker     if (&*BI == BCI)
2000*9880d681SAndroid Build Coastguard Worker       // Also skip over the bitcast.
2001*9880d681SAndroid Build Coastguard Worker       ++BI;
2002*9880d681SAndroid Build Coastguard Worker     if (&*BI != RI)
2003*9880d681SAndroid Build Coastguard Worker       return false;
2004*9880d681SAndroid Build Coastguard Worker   } else {
2005*9880d681SAndroid Build Coastguard Worker     BasicBlock::iterator BI = BB->begin();
2006*9880d681SAndroid Build Coastguard Worker     while (isa<DbgInfoIntrinsic>(BI)) ++BI;
2007*9880d681SAndroid Build Coastguard Worker     if (&*BI != RI)
2008*9880d681SAndroid Build Coastguard Worker       return false;
2009*9880d681SAndroid Build Coastguard Worker   }
2010*9880d681SAndroid Build Coastguard Worker 
2011*9880d681SAndroid Build Coastguard Worker   /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
2012*9880d681SAndroid Build Coastguard Worker   /// call.
2013*9880d681SAndroid Build Coastguard Worker   SmallVector<CallInst*, 4> TailCalls;
2014*9880d681SAndroid Build Coastguard Worker   if (PN) {
2015*9880d681SAndroid Build Coastguard Worker     for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) {
2016*9880d681SAndroid Build Coastguard Worker       CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I));
2017*9880d681SAndroid Build Coastguard Worker       // Make sure the phi value is indeed produced by the tail call.
2018*9880d681SAndroid Build Coastguard Worker       if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) &&
2019*9880d681SAndroid Build Coastguard Worker           TLI->mayBeEmittedAsTailCall(CI))
2020*9880d681SAndroid Build Coastguard Worker         TailCalls.push_back(CI);
2021*9880d681SAndroid Build Coastguard Worker     }
2022*9880d681SAndroid Build Coastguard Worker   } else {
2023*9880d681SAndroid Build Coastguard Worker     SmallPtrSet<BasicBlock*, 4> VisitedBBs;
2024*9880d681SAndroid Build Coastguard Worker     for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) {
2025*9880d681SAndroid Build Coastguard Worker       if (!VisitedBBs.insert(*PI).second)
2026*9880d681SAndroid Build Coastguard Worker         continue;
2027*9880d681SAndroid Build Coastguard Worker 
2028*9880d681SAndroid Build Coastguard Worker       BasicBlock::InstListType &InstList = (*PI)->getInstList();
2029*9880d681SAndroid Build Coastguard Worker       BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin();
2030*9880d681SAndroid Build Coastguard Worker       BasicBlock::InstListType::reverse_iterator RE = InstList.rend();
2031*9880d681SAndroid Build Coastguard Worker       do { ++RI; } while (RI != RE && isa<DbgInfoIntrinsic>(&*RI));
2032*9880d681SAndroid Build Coastguard Worker       if (RI == RE)
2033*9880d681SAndroid Build Coastguard Worker         continue;
2034*9880d681SAndroid Build Coastguard Worker 
2035*9880d681SAndroid Build Coastguard Worker       CallInst *CI = dyn_cast<CallInst>(&*RI);
2036*9880d681SAndroid Build Coastguard Worker       if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI))
2037*9880d681SAndroid Build Coastguard Worker         TailCalls.push_back(CI);
2038*9880d681SAndroid Build Coastguard Worker     }
2039*9880d681SAndroid Build Coastguard Worker   }
2040*9880d681SAndroid Build Coastguard Worker 
2041*9880d681SAndroid Build Coastguard Worker   bool Changed = false;
2042*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) {
2043*9880d681SAndroid Build Coastguard Worker     CallInst *CI = TailCalls[i];
2044*9880d681SAndroid Build Coastguard Worker     CallSite CS(CI);
2045*9880d681SAndroid Build Coastguard Worker 
2046*9880d681SAndroid Build Coastguard Worker     // Conservatively require the attributes of the call to match those of the
2047*9880d681SAndroid Build Coastguard Worker     // return. Ignore noalias because it doesn't affect the call sequence.
2048*9880d681SAndroid Build Coastguard Worker     AttributeSet CalleeAttrs = CS.getAttributes();
2049*9880d681SAndroid Build Coastguard Worker     if (AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex).
2050*9880d681SAndroid Build Coastguard Worker           removeAttribute(Attribute::NoAlias) !=
2051*9880d681SAndroid Build Coastguard Worker         AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex).
2052*9880d681SAndroid Build Coastguard Worker           removeAttribute(Attribute::NoAlias))
2053*9880d681SAndroid Build Coastguard Worker       continue;
2054*9880d681SAndroid Build Coastguard Worker 
2055*9880d681SAndroid Build Coastguard Worker     // Make sure the call instruction is followed by an unconditional branch to
2056*9880d681SAndroid Build Coastguard Worker     // the return block.
2057*9880d681SAndroid Build Coastguard Worker     BasicBlock *CallBB = CI->getParent();
2058*9880d681SAndroid Build Coastguard Worker     BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator());
2059*9880d681SAndroid Build Coastguard Worker     if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB)
2060*9880d681SAndroid Build Coastguard Worker       continue;
2061*9880d681SAndroid Build Coastguard Worker 
2062*9880d681SAndroid Build Coastguard Worker     // Duplicate the return into CallBB.
2063*9880d681SAndroid Build Coastguard Worker     (void)FoldReturnIntoUncondBranch(RI, BB, CallBB);
2064*9880d681SAndroid Build Coastguard Worker     ModifiedDT = Changed = true;
2065*9880d681SAndroid Build Coastguard Worker     ++NumRetsDup;
2066*9880d681SAndroid Build Coastguard Worker   }
2067*9880d681SAndroid Build Coastguard Worker 
2068*9880d681SAndroid Build Coastguard Worker   // If we eliminated all predecessors of the block, delete the block now.
2069*9880d681SAndroid Build Coastguard Worker   if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB))
2070*9880d681SAndroid Build Coastguard Worker     BB->eraseFromParent();
2071*9880d681SAndroid Build Coastguard Worker 
2072*9880d681SAndroid Build Coastguard Worker   return Changed;
2073*9880d681SAndroid Build Coastguard Worker }
2074*9880d681SAndroid Build Coastguard Worker 
2075*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
2076*9880d681SAndroid Build Coastguard Worker // Memory Optimization
2077*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
2078*9880d681SAndroid Build Coastguard Worker 
2079*9880d681SAndroid Build Coastguard Worker namespace {
2080*9880d681SAndroid Build Coastguard Worker 
2081*9880d681SAndroid Build Coastguard Worker /// This is an extended version of TargetLowering::AddrMode
2082*9880d681SAndroid Build Coastguard Worker /// which holds actual Value*'s for register values.
2083*9880d681SAndroid Build Coastguard Worker struct ExtAddrMode : public TargetLowering::AddrMode {
2084*9880d681SAndroid Build Coastguard Worker   Value *BaseReg;
2085*9880d681SAndroid Build Coastguard Worker   Value *ScaledReg;
ExtAddrMode__anon034c73f00211::ExtAddrMode2086*9880d681SAndroid Build Coastguard Worker   ExtAddrMode() : BaseReg(nullptr), ScaledReg(nullptr) {}
2087*9880d681SAndroid Build Coastguard Worker   void print(raw_ostream &OS) const;
2088*9880d681SAndroid Build Coastguard Worker   void dump() const;
2089*9880d681SAndroid Build Coastguard Worker 
operator ==__anon034c73f00211::ExtAddrMode2090*9880d681SAndroid Build Coastguard Worker   bool operator==(const ExtAddrMode& O) const {
2091*9880d681SAndroid Build Coastguard Worker     return (BaseReg == O.BaseReg) && (ScaledReg == O.ScaledReg) &&
2092*9880d681SAndroid Build Coastguard Worker            (BaseGV == O.BaseGV) && (BaseOffs == O.BaseOffs) &&
2093*9880d681SAndroid Build Coastguard Worker            (HasBaseReg == O.HasBaseReg) && (Scale == O.Scale);
2094*9880d681SAndroid Build Coastguard Worker   }
2095*9880d681SAndroid Build Coastguard Worker };
2096*9880d681SAndroid Build Coastguard Worker 
2097*9880d681SAndroid Build Coastguard Worker #ifndef NDEBUG
operator <<(raw_ostream & OS,const ExtAddrMode & AM)2098*9880d681SAndroid Build Coastguard Worker static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) {
2099*9880d681SAndroid Build Coastguard Worker   AM.print(OS);
2100*9880d681SAndroid Build Coastguard Worker   return OS;
2101*9880d681SAndroid Build Coastguard Worker }
2102*9880d681SAndroid Build Coastguard Worker #endif
2103*9880d681SAndroid Build Coastguard Worker 
print(raw_ostream & OS) const2104*9880d681SAndroid Build Coastguard Worker void ExtAddrMode::print(raw_ostream &OS) const {
2105*9880d681SAndroid Build Coastguard Worker   bool NeedPlus = false;
2106*9880d681SAndroid Build Coastguard Worker   OS << "[";
2107*9880d681SAndroid Build Coastguard Worker   if (BaseGV) {
2108*9880d681SAndroid Build Coastguard Worker     OS << (NeedPlus ? " + " : "")
2109*9880d681SAndroid Build Coastguard Worker        << "GV:";
2110*9880d681SAndroid Build Coastguard Worker     BaseGV->printAsOperand(OS, /*PrintType=*/false);
2111*9880d681SAndroid Build Coastguard Worker     NeedPlus = true;
2112*9880d681SAndroid Build Coastguard Worker   }
2113*9880d681SAndroid Build Coastguard Worker 
2114*9880d681SAndroid Build Coastguard Worker   if (BaseOffs) {
2115*9880d681SAndroid Build Coastguard Worker     OS << (NeedPlus ? " + " : "")
2116*9880d681SAndroid Build Coastguard Worker        << BaseOffs;
2117*9880d681SAndroid Build Coastguard Worker     NeedPlus = true;
2118*9880d681SAndroid Build Coastguard Worker   }
2119*9880d681SAndroid Build Coastguard Worker 
2120*9880d681SAndroid Build Coastguard Worker   if (BaseReg) {
2121*9880d681SAndroid Build Coastguard Worker     OS << (NeedPlus ? " + " : "")
2122*9880d681SAndroid Build Coastguard Worker        << "Base:";
2123*9880d681SAndroid Build Coastguard Worker     BaseReg->printAsOperand(OS, /*PrintType=*/false);
2124*9880d681SAndroid Build Coastguard Worker     NeedPlus = true;
2125*9880d681SAndroid Build Coastguard Worker   }
2126*9880d681SAndroid Build Coastguard Worker   if (Scale) {
2127*9880d681SAndroid Build Coastguard Worker     OS << (NeedPlus ? " + " : "")
2128*9880d681SAndroid Build Coastguard Worker        << Scale << "*";
2129*9880d681SAndroid Build Coastguard Worker     ScaledReg->printAsOperand(OS, /*PrintType=*/false);
2130*9880d681SAndroid Build Coastguard Worker   }
2131*9880d681SAndroid Build Coastguard Worker 
2132*9880d681SAndroid Build Coastguard Worker   OS << ']';
2133*9880d681SAndroid Build Coastguard Worker }
2134*9880d681SAndroid Build Coastguard Worker 
2135*9880d681SAndroid Build Coastguard Worker #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const2136*9880d681SAndroid Build Coastguard Worker LLVM_DUMP_METHOD void ExtAddrMode::dump() const {
2137*9880d681SAndroid Build Coastguard Worker   print(dbgs());
2138*9880d681SAndroid Build Coastguard Worker   dbgs() << '\n';
2139*9880d681SAndroid Build Coastguard Worker }
2140*9880d681SAndroid Build Coastguard Worker #endif
2141*9880d681SAndroid Build Coastguard Worker 
2142*9880d681SAndroid Build Coastguard Worker /// \brief This class provides transaction based operation on the IR.
2143*9880d681SAndroid Build Coastguard Worker /// Every change made through this class is recorded in the internal state and
2144*9880d681SAndroid Build Coastguard Worker /// can be undone (rollback) until commit is called.
2145*9880d681SAndroid Build Coastguard Worker class TypePromotionTransaction {
2146*9880d681SAndroid Build Coastguard Worker 
2147*9880d681SAndroid Build Coastguard Worker   /// \brief This represents the common interface of the individual transaction.
2148*9880d681SAndroid Build Coastguard Worker   /// Each class implements the logic for doing one specific modification on
2149*9880d681SAndroid Build Coastguard Worker   /// the IR via the TypePromotionTransaction.
2150*9880d681SAndroid Build Coastguard Worker   class TypePromotionAction {
2151*9880d681SAndroid Build Coastguard Worker   protected:
2152*9880d681SAndroid Build Coastguard Worker     /// The Instruction modified.
2153*9880d681SAndroid Build Coastguard Worker     Instruction *Inst;
2154*9880d681SAndroid Build Coastguard Worker 
2155*9880d681SAndroid Build Coastguard Worker   public:
2156*9880d681SAndroid Build Coastguard Worker     /// \brief Constructor of the action.
2157*9880d681SAndroid Build Coastguard Worker     /// The constructor performs the related action on the IR.
TypePromotionAction(Instruction * Inst)2158*9880d681SAndroid Build Coastguard Worker     TypePromotionAction(Instruction *Inst) : Inst(Inst) {}
2159*9880d681SAndroid Build Coastguard Worker 
~TypePromotionAction()2160*9880d681SAndroid Build Coastguard Worker     virtual ~TypePromotionAction() {}
2161*9880d681SAndroid Build Coastguard Worker 
2162*9880d681SAndroid Build Coastguard Worker     /// \brief Undo the modification done by this action.
2163*9880d681SAndroid Build Coastguard Worker     /// When this method is called, the IR must be in the same state as it was
2164*9880d681SAndroid Build Coastguard Worker     /// before this action was applied.
2165*9880d681SAndroid Build Coastguard Worker     /// \pre Undoing the action works if and only if the IR is in the exact same
2166*9880d681SAndroid Build Coastguard Worker     /// state as it was directly after this action was applied.
2167*9880d681SAndroid Build Coastguard Worker     virtual void undo() = 0;
2168*9880d681SAndroid Build Coastguard Worker 
2169*9880d681SAndroid Build Coastguard Worker     /// \brief Advocate every change made by this action.
2170*9880d681SAndroid Build Coastguard Worker     /// When the results on the IR of the action are to be kept, it is important
2171*9880d681SAndroid Build Coastguard Worker     /// to call this function, otherwise hidden information may be kept forever.
commit()2172*9880d681SAndroid Build Coastguard Worker     virtual void commit() {
2173*9880d681SAndroid Build Coastguard Worker       // Nothing to be done, this action is not doing anything.
2174*9880d681SAndroid Build Coastguard Worker     }
2175*9880d681SAndroid Build Coastguard Worker   };
2176*9880d681SAndroid Build Coastguard Worker 
2177*9880d681SAndroid Build Coastguard Worker   /// \brief Utility to remember the position of an instruction.
2178*9880d681SAndroid Build Coastguard Worker   class InsertionHandler {
2179*9880d681SAndroid Build Coastguard Worker     /// Position of an instruction.
2180*9880d681SAndroid Build Coastguard Worker     /// Either an instruction:
2181*9880d681SAndroid Build Coastguard Worker     /// - Is the first in a basic block: BB is used.
2182*9880d681SAndroid Build Coastguard Worker     /// - Has a previous instructon: PrevInst is used.
2183*9880d681SAndroid Build Coastguard Worker     union {
2184*9880d681SAndroid Build Coastguard Worker       Instruction *PrevInst;
2185*9880d681SAndroid Build Coastguard Worker       BasicBlock *BB;
2186*9880d681SAndroid Build Coastguard Worker     } Point;
2187*9880d681SAndroid Build Coastguard Worker     /// Remember whether or not the instruction had a previous instruction.
2188*9880d681SAndroid Build Coastguard Worker     bool HasPrevInstruction;
2189*9880d681SAndroid Build Coastguard Worker 
2190*9880d681SAndroid Build Coastguard Worker   public:
2191*9880d681SAndroid Build Coastguard Worker     /// \brief Record the position of \p Inst.
InsertionHandler(Instruction * Inst)2192*9880d681SAndroid Build Coastguard Worker     InsertionHandler(Instruction *Inst) {
2193*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator It = Inst->getIterator();
2194*9880d681SAndroid Build Coastguard Worker       HasPrevInstruction = (It != (Inst->getParent()->begin()));
2195*9880d681SAndroid Build Coastguard Worker       if (HasPrevInstruction)
2196*9880d681SAndroid Build Coastguard Worker         Point.PrevInst = &*--It;
2197*9880d681SAndroid Build Coastguard Worker       else
2198*9880d681SAndroid Build Coastguard Worker         Point.BB = Inst->getParent();
2199*9880d681SAndroid Build Coastguard Worker     }
2200*9880d681SAndroid Build Coastguard Worker 
2201*9880d681SAndroid Build Coastguard Worker     /// \brief Insert \p Inst at the recorded position.
insert(Instruction * Inst)2202*9880d681SAndroid Build Coastguard Worker     void insert(Instruction *Inst) {
2203*9880d681SAndroid Build Coastguard Worker       if (HasPrevInstruction) {
2204*9880d681SAndroid Build Coastguard Worker         if (Inst->getParent())
2205*9880d681SAndroid Build Coastguard Worker           Inst->removeFromParent();
2206*9880d681SAndroid Build Coastguard Worker         Inst->insertAfter(Point.PrevInst);
2207*9880d681SAndroid Build Coastguard Worker       } else {
2208*9880d681SAndroid Build Coastguard Worker         Instruction *Position = &*Point.BB->getFirstInsertionPt();
2209*9880d681SAndroid Build Coastguard Worker         if (Inst->getParent())
2210*9880d681SAndroid Build Coastguard Worker           Inst->moveBefore(Position);
2211*9880d681SAndroid Build Coastguard Worker         else
2212*9880d681SAndroid Build Coastguard Worker           Inst->insertBefore(Position);
2213*9880d681SAndroid Build Coastguard Worker       }
2214*9880d681SAndroid Build Coastguard Worker     }
2215*9880d681SAndroid Build Coastguard Worker   };
2216*9880d681SAndroid Build Coastguard Worker 
2217*9880d681SAndroid Build Coastguard Worker   /// \brief Move an instruction before another.
2218*9880d681SAndroid Build Coastguard Worker   class InstructionMoveBefore : public TypePromotionAction {
2219*9880d681SAndroid Build Coastguard Worker     /// Original position of the instruction.
2220*9880d681SAndroid Build Coastguard Worker     InsertionHandler Position;
2221*9880d681SAndroid Build Coastguard Worker 
2222*9880d681SAndroid Build Coastguard Worker   public:
2223*9880d681SAndroid Build Coastguard Worker     /// \brief Move \p Inst before \p Before.
InstructionMoveBefore(Instruction * Inst,Instruction * Before)2224*9880d681SAndroid Build Coastguard Worker     InstructionMoveBefore(Instruction *Inst, Instruction *Before)
2225*9880d681SAndroid Build Coastguard Worker         : TypePromotionAction(Inst), Position(Inst) {
2226*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before << "\n");
2227*9880d681SAndroid Build Coastguard Worker       Inst->moveBefore(Before);
2228*9880d681SAndroid Build Coastguard Worker     }
2229*9880d681SAndroid Build Coastguard Worker 
2230*9880d681SAndroid Build Coastguard Worker     /// \brief Move the instruction back to its original position.
undo()2231*9880d681SAndroid Build Coastguard Worker     void undo() override {
2232*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n");
2233*9880d681SAndroid Build Coastguard Worker       Position.insert(Inst);
2234*9880d681SAndroid Build Coastguard Worker     }
2235*9880d681SAndroid Build Coastguard Worker   };
2236*9880d681SAndroid Build Coastguard Worker 
2237*9880d681SAndroid Build Coastguard Worker   /// \brief Set the operand of an instruction with a new value.
2238*9880d681SAndroid Build Coastguard Worker   class OperandSetter : public TypePromotionAction {
2239*9880d681SAndroid Build Coastguard Worker     /// Original operand of the instruction.
2240*9880d681SAndroid Build Coastguard Worker     Value *Origin;
2241*9880d681SAndroid Build Coastguard Worker     /// Index of the modified instruction.
2242*9880d681SAndroid Build Coastguard Worker     unsigned Idx;
2243*9880d681SAndroid Build Coastguard Worker 
2244*9880d681SAndroid Build Coastguard Worker   public:
2245*9880d681SAndroid Build Coastguard Worker     /// \brief Set \p Idx operand of \p Inst with \p NewVal.
OperandSetter(Instruction * Inst,unsigned Idx,Value * NewVal)2246*9880d681SAndroid Build Coastguard Worker     OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal)
2247*9880d681SAndroid Build Coastguard Worker         : TypePromotionAction(Inst), Idx(Idx) {
2248*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n"
2249*9880d681SAndroid Build Coastguard Worker                    << "for:" << *Inst << "\n"
2250*9880d681SAndroid Build Coastguard Worker                    << "with:" << *NewVal << "\n");
2251*9880d681SAndroid Build Coastguard Worker       Origin = Inst->getOperand(Idx);
2252*9880d681SAndroid Build Coastguard Worker       Inst->setOperand(Idx, NewVal);
2253*9880d681SAndroid Build Coastguard Worker     }
2254*9880d681SAndroid Build Coastguard Worker 
2255*9880d681SAndroid Build Coastguard Worker     /// \brief Restore the original value of the instruction.
undo()2256*9880d681SAndroid Build Coastguard Worker     void undo() override {
2257*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n"
2258*9880d681SAndroid Build Coastguard Worker                    << "for: " << *Inst << "\n"
2259*9880d681SAndroid Build Coastguard Worker                    << "with: " << *Origin << "\n");
2260*9880d681SAndroid Build Coastguard Worker       Inst->setOperand(Idx, Origin);
2261*9880d681SAndroid Build Coastguard Worker     }
2262*9880d681SAndroid Build Coastguard Worker   };
2263*9880d681SAndroid Build Coastguard Worker 
2264*9880d681SAndroid Build Coastguard Worker   /// \brief Hide the operands of an instruction.
2265*9880d681SAndroid Build Coastguard Worker   /// Do as if this instruction was not using any of its operands.
2266*9880d681SAndroid Build Coastguard Worker   class OperandsHider : public TypePromotionAction {
2267*9880d681SAndroid Build Coastguard Worker     /// The list of original operands.
2268*9880d681SAndroid Build Coastguard Worker     SmallVector<Value *, 4> OriginalValues;
2269*9880d681SAndroid Build Coastguard Worker 
2270*9880d681SAndroid Build Coastguard Worker   public:
2271*9880d681SAndroid Build Coastguard Worker     /// \brief Remove \p Inst from the uses of the operands of \p Inst.
OperandsHider(Instruction * Inst)2272*9880d681SAndroid Build Coastguard Worker     OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) {
2273*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n");
2274*9880d681SAndroid Build Coastguard Worker       unsigned NumOpnds = Inst->getNumOperands();
2275*9880d681SAndroid Build Coastguard Worker       OriginalValues.reserve(NumOpnds);
2276*9880d681SAndroid Build Coastguard Worker       for (unsigned It = 0; It < NumOpnds; ++It) {
2277*9880d681SAndroid Build Coastguard Worker         // Save the current operand.
2278*9880d681SAndroid Build Coastguard Worker         Value *Val = Inst->getOperand(It);
2279*9880d681SAndroid Build Coastguard Worker         OriginalValues.push_back(Val);
2280*9880d681SAndroid Build Coastguard Worker         // Set a dummy one.
2281*9880d681SAndroid Build Coastguard Worker         // We could use OperandSetter here, but that would imply an overhead
2282*9880d681SAndroid Build Coastguard Worker         // that we are not willing to pay.
2283*9880d681SAndroid Build Coastguard Worker         Inst->setOperand(It, UndefValue::get(Val->getType()));
2284*9880d681SAndroid Build Coastguard Worker       }
2285*9880d681SAndroid Build Coastguard Worker     }
2286*9880d681SAndroid Build Coastguard Worker 
2287*9880d681SAndroid Build Coastguard Worker     /// \brief Restore the original list of uses.
undo()2288*9880d681SAndroid Build Coastguard Worker     void undo() override {
2289*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n");
2290*9880d681SAndroid Build Coastguard Worker       for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)
2291*9880d681SAndroid Build Coastguard Worker         Inst->setOperand(It, OriginalValues[It]);
2292*9880d681SAndroid Build Coastguard Worker     }
2293*9880d681SAndroid Build Coastguard Worker   };
2294*9880d681SAndroid Build Coastguard Worker 
2295*9880d681SAndroid Build Coastguard Worker   /// \brief Build a truncate instruction.
2296*9880d681SAndroid Build Coastguard Worker   class TruncBuilder : public TypePromotionAction {
2297*9880d681SAndroid Build Coastguard Worker     Value *Val;
2298*9880d681SAndroid Build Coastguard Worker   public:
2299*9880d681SAndroid Build Coastguard Worker     /// \brief Build a truncate instruction of \p Opnd producing a \p Ty
2300*9880d681SAndroid Build Coastguard Worker     /// result.
2301*9880d681SAndroid Build Coastguard Worker     /// trunc Opnd to Ty.
TruncBuilder(Instruction * Opnd,Type * Ty)2302*9880d681SAndroid Build Coastguard Worker     TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) {
2303*9880d681SAndroid Build Coastguard Worker       IRBuilder<> Builder(Opnd);
2304*9880d681SAndroid Build Coastguard Worker       Val = Builder.CreateTrunc(Opnd, Ty, "promoted");
2305*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n");
2306*9880d681SAndroid Build Coastguard Worker     }
2307*9880d681SAndroid Build Coastguard Worker 
2308*9880d681SAndroid Build Coastguard Worker     /// \brief Get the built value.
getBuiltValue()2309*9880d681SAndroid Build Coastguard Worker     Value *getBuiltValue() { return Val; }
2310*9880d681SAndroid Build Coastguard Worker 
2311*9880d681SAndroid Build Coastguard Worker     /// \brief Remove the built instruction.
undo()2312*9880d681SAndroid Build Coastguard Worker     void undo() override {
2313*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n");
2314*9880d681SAndroid Build Coastguard Worker       if (Instruction *IVal = dyn_cast<Instruction>(Val))
2315*9880d681SAndroid Build Coastguard Worker         IVal->eraseFromParent();
2316*9880d681SAndroid Build Coastguard Worker     }
2317*9880d681SAndroid Build Coastguard Worker   };
2318*9880d681SAndroid Build Coastguard Worker 
2319*9880d681SAndroid Build Coastguard Worker   /// \brief Build a sign extension instruction.
2320*9880d681SAndroid Build Coastguard Worker   class SExtBuilder : public TypePromotionAction {
2321*9880d681SAndroid Build Coastguard Worker     Value *Val;
2322*9880d681SAndroid Build Coastguard Worker   public:
2323*9880d681SAndroid Build Coastguard Worker     /// \brief Build a sign extension instruction of \p Opnd producing a \p Ty
2324*9880d681SAndroid Build Coastguard Worker     /// result.
2325*9880d681SAndroid Build Coastguard Worker     /// sext Opnd to Ty.
SExtBuilder(Instruction * InsertPt,Value * Opnd,Type * Ty)2326*9880d681SAndroid Build Coastguard Worker     SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
2327*9880d681SAndroid Build Coastguard Worker         : TypePromotionAction(InsertPt) {
2328*9880d681SAndroid Build Coastguard Worker       IRBuilder<> Builder(InsertPt);
2329*9880d681SAndroid Build Coastguard Worker       Val = Builder.CreateSExt(Opnd, Ty, "promoted");
2330*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n");
2331*9880d681SAndroid Build Coastguard Worker     }
2332*9880d681SAndroid Build Coastguard Worker 
2333*9880d681SAndroid Build Coastguard Worker     /// \brief Get the built value.
getBuiltValue()2334*9880d681SAndroid Build Coastguard Worker     Value *getBuiltValue() { return Val; }
2335*9880d681SAndroid Build Coastguard Worker 
2336*9880d681SAndroid Build Coastguard Worker     /// \brief Remove the built instruction.
undo()2337*9880d681SAndroid Build Coastguard Worker     void undo() override {
2338*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n");
2339*9880d681SAndroid Build Coastguard Worker       if (Instruction *IVal = dyn_cast<Instruction>(Val))
2340*9880d681SAndroid Build Coastguard Worker         IVal->eraseFromParent();
2341*9880d681SAndroid Build Coastguard Worker     }
2342*9880d681SAndroid Build Coastguard Worker   };
2343*9880d681SAndroid Build Coastguard Worker 
2344*9880d681SAndroid Build Coastguard Worker   /// \brief Build a zero extension instruction.
2345*9880d681SAndroid Build Coastguard Worker   class ZExtBuilder : public TypePromotionAction {
2346*9880d681SAndroid Build Coastguard Worker     Value *Val;
2347*9880d681SAndroid Build Coastguard Worker   public:
2348*9880d681SAndroid Build Coastguard Worker     /// \brief Build a zero extension instruction of \p Opnd producing a \p Ty
2349*9880d681SAndroid Build Coastguard Worker     /// result.
2350*9880d681SAndroid Build Coastguard Worker     /// zext Opnd to Ty.
ZExtBuilder(Instruction * InsertPt,Value * Opnd,Type * Ty)2351*9880d681SAndroid Build Coastguard Worker     ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
2352*9880d681SAndroid Build Coastguard Worker         : TypePromotionAction(InsertPt) {
2353*9880d681SAndroid Build Coastguard Worker       IRBuilder<> Builder(InsertPt);
2354*9880d681SAndroid Build Coastguard Worker       Val = Builder.CreateZExt(Opnd, Ty, "promoted");
2355*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n");
2356*9880d681SAndroid Build Coastguard Worker     }
2357*9880d681SAndroid Build Coastguard Worker 
2358*9880d681SAndroid Build Coastguard Worker     /// \brief Get the built value.
getBuiltValue()2359*9880d681SAndroid Build Coastguard Worker     Value *getBuiltValue() { return Val; }
2360*9880d681SAndroid Build Coastguard Worker 
2361*9880d681SAndroid Build Coastguard Worker     /// \brief Remove the built instruction.
undo()2362*9880d681SAndroid Build Coastguard Worker     void undo() override {
2363*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n");
2364*9880d681SAndroid Build Coastguard Worker       if (Instruction *IVal = dyn_cast<Instruction>(Val))
2365*9880d681SAndroid Build Coastguard Worker         IVal->eraseFromParent();
2366*9880d681SAndroid Build Coastguard Worker     }
2367*9880d681SAndroid Build Coastguard Worker   };
2368*9880d681SAndroid Build Coastguard Worker 
2369*9880d681SAndroid Build Coastguard Worker   /// \brief Mutate an instruction to another type.
2370*9880d681SAndroid Build Coastguard Worker   class TypeMutator : public TypePromotionAction {
2371*9880d681SAndroid Build Coastguard Worker     /// Record the original type.
2372*9880d681SAndroid Build Coastguard Worker     Type *OrigTy;
2373*9880d681SAndroid Build Coastguard Worker 
2374*9880d681SAndroid Build Coastguard Worker   public:
2375*9880d681SAndroid Build Coastguard Worker     /// \brief Mutate the type of \p Inst into \p NewTy.
TypeMutator(Instruction * Inst,Type * NewTy)2376*9880d681SAndroid Build Coastguard Worker     TypeMutator(Instruction *Inst, Type *NewTy)
2377*9880d681SAndroid Build Coastguard Worker         : TypePromotionAction(Inst), OrigTy(Inst->getType()) {
2378*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy
2379*9880d681SAndroid Build Coastguard Worker                    << "\n");
2380*9880d681SAndroid Build Coastguard Worker       Inst->mutateType(NewTy);
2381*9880d681SAndroid Build Coastguard Worker     }
2382*9880d681SAndroid Build Coastguard Worker 
2383*9880d681SAndroid Build Coastguard Worker     /// \brief Mutate the instruction back to its original type.
undo()2384*9880d681SAndroid Build Coastguard Worker     void undo() override {
2385*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy
2386*9880d681SAndroid Build Coastguard Worker                    << "\n");
2387*9880d681SAndroid Build Coastguard Worker       Inst->mutateType(OrigTy);
2388*9880d681SAndroid Build Coastguard Worker     }
2389*9880d681SAndroid Build Coastguard Worker   };
2390*9880d681SAndroid Build Coastguard Worker 
2391*9880d681SAndroid Build Coastguard Worker   /// \brief Replace the uses of an instruction by another instruction.
2392*9880d681SAndroid Build Coastguard Worker   class UsesReplacer : public TypePromotionAction {
2393*9880d681SAndroid Build Coastguard Worker     /// Helper structure to keep track of the replaced uses.
2394*9880d681SAndroid Build Coastguard Worker     struct InstructionAndIdx {
2395*9880d681SAndroid Build Coastguard Worker       /// The instruction using the instruction.
2396*9880d681SAndroid Build Coastguard Worker       Instruction *Inst;
2397*9880d681SAndroid Build Coastguard Worker       /// The index where this instruction is used for Inst.
2398*9880d681SAndroid Build Coastguard Worker       unsigned Idx;
InstructionAndIdx__anon034c73f00211::TypePromotionTransaction::UsesReplacer::InstructionAndIdx2399*9880d681SAndroid Build Coastguard Worker       InstructionAndIdx(Instruction *Inst, unsigned Idx)
2400*9880d681SAndroid Build Coastguard Worker           : Inst(Inst), Idx(Idx) {}
2401*9880d681SAndroid Build Coastguard Worker     };
2402*9880d681SAndroid Build Coastguard Worker 
2403*9880d681SAndroid Build Coastguard Worker     /// Keep track of the original uses (pair Instruction, Index).
2404*9880d681SAndroid Build Coastguard Worker     SmallVector<InstructionAndIdx, 4> OriginalUses;
2405*9880d681SAndroid Build Coastguard Worker     typedef SmallVectorImpl<InstructionAndIdx>::iterator use_iterator;
2406*9880d681SAndroid Build Coastguard Worker 
2407*9880d681SAndroid Build Coastguard Worker   public:
2408*9880d681SAndroid Build Coastguard Worker     /// \brief Replace all the use of \p Inst by \p New.
UsesReplacer(Instruction * Inst,Value * New)2409*9880d681SAndroid Build Coastguard Worker     UsesReplacer(Instruction *Inst, Value *New) : TypePromotionAction(Inst) {
2410*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New
2411*9880d681SAndroid Build Coastguard Worker                    << "\n");
2412*9880d681SAndroid Build Coastguard Worker       // Record the original uses.
2413*9880d681SAndroid Build Coastguard Worker       for (Use &U : Inst->uses()) {
2414*9880d681SAndroid Build Coastguard Worker         Instruction *UserI = cast<Instruction>(U.getUser());
2415*9880d681SAndroid Build Coastguard Worker         OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));
2416*9880d681SAndroid Build Coastguard Worker       }
2417*9880d681SAndroid Build Coastguard Worker       // Now, we can replace the uses.
2418*9880d681SAndroid Build Coastguard Worker       Inst->replaceAllUsesWith(New);
2419*9880d681SAndroid Build Coastguard Worker     }
2420*9880d681SAndroid Build Coastguard Worker 
2421*9880d681SAndroid Build Coastguard Worker     /// \brief Reassign the original uses of Inst to Inst.
undo()2422*9880d681SAndroid Build Coastguard Worker     void undo() override {
2423*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n");
2424*9880d681SAndroid Build Coastguard Worker       for (use_iterator UseIt = OriginalUses.begin(),
2425*9880d681SAndroid Build Coastguard Worker                         EndIt = OriginalUses.end();
2426*9880d681SAndroid Build Coastguard Worker            UseIt != EndIt; ++UseIt) {
2427*9880d681SAndroid Build Coastguard Worker         UseIt->Inst->setOperand(UseIt->Idx, Inst);
2428*9880d681SAndroid Build Coastguard Worker       }
2429*9880d681SAndroid Build Coastguard Worker     }
2430*9880d681SAndroid Build Coastguard Worker   };
2431*9880d681SAndroid Build Coastguard Worker 
2432*9880d681SAndroid Build Coastguard Worker   /// \brief Remove an instruction from the IR.
2433*9880d681SAndroid Build Coastguard Worker   class InstructionRemover : public TypePromotionAction {
2434*9880d681SAndroid Build Coastguard Worker     /// Original position of the instruction.
2435*9880d681SAndroid Build Coastguard Worker     InsertionHandler Inserter;
2436*9880d681SAndroid Build Coastguard Worker     /// Helper structure to hide all the link to the instruction. In other
2437*9880d681SAndroid Build Coastguard Worker     /// words, this helps to do as if the instruction was removed.
2438*9880d681SAndroid Build Coastguard Worker     OperandsHider Hider;
2439*9880d681SAndroid Build Coastguard Worker     /// Keep track of the uses replaced, if any.
2440*9880d681SAndroid Build Coastguard Worker     UsesReplacer *Replacer;
2441*9880d681SAndroid Build Coastguard Worker 
2442*9880d681SAndroid Build Coastguard Worker   public:
2443*9880d681SAndroid Build Coastguard Worker     /// \brief Remove all reference of \p Inst and optinally replace all its
2444*9880d681SAndroid Build Coastguard Worker     /// uses with New.
2445*9880d681SAndroid Build Coastguard Worker     /// \pre If !Inst->use_empty(), then New != nullptr
InstructionRemover(Instruction * Inst,Value * New=nullptr)2446*9880d681SAndroid Build Coastguard Worker     InstructionRemover(Instruction *Inst, Value *New = nullptr)
2447*9880d681SAndroid Build Coastguard Worker         : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),
2448*9880d681SAndroid Build Coastguard Worker           Replacer(nullptr) {
2449*9880d681SAndroid Build Coastguard Worker       if (New)
2450*9880d681SAndroid Build Coastguard Worker         Replacer = new UsesReplacer(Inst, New);
2451*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n");
2452*9880d681SAndroid Build Coastguard Worker       Inst->removeFromParent();
2453*9880d681SAndroid Build Coastguard Worker     }
2454*9880d681SAndroid Build Coastguard Worker 
~InstructionRemover()2455*9880d681SAndroid Build Coastguard Worker     ~InstructionRemover() override { delete Replacer; }
2456*9880d681SAndroid Build Coastguard Worker 
2457*9880d681SAndroid Build Coastguard Worker     /// \brief Really remove the instruction.
commit()2458*9880d681SAndroid Build Coastguard Worker     void commit() override { delete Inst; }
2459*9880d681SAndroid Build Coastguard Worker 
2460*9880d681SAndroid Build Coastguard Worker     /// \brief Resurrect the instruction and reassign it to the proper uses if
2461*9880d681SAndroid Build Coastguard Worker     /// new value was provided when build this action.
undo()2462*9880d681SAndroid Build Coastguard Worker     void undo() override {
2463*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n");
2464*9880d681SAndroid Build Coastguard Worker       Inserter.insert(Inst);
2465*9880d681SAndroid Build Coastguard Worker       if (Replacer)
2466*9880d681SAndroid Build Coastguard Worker         Replacer->undo();
2467*9880d681SAndroid Build Coastguard Worker       Hider.undo();
2468*9880d681SAndroid Build Coastguard Worker     }
2469*9880d681SAndroid Build Coastguard Worker   };
2470*9880d681SAndroid Build Coastguard Worker 
2471*9880d681SAndroid Build Coastguard Worker public:
2472*9880d681SAndroid Build Coastguard Worker   /// Restoration point.
2473*9880d681SAndroid Build Coastguard Worker   /// The restoration point is a pointer to an action instead of an iterator
2474*9880d681SAndroid Build Coastguard Worker   /// because the iterator may be invalidated but not the pointer.
2475*9880d681SAndroid Build Coastguard Worker   typedef const TypePromotionAction *ConstRestorationPt;
2476*9880d681SAndroid Build Coastguard Worker   /// Advocate every changes made in that transaction.
2477*9880d681SAndroid Build Coastguard Worker   void commit();
2478*9880d681SAndroid Build Coastguard Worker   /// Undo all the changes made after the given point.
2479*9880d681SAndroid Build Coastguard Worker   void rollback(ConstRestorationPt Point);
2480*9880d681SAndroid Build Coastguard Worker   /// Get the current restoration point.
2481*9880d681SAndroid Build Coastguard Worker   ConstRestorationPt getRestorationPoint() const;
2482*9880d681SAndroid Build Coastguard Worker 
2483*9880d681SAndroid Build Coastguard Worker   /// \name API for IR modification with state keeping to support rollback.
2484*9880d681SAndroid Build Coastguard Worker   /// @{
2485*9880d681SAndroid Build Coastguard Worker   /// Same as Instruction::setOperand.
2486*9880d681SAndroid Build Coastguard Worker   void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal);
2487*9880d681SAndroid Build Coastguard Worker   /// Same as Instruction::eraseFromParent.
2488*9880d681SAndroid Build Coastguard Worker   void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr);
2489*9880d681SAndroid Build Coastguard Worker   /// Same as Value::replaceAllUsesWith.
2490*9880d681SAndroid Build Coastguard Worker   void replaceAllUsesWith(Instruction *Inst, Value *New);
2491*9880d681SAndroid Build Coastguard Worker   /// Same as Value::mutateType.
2492*9880d681SAndroid Build Coastguard Worker   void mutateType(Instruction *Inst, Type *NewTy);
2493*9880d681SAndroid Build Coastguard Worker   /// Same as IRBuilder::createTrunc.
2494*9880d681SAndroid Build Coastguard Worker   Value *createTrunc(Instruction *Opnd, Type *Ty);
2495*9880d681SAndroid Build Coastguard Worker   /// Same as IRBuilder::createSExt.
2496*9880d681SAndroid Build Coastguard Worker   Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty);
2497*9880d681SAndroid Build Coastguard Worker   /// Same as IRBuilder::createZExt.
2498*9880d681SAndroid Build Coastguard Worker   Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty);
2499*9880d681SAndroid Build Coastguard Worker   /// Same as Instruction::moveBefore.
2500*9880d681SAndroid Build Coastguard Worker   void moveBefore(Instruction *Inst, Instruction *Before);
2501*9880d681SAndroid Build Coastguard Worker   /// @}
2502*9880d681SAndroid Build Coastguard Worker 
2503*9880d681SAndroid Build Coastguard Worker private:
2504*9880d681SAndroid Build Coastguard Worker   /// The ordered list of actions made so far.
2505*9880d681SAndroid Build Coastguard Worker   SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions;
2506*9880d681SAndroid Build Coastguard Worker   typedef SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator CommitPt;
2507*9880d681SAndroid Build Coastguard Worker };
2508*9880d681SAndroid Build Coastguard Worker 
setOperand(Instruction * Inst,unsigned Idx,Value * NewVal)2509*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx,
2510*9880d681SAndroid Build Coastguard Worker                                           Value *NewVal) {
2511*9880d681SAndroid Build Coastguard Worker   Actions.push_back(
2512*9880d681SAndroid Build Coastguard Worker       make_unique<TypePromotionTransaction::OperandSetter>(Inst, Idx, NewVal));
2513*9880d681SAndroid Build Coastguard Worker }
2514*9880d681SAndroid Build Coastguard Worker 
eraseInstruction(Instruction * Inst,Value * NewVal)2515*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::eraseInstruction(Instruction *Inst,
2516*9880d681SAndroid Build Coastguard Worker                                                 Value *NewVal) {
2517*9880d681SAndroid Build Coastguard Worker   Actions.push_back(
2518*9880d681SAndroid Build Coastguard Worker       make_unique<TypePromotionTransaction::InstructionRemover>(Inst, NewVal));
2519*9880d681SAndroid Build Coastguard Worker }
2520*9880d681SAndroid Build Coastguard Worker 
replaceAllUsesWith(Instruction * Inst,Value * New)2521*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst,
2522*9880d681SAndroid Build Coastguard Worker                                                   Value *New) {
2523*9880d681SAndroid Build Coastguard Worker   Actions.push_back(make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
2524*9880d681SAndroid Build Coastguard Worker }
2525*9880d681SAndroid Build Coastguard Worker 
mutateType(Instruction * Inst,Type * NewTy)2526*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) {
2527*9880d681SAndroid Build Coastguard Worker   Actions.push_back(make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
2528*9880d681SAndroid Build Coastguard Worker }
2529*9880d681SAndroid Build Coastguard Worker 
createTrunc(Instruction * Opnd,Type * Ty)2530*9880d681SAndroid Build Coastguard Worker Value *TypePromotionTransaction::createTrunc(Instruction *Opnd,
2531*9880d681SAndroid Build Coastguard Worker                                              Type *Ty) {
2532*9880d681SAndroid Build Coastguard Worker   std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty));
2533*9880d681SAndroid Build Coastguard Worker   Value *Val = Ptr->getBuiltValue();
2534*9880d681SAndroid Build Coastguard Worker   Actions.push_back(std::move(Ptr));
2535*9880d681SAndroid Build Coastguard Worker   return Val;
2536*9880d681SAndroid Build Coastguard Worker }
2537*9880d681SAndroid Build Coastguard Worker 
createSExt(Instruction * Inst,Value * Opnd,Type * Ty)2538*9880d681SAndroid Build Coastguard Worker Value *TypePromotionTransaction::createSExt(Instruction *Inst,
2539*9880d681SAndroid Build Coastguard Worker                                             Value *Opnd, Type *Ty) {
2540*9880d681SAndroid Build Coastguard Worker   std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty));
2541*9880d681SAndroid Build Coastguard Worker   Value *Val = Ptr->getBuiltValue();
2542*9880d681SAndroid Build Coastguard Worker   Actions.push_back(std::move(Ptr));
2543*9880d681SAndroid Build Coastguard Worker   return Val;
2544*9880d681SAndroid Build Coastguard Worker }
2545*9880d681SAndroid Build Coastguard Worker 
createZExt(Instruction * Inst,Value * Opnd,Type * Ty)2546*9880d681SAndroid Build Coastguard Worker Value *TypePromotionTransaction::createZExt(Instruction *Inst,
2547*9880d681SAndroid Build Coastguard Worker                                             Value *Opnd, Type *Ty) {
2548*9880d681SAndroid Build Coastguard Worker   std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty));
2549*9880d681SAndroid Build Coastguard Worker   Value *Val = Ptr->getBuiltValue();
2550*9880d681SAndroid Build Coastguard Worker   Actions.push_back(std::move(Ptr));
2551*9880d681SAndroid Build Coastguard Worker   return Val;
2552*9880d681SAndroid Build Coastguard Worker }
2553*9880d681SAndroid Build Coastguard Worker 
moveBefore(Instruction * Inst,Instruction * Before)2554*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::moveBefore(Instruction *Inst,
2555*9880d681SAndroid Build Coastguard Worker                                           Instruction *Before) {
2556*9880d681SAndroid Build Coastguard Worker   Actions.push_back(
2557*9880d681SAndroid Build Coastguard Worker       make_unique<TypePromotionTransaction::InstructionMoveBefore>(Inst, Before));
2558*9880d681SAndroid Build Coastguard Worker }
2559*9880d681SAndroid Build Coastguard Worker 
2560*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt
getRestorationPoint() const2561*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::getRestorationPoint() const {
2562*9880d681SAndroid Build Coastguard Worker   return !Actions.empty() ? Actions.back().get() : nullptr;
2563*9880d681SAndroid Build Coastguard Worker }
2564*9880d681SAndroid Build Coastguard Worker 
commit()2565*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::commit() {
2566*9880d681SAndroid Build Coastguard Worker   for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt;
2567*9880d681SAndroid Build Coastguard Worker        ++It)
2568*9880d681SAndroid Build Coastguard Worker     (*It)->commit();
2569*9880d681SAndroid Build Coastguard Worker   Actions.clear();
2570*9880d681SAndroid Build Coastguard Worker }
2571*9880d681SAndroid Build Coastguard Worker 
rollback(TypePromotionTransaction::ConstRestorationPt Point)2572*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::rollback(
2573*9880d681SAndroid Build Coastguard Worker     TypePromotionTransaction::ConstRestorationPt Point) {
2574*9880d681SAndroid Build Coastguard Worker   while (!Actions.empty() && Point != Actions.back().get()) {
2575*9880d681SAndroid Build Coastguard Worker     std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
2576*9880d681SAndroid Build Coastguard Worker     Curr->undo();
2577*9880d681SAndroid Build Coastguard Worker   }
2578*9880d681SAndroid Build Coastguard Worker }
2579*9880d681SAndroid Build Coastguard Worker 
2580*9880d681SAndroid Build Coastguard Worker /// \brief A helper class for matching addressing modes.
2581*9880d681SAndroid Build Coastguard Worker ///
2582*9880d681SAndroid Build Coastguard Worker /// This encapsulates the logic for matching the target-legal addressing modes.
2583*9880d681SAndroid Build Coastguard Worker class AddressingModeMatcher {
2584*9880d681SAndroid Build Coastguard Worker   SmallVectorImpl<Instruction*> &AddrModeInsts;
2585*9880d681SAndroid Build Coastguard Worker   const TargetMachine &TM;
2586*9880d681SAndroid Build Coastguard Worker   const TargetLowering &TLI;
2587*9880d681SAndroid Build Coastguard Worker   const DataLayout &DL;
2588*9880d681SAndroid Build Coastguard Worker 
2589*9880d681SAndroid Build Coastguard Worker   /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
2590*9880d681SAndroid Build Coastguard Worker   /// the memory instruction that we're computing this address for.
2591*9880d681SAndroid Build Coastguard Worker   Type *AccessTy;
2592*9880d681SAndroid Build Coastguard Worker   unsigned AddrSpace;
2593*9880d681SAndroid Build Coastguard Worker   Instruction *MemoryInst;
2594*9880d681SAndroid Build Coastguard Worker 
2595*9880d681SAndroid Build Coastguard Worker   /// This is the addressing mode that we're building up. This is
2596*9880d681SAndroid Build Coastguard Worker   /// part of the return value of this addressing mode matching stuff.
2597*9880d681SAndroid Build Coastguard Worker   ExtAddrMode &AddrMode;
2598*9880d681SAndroid Build Coastguard Worker 
2599*9880d681SAndroid Build Coastguard Worker   /// The instructions inserted by other CodeGenPrepare optimizations.
2600*9880d681SAndroid Build Coastguard Worker   const SetOfInstrs &InsertedInsts;
2601*9880d681SAndroid Build Coastguard Worker   /// A map from the instructions to their type before promotion.
2602*9880d681SAndroid Build Coastguard Worker   InstrToOrigTy &PromotedInsts;
2603*9880d681SAndroid Build Coastguard Worker   /// The ongoing transaction where every action should be registered.
2604*9880d681SAndroid Build Coastguard Worker   TypePromotionTransaction &TPT;
2605*9880d681SAndroid Build Coastguard Worker 
2606*9880d681SAndroid Build Coastguard Worker   /// This is set to true when we should not do profitability checks.
2607*9880d681SAndroid Build Coastguard Worker   /// When true, IsProfitableToFoldIntoAddressingMode always returns true.
2608*9880d681SAndroid Build Coastguard Worker   bool IgnoreProfitability;
2609*9880d681SAndroid Build Coastguard Worker 
AddressingModeMatcher(SmallVectorImpl<Instruction * > & AMI,const TargetMachine & TM,Type * AT,unsigned AS,Instruction * MI,ExtAddrMode & AM,const SetOfInstrs & InsertedInsts,InstrToOrigTy & PromotedInsts,TypePromotionTransaction & TPT)2610*9880d681SAndroid Build Coastguard Worker   AddressingModeMatcher(SmallVectorImpl<Instruction *> &AMI,
2611*9880d681SAndroid Build Coastguard Worker                         const TargetMachine &TM, Type *AT, unsigned AS,
2612*9880d681SAndroid Build Coastguard Worker                         Instruction *MI, ExtAddrMode &AM,
2613*9880d681SAndroid Build Coastguard Worker                         const SetOfInstrs &InsertedInsts,
2614*9880d681SAndroid Build Coastguard Worker                         InstrToOrigTy &PromotedInsts,
2615*9880d681SAndroid Build Coastguard Worker                         TypePromotionTransaction &TPT)
2616*9880d681SAndroid Build Coastguard Worker       : AddrModeInsts(AMI), TM(TM),
2617*9880d681SAndroid Build Coastguard Worker         TLI(*TM.getSubtargetImpl(*MI->getParent()->getParent())
2618*9880d681SAndroid Build Coastguard Worker                  ->getTargetLowering()),
2619*9880d681SAndroid Build Coastguard Worker         DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS),
2620*9880d681SAndroid Build Coastguard Worker         MemoryInst(MI), AddrMode(AM), InsertedInsts(InsertedInsts),
2621*9880d681SAndroid Build Coastguard Worker         PromotedInsts(PromotedInsts), TPT(TPT) {
2622*9880d681SAndroid Build Coastguard Worker     IgnoreProfitability = false;
2623*9880d681SAndroid Build Coastguard Worker   }
2624*9880d681SAndroid Build Coastguard Worker public:
2625*9880d681SAndroid Build Coastguard Worker 
2626*9880d681SAndroid Build Coastguard Worker   /// Find the maximal addressing mode that a load/store of V can fold,
2627*9880d681SAndroid Build Coastguard Worker   /// give an access type of AccessTy.  This returns a list of involved
2628*9880d681SAndroid Build Coastguard Worker   /// instructions in AddrModeInsts.
2629*9880d681SAndroid Build Coastguard Worker   /// \p InsertedInsts The instructions inserted by other CodeGenPrepare
2630*9880d681SAndroid Build Coastguard Worker   /// optimizations.
2631*9880d681SAndroid Build Coastguard Worker   /// \p PromotedInsts maps the instructions to their type before promotion.
2632*9880d681SAndroid Build Coastguard Worker   /// \p The ongoing transaction where every action should be registered.
Match(Value * V,Type * AccessTy,unsigned AS,Instruction * MemoryInst,SmallVectorImpl<Instruction * > & AddrModeInsts,const TargetMachine & TM,const SetOfInstrs & InsertedInsts,InstrToOrigTy & PromotedInsts,TypePromotionTransaction & TPT)2633*9880d681SAndroid Build Coastguard Worker   static ExtAddrMode Match(Value *V, Type *AccessTy, unsigned AS,
2634*9880d681SAndroid Build Coastguard Worker                            Instruction *MemoryInst,
2635*9880d681SAndroid Build Coastguard Worker                            SmallVectorImpl<Instruction*> &AddrModeInsts,
2636*9880d681SAndroid Build Coastguard Worker                            const TargetMachine &TM,
2637*9880d681SAndroid Build Coastguard Worker                            const SetOfInstrs &InsertedInsts,
2638*9880d681SAndroid Build Coastguard Worker                            InstrToOrigTy &PromotedInsts,
2639*9880d681SAndroid Build Coastguard Worker                            TypePromotionTransaction &TPT) {
2640*9880d681SAndroid Build Coastguard Worker     ExtAddrMode Result;
2641*9880d681SAndroid Build Coastguard Worker 
2642*9880d681SAndroid Build Coastguard Worker     bool Success = AddressingModeMatcher(AddrModeInsts, TM, AccessTy, AS,
2643*9880d681SAndroid Build Coastguard Worker                                          MemoryInst, Result, InsertedInsts,
2644*9880d681SAndroid Build Coastguard Worker                                          PromotedInsts, TPT).matchAddr(V, 0);
2645*9880d681SAndroid Build Coastguard Worker     (void)Success; assert(Success && "Couldn't select *anything*?");
2646*9880d681SAndroid Build Coastguard Worker     return Result;
2647*9880d681SAndroid Build Coastguard Worker   }
2648*9880d681SAndroid Build Coastguard Worker private:
2649*9880d681SAndroid Build Coastguard Worker   bool matchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth);
2650*9880d681SAndroid Build Coastguard Worker   bool matchAddr(Value *V, unsigned Depth);
2651*9880d681SAndroid Build Coastguard Worker   bool matchOperationAddr(User *Operation, unsigned Opcode, unsigned Depth,
2652*9880d681SAndroid Build Coastguard Worker                           bool *MovedAway = nullptr);
2653*9880d681SAndroid Build Coastguard Worker   bool isProfitableToFoldIntoAddressingMode(Instruction *I,
2654*9880d681SAndroid Build Coastguard Worker                                             ExtAddrMode &AMBefore,
2655*9880d681SAndroid Build Coastguard Worker                                             ExtAddrMode &AMAfter);
2656*9880d681SAndroid Build Coastguard Worker   bool valueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2);
2657*9880d681SAndroid Build Coastguard Worker   bool isPromotionProfitable(unsigned NewCost, unsigned OldCost,
2658*9880d681SAndroid Build Coastguard Worker                              Value *PromotedOperand) const;
2659*9880d681SAndroid Build Coastguard Worker };
2660*9880d681SAndroid Build Coastguard Worker 
2661*9880d681SAndroid Build Coastguard Worker /// Try adding ScaleReg*Scale to the current addressing mode.
2662*9880d681SAndroid Build Coastguard Worker /// Return true and update AddrMode if this addr mode is legal for the target,
2663*9880d681SAndroid Build Coastguard Worker /// false if not.
matchScaledValue(Value * ScaleReg,int64_t Scale,unsigned Depth)2664*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale,
2665*9880d681SAndroid Build Coastguard Worker                                              unsigned Depth) {
2666*9880d681SAndroid Build Coastguard Worker   // If Scale is 1, then this is the same as adding ScaleReg to the addressing
2667*9880d681SAndroid Build Coastguard Worker   // mode.  Just process that directly.
2668*9880d681SAndroid Build Coastguard Worker   if (Scale == 1)
2669*9880d681SAndroid Build Coastguard Worker     return matchAddr(ScaleReg, Depth);
2670*9880d681SAndroid Build Coastguard Worker 
2671*9880d681SAndroid Build Coastguard Worker   // If the scale is 0, it takes nothing to add this.
2672*9880d681SAndroid Build Coastguard Worker   if (Scale == 0)
2673*9880d681SAndroid Build Coastguard Worker     return true;
2674*9880d681SAndroid Build Coastguard Worker 
2675*9880d681SAndroid Build Coastguard Worker   // If we already have a scale of this value, we can add to it, otherwise, we
2676*9880d681SAndroid Build Coastguard Worker   // need an available scale field.
2677*9880d681SAndroid Build Coastguard Worker   if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg)
2678*9880d681SAndroid Build Coastguard Worker     return false;
2679*9880d681SAndroid Build Coastguard Worker 
2680*9880d681SAndroid Build Coastguard Worker   ExtAddrMode TestAddrMode = AddrMode;
2681*9880d681SAndroid Build Coastguard Worker 
2682*9880d681SAndroid Build Coastguard Worker   // Add scale to turn X*4+X*3 -> X*7.  This could also do things like
2683*9880d681SAndroid Build Coastguard Worker   // [A+B + A*7] -> [B+A*8].
2684*9880d681SAndroid Build Coastguard Worker   TestAddrMode.Scale += Scale;
2685*9880d681SAndroid Build Coastguard Worker   TestAddrMode.ScaledReg = ScaleReg;
2686*9880d681SAndroid Build Coastguard Worker 
2687*9880d681SAndroid Build Coastguard Worker   // If the new address isn't legal, bail out.
2688*9880d681SAndroid Build Coastguard Worker   if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))
2689*9880d681SAndroid Build Coastguard Worker     return false;
2690*9880d681SAndroid Build Coastguard Worker 
2691*9880d681SAndroid Build Coastguard Worker   // It was legal, so commit it.
2692*9880d681SAndroid Build Coastguard Worker   AddrMode = TestAddrMode;
2693*9880d681SAndroid Build Coastguard Worker 
2694*9880d681SAndroid Build Coastguard Worker   // Okay, we decided that we can add ScaleReg+Scale to AddrMode.  Check now
2695*9880d681SAndroid Build Coastguard Worker   // to see if ScaleReg is actually X+C.  If so, we can turn this into adding
2696*9880d681SAndroid Build Coastguard Worker   // X*Scale + C*Scale to addr mode.
2697*9880d681SAndroid Build Coastguard Worker   ConstantInt *CI = nullptr; Value *AddLHS = nullptr;
2698*9880d681SAndroid Build Coastguard Worker   if (isa<Instruction>(ScaleReg) &&  // not a constant expr.
2699*9880d681SAndroid Build Coastguard Worker       match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) {
2700*9880d681SAndroid Build Coastguard Worker     TestAddrMode.ScaledReg = AddLHS;
2701*9880d681SAndroid Build Coastguard Worker     TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale;
2702*9880d681SAndroid Build Coastguard Worker 
2703*9880d681SAndroid Build Coastguard Worker     // If this addressing mode is legal, commit it and remember that we folded
2704*9880d681SAndroid Build Coastguard Worker     // this instruction.
2705*9880d681SAndroid Build Coastguard Worker     if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {
2706*9880d681SAndroid Build Coastguard Worker       AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
2707*9880d681SAndroid Build Coastguard Worker       AddrMode = TestAddrMode;
2708*9880d681SAndroid Build Coastguard Worker       return true;
2709*9880d681SAndroid Build Coastguard Worker     }
2710*9880d681SAndroid Build Coastguard Worker   }
2711*9880d681SAndroid Build Coastguard Worker 
2712*9880d681SAndroid Build Coastguard Worker   // Otherwise, not (x+c)*scale, just return what we have.
2713*9880d681SAndroid Build Coastguard Worker   return true;
2714*9880d681SAndroid Build Coastguard Worker }
2715*9880d681SAndroid Build Coastguard Worker 
2716*9880d681SAndroid Build Coastguard Worker /// This is a little filter, which returns true if an addressing computation
2717*9880d681SAndroid Build Coastguard Worker /// involving I might be folded into a load/store accessing it.
2718*9880d681SAndroid Build Coastguard Worker /// This doesn't need to be perfect, but needs to accept at least
2719*9880d681SAndroid Build Coastguard Worker /// the set of instructions that MatchOperationAddr can.
MightBeFoldableInst(Instruction * I)2720*9880d681SAndroid Build Coastguard Worker static bool MightBeFoldableInst(Instruction *I) {
2721*9880d681SAndroid Build Coastguard Worker   switch (I->getOpcode()) {
2722*9880d681SAndroid Build Coastguard Worker   case Instruction::BitCast:
2723*9880d681SAndroid Build Coastguard Worker   case Instruction::AddrSpaceCast:
2724*9880d681SAndroid Build Coastguard Worker     // Don't touch identity bitcasts.
2725*9880d681SAndroid Build Coastguard Worker     if (I->getType() == I->getOperand(0)->getType())
2726*9880d681SAndroid Build Coastguard Worker       return false;
2727*9880d681SAndroid Build Coastguard Worker     return I->getType()->isPointerTy() || I->getType()->isIntegerTy();
2728*9880d681SAndroid Build Coastguard Worker   case Instruction::PtrToInt:
2729*9880d681SAndroid Build Coastguard Worker     // PtrToInt is always a noop, as we know that the int type is pointer sized.
2730*9880d681SAndroid Build Coastguard Worker     return true;
2731*9880d681SAndroid Build Coastguard Worker   case Instruction::IntToPtr:
2732*9880d681SAndroid Build Coastguard Worker     // We know the input is intptr_t, so this is foldable.
2733*9880d681SAndroid Build Coastguard Worker     return true;
2734*9880d681SAndroid Build Coastguard Worker   case Instruction::Add:
2735*9880d681SAndroid Build Coastguard Worker     return true;
2736*9880d681SAndroid Build Coastguard Worker   case Instruction::Mul:
2737*9880d681SAndroid Build Coastguard Worker   case Instruction::Shl:
2738*9880d681SAndroid Build Coastguard Worker     // Can only handle X*C and X << C.
2739*9880d681SAndroid Build Coastguard Worker     return isa<ConstantInt>(I->getOperand(1));
2740*9880d681SAndroid Build Coastguard Worker   case Instruction::GetElementPtr:
2741*9880d681SAndroid Build Coastguard Worker     return true;
2742*9880d681SAndroid Build Coastguard Worker   default:
2743*9880d681SAndroid Build Coastguard Worker     return false;
2744*9880d681SAndroid Build Coastguard Worker   }
2745*9880d681SAndroid Build Coastguard Worker }
2746*9880d681SAndroid Build Coastguard Worker 
2747*9880d681SAndroid Build Coastguard Worker /// \brief Check whether or not \p Val is a legal instruction for \p TLI.
2748*9880d681SAndroid Build Coastguard Worker /// \note \p Val is assumed to be the product of some type promotion.
2749*9880d681SAndroid Build Coastguard Worker /// Therefore if \p Val has an undefined state in \p TLI, this is assumed
2750*9880d681SAndroid Build Coastguard Worker /// to be legal, as the non-promoted value would have had the same state.
isPromotedInstructionLegal(const TargetLowering & TLI,const DataLayout & DL,Value * Val)2751*9880d681SAndroid Build Coastguard Worker static bool isPromotedInstructionLegal(const TargetLowering &TLI,
2752*9880d681SAndroid Build Coastguard Worker                                        const DataLayout &DL, Value *Val) {
2753*9880d681SAndroid Build Coastguard Worker   Instruction *PromotedInst = dyn_cast<Instruction>(Val);
2754*9880d681SAndroid Build Coastguard Worker   if (!PromotedInst)
2755*9880d681SAndroid Build Coastguard Worker     return false;
2756*9880d681SAndroid Build Coastguard Worker   int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode());
2757*9880d681SAndroid Build Coastguard Worker   // If the ISDOpcode is undefined, it was undefined before the promotion.
2758*9880d681SAndroid Build Coastguard Worker   if (!ISDOpcode)
2759*9880d681SAndroid Build Coastguard Worker     return true;
2760*9880d681SAndroid Build Coastguard Worker   // Otherwise, check if the promoted instruction is legal or not.
2761*9880d681SAndroid Build Coastguard Worker   return TLI.isOperationLegalOrCustom(
2762*9880d681SAndroid Build Coastguard Worker       ISDOpcode, TLI.getValueType(DL, PromotedInst->getType()));
2763*9880d681SAndroid Build Coastguard Worker }
2764*9880d681SAndroid Build Coastguard Worker 
2765*9880d681SAndroid Build Coastguard Worker /// \brief Hepler class to perform type promotion.
2766*9880d681SAndroid Build Coastguard Worker class TypePromotionHelper {
2767*9880d681SAndroid Build Coastguard Worker   /// \brief Utility function to check whether or not a sign or zero extension
2768*9880d681SAndroid Build Coastguard Worker   /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by
2769*9880d681SAndroid Build Coastguard Worker   /// either using the operands of \p Inst or promoting \p Inst.
2770*9880d681SAndroid Build Coastguard Worker   /// The type of the extension is defined by \p IsSExt.
2771*9880d681SAndroid Build Coastguard Worker   /// In other words, check if:
2772*9880d681SAndroid Build Coastguard Worker   /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType.
2773*9880d681SAndroid Build Coastguard Worker   /// #1 Promotion applies:
2774*9880d681SAndroid Build Coastguard Worker   /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...).
2775*9880d681SAndroid Build Coastguard Worker   /// #2 Operand reuses:
2776*9880d681SAndroid Build Coastguard Worker   /// ext opnd1 to ConsideredExtType.
2777*9880d681SAndroid Build Coastguard Worker   /// \p PromotedInsts maps the instructions to their type before promotion.
2778*9880d681SAndroid Build Coastguard Worker   static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType,
2779*9880d681SAndroid Build Coastguard Worker                             const InstrToOrigTy &PromotedInsts, bool IsSExt);
2780*9880d681SAndroid Build Coastguard Worker 
2781*9880d681SAndroid Build Coastguard Worker   /// \brief Utility function to determine if \p OpIdx should be promoted when
2782*9880d681SAndroid Build Coastguard Worker   /// promoting \p Inst.
shouldExtOperand(const Instruction * Inst,int OpIdx)2783*9880d681SAndroid Build Coastguard Worker   static bool shouldExtOperand(const Instruction *Inst, int OpIdx) {
2784*9880d681SAndroid Build Coastguard Worker     return !(isa<SelectInst>(Inst) && OpIdx == 0);
2785*9880d681SAndroid Build Coastguard Worker   }
2786*9880d681SAndroid Build Coastguard Worker 
2787*9880d681SAndroid Build Coastguard Worker   /// \brief Utility function to promote the operand of \p Ext when this
2788*9880d681SAndroid Build Coastguard Worker   /// operand is a promotable trunc or sext or zext.
2789*9880d681SAndroid Build Coastguard Worker   /// \p PromotedInsts maps the instructions to their type before promotion.
2790*9880d681SAndroid Build Coastguard Worker   /// \p CreatedInstsCost[out] contains the cost of all instructions
2791*9880d681SAndroid Build Coastguard Worker   /// created to promote the operand of Ext.
2792*9880d681SAndroid Build Coastguard Worker   /// Newly added extensions are inserted in \p Exts.
2793*9880d681SAndroid Build Coastguard Worker   /// Newly added truncates are inserted in \p Truncs.
2794*9880d681SAndroid Build Coastguard Worker   /// Should never be called directly.
2795*9880d681SAndroid Build Coastguard Worker   /// \return The promoted value which is used instead of Ext.
2796*9880d681SAndroid Build Coastguard Worker   static Value *promoteOperandForTruncAndAnyExt(
2797*9880d681SAndroid Build Coastguard Worker       Instruction *Ext, TypePromotionTransaction &TPT,
2798*9880d681SAndroid Build Coastguard Worker       InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2799*9880d681SAndroid Build Coastguard Worker       SmallVectorImpl<Instruction *> *Exts,
2800*9880d681SAndroid Build Coastguard Worker       SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI);
2801*9880d681SAndroid Build Coastguard Worker 
2802*9880d681SAndroid Build Coastguard Worker   /// \brief Utility function to promote the operand of \p Ext when this
2803*9880d681SAndroid Build Coastguard Worker   /// operand is promotable and is not a supported trunc or sext.
2804*9880d681SAndroid Build Coastguard Worker   /// \p PromotedInsts maps the instructions to their type before promotion.
2805*9880d681SAndroid Build Coastguard Worker   /// \p CreatedInstsCost[out] contains the cost of all the instructions
2806*9880d681SAndroid Build Coastguard Worker   /// created to promote the operand of Ext.
2807*9880d681SAndroid Build Coastguard Worker   /// Newly added extensions are inserted in \p Exts.
2808*9880d681SAndroid Build Coastguard Worker   /// Newly added truncates are inserted in \p Truncs.
2809*9880d681SAndroid Build Coastguard Worker   /// Should never be called directly.
2810*9880d681SAndroid Build Coastguard Worker   /// \return The promoted value which is used instead of Ext.
2811*9880d681SAndroid Build Coastguard Worker   static Value *promoteOperandForOther(Instruction *Ext,
2812*9880d681SAndroid Build Coastguard Worker                                        TypePromotionTransaction &TPT,
2813*9880d681SAndroid Build Coastguard Worker                                        InstrToOrigTy &PromotedInsts,
2814*9880d681SAndroid Build Coastguard Worker                                        unsigned &CreatedInstsCost,
2815*9880d681SAndroid Build Coastguard Worker                                        SmallVectorImpl<Instruction *> *Exts,
2816*9880d681SAndroid Build Coastguard Worker                                        SmallVectorImpl<Instruction *> *Truncs,
2817*9880d681SAndroid Build Coastguard Worker                                        const TargetLowering &TLI, bool IsSExt);
2818*9880d681SAndroid Build Coastguard Worker 
2819*9880d681SAndroid Build Coastguard Worker   /// \see promoteOperandForOther.
signExtendOperandForOther(Instruction * Ext,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI)2820*9880d681SAndroid Build Coastguard Worker   static Value *signExtendOperandForOther(
2821*9880d681SAndroid Build Coastguard Worker       Instruction *Ext, TypePromotionTransaction &TPT,
2822*9880d681SAndroid Build Coastguard Worker       InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2823*9880d681SAndroid Build Coastguard Worker       SmallVectorImpl<Instruction *> *Exts,
2824*9880d681SAndroid Build Coastguard Worker       SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
2825*9880d681SAndroid Build Coastguard Worker     return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
2826*9880d681SAndroid Build Coastguard Worker                                   Exts, Truncs, TLI, true);
2827*9880d681SAndroid Build Coastguard Worker   }
2828*9880d681SAndroid Build Coastguard Worker 
2829*9880d681SAndroid Build Coastguard Worker   /// \see promoteOperandForOther.
zeroExtendOperandForOther(Instruction * Ext,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI)2830*9880d681SAndroid Build Coastguard Worker   static Value *zeroExtendOperandForOther(
2831*9880d681SAndroid Build Coastguard Worker       Instruction *Ext, TypePromotionTransaction &TPT,
2832*9880d681SAndroid Build Coastguard Worker       InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2833*9880d681SAndroid Build Coastguard Worker       SmallVectorImpl<Instruction *> *Exts,
2834*9880d681SAndroid Build Coastguard Worker       SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
2835*9880d681SAndroid Build Coastguard Worker     return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
2836*9880d681SAndroid Build Coastguard Worker                                   Exts, Truncs, TLI, false);
2837*9880d681SAndroid Build Coastguard Worker   }
2838*9880d681SAndroid Build Coastguard Worker 
2839*9880d681SAndroid Build Coastguard Worker public:
2840*9880d681SAndroid Build Coastguard Worker   /// Type for the utility function that promotes the operand of Ext.
2841*9880d681SAndroid Build Coastguard Worker   typedef Value *(*Action)(Instruction *Ext, TypePromotionTransaction &TPT,
2842*9880d681SAndroid Build Coastguard Worker                            InstrToOrigTy &PromotedInsts,
2843*9880d681SAndroid Build Coastguard Worker                            unsigned &CreatedInstsCost,
2844*9880d681SAndroid Build Coastguard Worker                            SmallVectorImpl<Instruction *> *Exts,
2845*9880d681SAndroid Build Coastguard Worker                            SmallVectorImpl<Instruction *> *Truncs,
2846*9880d681SAndroid Build Coastguard Worker                            const TargetLowering &TLI);
2847*9880d681SAndroid Build Coastguard Worker   /// \brief Given a sign/zero extend instruction \p Ext, return the approriate
2848*9880d681SAndroid Build Coastguard Worker   /// action to promote the operand of \p Ext instead of using Ext.
2849*9880d681SAndroid Build Coastguard Worker   /// \return NULL if no promotable action is possible with the current
2850*9880d681SAndroid Build Coastguard Worker   /// sign extension.
2851*9880d681SAndroid Build Coastguard Worker   /// \p InsertedInsts keeps track of all the instructions inserted by the
2852*9880d681SAndroid Build Coastguard Worker   /// other CodeGenPrepare optimizations. This information is important
2853*9880d681SAndroid Build Coastguard Worker   /// because we do not want to promote these instructions as CodeGenPrepare
2854*9880d681SAndroid Build Coastguard Worker   /// will reinsert them later. Thus creating an infinite loop: create/remove.
2855*9880d681SAndroid Build Coastguard Worker   /// \p PromotedInsts maps the instructions to their type before promotion.
2856*9880d681SAndroid Build Coastguard Worker   static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts,
2857*9880d681SAndroid Build Coastguard Worker                           const TargetLowering &TLI,
2858*9880d681SAndroid Build Coastguard Worker                           const InstrToOrigTy &PromotedInsts);
2859*9880d681SAndroid Build Coastguard Worker };
2860*9880d681SAndroid Build Coastguard Worker 
canGetThrough(const Instruction * Inst,Type * ConsideredExtType,const InstrToOrigTy & PromotedInsts,bool IsSExt)2861*9880d681SAndroid Build Coastguard Worker bool TypePromotionHelper::canGetThrough(const Instruction *Inst,
2862*9880d681SAndroid Build Coastguard Worker                                         Type *ConsideredExtType,
2863*9880d681SAndroid Build Coastguard Worker                                         const InstrToOrigTy &PromotedInsts,
2864*9880d681SAndroid Build Coastguard Worker                                         bool IsSExt) {
2865*9880d681SAndroid Build Coastguard Worker   // The promotion helper does not know how to deal with vector types yet.
2866*9880d681SAndroid Build Coastguard Worker   // To be able to fix that, we would need to fix the places where we
2867*9880d681SAndroid Build Coastguard Worker   // statically extend, e.g., constants and such.
2868*9880d681SAndroid Build Coastguard Worker   if (Inst->getType()->isVectorTy())
2869*9880d681SAndroid Build Coastguard Worker     return false;
2870*9880d681SAndroid Build Coastguard Worker 
2871*9880d681SAndroid Build Coastguard Worker   // We can always get through zext.
2872*9880d681SAndroid Build Coastguard Worker   if (isa<ZExtInst>(Inst))
2873*9880d681SAndroid Build Coastguard Worker     return true;
2874*9880d681SAndroid Build Coastguard Worker 
2875*9880d681SAndroid Build Coastguard Worker   // sext(sext) is ok too.
2876*9880d681SAndroid Build Coastguard Worker   if (IsSExt && isa<SExtInst>(Inst))
2877*9880d681SAndroid Build Coastguard Worker     return true;
2878*9880d681SAndroid Build Coastguard Worker 
2879*9880d681SAndroid Build Coastguard Worker   // We can get through binary operator, if it is legal. In other words, the
2880*9880d681SAndroid Build Coastguard Worker   // binary operator must have a nuw or nsw flag.
2881*9880d681SAndroid Build Coastguard Worker   const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst);
2882*9880d681SAndroid Build Coastguard Worker   if (BinOp && isa<OverflowingBinaryOperator>(BinOp) &&
2883*9880d681SAndroid Build Coastguard Worker       ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||
2884*9880d681SAndroid Build Coastguard Worker        (IsSExt && BinOp->hasNoSignedWrap())))
2885*9880d681SAndroid Build Coastguard Worker     return true;
2886*9880d681SAndroid Build Coastguard Worker 
2887*9880d681SAndroid Build Coastguard Worker   // Check if we can do the following simplification.
2888*9880d681SAndroid Build Coastguard Worker   // ext(trunc(opnd)) --> ext(opnd)
2889*9880d681SAndroid Build Coastguard Worker   if (!isa<TruncInst>(Inst))
2890*9880d681SAndroid Build Coastguard Worker     return false;
2891*9880d681SAndroid Build Coastguard Worker 
2892*9880d681SAndroid Build Coastguard Worker   Value *OpndVal = Inst->getOperand(0);
2893*9880d681SAndroid Build Coastguard Worker   // Check if we can use this operand in the extension.
2894*9880d681SAndroid Build Coastguard Worker   // If the type is larger than the result type of the extension, we cannot.
2895*9880d681SAndroid Build Coastguard Worker   if (!OpndVal->getType()->isIntegerTy() ||
2896*9880d681SAndroid Build Coastguard Worker       OpndVal->getType()->getIntegerBitWidth() >
2897*9880d681SAndroid Build Coastguard Worker           ConsideredExtType->getIntegerBitWidth())
2898*9880d681SAndroid Build Coastguard Worker     return false;
2899*9880d681SAndroid Build Coastguard Worker 
2900*9880d681SAndroid Build Coastguard Worker   // If the operand of the truncate is not an instruction, we will not have
2901*9880d681SAndroid Build Coastguard Worker   // any information on the dropped bits.
2902*9880d681SAndroid Build Coastguard Worker   // (Actually we could for constant but it is not worth the extra logic).
2903*9880d681SAndroid Build Coastguard Worker   Instruction *Opnd = dyn_cast<Instruction>(OpndVal);
2904*9880d681SAndroid Build Coastguard Worker   if (!Opnd)
2905*9880d681SAndroid Build Coastguard Worker     return false;
2906*9880d681SAndroid Build Coastguard Worker 
2907*9880d681SAndroid Build Coastguard Worker   // Check if the source of the type is narrow enough.
2908*9880d681SAndroid Build Coastguard Worker   // I.e., check that trunc just drops extended bits of the same kind of
2909*9880d681SAndroid Build Coastguard Worker   // the extension.
2910*9880d681SAndroid Build Coastguard Worker   // #1 get the type of the operand and check the kind of the extended bits.
2911*9880d681SAndroid Build Coastguard Worker   const Type *OpndType;
2912*9880d681SAndroid Build Coastguard Worker   InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
2913*9880d681SAndroid Build Coastguard Worker   if (It != PromotedInsts.end() && It->second.getInt() == IsSExt)
2914*9880d681SAndroid Build Coastguard Worker     OpndType = It->second.getPointer();
2915*9880d681SAndroid Build Coastguard Worker   else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
2916*9880d681SAndroid Build Coastguard Worker     OpndType = Opnd->getOperand(0)->getType();
2917*9880d681SAndroid Build Coastguard Worker   else
2918*9880d681SAndroid Build Coastguard Worker     return false;
2919*9880d681SAndroid Build Coastguard Worker 
2920*9880d681SAndroid Build Coastguard Worker   // #2 check that the truncate just drops extended bits.
2921*9880d681SAndroid Build Coastguard Worker   return Inst->getType()->getIntegerBitWidth() >=
2922*9880d681SAndroid Build Coastguard Worker          OpndType->getIntegerBitWidth();
2923*9880d681SAndroid Build Coastguard Worker }
2924*9880d681SAndroid Build Coastguard Worker 
getAction(Instruction * Ext,const SetOfInstrs & InsertedInsts,const TargetLowering & TLI,const InstrToOrigTy & PromotedInsts)2925*9880d681SAndroid Build Coastguard Worker TypePromotionHelper::Action TypePromotionHelper::getAction(
2926*9880d681SAndroid Build Coastguard Worker     Instruction *Ext, const SetOfInstrs &InsertedInsts,
2927*9880d681SAndroid Build Coastguard Worker     const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) {
2928*9880d681SAndroid Build Coastguard Worker   assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2929*9880d681SAndroid Build Coastguard Worker          "Unexpected instruction type");
2930*9880d681SAndroid Build Coastguard Worker   Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0));
2931*9880d681SAndroid Build Coastguard Worker   Type *ExtTy = Ext->getType();
2932*9880d681SAndroid Build Coastguard Worker   bool IsSExt = isa<SExtInst>(Ext);
2933*9880d681SAndroid Build Coastguard Worker   // If the operand of the extension is not an instruction, we cannot
2934*9880d681SAndroid Build Coastguard Worker   // get through.
2935*9880d681SAndroid Build Coastguard Worker   // If it, check we can get through.
2936*9880d681SAndroid Build Coastguard Worker   if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
2937*9880d681SAndroid Build Coastguard Worker     return nullptr;
2938*9880d681SAndroid Build Coastguard Worker 
2939*9880d681SAndroid Build Coastguard Worker   // Do not promote if the operand has been added by codegenprepare.
2940*9880d681SAndroid Build Coastguard Worker   // Otherwise, it means we are undoing an optimization that is likely to be
2941*9880d681SAndroid Build Coastguard Worker   // redone, thus causing potential infinite loop.
2942*9880d681SAndroid Build Coastguard Worker   if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
2943*9880d681SAndroid Build Coastguard Worker     return nullptr;
2944*9880d681SAndroid Build Coastguard Worker 
2945*9880d681SAndroid Build Coastguard Worker   // SExt or Trunc instructions.
2946*9880d681SAndroid Build Coastguard Worker   // Return the related handler.
2947*9880d681SAndroid Build Coastguard Worker   if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||
2948*9880d681SAndroid Build Coastguard Worker       isa<ZExtInst>(ExtOpnd))
2949*9880d681SAndroid Build Coastguard Worker     return promoteOperandForTruncAndAnyExt;
2950*9880d681SAndroid Build Coastguard Worker 
2951*9880d681SAndroid Build Coastguard Worker   // Regular instruction.
2952*9880d681SAndroid Build Coastguard Worker   // Abort early if we will have to insert non-free instructions.
2953*9880d681SAndroid Build Coastguard Worker   if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType()))
2954*9880d681SAndroid Build Coastguard Worker     return nullptr;
2955*9880d681SAndroid Build Coastguard Worker   return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
2956*9880d681SAndroid Build Coastguard Worker }
2957*9880d681SAndroid Build Coastguard Worker 
promoteOperandForTruncAndAnyExt(llvm::Instruction * SExt,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI)2958*9880d681SAndroid Build Coastguard Worker Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
2959*9880d681SAndroid Build Coastguard Worker     llvm::Instruction *SExt, TypePromotionTransaction &TPT,
2960*9880d681SAndroid Build Coastguard Worker     InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2961*9880d681SAndroid Build Coastguard Worker     SmallVectorImpl<Instruction *> *Exts,
2962*9880d681SAndroid Build Coastguard Worker     SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
2963*9880d681SAndroid Build Coastguard Worker   // By construction, the operand of SExt is an instruction. Otherwise we cannot
2964*9880d681SAndroid Build Coastguard Worker   // get through it and this method should not be called.
2965*9880d681SAndroid Build Coastguard Worker   Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0));
2966*9880d681SAndroid Build Coastguard Worker   Value *ExtVal = SExt;
2967*9880d681SAndroid Build Coastguard Worker   bool HasMergedNonFreeExt = false;
2968*9880d681SAndroid Build Coastguard Worker   if (isa<ZExtInst>(SExtOpnd)) {
2969*9880d681SAndroid Build Coastguard Worker     // Replace s|zext(zext(opnd))
2970*9880d681SAndroid Build Coastguard Worker     // => zext(opnd).
2971*9880d681SAndroid Build Coastguard Worker     HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd);
2972*9880d681SAndroid Build Coastguard Worker     Value *ZExt =
2973*9880d681SAndroid Build Coastguard Worker         TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType());
2974*9880d681SAndroid Build Coastguard Worker     TPT.replaceAllUsesWith(SExt, ZExt);
2975*9880d681SAndroid Build Coastguard Worker     TPT.eraseInstruction(SExt);
2976*9880d681SAndroid Build Coastguard Worker     ExtVal = ZExt;
2977*9880d681SAndroid Build Coastguard Worker   } else {
2978*9880d681SAndroid Build Coastguard Worker     // Replace z|sext(trunc(opnd)) or sext(sext(opnd))
2979*9880d681SAndroid Build Coastguard Worker     // => z|sext(opnd).
2980*9880d681SAndroid Build Coastguard Worker     TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0));
2981*9880d681SAndroid Build Coastguard Worker   }
2982*9880d681SAndroid Build Coastguard Worker   CreatedInstsCost = 0;
2983*9880d681SAndroid Build Coastguard Worker 
2984*9880d681SAndroid Build Coastguard Worker   // Remove dead code.
2985*9880d681SAndroid Build Coastguard Worker   if (SExtOpnd->use_empty())
2986*9880d681SAndroid Build Coastguard Worker     TPT.eraseInstruction(SExtOpnd);
2987*9880d681SAndroid Build Coastguard Worker 
2988*9880d681SAndroid Build Coastguard Worker   // Check if the extension is still needed.
2989*9880d681SAndroid Build Coastguard Worker   Instruction *ExtInst = dyn_cast<Instruction>(ExtVal);
2990*9880d681SAndroid Build Coastguard Worker   if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) {
2991*9880d681SAndroid Build Coastguard Worker     if (ExtInst) {
2992*9880d681SAndroid Build Coastguard Worker       if (Exts)
2993*9880d681SAndroid Build Coastguard Worker         Exts->push_back(ExtInst);
2994*9880d681SAndroid Build Coastguard Worker       CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt;
2995*9880d681SAndroid Build Coastguard Worker     }
2996*9880d681SAndroid Build Coastguard Worker     return ExtVal;
2997*9880d681SAndroid Build Coastguard Worker   }
2998*9880d681SAndroid Build Coastguard Worker 
2999*9880d681SAndroid Build Coastguard Worker   // At this point we have: ext ty opnd to ty.
3000*9880d681SAndroid Build Coastguard Worker   // Reassign the uses of ExtInst to the opnd and remove ExtInst.
3001*9880d681SAndroid Build Coastguard Worker   Value *NextVal = ExtInst->getOperand(0);
3002*9880d681SAndroid Build Coastguard Worker   TPT.eraseInstruction(ExtInst, NextVal);
3003*9880d681SAndroid Build Coastguard Worker   return NextVal;
3004*9880d681SAndroid Build Coastguard Worker }
3005*9880d681SAndroid Build Coastguard Worker 
promoteOperandForOther(Instruction * Ext,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI,bool IsSExt)3006*9880d681SAndroid Build Coastguard Worker Value *TypePromotionHelper::promoteOperandForOther(
3007*9880d681SAndroid Build Coastguard Worker     Instruction *Ext, TypePromotionTransaction &TPT,
3008*9880d681SAndroid Build Coastguard Worker     InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
3009*9880d681SAndroid Build Coastguard Worker     SmallVectorImpl<Instruction *> *Exts,
3010*9880d681SAndroid Build Coastguard Worker     SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI,
3011*9880d681SAndroid Build Coastguard Worker     bool IsSExt) {
3012*9880d681SAndroid Build Coastguard Worker   // By construction, the operand of Ext is an instruction. Otherwise we cannot
3013*9880d681SAndroid Build Coastguard Worker   // get through it and this method should not be called.
3014*9880d681SAndroid Build Coastguard Worker   Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0));
3015*9880d681SAndroid Build Coastguard Worker   CreatedInstsCost = 0;
3016*9880d681SAndroid Build Coastguard Worker   if (!ExtOpnd->hasOneUse()) {
3017*9880d681SAndroid Build Coastguard Worker     // ExtOpnd will be promoted.
3018*9880d681SAndroid Build Coastguard Worker     // All its uses, but Ext, will need to use a truncated value of the
3019*9880d681SAndroid Build Coastguard Worker     // promoted version.
3020*9880d681SAndroid Build Coastguard Worker     // Create the truncate now.
3021*9880d681SAndroid Build Coastguard Worker     Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType());
3022*9880d681SAndroid Build Coastguard Worker     if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {
3023*9880d681SAndroid Build Coastguard Worker       ITrunc->removeFromParent();
3024*9880d681SAndroid Build Coastguard Worker       // Insert it just after the definition.
3025*9880d681SAndroid Build Coastguard Worker       ITrunc->insertAfter(ExtOpnd);
3026*9880d681SAndroid Build Coastguard Worker       if (Truncs)
3027*9880d681SAndroid Build Coastguard Worker         Truncs->push_back(ITrunc);
3028*9880d681SAndroid Build Coastguard Worker     }
3029*9880d681SAndroid Build Coastguard Worker 
3030*9880d681SAndroid Build Coastguard Worker     TPT.replaceAllUsesWith(ExtOpnd, Trunc);
3031*9880d681SAndroid Build Coastguard Worker     // Restore the operand of Ext (which has been replaced by the previous call
3032*9880d681SAndroid Build Coastguard Worker     // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext.
3033*9880d681SAndroid Build Coastguard Worker     TPT.setOperand(Ext, 0, ExtOpnd);
3034*9880d681SAndroid Build Coastguard Worker   }
3035*9880d681SAndroid Build Coastguard Worker 
3036*9880d681SAndroid Build Coastguard Worker   // Get through the Instruction:
3037*9880d681SAndroid Build Coastguard Worker   // 1. Update its type.
3038*9880d681SAndroid Build Coastguard Worker   // 2. Replace the uses of Ext by Inst.
3039*9880d681SAndroid Build Coastguard Worker   // 3. Extend each operand that needs to be extended.
3040*9880d681SAndroid Build Coastguard Worker 
3041*9880d681SAndroid Build Coastguard Worker   // Remember the original type of the instruction before promotion.
3042*9880d681SAndroid Build Coastguard Worker   // This is useful to know that the high bits are sign extended bits.
3043*9880d681SAndroid Build Coastguard Worker   PromotedInsts.insert(std::pair<Instruction *, TypeIsSExt>(
3044*9880d681SAndroid Build Coastguard Worker       ExtOpnd, TypeIsSExt(ExtOpnd->getType(), IsSExt)));
3045*9880d681SAndroid Build Coastguard Worker   // Step #1.
3046*9880d681SAndroid Build Coastguard Worker   TPT.mutateType(ExtOpnd, Ext->getType());
3047*9880d681SAndroid Build Coastguard Worker   // Step #2.
3048*9880d681SAndroid Build Coastguard Worker   TPT.replaceAllUsesWith(Ext, ExtOpnd);
3049*9880d681SAndroid Build Coastguard Worker   // Step #3.
3050*9880d681SAndroid Build Coastguard Worker   Instruction *ExtForOpnd = Ext;
3051*9880d681SAndroid Build Coastguard Worker 
3052*9880d681SAndroid Build Coastguard Worker   DEBUG(dbgs() << "Propagate Ext to operands\n");
3053*9880d681SAndroid Build Coastguard Worker   for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx;
3054*9880d681SAndroid Build Coastguard Worker        ++OpIdx) {
3055*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n');
3056*9880d681SAndroid Build Coastguard Worker     if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() ||
3057*9880d681SAndroid Build Coastguard Worker         !shouldExtOperand(ExtOpnd, OpIdx)) {
3058*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "No need to propagate\n");
3059*9880d681SAndroid Build Coastguard Worker       continue;
3060*9880d681SAndroid Build Coastguard Worker     }
3061*9880d681SAndroid Build Coastguard Worker     // Check if we can statically extend the operand.
3062*9880d681SAndroid Build Coastguard Worker     Value *Opnd = ExtOpnd->getOperand(OpIdx);
3063*9880d681SAndroid Build Coastguard Worker     if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {
3064*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Statically extend\n");
3065*9880d681SAndroid Build Coastguard Worker       unsigned BitWidth = Ext->getType()->getIntegerBitWidth();
3066*9880d681SAndroid Build Coastguard Worker       APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth)
3067*9880d681SAndroid Build Coastguard Worker                             : Cst->getValue().zext(BitWidth);
3068*9880d681SAndroid Build Coastguard Worker       TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal));
3069*9880d681SAndroid Build Coastguard Worker       continue;
3070*9880d681SAndroid Build Coastguard Worker     }
3071*9880d681SAndroid Build Coastguard Worker     // UndefValue are typed, so we have to statically sign extend them.
3072*9880d681SAndroid Build Coastguard Worker     if (isa<UndefValue>(Opnd)) {
3073*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Statically extend\n");
3074*9880d681SAndroid Build Coastguard Worker       TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType()));
3075*9880d681SAndroid Build Coastguard Worker       continue;
3076*9880d681SAndroid Build Coastguard Worker     }
3077*9880d681SAndroid Build Coastguard Worker 
3078*9880d681SAndroid Build Coastguard Worker     // Otherwise we have to explicity sign extend the operand.
3079*9880d681SAndroid Build Coastguard Worker     // Check if Ext was reused to extend an operand.
3080*9880d681SAndroid Build Coastguard Worker     if (!ExtForOpnd) {
3081*9880d681SAndroid Build Coastguard Worker       // If yes, create a new one.
3082*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "More operands to ext\n");
3083*9880d681SAndroid Build Coastguard Worker       Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->getType())
3084*9880d681SAndroid Build Coastguard Worker         : TPT.createZExt(Ext, Opnd, Ext->getType());
3085*9880d681SAndroid Build Coastguard Worker       if (!isa<Instruction>(ValForExtOpnd)) {
3086*9880d681SAndroid Build Coastguard Worker         TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
3087*9880d681SAndroid Build Coastguard Worker         continue;
3088*9880d681SAndroid Build Coastguard Worker       }
3089*9880d681SAndroid Build Coastguard Worker       ExtForOpnd = cast<Instruction>(ValForExtOpnd);
3090*9880d681SAndroid Build Coastguard Worker     }
3091*9880d681SAndroid Build Coastguard Worker     if (Exts)
3092*9880d681SAndroid Build Coastguard Worker       Exts->push_back(ExtForOpnd);
3093*9880d681SAndroid Build Coastguard Worker     TPT.setOperand(ExtForOpnd, 0, Opnd);
3094*9880d681SAndroid Build Coastguard Worker 
3095*9880d681SAndroid Build Coastguard Worker     // Move the sign extension before the insertion point.
3096*9880d681SAndroid Build Coastguard Worker     TPT.moveBefore(ExtForOpnd, ExtOpnd);
3097*9880d681SAndroid Build Coastguard Worker     TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd);
3098*9880d681SAndroid Build Coastguard Worker     CreatedInstsCost += !TLI.isExtFree(ExtForOpnd);
3099*9880d681SAndroid Build Coastguard Worker     // If more sext are required, new instructions will have to be created.
3100*9880d681SAndroid Build Coastguard Worker     ExtForOpnd = nullptr;
3101*9880d681SAndroid Build Coastguard Worker   }
3102*9880d681SAndroid Build Coastguard Worker   if (ExtForOpnd == Ext) {
3103*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Extension is useless now\n");
3104*9880d681SAndroid Build Coastguard Worker     TPT.eraseInstruction(Ext);
3105*9880d681SAndroid Build Coastguard Worker   }
3106*9880d681SAndroid Build Coastguard Worker   return ExtOpnd;
3107*9880d681SAndroid Build Coastguard Worker }
3108*9880d681SAndroid Build Coastguard Worker 
3109*9880d681SAndroid Build Coastguard Worker /// Check whether or not promoting an instruction to a wider type is profitable.
3110*9880d681SAndroid Build Coastguard Worker /// \p NewCost gives the cost of extension instructions created by the
3111*9880d681SAndroid Build Coastguard Worker /// promotion.
3112*9880d681SAndroid Build Coastguard Worker /// \p OldCost gives the cost of extension instructions before the promotion
3113*9880d681SAndroid Build Coastguard Worker /// plus the number of instructions that have been
3114*9880d681SAndroid Build Coastguard Worker /// matched in the addressing mode the promotion.
3115*9880d681SAndroid Build Coastguard Worker /// \p PromotedOperand is the value that has been promoted.
3116*9880d681SAndroid Build Coastguard Worker /// \return True if the promotion is profitable, false otherwise.
isPromotionProfitable(unsigned NewCost,unsigned OldCost,Value * PromotedOperand) const3117*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::isPromotionProfitable(
3118*9880d681SAndroid Build Coastguard Worker     unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const {
3119*9880d681SAndroid Build Coastguard Worker   DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost << '\n');
3120*9880d681SAndroid Build Coastguard Worker   // The cost of the new extensions is greater than the cost of the
3121*9880d681SAndroid Build Coastguard Worker   // old extension plus what we folded.
3122*9880d681SAndroid Build Coastguard Worker   // This is not profitable.
3123*9880d681SAndroid Build Coastguard Worker   if (NewCost > OldCost)
3124*9880d681SAndroid Build Coastguard Worker     return false;
3125*9880d681SAndroid Build Coastguard Worker   if (NewCost < OldCost)
3126*9880d681SAndroid Build Coastguard Worker     return true;
3127*9880d681SAndroid Build Coastguard Worker   // The promotion is neutral but it may help folding the sign extension in
3128*9880d681SAndroid Build Coastguard Worker   // loads for instance.
3129*9880d681SAndroid Build Coastguard Worker   // Check that we did not create an illegal instruction.
3130*9880d681SAndroid Build Coastguard Worker   return isPromotedInstructionLegal(TLI, DL, PromotedOperand);
3131*9880d681SAndroid Build Coastguard Worker }
3132*9880d681SAndroid Build Coastguard Worker 
3133*9880d681SAndroid Build Coastguard Worker /// Given an instruction or constant expr, see if we can fold the operation
3134*9880d681SAndroid Build Coastguard Worker /// into the addressing mode. If so, update the addressing mode and return
3135*9880d681SAndroid Build Coastguard Worker /// true, otherwise return false without modifying AddrMode.
3136*9880d681SAndroid Build Coastguard Worker /// If \p MovedAway is not NULL, it contains the information of whether or
3137*9880d681SAndroid Build Coastguard Worker /// not AddrInst has to be folded into the addressing mode on success.
3138*9880d681SAndroid Build Coastguard Worker /// If \p MovedAway == true, \p AddrInst will not be part of the addressing
3139*9880d681SAndroid Build Coastguard Worker /// because it has been moved away.
3140*9880d681SAndroid Build Coastguard Worker /// Thus AddrInst must not be added in the matched instructions.
3141*9880d681SAndroid Build Coastguard Worker /// This state can happen when AddrInst is a sext, since it may be moved away.
3142*9880d681SAndroid Build Coastguard Worker /// Therefore, AddrInst may not be valid when MovedAway is true and it must
3143*9880d681SAndroid Build Coastguard Worker /// not be referenced anymore.
matchOperationAddr(User * AddrInst,unsigned Opcode,unsigned Depth,bool * MovedAway)3144*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
3145*9880d681SAndroid Build Coastguard Worker                                                unsigned Depth,
3146*9880d681SAndroid Build Coastguard Worker                                                bool *MovedAway) {
3147*9880d681SAndroid Build Coastguard Worker   // Avoid exponential behavior on extremely deep expression trees.
3148*9880d681SAndroid Build Coastguard Worker   if (Depth >= 5) return false;
3149*9880d681SAndroid Build Coastguard Worker 
3150*9880d681SAndroid Build Coastguard Worker   // By default, all matched instructions stay in place.
3151*9880d681SAndroid Build Coastguard Worker   if (MovedAway)
3152*9880d681SAndroid Build Coastguard Worker     *MovedAway = false;
3153*9880d681SAndroid Build Coastguard Worker 
3154*9880d681SAndroid Build Coastguard Worker   switch (Opcode) {
3155*9880d681SAndroid Build Coastguard Worker   case Instruction::PtrToInt:
3156*9880d681SAndroid Build Coastguard Worker     // PtrToInt is always a noop, as we know that the int type is pointer sized.
3157*9880d681SAndroid Build Coastguard Worker     return matchAddr(AddrInst->getOperand(0), Depth);
3158*9880d681SAndroid Build Coastguard Worker   case Instruction::IntToPtr: {
3159*9880d681SAndroid Build Coastguard Worker     auto AS = AddrInst->getType()->getPointerAddressSpace();
3160*9880d681SAndroid Build Coastguard Worker     auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
3161*9880d681SAndroid Build Coastguard Worker     // This inttoptr is a no-op if the integer type is pointer sized.
3162*9880d681SAndroid Build Coastguard Worker     if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy)
3163*9880d681SAndroid Build Coastguard Worker       return matchAddr(AddrInst->getOperand(0), Depth);
3164*9880d681SAndroid Build Coastguard Worker     return false;
3165*9880d681SAndroid Build Coastguard Worker   }
3166*9880d681SAndroid Build Coastguard Worker   case Instruction::BitCast:
3167*9880d681SAndroid Build Coastguard Worker     // BitCast is always a noop, and we can handle it as long as it is
3168*9880d681SAndroid Build Coastguard Worker     // int->int or pointer->pointer (we don't want int<->fp or something).
3169*9880d681SAndroid Build Coastguard Worker     if ((AddrInst->getOperand(0)->getType()->isPointerTy() ||
3170*9880d681SAndroid Build Coastguard Worker          AddrInst->getOperand(0)->getType()->isIntegerTy()) &&
3171*9880d681SAndroid Build Coastguard Worker         // Don't touch identity bitcasts.  These were probably put here by LSR,
3172*9880d681SAndroid Build Coastguard Worker         // and we don't want to mess around with them.  Assume it knows what it
3173*9880d681SAndroid Build Coastguard Worker         // is doing.
3174*9880d681SAndroid Build Coastguard Worker         AddrInst->getOperand(0)->getType() != AddrInst->getType())
3175*9880d681SAndroid Build Coastguard Worker       return matchAddr(AddrInst->getOperand(0), Depth);
3176*9880d681SAndroid Build Coastguard Worker     return false;
3177*9880d681SAndroid Build Coastguard Worker   case Instruction::AddrSpaceCast: {
3178*9880d681SAndroid Build Coastguard Worker     unsigned SrcAS
3179*9880d681SAndroid Build Coastguard Worker       = AddrInst->getOperand(0)->getType()->getPointerAddressSpace();
3180*9880d681SAndroid Build Coastguard Worker     unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();
3181*9880d681SAndroid Build Coastguard Worker     if (TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3182*9880d681SAndroid Build Coastguard Worker       return matchAddr(AddrInst->getOperand(0), Depth);
3183*9880d681SAndroid Build Coastguard Worker     return false;
3184*9880d681SAndroid Build Coastguard Worker   }
3185*9880d681SAndroid Build Coastguard Worker   case Instruction::Add: {
3186*9880d681SAndroid Build Coastguard Worker     // Check to see if we can merge in the RHS then the LHS.  If so, we win.
3187*9880d681SAndroid Build Coastguard Worker     ExtAddrMode BackupAddrMode = AddrMode;
3188*9880d681SAndroid Build Coastguard Worker     unsigned OldSize = AddrModeInsts.size();
3189*9880d681SAndroid Build Coastguard Worker     // Start a transaction at this point.
3190*9880d681SAndroid Build Coastguard Worker     // The LHS may match but not the RHS.
3191*9880d681SAndroid Build Coastguard Worker     // Therefore, we need a higher level restoration point to undo partially
3192*9880d681SAndroid Build Coastguard Worker     // matched operation.
3193*9880d681SAndroid Build Coastguard Worker     TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3194*9880d681SAndroid Build Coastguard Worker         TPT.getRestorationPoint();
3195*9880d681SAndroid Build Coastguard Worker 
3196*9880d681SAndroid Build Coastguard Worker     if (matchAddr(AddrInst->getOperand(1), Depth+1) &&
3197*9880d681SAndroid Build Coastguard Worker         matchAddr(AddrInst->getOperand(0), Depth+1))
3198*9880d681SAndroid Build Coastguard Worker       return true;
3199*9880d681SAndroid Build Coastguard Worker 
3200*9880d681SAndroid Build Coastguard Worker     // Restore the old addr mode info.
3201*9880d681SAndroid Build Coastguard Worker     AddrMode = BackupAddrMode;
3202*9880d681SAndroid Build Coastguard Worker     AddrModeInsts.resize(OldSize);
3203*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
3204*9880d681SAndroid Build Coastguard Worker 
3205*9880d681SAndroid Build Coastguard Worker     // Otherwise this was over-aggressive.  Try merging in the LHS then the RHS.
3206*9880d681SAndroid Build Coastguard Worker     if (matchAddr(AddrInst->getOperand(0), Depth+1) &&
3207*9880d681SAndroid Build Coastguard Worker         matchAddr(AddrInst->getOperand(1), Depth+1))
3208*9880d681SAndroid Build Coastguard Worker       return true;
3209*9880d681SAndroid Build Coastguard Worker 
3210*9880d681SAndroid Build Coastguard Worker     // Otherwise we definitely can't merge the ADD in.
3211*9880d681SAndroid Build Coastguard Worker     AddrMode = BackupAddrMode;
3212*9880d681SAndroid Build Coastguard Worker     AddrModeInsts.resize(OldSize);
3213*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
3214*9880d681SAndroid Build Coastguard Worker     break;
3215*9880d681SAndroid Build Coastguard Worker   }
3216*9880d681SAndroid Build Coastguard Worker   //case Instruction::Or:
3217*9880d681SAndroid Build Coastguard Worker   // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD.
3218*9880d681SAndroid Build Coastguard Worker   //break;
3219*9880d681SAndroid Build Coastguard Worker   case Instruction::Mul:
3220*9880d681SAndroid Build Coastguard Worker   case Instruction::Shl: {
3221*9880d681SAndroid Build Coastguard Worker     // Can only handle X*C and X << C.
3222*9880d681SAndroid Build Coastguard Worker     ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1));
3223*9880d681SAndroid Build Coastguard Worker     if (!RHS)
3224*9880d681SAndroid Build Coastguard Worker       return false;
3225*9880d681SAndroid Build Coastguard Worker     int64_t Scale = RHS->getSExtValue();
3226*9880d681SAndroid Build Coastguard Worker     if (Opcode == Instruction::Shl)
3227*9880d681SAndroid Build Coastguard Worker       Scale = 1LL << Scale;
3228*9880d681SAndroid Build Coastguard Worker 
3229*9880d681SAndroid Build Coastguard Worker     return matchScaledValue(AddrInst->getOperand(0), Scale, Depth);
3230*9880d681SAndroid Build Coastguard Worker   }
3231*9880d681SAndroid Build Coastguard Worker   case Instruction::GetElementPtr: {
3232*9880d681SAndroid Build Coastguard Worker     // Scan the GEP.  We check it if it contains constant offsets and at most
3233*9880d681SAndroid Build Coastguard Worker     // one variable offset.
3234*9880d681SAndroid Build Coastguard Worker     int VariableOperand = -1;
3235*9880d681SAndroid Build Coastguard Worker     unsigned VariableScale = 0;
3236*9880d681SAndroid Build Coastguard Worker 
3237*9880d681SAndroid Build Coastguard Worker     int64_t ConstantOffset = 0;
3238*9880d681SAndroid Build Coastguard Worker     gep_type_iterator GTI = gep_type_begin(AddrInst);
3239*9880d681SAndroid Build Coastguard Worker     for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
3240*9880d681SAndroid Build Coastguard Worker       if (StructType *STy = dyn_cast<StructType>(*GTI)) {
3241*9880d681SAndroid Build Coastguard Worker         const StructLayout *SL = DL.getStructLayout(STy);
3242*9880d681SAndroid Build Coastguard Worker         unsigned Idx =
3243*9880d681SAndroid Build Coastguard Worker           cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
3244*9880d681SAndroid Build Coastguard Worker         ConstantOffset += SL->getElementOffset(Idx);
3245*9880d681SAndroid Build Coastguard Worker       } else {
3246*9880d681SAndroid Build Coastguard Worker         uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
3247*9880d681SAndroid Build Coastguard Worker         if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
3248*9880d681SAndroid Build Coastguard Worker           ConstantOffset += CI->getSExtValue()*TypeSize;
3249*9880d681SAndroid Build Coastguard Worker         } else if (TypeSize) {  // Scales of zero don't do anything.
3250*9880d681SAndroid Build Coastguard Worker           // We only allow one variable index at the moment.
3251*9880d681SAndroid Build Coastguard Worker           if (VariableOperand != -1)
3252*9880d681SAndroid Build Coastguard Worker             return false;
3253*9880d681SAndroid Build Coastguard Worker 
3254*9880d681SAndroid Build Coastguard Worker           // Remember the variable index.
3255*9880d681SAndroid Build Coastguard Worker           VariableOperand = i;
3256*9880d681SAndroid Build Coastguard Worker           VariableScale = TypeSize;
3257*9880d681SAndroid Build Coastguard Worker         }
3258*9880d681SAndroid Build Coastguard Worker       }
3259*9880d681SAndroid Build Coastguard Worker     }
3260*9880d681SAndroid Build Coastguard Worker 
3261*9880d681SAndroid Build Coastguard Worker     // A common case is for the GEP to only do a constant offset.  In this case,
3262*9880d681SAndroid Build Coastguard Worker     // just add it to the disp field and check validity.
3263*9880d681SAndroid Build Coastguard Worker     if (VariableOperand == -1) {
3264*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseOffs += ConstantOffset;
3265*9880d681SAndroid Build Coastguard Worker       if (ConstantOffset == 0 ||
3266*9880d681SAndroid Build Coastguard Worker           TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) {
3267*9880d681SAndroid Build Coastguard Worker         // Check to see if we can fold the base pointer in too.
3268*9880d681SAndroid Build Coastguard Worker         if (matchAddr(AddrInst->getOperand(0), Depth+1))
3269*9880d681SAndroid Build Coastguard Worker           return true;
3270*9880d681SAndroid Build Coastguard Worker       }
3271*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseOffs -= ConstantOffset;
3272*9880d681SAndroid Build Coastguard Worker       return false;
3273*9880d681SAndroid Build Coastguard Worker     }
3274*9880d681SAndroid Build Coastguard Worker 
3275*9880d681SAndroid Build Coastguard Worker     // Save the valid addressing mode in case we can't match.
3276*9880d681SAndroid Build Coastguard Worker     ExtAddrMode BackupAddrMode = AddrMode;
3277*9880d681SAndroid Build Coastguard Worker     unsigned OldSize = AddrModeInsts.size();
3278*9880d681SAndroid Build Coastguard Worker 
3279*9880d681SAndroid Build Coastguard Worker     // See if the scale and offset amount is valid for this target.
3280*9880d681SAndroid Build Coastguard Worker     AddrMode.BaseOffs += ConstantOffset;
3281*9880d681SAndroid Build Coastguard Worker 
3282*9880d681SAndroid Build Coastguard Worker     // Match the base operand of the GEP.
3283*9880d681SAndroid Build Coastguard Worker     if (!matchAddr(AddrInst->getOperand(0), Depth+1)) {
3284*9880d681SAndroid Build Coastguard Worker       // If it couldn't be matched, just stuff the value in a register.
3285*9880d681SAndroid Build Coastguard Worker       if (AddrMode.HasBaseReg) {
3286*9880d681SAndroid Build Coastguard Worker         AddrMode = BackupAddrMode;
3287*9880d681SAndroid Build Coastguard Worker         AddrModeInsts.resize(OldSize);
3288*9880d681SAndroid Build Coastguard Worker         return false;
3289*9880d681SAndroid Build Coastguard Worker       }
3290*9880d681SAndroid Build Coastguard Worker       AddrMode.HasBaseReg = true;
3291*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseReg = AddrInst->getOperand(0);
3292*9880d681SAndroid Build Coastguard Worker     }
3293*9880d681SAndroid Build Coastguard Worker 
3294*9880d681SAndroid Build Coastguard Worker     // Match the remaining variable portion of the GEP.
3295*9880d681SAndroid Build Coastguard Worker     if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale,
3296*9880d681SAndroid Build Coastguard Worker                           Depth)) {
3297*9880d681SAndroid Build Coastguard Worker       // If it couldn't be matched, try stuffing the base into a register
3298*9880d681SAndroid Build Coastguard Worker       // instead of matching it, and retrying the match of the scale.
3299*9880d681SAndroid Build Coastguard Worker       AddrMode = BackupAddrMode;
3300*9880d681SAndroid Build Coastguard Worker       AddrModeInsts.resize(OldSize);
3301*9880d681SAndroid Build Coastguard Worker       if (AddrMode.HasBaseReg)
3302*9880d681SAndroid Build Coastguard Worker         return false;
3303*9880d681SAndroid Build Coastguard Worker       AddrMode.HasBaseReg = true;
3304*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseReg = AddrInst->getOperand(0);
3305*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseOffs += ConstantOffset;
3306*9880d681SAndroid Build Coastguard Worker       if (!matchScaledValue(AddrInst->getOperand(VariableOperand),
3307*9880d681SAndroid Build Coastguard Worker                             VariableScale, Depth)) {
3308*9880d681SAndroid Build Coastguard Worker         // If even that didn't work, bail.
3309*9880d681SAndroid Build Coastguard Worker         AddrMode = BackupAddrMode;
3310*9880d681SAndroid Build Coastguard Worker         AddrModeInsts.resize(OldSize);
3311*9880d681SAndroid Build Coastguard Worker         return false;
3312*9880d681SAndroid Build Coastguard Worker       }
3313*9880d681SAndroid Build Coastguard Worker     }
3314*9880d681SAndroid Build Coastguard Worker 
3315*9880d681SAndroid Build Coastguard Worker     return true;
3316*9880d681SAndroid Build Coastguard Worker   }
3317*9880d681SAndroid Build Coastguard Worker   case Instruction::SExt:
3318*9880d681SAndroid Build Coastguard Worker   case Instruction::ZExt: {
3319*9880d681SAndroid Build Coastguard Worker     Instruction *Ext = dyn_cast<Instruction>(AddrInst);
3320*9880d681SAndroid Build Coastguard Worker     if (!Ext)
3321*9880d681SAndroid Build Coastguard Worker       return false;
3322*9880d681SAndroid Build Coastguard Worker 
3323*9880d681SAndroid Build Coastguard Worker     // Try to move this ext out of the way of the addressing mode.
3324*9880d681SAndroid Build Coastguard Worker     // Ask for a method for doing so.
3325*9880d681SAndroid Build Coastguard Worker     TypePromotionHelper::Action TPH =
3326*9880d681SAndroid Build Coastguard Worker         TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);
3327*9880d681SAndroid Build Coastguard Worker     if (!TPH)
3328*9880d681SAndroid Build Coastguard Worker       return false;
3329*9880d681SAndroid Build Coastguard Worker 
3330*9880d681SAndroid Build Coastguard Worker     TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3331*9880d681SAndroid Build Coastguard Worker         TPT.getRestorationPoint();
3332*9880d681SAndroid Build Coastguard Worker     unsigned CreatedInstsCost = 0;
3333*9880d681SAndroid Build Coastguard Worker     unsigned ExtCost = !TLI.isExtFree(Ext);
3334*9880d681SAndroid Build Coastguard Worker     Value *PromotedOperand =
3335*9880d681SAndroid Build Coastguard Worker         TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI);
3336*9880d681SAndroid Build Coastguard Worker     // SExt has been moved away.
3337*9880d681SAndroid Build Coastguard Worker     // Thus either it will be rematched later in the recursive calls or it is
3338*9880d681SAndroid Build Coastguard Worker     // gone. Anyway, we must not fold it into the addressing mode at this point.
3339*9880d681SAndroid Build Coastguard Worker     // E.g.,
3340*9880d681SAndroid Build Coastguard Worker     // op = add opnd, 1
3341*9880d681SAndroid Build Coastguard Worker     // idx = ext op
3342*9880d681SAndroid Build Coastguard Worker     // addr = gep base, idx
3343*9880d681SAndroid Build Coastguard Worker     // is now:
3344*9880d681SAndroid Build Coastguard Worker     // promotedOpnd = ext opnd            <- no match here
3345*9880d681SAndroid Build Coastguard Worker     // op = promoted_add promotedOpnd, 1  <- match (later in recursive calls)
3346*9880d681SAndroid Build Coastguard Worker     // addr = gep base, op                <- match
3347*9880d681SAndroid Build Coastguard Worker     if (MovedAway)
3348*9880d681SAndroid Build Coastguard Worker       *MovedAway = true;
3349*9880d681SAndroid Build Coastguard Worker 
3350*9880d681SAndroid Build Coastguard Worker     assert(PromotedOperand &&
3351*9880d681SAndroid Build Coastguard Worker            "TypePromotionHelper should have filtered out those cases");
3352*9880d681SAndroid Build Coastguard Worker 
3353*9880d681SAndroid Build Coastguard Worker     ExtAddrMode BackupAddrMode = AddrMode;
3354*9880d681SAndroid Build Coastguard Worker     unsigned OldSize = AddrModeInsts.size();
3355*9880d681SAndroid Build Coastguard Worker 
3356*9880d681SAndroid Build Coastguard Worker     if (!matchAddr(PromotedOperand, Depth) ||
3357*9880d681SAndroid Build Coastguard Worker         // The total of the new cost is equal to the cost of the created
3358*9880d681SAndroid Build Coastguard Worker         // instructions.
3359*9880d681SAndroid Build Coastguard Worker         // The total of the old cost is equal to the cost of the extension plus
3360*9880d681SAndroid Build Coastguard Worker         // what we have saved in the addressing mode.
3361*9880d681SAndroid Build Coastguard Worker         !isPromotionProfitable(CreatedInstsCost,
3362*9880d681SAndroid Build Coastguard Worker                                ExtCost + (AddrModeInsts.size() - OldSize),
3363*9880d681SAndroid Build Coastguard Worker                                PromotedOperand)) {
3364*9880d681SAndroid Build Coastguard Worker       AddrMode = BackupAddrMode;
3365*9880d681SAndroid Build Coastguard Worker       AddrModeInsts.resize(OldSize);
3366*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Sign extension does not pay off: rollback\n");
3367*9880d681SAndroid Build Coastguard Worker       TPT.rollback(LastKnownGood);
3368*9880d681SAndroid Build Coastguard Worker       return false;
3369*9880d681SAndroid Build Coastguard Worker     }
3370*9880d681SAndroid Build Coastguard Worker     return true;
3371*9880d681SAndroid Build Coastguard Worker   }
3372*9880d681SAndroid Build Coastguard Worker   }
3373*9880d681SAndroid Build Coastguard Worker   return false;
3374*9880d681SAndroid Build Coastguard Worker }
3375*9880d681SAndroid Build Coastguard Worker 
3376*9880d681SAndroid Build Coastguard Worker /// If we can, try to add the value of 'Addr' into the current addressing mode.
3377*9880d681SAndroid Build Coastguard Worker /// If Addr can't be added to AddrMode this returns false and leaves AddrMode
3378*9880d681SAndroid Build Coastguard Worker /// unmodified. This assumes that Addr is either a pointer type or intptr_t
3379*9880d681SAndroid Build Coastguard Worker /// for the target.
3380*9880d681SAndroid Build Coastguard Worker ///
matchAddr(Value * Addr,unsigned Depth)3381*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) {
3382*9880d681SAndroid Build Coastguard Worker   // Start a transaction at this point that we will rollback if the matching
3383*9880d681SAndroid Build Coastguard Worker   // fails.
3384*9880d681SAndroid Build Coastguard Worker   TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3385*9880d681SAndroid Build Coastguard Worker       TPT.getRestorationPoint();
3386*9880d681SAndroid Build Coastguard Worker   if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
3387*9880d681SAndroid Build Coastguard Worker     // Fold in immediates if legal for the target.
3388*9880d681SAndroid Build Coastguard Worker     AddrMode.BaseOffs += CI->getSExtValue();
3389*9880d681SAndroid Build Coastguard Worker     if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3390*9880d681SAndroid Build Coastguard Worker       return true;
3391*9880d681SAndroid Build Coastguard Worker     AddrMode.BaseOffs -= CI->getSExtValue();
3392*9880d681SAndroid Build Coastguard Worker   } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
3393*9880d681SAndroid Build Coastguard Worker     // If this is a global variable, try to fold it into the addressing mode.
3394*9880d681SAndroid Build Coastguard Worker     if (!AddrMode.BaseGV) {
3395*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseGV = GV;
3396*9880d681SAndroid Build Coastguard Worker       if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3397*9880d681SAndroid Build Coastguard Worker         return true;
3398*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseGV = nullptr;
3399*9880d681SAndroid Build Coastguard Worker     }
3400*9880d681SAndroid Build Coastguard Worker   } else if (Instruction *I = dyn_cast<Instruction>(Addr)) {
3401*9880d681SAndroid Build Coastguard Worker     ExtAddrMode BackupAddrMode = AddrMode;
3402*9880d681SAndroid Build Coastguard Worker     unsigned OldSize = AddrModeInsts.size();
3403*9880d681SAndroid Build Coastguard Worker 
3404*9880d681SAndroid Build Coastguard Worker     // Check to see if it is possible to fold this operation.
3405*9880d681SAndroid Build Coastguard Worker     bool MovedAway = false;
3406*9880d681SAndroid Build Coastguard Worker     if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) {
3407*9880d681SAndroid Build Coastguard Worker       // This instruction may have been moved away. If so, there is nothing
3408*9880d681SAndroid Build Coastguard Worker       // to check here.
3409*9880d681SAndroid Build Coastguard Worker       if (MovedAway)
3410*9880d681SAndroid Build Coastguard Worker         return true;
3411*9880d681SAndroid Build Coastguard Worker       // Okay, it's possible to fold this.  Check to see if it is actually
3412*9880d681SAndroid Build Coastguard Worker       // *profitable* to do so.  We use a simple cost model to avoid increasing
3413*9880d681SAndroid Build Coastguard Worker       // register pressure too much.
3414*9880d681SAndroid Build Coastguard Worker       if (I->hasOneUse() ||
3415*9880d681SAndroid Build Coastguard Worker           isProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) {
3416*9880d681SAndroid Build Coastguard Worker         AddrModeInsts.push_back(I);
3417*9880d681SAndroid Build Coastguard Worker         return true;
3418*9880d681SAndroid Build Coastguard Worker       }
3419*9880d681SAndroid Build Coastguard Worker 
3420*9880d681SAndroid Build Coastguard Worker       // It isn't profitable to do this, roll back.
3421*9880d681SAndroid Build Coastguard Worker       //cerr << "NOT FOLDING: " << *I;
3422*9880d681SAndroid Build Coastguard Worker       AddrMode = BackupAddrMode;
3423*9880d681SAndroid Build Coastguard Worker       AddrModeInsts.resize(OldSize);
3424*9880d681SAndroid Build Coastguard Worker       TPT.rollback(LastKnownGood);
3425*9880d681SAndroid Build Coastguard Worker     }
3426*9880d681SAndroid Build Coastguard Worker   } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) {
3427*9880d681SAndroid Build Coastguard Worker     if (matchOperationAddr(CE, CE->getOpcode(), Depth))
3428*9880d681SAndroid Build Coastguard Worker       return true;
3429*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
3430*9880d681SAndroid Build Coastguard Worker   } else if (isa<ConstantPointerNull>(Addr)) {
3431*9880d681SAndroid Build Coastguard Worker     // Null pointer gets folded without affecting the addressing mode.
3432*9880d681SAndroid Build Coastguard Worker     return true;
3433*9880d681SAndroid Build Coastguard Worker   }
3434*9880d681SAndroid Build Coastguard Worker 
3435*9880d681SAndroid Build Coastguard Worker   // Worse case, the target should support [reg] addressing modes. :)
3436*9880d681SAndroid Build Coastguard Worker   if (!AddrMode.HasBaseReg) {
3437*9880d681SAndroid Build Coastguard Worker     AddrMode.HasBaseReg = true;
3438*9880d681SAndroid Build Coastguard Worker     AddrMode.BaseReg = Addr;
3439*9880d681SAndroid Build Coastguard Worker     // Still check for legality in case the target supports [imm] but not [i+r].
3440*9880d681SAndroid Build Coastguard Worker     if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3441*9880d681SAndroid Build Coastguard Worker       return true;
3442*9880d681SAndroid Build Coastguard Worker     AddrMode.HasBaseReg = false;
3443*9880d681SAndroid Build Coastguard Worker     AddrMode.BaseReg = nullptr;
3444*9880d681SAndroid Build Coastguard Worker   }
3445*9880d681SAndroid Build Coastguard Worker 
3446*9880d681SAndroid Build Coastguard Worker   // If the base register is already taken, see if we can do [r+r].
3447*9880d681SAndroid Build Coastguard Worker   if (AddrMode.Scale == 0) {
3448*9880d681SAndroid Build Coastguard Worker     AddrMode.Scale = 1;
3449*9880d681SAndroid Build Coastguard Worker     AddrMode.ScaledReg = Addr;
3450*9880d681SAndroid Build Coastguard Worker     if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3451*9880d681SAndroid Build Coastguard Worker       return true;
3452*9880d681SAndroid Build Coastguard Worker     AddrMode.Scale = 0;
3453*9880d681SAndroid Build Coastguard Worker     AddrMode.ScaledReg = nullptr;
3454*9880d681SAndroid Build Coastguard Worker   }
3455*9880d681SAndroid Build Coastguard Worker   // Couldn't match.
3456*9880d681SAndroid Build Coastguard Worker   TPT.rollback(LastKnownGood);
3457*9880d681SAndroid Build Coastguard Worker   return false;
3458*9880d681SAndroid Build Coastguard Worker }
3459*9880d681SAndroid Build Coastguard Worker 
3460*9880d681SAndroid Build Coastguard Worker /// Check to see if all uses of OpVal by the specified inline asm call are due
3461*9880d681SAndroid Build Coastguard Worker /// to memory operands. If so, return true, otherwise return false.
IsOperandAMemoryOperand(CallInst * CI,InlineAsm * IA,Value * OpVal,const TargetMachine & TM)3462*9880d681SAndroid Build Coastguard Worker static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal,
3463*9880d681SAndroid Build Coastguard Worker                                     const TargetMachine &TM) {
3464*9880d681SAndroid Build Coastguard Worker   const Function *F = CI->getParent()->getParent();
3465*9880d681SAndroid Build Coastguard Worker   const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
3466*9880d681SAndroid Build Coastguard Worker   const TargetRegisterInfo *TRI = TM.getSubtargetImpl(*F)->getRegisterInfo();
3467*9880d681SAndroid Build Coastguard Worker   TargetLowering::AsmOperandInfoVector TargetConstraints =
3468*9880d681SAndroid Build Coastguard Worker       TLI->ParseConstraints(F->getParent()->getDataLayout(), TRI,
3469*9880d681SAndroid Build Coastguard Worker                             ImmutableCallSite(CI));
3470*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
3471*9880d681SAndroid Build Coastguard Worker     TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
3472*9880d681SAndroid Build Coastguard Worker 
3473*9880d681SAndroid Build Coastguard Worker     // Compute the constraint code and ConstraintType to use.
3474*9880d681SAndroid Build Coastguard Worker     TLI->ComputeConstraintToUse(OpInfo, SDValue());
3475*9880d681SAndroid Build Coastguard Worker 
3476*9880d681SAndroid Build Coastguard Worker     // If this asm operand is our Value*, and if it isn't an indirect memory
3477*9880d681SAndroid Build Coastguard Worker     // operand, we can't fold it!
3478*9880d681SAndroid Build Coastguard Worker     if (OpInfo.CallOperandVal == OpVal &&
3479*9880d681SAndroid Build Coastguard Worker         (OpInfo.ConstraintType != TargetLowering::C_Memory ||
3480*9880d681SAndroid Build Coastguard Worker          !OpInfo.isIndirect))
3481*9880d681SAndroid Build Coastguard Worker       return false;
3482*9880d681SAndroid Build Coastguard Worker   }
3483*9880d681SAndroid Build Coastguard Worker 
3484*9880d681SAndroid Build Coastguard Worker   return true;
3485*9880d681SAndroid Build Coastguard Worker }
3486*9880d681SAndroid Build Coastguard Worker 
3487*9880d681SAndroid Build Coastguard Worker /// Recursively walk all the uses of I until we find a memory use.
3488*9880d681SAndroid Build Coastguard Worker /// If we find an obviously non-foldable instruction, return true.
3489*9880d681SAndroid Build Coastguard Worker /// Add the ultimately found memory instructions to MemoryUses.
FindAllMemoryUses(Instruction * I,SmallVectorImpl<std::pair<Instruction *,unsigned>> & MemoryUses,SmallPtrSetImpl<Instruction * > & ConsideredInsts,const TargetMachine & TM)3490*9880d681SAndroid Build Coastguard Worker static bool FindAllMemoryUses(
3491*9880d681SAndroid Build Coastguard Worker     Instruction *I,
3492*9880d681SAndroid Build Coastguard Worker     SmallVectorImpl<std::pair<Instruction *, unsigned>> &MemoryUses,
3493*9880d681SAndroid Build Coastguard Worker     SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetMachine &TM) {
3494*9880d681SAndroid Build Coastguard Worker   // If we already considered this instruction, we're done.
3495*9880d681SAndroid Build Coastguard Worker   if (!ConsideredInsts.insert(I).second)
3496*9880d681SAndroid Build Coastguard Worker     return false;
3497*9880d681SAndroid Build Coastguard Worker 
3498*9880d681SAndroid Build Coastguard Worker   // If this is an obviously unfoldable instruction, bail out.
3499*9880d681SAndroid Build Coastguard Worker   if (!MightBeFoldableInst(I))
3500*9880d681SAndroid Build Coastguard Worker     return true;
3501*9880d681SAndroid Build Coastguard Worker 
3502*9880d681SAndroid Build Coastguard Worker   const bool OptSize = I->getFunction()->optForSize();
3503*9880d681SAndroid Build Coastguard Worker 
3504*9880d681SAndroid Build Coastguard Worker   // Loop over all the uses, recursively processing them.
3505*9880d681SAndroid Build Coastguard Worker   for (Use &U : I->uses()) {
3506*9880d681SAndroid Build Coastguard Worker     Instruction *UserI = cast<Instruction>(U.getUser());
3507*9880d681SAndroid Build Coastguard Worker 
3508*9880d681SAndroid Build Coastguard Worker     if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) {
3509*9880d681SAndroid Build Coastguard Worker       MemoryUses.push_back(std::make_pair(LI, U.getOperandNo()));
3510*9880d681SAndroid Build Coastguard Worker       continue;
3511*9880d681SAndroid Build Coastguard Worker     }
3512*9880d681SAndroid Build Coastguard Worker 
3513*9880d681SAndroid Build Coastguard Worker     if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) {
3514*9880d681SAndroid Build Coastguard Worker       unsigned opNo = U.getOperandNo();
3515*9880d681SAndroid Build Coastguard Worker       if (opNo == 0) return true; // Storing addr, not into addr.
3516*9880d681SAndroid Build Coastguard Worker       MemoryUses.push_back(std::make_pair(SI, opNo));
3517*9880d681SAndroid Build Coastguard Worker       continue;
3518*9880d681SAndroid Build Coastguard Worker     }
3519*9880d681SAndroid Build Coastguard Worker 
3520*9880d681SAndroid Build Coastguard Worker     if (CallInst *CI = dyn_cast<CallInst>(UserI)) {
3521*9880d681SAndroid Build Coastguard Worker       // If this is a cold call, we can sink the addressing calculation into
3522*9880d681SAndroid Build Coastguard Worker       // the cold path.  See optimizeCallInst
3523*9880d681SAndroid Build Coastguard Worker       if (!OptSize && CI->hasFnAttr(Attribute::Cold))
3524*9880d681SAndroid Build Coastguard Worker         continue;
3525*9880d681SAndroid Build Coastguard Worker 
3526*9880d681SAndroid Build Coastguard Worker       InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue());
3527*9880d681SAndroid Build Coastguard Worker       if (!IA) return true;
3528*9880d681SAndroid Build Coastguard Worker 
3529*9880d681SAndroid Build Coastguard Worker       // If this is a memory operand, we're cool, otherwise bail out.
3530*9880d681SAndroid Build Coastguard Worker       if (!IsOperandAMemoryOperand(CI, IA, I, TM))
3531*9880d681SAndroid Build Coastguard Worker         return true;
3532*9880d681SAndroid Build Coastguard Worker       continue;
3533*9880d681SAndroid Build Coastguard Worker     }
3534*9880d681SAndroid Build Coastguard Worker 
3535*9880d681SAndroid Build Coastguard Worker     if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TM))
3536*9880d681SAndroid Build Coastguard Worker       return true;
3537*9880d681SAndroid Build Coastguard Worker   }
3538*9880d681SAndroid Build Coastguard Worker 
3539*9880d681SAndroid Build Coastguard Worker   return false;
3540*9880d681SAndroid Build Coastguard Worker }
3541*9880d681SAndroid Build Coastguard Worker 
3542*9880d681SAndroid Build Coastguard Worker /// Return true if Val is already known to be live at the use site that we're
3543*9880d681SAndroid Build Coastguard Worker /// folding it into. If so, there is no cost to include it in the addressing
3544*9880d681SAndroid Build Coastguard Worker /// mode. KnownLive1 and KnownLive2 are two values that we know are live at the
3545*9880d681SAndroid Build Coastguard Worker /// instruction already.
valueAlreadyLiveAtInst(Value * Val,Value * KnownLive1,Value * KnownLive2)3546*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,Value *KnownLive1,
3547*9880d681SAndroid Build Coastguard Worker                                                    Value *KnownLive2) {
3548*9880d681SAndroid Build Coastguard Worker   // If Val is either of the known-live values, we know it is live!
3549*9880d681SAndroid Build Coastguard Worker   if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2)
3550*9880d681SAndroid Build Coastguard Worker     return true;
3551*9880d681SAndroid Build Coastguard Worker 
3552*9880d681SAndroid Build Coastguard Worker   // All values other than instructions and arguments (e.g. constants) are live.
3553*9880d681SAndroid Build Coastguard Worker   if (!isa<Instruction>(Val) && !isa<Argument>(Val)) return true;
3554*9880d681SAndroid Build Coastguard Worker 
3555*9880d681SAndroid Build Coastguard Worker   // If Val is a constant sized alloca in the entry block, it is live, this is
3556*9880d681SAndroid Build Coastguard Worker   // true because it is just a reference to the stack/frame pointer, which is
3557*9880d681SAndroid Build Coastguard Worker   // live for the whole function.
3558*9880d681SAndroid Build Coastguard Worker   if (AllocaInst *AI = dyn_cast<AllocaInst>(Val))
3559*9880d681SAndroid Build Coastguard Worker     if (AI->isStaticAlloca())
3560*9880d681SAndroid Build Coastguard Worker       return true;
3561*9880d681SAndroid Build Coastguard Worker 
3562*9880d681SAndroid Build Coastguard Worker   // Check to see if this value is already used in the memory instruction's
3563*9880d681SAndroid Build Coastguard Worker   // block.  If so, it's already live into the block at the very least, so we
3564*9880d681SAndroid Build Coastguard Worker   // can reasonably fold it.
3565*9880d681SAndroid Build Coastguard Worker   return Val->isUsedInBasicBlock(MemoryInst->getParent());
3566*9880d681SAndroid Build Coastguard Worker }
3567*9880d681SAndroid Build Coastguard Worker 
3568*9880d681SAndroid Build Coastguard Worker /// It is possible for the addressing mode of the machine to fold the specified
3569*9880d681SAndroid Build Coastguard Worker /// instruction into a load or store that ultimately uses it.
3570*9880d681SAndroid Build Coastguard Worker /// However, the specified instruction has multiple uses.
3571*9880d681SAndroid Build Coastguard Worker /// Given this, it may actually increase register pressure to fold it
3572*9880d681SAndroid Build Coastguard Worker /// into the load. For example, consider this code:
3573*9880d681SAndroid Build Coastguard Worker ///
3574*9880d681SAndroid Build Coastguard Worker ///     X = ...
3575*9880d681SAndroid Build Coastguard Worker ///     Y = X+1
3576*9880d681SAndroid Build Coastguard Worker ///     use(Y)   -> nonload/store
3577*9880d681SAndroid Build Coastguard Worker ///     Z = Y+1
3578*9880d681SAndroid Build Coastguard Worker ///     load Z
3579*9880d681SAndroid Build Coastguard Worker ///
3580*9880d681SAndroid Build Coastguard Worker /// In this case, Y has multiple uses, and can be folded into the load of Z
3581*9880d681SAndroid Build Coastguard Worker /// (yielding load [X+2]).  However, doing this will cause both "X" and "X+1" to
3582*9880d681SAndroid Build Coastguard Worker /// be live at the use(Y) line.  If we don't fold Y into load Z, we use one
3583*9880d681SAndroid Build Coastguard Worker /// fewer register.  Since Y can't be folded into "use(Y)" we don't increase the
3584*9880d681SAndroid Build Coastguard Worker /// number of computations either.
3585*9880d681SAndroid Build Coastguard Worker ///
3586*9880d681SAndroid Build Coastguard Worker /// Note that this (like most of CodeGenPrepare) is just a rough heuristic.  If
3587*9880d681SAndroid Build Coastguard Worker /// X was live across 'load Z' for other reasons, we actually *would* want to
3588*9880d681SAndroid Build Coastguard Worker /// fold the addressing mode in the Z case.  This would make Y die earlier.
3589*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::
isProfitableToFoldIntoAddressingMode(Instruction * I,ExtAddrMode & AMBefore,ExtAddrMode & AMAfter)3590*9880d681SAndroid Build Coastguard Worker isProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore,
3591*9880d681SAndroid Build Coastguard Worker                                      ExtAddrMode &AMAfter) {
3592*9880d681SAndroid Build Coastguard Worker   if (IgnoreProfitability) return true;
3593*9880d681SAndroid Build Coastguard Worker 
3594*9880d681SAndroid Build Coastguard Worker   // AMBefore is the addressing mode before this instruction was folded into it,
3595*9880d681SAndroid Build Coastguard Worker   // and AMAfter is the addressing mode after the instruction was folded.  Get
3596*9880d681SAndroid Build Coastguard Worker   // the set of registers referenced by AMAfter and subtract out those
3597*9880d681SAndroid Build Coastguard Worker   // referenced by AMBefore: this is the set of values which folding in this
3598*9880d681SAndroid Build Coastguard Worker   // address extends the lifetime of.
3599*9880d681SAndroid Build Coastguard Worker   //
3600*9880d681SAndroid Build Coastguard Worker   // Note that there are only two potential values being referenced here,
3601*9880d681SAndroid Build Coastguard Worker   // BaseReg and ScaleReg (global addresses are always available, as are any
3602*9880d681SAndroid Build Coastguard Worker   // folded immediates).
3603*9880d681SAndroid Build Coastguard Worker   Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg;
3604*9880d681SAndroid Build Coastguard Worker 
3605*9880d681SAndroid Build Coastguard Worker   // If the BaseReg or ScaledReg was referenced by the previous addrmode, their
3606*9880d681SAndroid Build Coastguard Worker   // lifetime wasn't extended by adding this instruction.
3607*9880d681SAndroid Build Coastguard Worker   if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg))
3608*9880d681SAndroid Build Coastguard Worker     BaseReg = nullptr;
3609*9880d681SAndroid Build Coastguard Worker   if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg))
3610*9880d681SAndroid Build Coastguard Worker     ScaledReg = nullptr;
3611*9880d681SAndroid Build Coastguard Worker 
3612*9880d681SAndroid Build Coastguard Worker   // If folding this instruction (and it's subexprs) didn't extend any live
3613*9880d681SAndroid Build Coastguard Worker   // ranges, we're ok with it.
3614*9880d681SAndroid Build Coastguard Worker   if (!BaseReg && !ScaledReg)
3615*9880d681SAndroid Build Coastguard Worker     return true;
3616*9880d681SAndroid Build Coastguard Worker 
3617*9880d681SAndroid Build Coastguard Worker   // If all uses of this instruction can have the address mode sunk into them,
3618*9880d681SAndroid Build Coastguard Worker   // we can remove the addressing mode and effectively trade one live register
3619*9880d681SAndroid Build Coastguard Worker   // for another (at worst.)  In this context, folding an addressing mode into
3620*9880d681SAndroid Build Coastguard Worker   // the use is just a particularly nice way of sinking it.
3621*9880d681SAndroid Build Coastguard Worker   SmallVector<std::pair<Instruction*,unsigned>, 16> MemoryUses;
3622*9880d681SAndroid Build Coastguard Worker   SmallPtrSet<Instruction*, 16> ConsideredInsts;
3623*9880d681SAndroid Build Coastguard Worker   if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TM))
3624*9880d681SAndroid Build Coastguard Worker     return false;  // Has a non-memory, non-foldable use!
3625*9880d681SAndroid Build Coastguard Worker 
3626*9880d681SAndroid Build Coastguard Worker   // Now that we know that all uses of this instruction are part of a chain of
3627*9880d681SAndroid Build Coastguard Worker   // computation involving only operations that could theoretically be folded
3628*9880d681SAndroid Build Coastguard Worker   // into a memory use, loop over each of these memory operation uses and see
3629*9880d681SAndroid Build Coastguard Worker   // if they could  *actually* fold the instruction.  The assumption is that
3630*9880d681SAndroid Build Coastguard Worker   // addressing modes are cheap and that duplicating the computation involved
3631*9880d681SAndroid Build Coastguard Worker   // many times is worthwhile, even on a fastpath. For sinking candidates
3632*9880d681SAndroid Build Coastguard Worker   // (i.e. cold call sites), this serves as a way to prevent excessive code
3633*9880d681SAndroid Build Coastguard Worker   // growth since most architectures have some reasonable small and fast way to
3634*9880d681SAndroid Build Coastguard Worker   // compute an effective address.  (i.e LEA on x86)
3635*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction*, 32> MatchedAddrModeInsts;
3636*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0, e = MemoryUses.size(); i != e; ++i) {
3637*9880d681SAndroid Build Coastguard Worker     Instruction *User = MemoryUses[i].first;
3638*9880d681SAndroid Build Coastguard Worker     unsigned OpNo = MemoryUses[i].second;
3639*9880d681SAndroid Build Coastguard Worker 
3640*9880d681SAndroid Build Coastguard Worker     // Get the access type of this use.  If the use isn't a pointer, we don't
3641*9880d681SAndroid Build Coastguard Worker     // know what it accesses.
3642*9880d681SAndroid Build Coastguard Worker     Value *Address = User->getOperand(OpNo);
3643*9880d681SAndroid Build Coastguard Worker     PointerType *AddrTy = dyn_cast<PointerType>(Address->getType());
3644*9880d681SAndroid Build Coastguard Worker     if (!AddrTy)
3645*9880d681SAndroid Build Coastguard Worker       return false;
3646*9880d681SAndroid Build Coastguard Worker     Type *AddressAccessTy = AddrTy->getElementType();
3647*9880d681SAndroid Build Coastguard Worker     unsigned AS = AddrTy->getAddressSpace();
3648*9880d681SAndroid Build Coastguard Worker 
3649*9880d681SAndroid Build Coastguard Worker     // Do a match against the root of this address, ignoring profitability. This
3650*9880d681SAndroid Build Coastguard Worker     // will tell us if the addressing mode for the memory operation will
3651*9880d681SAndroid Build Coastguard Worker     // *actually* cover the shared instruction.
3652*9880d681SAndroid Build Coastguard Worker     ExtAddrMode Result;
3653*9880d681SAndroid Build Coastguard Worker     TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3654*9880d681SAndroid Build Coastguard Worker         TPT.getRestorationPoint();
3655*9880d681SAndroid Build Coastguard Worker     AddressingModeMatcher Matcher(MatchedAddrModeInsts, TM, AddressAccessTy, AS,
3656*9880d681SAndroid Build Coastguard Worker                                   MemoryInst, Result, InsertedInsts,
3657*9880d681SAndroid Build Coastguard Worker                                   PromotedInsts, TPT);
3658*9880d681SAndroid Build Coastguard Worker     Matcher.IgnoreProfitability = true;
3659*9880d681SAndroid Build Coastguard Worker     bool Success = Matcher.matchAddr(Address, 0);
3660*9880d681SAndroid Build Coastguard Worker     (void)Success; assert(Success && "Couldn't select *anything*?");
3661*9880d681SAndroid Build Coastguard Worker 
3662*9880d681SAndroid Build Coastguard Worker     // The match was to check the profitability, the changes made are not
3663*9880d681SAndroid Build Coastguard Worker     // part of the original matcher. Therefore, they should be dropped
3664*9880d681SAndroid Build Coastguard Worker     // otherwise the original matcher will not present the right state.
3665*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
3666*9880d681SAndroid Build Coastguard Worker 
3667*9880d681SAndroid Build Coastguard Worker     // If the match didn't cover I, then it won't be shared by it.
3668*9880d681SAndroid Build Coastguard Worker     if (std::find(MatchedAddrModeInsts.begin(), MatchedAddrModeInsts.end(),
3669*9880d681SAndroid Build Coastguard Worker                   I) == MatchedAddrModeInsts.end())
3670*9880d681SAndroid Build Coastguard Worker       return false;
3671*9880d681SAndroid Build Coastguard Worker 
3672*9880d681SAndroid Build Coastguard Worker     MatchedAddrModeInsts.clear();
3673*9880d681SAndroid Build Coastguard Worker   }
3674*9880d681SAndroid Build Coastguard Worker 
3675*9880d681SAndroid Build Coastguard Worker   return true;
3676*9880d681SAndroid Build Coastguard Worker }
3677*9880d681SAndroid Build Coastguard Worker 
3678*9880d681SAndroid Build Coastguard Worker } // end anonymous namespace
3679*9880d681SAndroid Build Coastguard Worker 
3680*9880d681SAndroid Build Coastguard Worker /// Return true if the specified values are defined in a
3681*9880d681SAndroid Build Coastguard Worker /// different basic block than BB.
IsNonLocalValue(Value * V,BasicBlock * BB)3682*9880d681SAndroid Build Coastguard Worker static bool IsNonLocalValue(Value *V, BasicBlock *BB) {
3683*9880d681SAndroid Build Coastguard Worker   if (Instruction *I = dyn_cast<Instruction>(V))
3684*9880d681SAndroid Build Coastguard Worker     return I->getParent() != BB;
3685*9880d681SAndroid Build Coastguard Worker   return false;
3686*9880d681SAndroid Build Coastguard Worker }
3687*9880d681SAndroid Build Coastguard Worker 
3688*9880d681SAndroid Build Coastguard Worker /// Sink addressing mode computation immediate before MemoryInst if doing so
3689*9880d681SAndroid Build Coastguard Worker /// can be done without increasing register pressure.  The need for the
3690*9880d681SAndroid Build Coastguard Worker /// register pressure constraint means this can end up being an all or nothing
3691*9880d681SAndroid Build Coastguard Worker /// decision for all uses of the same addressing computation.
3692*9880d681SAndroid Build Coastguard Worker ///
3693*9880d681SAndroid Build Coastguard Worker /// Load and Store Instructions often have addressing modes that can do
3694*9880d681SAndroid Build Coastguard Worker /// significant amounts of computation. As such, instruction selection will try
3695*9880d681SAndroid Build Coastguard Worker /// to get the load or store to do as much computation as possible for the
3696*9880d681SAndroid Build Coastguard Worker /// program. The problem is that isel can only see within a single block. As
3697*9880d681SAndroid Build Coastguard Worker /// such, we sink as much legal addressing mode work into the block as possible.
3698*9880d681SAndroid Build Coastguard Worker ///
3699*9880d681SAndroid Build Coastguard Worker /// This method is used to optimize both load/store and inline asms with memory
3700*9880d681SAndroid Build Coastguard Worker /// operands.  It's also used to sink addressing computations feeding into cold
3701*9880d681SAndroid Build Coastguard Worker /// call sites into their (cold) basic block.
3702*9880d681SAndroid Build Coastguard Worker ///
3703*9880d681SAndroid Build Coastguard Worker /// The motivation for handling sinking into cold blocks is that doing so can
3704*9880d681SAndroid Build Coastguard Worker /// both enable other address mode sinking (by satisfying the register pressure
3705*9880d681SAndroid Build Coastguard Worker /// constraint above), and reduce register pressure globally (by removing the
3706*9880d681SAndroid Build Coastguard Worker /// addressing mode computation from the fast path entirely.).
optimizeMemoryInst(Instruction * MemoryInst,Value * Addr,Type * AccessTy,unsigned AddrSpace)3707*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
3708*9880d681SAndroid Build Coastguard Worker                                         Type *AccessTy, unsigned AddrSpace) {
3709*9880d681SAndroid Build Coastguard Worker   Value *Repl = Addr;
3710*9880d681SAndroid Build Coastguard Worker 
3711*9880d681SAndroid Build Coastguard Worker   // Try to collapse single-value PHI nodes.  This is necessary to undo
3712*9880d681SAndroid Build Coastguard Worker   // unprofitable PRE transformations.
3713*9880d681SAndroid Build Coastguard Worker   SmallVector<Value*, 8> worklist;
3714*9880d681SAndroid Build Coastguard Worker   SmallPtrSet<Value*, 16> Visited;
3715*9880d681SAndroid Build Coastguard Worker   worklist.push_back(Addr);
3716*9880d681SAndroid Build Coastguard Worker 
3717*9880d681SAndroid Build Coastguard Worker   // Use a worklist to iteratively look through PHI nodes, and ensure that
3718*9880d681SAndroid Build Coastguard Worker   // the addressing mode obtained from the non-PHI roots of the graph
3719*9880d681SAndroid Build Coastguard Worker   // are equivalent.
3720*9880d681SAndroid Build Coastguard Worker   Value *Consensus = nullptr;
3721*9880d681SAndroid Build Coastguard Worker   unsigned NumUsesConsensus = 0;
3722*9880d681SAndroid Build Coastguard Worker   bool IsNumUsesConsensusValid = false;
3723*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction*, 16> AddrModeInsts;
3724*9880d681SAndroid Build Coastguard Worker   ExtAddrMode AddrMode;
3725*9880d681SAndroid Build Coastguard Worker   TypePromotionTransaction TPT;
3726*9880d681SAndroid Build Coastguard Worker   TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3727*9880d681SAndroid Build Coastguard Worker       TPT.getRestorationPoint();
3728*9880d681SAndroid Build Coastguard Worker   while (!worklist.empty()) {
3729*9880d681SAndroid Build Coastguard Worker     Value *V = worklist.back();
3730*9880d681SAndroid Build Coastguard Worker     worklist.pop_back();
3731*9880d681SAndroid Build Coastguard Worker 
3732*9880d681SAndroid Build Coastguard Worker     // Break use-def graph loops.
3733*9880d681SAndroid Build Coastguard Worker     if (!Visited.insert(V).second) {
3734*9880d681SAndroid Build Coastguard Worker       Consensus = nullptr;
3735*9880d681SAndroid Build Coastguard Worker       break;
3736*9880d681SAndroid Build Coastguard Worker     }
3737*9880d681SAndroid Build Coastguard Worker 
3738*9880d681SAndroid Build Coastguard Worker     // For a PHI node, push all of its incoming values.
3739*9880d681SAndroid Build Coastguard Worker     if (PHINode *P = dyn_cast<PHINode>(V)) {
3740*9880d681SAndroid Build Coastguard Worker       for (Value *IncValue : P->incoming_values())
3741*9880d681SAndroid Build Coastguard Worker         worklist.push_back(IncValue);
3742*9880d681SAndroid Build Coastguard Worker       continue;
3743*9880d681SAndroid Build Coastguard Worker     }
3744*9880d681SAndroid Build Coastguard Worker 
3745*9880d681SAndroid Build Coastguard Worker     // For non-PHIs, determine the addressing mode being computed.  Note that
3746*9880d681SAndroid Build Coastguard Worker     // the result may differ depending on what other uses our candidate
3747*9880d681SAndroid Build Coastguard Worker     // addressing instructions might have.
3748*9880d681SAndroid Build Coastguard Worker     SmallVector<Instruction*, 16> NewAddrModeInsts;
3749*9880d681SAndroid Build Coastguard Worker     ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
3750*9880d681SAndroid Build Coastguard Worker       V, AccessTy, AddrSpace, MemoryInst, NewAddrModeInsts, *TM,
3751*9880d681SAndroid Build Coastguard Worker       InsertedInsts, PromotedInsts, TPT);
3752*9880d681SAndroid Build Coastguard Worker 
3753*9880d681SAndroid Build Coastguard Worker     // This check is broken into two cases with very similar code to avoid using
3754*9880d681SAndroid Build Coastguard Worker     // getNumUses() as much as possible. Some values have a lot of uses, so
3755*9880d681SAndroid Build Coastguard Worker     // calling getNumUses() unconditionally caused a significant compile-time
3756*9880d681SAndroid Build Coastguard Worker     // regression.
3757*9880d681SAndroid Build Coastguard Worker     if (!Consensus) {
3758*9880d681SAndroid Build Coastguard Worker       Consensus = V;
3759*9880d681SAndroid Build Coastguard Worker       AddrMode = NewAddrMode;
3760*9880d681SAndroid Build Coastguard Worker       AddrModeInsts = NewAddrModeInsts;
3761*9880d681SAndroid Build Coastguard Worker       continue;
3762*9880d681SAndroid Build Coastguard Worker     } else if (NewAddrMode == AddrMode) {
3763*9880d681SAndroid Build Coastguard Worker       if (!IsNumUsesConsensusValid) {
3764*9880d681SAndroid Build Coastguard Worker         NumUsesConsensus = Consensus->getNumUses();
3765*9880d681SAndroid Build Coastguard Worker         IsNumUsesConsensusValid = true;
3766*9880d681SAndroid Build Coastguard Worker       }
3767*9880d681SAndroid Build Coastguard Worker 
3768*9880d681SAndroid Build Coastguard Worker       // Ensure that the obtained addressing mode is equivalent to that obtained
3769*9880d681SAndroid Build Coastguard Worker       // for all other roots of the PHI traversal.  Also, when choosing one
3770*9880d681SAndroid Build Coastguard Worker       // such root as representative, select the one with the most uses in order
3771*9880d681SAndroid Build Coastguard Worker       // to keep the cost modeling heuristics in AddressingModeMatcher
3772*9880d681SAndroid Build Coastguard Worker       // applicable.
3773*9880d681SAndroid Build Coastguard Worker       unsigned NumUses = V->getNumUses();
3774*9880d681SAndroid Build Coastguard Worker       if (NumUses > NumUsesConsensus) {
3775*9880d681SAndroid Build Coastguard Worker         Consensus = V;
3776*9880d681SAndroid Build Coastguard Worker         NumUsesConsensus = NumUses;
3777*9880d681SAndroid Build Coastguard Worker         AddrModeInsts = NewAddrModeInsts;
3778*9880d681SAndroid Build Coastguard Worker       }
3779*9880d681SAndroid Build Coastguard Worker       continue;
3780*9880d681SAndroid Build Coastguard Worker     }
3781*9880d681SAndroid Build Coastguard Worker 
3782*9880d681SAndroid Build Coastguard Worker     Consensus = nullptr;
3783*9880d681SAndroid Build Coastguard Worker     break;
3784*9880d681SAndroid Build Coastguard Worker   }
3785*9880d681SAndroid Build Coastguard Worker 
3786*9880d681SAndroid Build Coastguard Worker   // If the addressing mode couldn't be determined, or if multiple different
3787*9880d681SAndroid Build Coastguard Worker   // ones were determined, bail out now.
3788*9880d681SAndroid Build Coastguard Worker   if (!Consensus) {
3789*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
3790*9880d681SAndroid Build Coastguard Worker     return false;
3791*9880d681SAndroid Build Coastguard Worker   }
3792*9880d681SAndroid Build Coastguard Worker   TPT.commit();
3793*9880d681SAndroid Build Coastguard Worker 
3794*9880d681SAndroid Build Coastguard Worker   // Check to see if any of the instructions supersumed by this addr mode are
3795*9880d681SAndroid Build Coastguard Worker   // non-local to I's BB.
3796*9880d681SAndroid Build Coastguard Worker   bool AnyNonLocal = false;
3797*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0, e = AddrModeInsts.size(); i != e; ++i) {
3798*9880d681SAndroid Build Coastguard Worker     if (IsNonLocalValue(AddrModeInsts[i], MemoryInst->getParent())) {
3799*9880d681SAndroid Build Coastguard Worker       AnyNonLocal = true;
3800*9880d681SAndroid Build Coastguard Worker       break;
3801*9880d681SAndroid Build Coastguard Worker     }
3802*9880d681SAndroid Build Coastguard Worker   }
3803*9880d681SAndroid Build Coastguard Worker 
3804*9880d681SAndroid Build Coastguard Worker   // If all the instructions matched are already in this BB, don't do anything.
3805*9880d681SAndroid Build Coastguard Worker   if (!AnyNonLocal) {
3806*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "CGP: Found      local addrmode: " << AddrMode << "\n");
3807*9880d681SAndroid Build Coastguard Worker     return false;
3808*9880d681SAndroid Build Coastguard Worker   }
3809*9880d681SAndroid Build Coastguard Worker 
3810*9880d681SAndroid Build Coastguard Worker   // Insert this computation right after this user.  Since our caller is
3811*9880d681SAndroid Build Coastguard Worker   // scanning from the top of the BB to the bottom, reuse of the expr are
3812*9880d681SAndroid Build Coastguard Worker   // guaranteed to happen later.
3813*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(MemoryInst);
3814*9880d681SAndroid Build Coastguard Worker 
3815*9880d681SAndroid Build Coastguard Worker   // Now that we determined the addressing expression we want to use and know
3816*9880d681SAndroid Build Coastguard Worker   // that we have to sink it into this block.  Check to see if we have already
3817*9880d681SAndroid Build Coastguard Worker   // done this for some other load/store instr in this block.  If so, reuse the
3818*9880d681SAndroid Build Coastguard Worker   // computation.
3819*9880d681SAndroid Build Coastguard Worker   Value *&SunkAddr = SunkAddrs[Addr];
3820*9880d681SAndroid Build Coastguard Worker   if (SunkAddr) {
3821*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for "
3822*9880d681SAndroid Build Coastguard Worker                  << *MemoryInst << "\n");
3823*9880d681SAndroid Build Coastguard Worker     if (SunkAddr->getType() != Addr->getType())
3824*9880d681SAndroid Build Coastguard Worker       SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType());
3825*9880d681SAndroid Build Coastguard Worker   } else if (AddrSinkUsingGEPs ||
3826*9880d681SAndroid Build Coastguard Worker              (!AddrSinkUsingGEPs.getNumOccurrences() && TM &&
3827*9880d681SAndroid Build Coastguard Worker               TM->getSubtargetImpl(*MemoryInst->getParent()->getParent())
3828*9880d681SAndroid Build Coastguard Worker                   ->useAA())) {
3829*9880d681SAndroid Build Coastguard Worker     // By default, we use the GEP-based method when AA is used later. This
3830*9880d681SAndroid Build Coastguard Worker     // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities.
3831*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
3832*9880d681SAndroid Build Coastguard Worker                  << *MemoryInst << "\n");
3833*9880d681SAndroid Build Coastguard Worker     Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
3834*9880d681SAndroid Build Coastguard Worker     Value *ResultPtr = nullptr, *ResultIndex = nullptr;
3835*9880d681SAndroid Build Coastguard Worker 
3836*9880d681SAndroid Build Coastguard Worker     // First, find the pointer.
3837*9880d681SAndroid Build Coastguard Worker     if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) {
3838*9880d681SAndroid Build Coastguard Worker       ResultPtr = AddrMode.BaseReg;
3839*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseReg = nullptr;
3840*9880d681SAndroid Build Coastguard Worker     }
3841*9880d681SAndroid Build Coastguard Worker 
3842*9880d681SAndroid Build Coastguard Worker     if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) {
3843*9880d681SAndroid Build Coastguard Worker       // We can't add more than one pointer together, nor can we scale a
3844*9880d681SAndroid Build Coastguard Worker       // pointer (both of which seem meaningless).
3845*9880d681SAndroid Build Coastguard Worker       if (ResultPtr || AddrMode.Scale != 1)
3846*9880d681SAndroid Build Coastguard Worker         return false;
3847*9880d681SAndroid Build Coastguard Worker 
3848*9880d681SAndroid Build Coastguard Worker       ResultPtr = AddrMode.ScaledReg;
3849*9880d681SAndroid Build Coastguard Worker       AddrMode.Scale = 0;
3850*9880d681SAndroid Build Coastguard Worker     }
3851*9880d681SAndroid Build Coastguard Worker 
3852*9880d681SAndroid Build Coastguard Worker     if (AddrMode.BaseGV) {
3853*9880d681SAndroid Build Coastguard Worker       if (ResultPtr)
3854*9880d681SAndroid Build Coastguard Worker         return false;
3855*9880d681SAndroid Build Coastguard Worker 
3856*9880d681SAndroid Build Coastguard Worker       ResultPtr = AddrMode.BaseGV;
3857*9880d681SAndroid Build Coastguard Worker     }
3858*9880d681SAndroid Build Coastguard Worker 
3859*9880d681SAndroid Build Coastguard Worker     // If the real base value actually came from an inttoptr, then the matcher
3860*9880d681SAndroid Build Coastguard Worker     // will look through it and provide only the integer value. In that case,
3861*9880d681SAndroid Build Coastguard Worker     // use it here.
3862*9880d681SAndroid Build Coastguard Worker     if (!ResultPtr && AddrMode.BaseReg) {
3863*9880d681SAndroid Build Coastguard Worker       ResultPtr =
3864*9880d681SAndroid Build Coastguard Worker         Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), "sunkaddr");
3865*9880d681SAndroid Build Coastguard Worker       AddrMode.BaseReg = nullptr;
3866*9880d681SAndroid Build Coastguard Worker     } else if (!ResultPtr && AddrMode.Scale == 1) {
3867*9880d681SAndroid Build Coastguard Worker       ResultPtr =
3868*9880d681SAndroid Build Coastguard Worker         Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), "sunkaddr");
3869*9880d681SAndroid Build Coastguard Worker       AddrMode.Scale = 0;
3870*9880d681SAndroid Build Coastguard Worker     }
3871*9880d681SAndroid Build Coastguard Worker 
3872*9880d681SAndroid Build Coastguard Worker     if (!ResultPtr &&
3873*9880d681SAndroid Build Coastguard Worker         !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) {
3874*9880d681SAndroid Build Coastguard Worker       SunkAddr = Constant::getNullValue(Addr->getType());
3875*9880d681SAndroid Build Coastguard Worker     } else if (!ResultPtr) {
3876*9880d681SAndroid Build Coastguard Worker       return false;
3877*9880d681SAndroid Build Coastguard Worker     } else {
3878*9880d681SAndroid Build Coastguard Worker       Type *I8PtrTy =
3879*9880d681SAndroid Build Coastguard Worker           Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace());
3880*9880d681SAndroid Build Coastguard Worker       Type *I8Ty = Builder.getInt8Ty();
3881*9880d681SAndroid Build Coastguard Worker 
3882*9880d681SAndroid Build Coastguard Worker       // Start with the base register. Do this first so that subsequent address
3883*9880d681SAndroid Build Coastguard Worker       // matching finds it last, which will prevent it from trying to match it
3884*9880d681SAndroid Build Coastguard Worker       // as the scaled value in case it happens to be a mul. That would be
3885*9880d681SAndroid Build Coastguard Worker       // problematic if we've sunk a different mul for the scale, because then
3886*9880d681SAndroid Build Coastguard Worker       // we'd end up sinking both muls.
3887*9880d681SAndroid Build Coastguard Worker       if (AddrMode.BaseReg) {
3888*9880d681SAndroid Build Coastguard Worker         Value *V = AddrMode.BaseReg;
3889*9880d681SAndroid Build Coastguard Worker         if (V->getType() != IntPtrTy)
3890*9880d681SAndroid Build Coastguard Worker           V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
3891*9880d681SAndroid Build Coastguard Worker 
3892*9880d681SAndroid Build Coastguard Worker         ResultIndex = V;
3893*9880d681SAndroid Build Coastguard Worker       }
3894*9880d681SAndroid Build Coastguard Worker 
3895*9880d681SAndroid Build Coastguard Worker       // Add the scale value.
3896*9880d681SAndroid Build Coastguard Worker       if (AddrMode.Scale) {
3897*9880d681SAndroid Build Coastguard Worker         Value *V = AddrMode.ScaledReg;
3898*9880d681SAndroid Build Coastguard Worker         if (V->getType() == IntPtrTy) {
3899*9880d681SAndroid Build Coastguard Worker           // done.
3900*9880d681SAndroid Build Coastguard Worker         } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
3901*9880d681SAndroid Build Coastguard Worker                    cast<IntegerType>(V->getType())->getBitWidth()) {
3902*9880d681SAndroid Build Coastguard Worker           V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
3903*9880d681SAndroid Build Coastguard Worker         } else {
3904*9880d681SAndroid Build Coastguard Worker           // It is only safe to sign extend the BaseReg if we know that the math
3905*9880d681SAndroid Build Coastguard Worker           // required to create it did not overflow before we extend it. Since
3906*9880d681SAndroid Build Coastguard Worker           // the original IR value was tossed in favor of a constant back when
3907*9880d681SAndroid Build Coastguard Worker           // the AddrMode was created we need to bail out gracefully if widths
3908*9880d681SAndroid Build Coastguard Worker           // do not match instead of extending it.
3909*9880d681SAndroid Build Coastguard Worker           Instruction *I = dyn_cast_or_null<Instruction>(ResultIndex);
3910*9880d681SAndroid Build Coastguard Worker           if (I && (ResultIndex != AddrMode.BaseReg))
3911*9880d681SAndroid Build Coastguard Worker             I->eraseFromParent();
3912*9880d681SAndroid Build Coastguard Worker           return false;
3913*9880d681SAndroid Build Coastguard Worker         }
3914*9880d681SAndroid Build Coastguard Worker 
3915*9880d681SAndroid Build Coastguard Worker         if (AddrMode.Scale != 1)
3916*9880d681SAndroid Build Coastguard Worker           V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
3917*9880d681SAndroid Build Coastguard Worker                                 "sunkaddr");
3918*9880d681SAndroid Build Coastguard Worker         if (ResultIndex)
3919*9880d681SAndroid Build Coastguard Worker           ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr");
3920*9880d681SAndroid Build Coastguard Worker         else
3921*9880d681SAndroid Build Coastguard Worker           ResultIndex = V;
3922*9880d681SAndroid Build Coastguard Worker       }
3923*9880d681SAndroid Build Coastguard Worker 
3924*9880d681SAndroid Build Coastguard Worker       // Add in the Base Offset if present.
3925*9880d681SAndroid Build Coastguard Worker       if (AddrMode.BaseOffs) {
3926*9880d681SAndroid Build Coastguard Worker         Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
3927*9880d681SAndroid Build Coastguard Worker         if (ResultIndex) {
3928*9880d681SAndroid Build Coastguard Worker           // We need to add this separately from the scale above to help with
3929*9880d681SAndroid Build Coastguard Worker           // SDAG consecutive load/store merging.
3930*9880d681SAndroid Build Coastguard Worker           if (ResultPtr->getType() != I8PtrTy)
3931*9880d681SAndroid Build Coastguard Worker             ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
3932*9880d681SAndroid Build Coastguard Worker           ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr");
3933*9880d681SAndroid Build Coastguard Worker         }
3934*9880d681SAndroid Build Coastguard Worker 
3935*9880d681SAndroid Build Coastguard Worker         ResultIndex = V;
3936*9880d681SAndroid Build Coastguard Worker       }
3937*9880d681SAndroid Build Coastguard Worker 
3938*9880d681SAndroid Build Coastguard Worker       if (!ResultIndex) {
3939*9880d681SAndroid Build Coastguard Worker         SunkAddr = ResultPtr;
3940*9880d681SAndroid Build Coastguard Worker       } else {
3941*9880d681SAndroid Build Coastguard Worker         if (ResultPtr->getType() != I8PtrTy)
3942*9880d681SAndroid Build Coastguard Worker           ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
3943*9880d681SAndroid Build Coastguard Worker         SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr");
3944*9880d681SAndroid Build Coastguard Worker       }
3945*9880d681SAndroid Build Coastguard Worker 
3946*9880d681SAndroid Build Coastguard Worker       if (SunkAddr->getType() != Addr->getType())
3947*9880d681SAndroid Build Coastguard Worker         SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType());
3948*9880d681SAndroid Build Coastguard Worker     }
3949*9880d681SAndroid Build Coastguard Worker   } else {
3950*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
3951*9880d681SAndroid Build Coastguard Worker                  << *MemoryInst << "\n");
3952*9880d681SAndroid Build Coastguard Worker     Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
3953*9880d681SAndroid Build Coastguard Worker     Value *Result = nullptr;
3954*9880d681SAndroid Build Coastguard Worker 
3955*9880d681SAndroid Build Coastguard Worker     // Start with the base register. Do this first so that subsequent address
3956*9880d681SAndroid Build Coastguard Worker     // matching finds it last, which will prevent it from trying to match it
3957*9880d681SAndroid Build Coastguard Worker     // as the scaled value in case it happens to be a mul. That would be
3958*9880d681SAndroid Build Coastguard Worker     // problematic if we've sunk a different mul for the scale, because then
3959*9880d681SAndroid Build Coastguard Worker     // we'd end up sinking both muls.
3960*9880d681SAndroid Build Coastguard Worker     if (AddrMode.BaseReg) {
3961*9880d681SAndroid Build Coastguard Worker       Value *V = AddrMode.BaseReg;
3962*9880d681SAndroid Build Coastguard Worker       if (V->getType()->isPointerTy())
3963*9880d681SAndroid Build Coastguard Worker         V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
3964*9880d681SAndroid Build Coastguard Worker       if (V->getType() != IntPtrTy)
3965*9880d681SAndroid Build Coastguard Worker         V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
3966*9880d681SAndroid Build Coastguard Worker       Result = V;
3967*9880d681SAndroid Build Coastguard Worker     }
3968*9880d681SAndroid Build Coastguard Worker 
3969*9880d681SAndroid Build Coastguard Worker     // Add the scale value.
3970*9880d681SAndroid Build Coastguard Worker     if (AddrMode.Scale) {
3971*9880d681SAndroid Build Coastguard Worker       Value *V = AddrMode.ScaledReg;
3972*9880d681SAndroid Build Coastguard Worker       if (V->getType() == IntPtrTy) {
3973*9880d681SAndroid Build Coastguard Worker         // done.
3974*9880d681SAndroid Build Coastguard Worker       } else if (V->getType()->isPointerTy()) {
3975*9880d681SAndroid Build Coastguard Worker         V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
3976*9880d681SAndroid Build Coastguard Worker       } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
3977*9880d681SAndroid Build Coastguard Worker                  cast<IntegerType>(V->getType())->getBitWidth()) {
3978*9880d681SAndroid Build Coastguard Worker         V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
3979*9880d681SAndroid Build Coastguard Worker       } else {
3980*9880d681SAndroid Build Coastguard Worker         // It is only safe to sign extend the BaseReg if we know that the math
3981*9880d681SAndroid Build Coastguard Worker         // required to create it did not overflow before we extend it. Since
3982*9880d681SAndroid Build Coastguard Worker         // the original IR value was tossed in favor of a constant back when
3983*9880d681SAndroid Build Coastguard Worker         // the AddrMode was created we need to bail out gracefully if widths
3984*9880d681SAndroid Build Coastguard Worker         // do not match instead of extending it.
3985*9880d681SAndroid Build Coastguard Worker         Instruction *I = dyn_cast_or_null<Instruction>(Result);
3986*9880d681SAndroid Build Coastguard Worker         if (I && (Result != AddrMode.BaseReg))
3987*9880d681SAndroid Build Coastguard Worker           I->eraseFromParent();
3988*9880d681SAndroid Build Coastguard Worker         return false;
3989*9880d681SAndroid Build Coastguard Worker       }
3990*9880d681SAndroid Build Coastguard Worker       if (AddrMode.Scale != 1)
3991*9880d681SAndroid Build Coastguard Worker         V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
3992*9880d681SAndroid Build Coastguard Worker                               "sunkaddr");
3993*9880d681SAndroid Build Coastguard Worker       if (Result)
3994*9880d681SAndroid Build Coastguard Worker         Result = Builder.CreateAdd(Result, V, "sunkaddr");
3995*9880d681SAndroid Build Coastguard Worker       else
3996*9880d681SAndroid Build Coastguard Worker         Result = V;
3997*9880d681SAndroid Build Coastguard Worker     }
3998*9880d681SAndroid Build Coastguard Worker 
3999*9880d681SAndroid Build Coastguard Worker     // Add in the BaseGV if present.
4000*9880d681SAndroid Build Coastguard Worker     if (AddrMode.BaseGV) {
4001*9880d681SAndroid Build Coastguard Worker       Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr");
4002*9880d681SAndroid Build Coastguard Worker       if (Result)
4003*9880d681SAndroid Build Coastguard Worker         Result = Builder.CreateAdd(Result, V, "sunkaddr");
4004*9880d681SAndroid Build Coastguard Worker       else
4005*9880d681SAndroid Build Coastguard Worker         Result = V;
4006*9880d681SAndroid Build Coastguard Worker     }
4007*9880d681SAndroid Build Coastguard Worker 
4008*9880d681SAndroid Build Coastguard Worker     // Add in the Base Offset if present.
4009*9880d681SAndroid Build Coastguard Worker     if (AddrMode.BaseOffs) {
4010*9880d681SAndroid Build Coastguard Worker       Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
4011*9880d681SAndroid Build Coastguard Worker       if (Result)
4012*9880d681SAndroid Build Coastguard Worker         Result = Builder.CreateAdd(Result, V, "sunkaddr");
4013*9880d681SAndroid Build Coastguard Worker       else
4014*9880d681SAndroid Build Coastguard Worker         Result = V;
4015*9880d681SAndroid Build Coastguard Worker     }
4016*9880d681SAndroid Build Coastguard Worker 
4017*9880d681SAndroid Build Coastguard Worker     if (!Result)
4018*9880d681SAndroid Build Coastguard Worker       SunkAddr = Constant::getNullValue(Addr->getType());
4019*9880d681SAndroid Build Coastguard Worker     else
4020*9880d681SAndroid Build Coastguard Worker       SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr");
4021*9880d681SAndroid Build Coastguard Worker   }
4022*9880d681SAndroid Build Coastguard Worker 
4023*9880d681SAndroid Build Coastguard Worker   MemoryInst->replaceUsesOfWith(Repl, SunkAddr);
4024*9880d681SAndroid Build Coastguard Worker 
4025*9880d681SAndroid Build Coastguard Worker   // If we have no uses, recursively delete the value and all dead instructions
4026*9880d681SAndroid Build Coastguard Worker   // using it.
4027*9880d681SAndroid Build Coastguard Worker   if (Repl->use_empty()) {
4028*9880d681SAndroid Build Coastguard Worker     // This can cause recursive deletion, which can invalidate our iterator.
4029*9880d681SAndroid Build Coastguard Worker     // Use a WeakVH to hold onto it in case this happens.
4030*9880d681SAndroid Build Coastguard Worker     Value *CurValue = &*CurInstIterator;
4031*9880d681SAndroid Build Coastguard Worker     WeakVH IterHandle(CurValue);
4032*9880d681SAndroid Build Coastguard Worker     BasicBlock *BB = CurInstIterator->getParent();
4033*9880d681SAndroid Build Coastguard Worker 
4034*9880d681SAndroid Build Coastguard Worker     RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo);
4035*9880d681SAndroid Build Coastguard Worker 
4036*9880d681SAndroid Build Coastguard Worker     if (IterHandle != CurValue) {
4037*9880d681SAndroid Build Coastguard Worker       // If the iterator instruction was recursively deleted, start over at the
4038*9880d681SAndroid Build Coastguard Worker       // start of the block.
4039*9880d681SAndroid Build Coastguard Worker       CurInstIterator = BB->begin();
4040*9880d681SAndroid Build Coastguard Worker       SunkAddrs.clear();
4041*9880d681SAndroid Build Coastguard Worker     }
4042*9880d681SAndroid Build Coastguard Worker   }
4043*9880d681SAndroid Build Coastguard Worker   ++NumMemoryInsts;
4044*9880d681SAndroid Build Coastguard Worker   return true;
4045*9880d681SAndroid Build Coastguard Worker }
4046*9880d681SAndroid Build Coastguard Worker 
4047*9880d681SAndroid Build Coastguard Worker /// If there are any memory operands, use OptimizeMemoryInst to sink their
4048*9880d681SAndroid Build Coastguard Worker /// address computing into the block when possible / profitable.
optimizeInlineAsmInst(CallInst * CS)4049*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
4050*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
4051*9880d681SAndroid Build Coastguard Worker 
4052*9880d681SAndroid Build Coastguard Worker   const TargetRegisterInfo *TRI =
4053*9880d681SAndroid Build Coastguard Worker       TM->getSubtargetImpl(*CS->getParent()->getParent())->getRegisterInfo();
4054*9880d681SAndroid Build Coastguard Worker   TargetLowering::AsmOperandInfoVector TargetConstraints =
4055*9880d681SAndroid Build Coastguard Worker       TLI->ParseConstraints(*DL, TRI, CS);
4056*9880d681SAndroid Build Coastguard Worker   unsigned ArgNo = 0;
4057*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
4058*9880d681SAndroid Build Coastguard Worker     TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
4059*9880d681SAndroid Build Coastguard Worker 
4060*9880d681SAndroid Build Coastguard Worker     // Compute the constraint code and ConstraintType to use.
4061*9880d681SAndroid Build Coastguard Worker     TLI->ComputeConstraintToUse(OpInfo, SDValue());
4062*9880d681SAndroid Build Coastguard Worker 
4063*9880d681SAndroid Build Coastguard Worker     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
4064*9880d681SAndroid Build Coastguard Worker         OpInfo.isIndirect) {
4065*9880d681SAndroid Build Coastguard Worker       Value *OpVal = CS->getArgOperand(ArgNo++);
4066*9880d681SAndroid Build Coastguard Worker       MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u);
4067*9880d681SAndroid Build Coastguard Worker     } else if (OpInfo.Type == InlineAsm::isInput)
4068*9880d681SAndroid Build Coastguard Worker       ArgNo++;
4069*9880d681SAndroid Build Coastguard Worker   }
4070*9880d681SAndroid Build Coastguard Worker 
4071*9880d681SAndroid Build Coastguard Worker   return MadeChange;
4072*9880d681SAndroid Build Coastguard Worker }
4073*9880d681SAndroid Build Coastguard Worker 
4074*9880d681SAndroid Build Coastguard Worker /// \brief Check if all the uses of \p Inst are equivalent (or free) zero or
4075*9880d681SAndroid Build Coastguard Worker /// sign extensions.
hasSameExtUse(Instruction * Inst,const TargetLowering & TLI)4076*9880d681SAndroid Build Coastguard Worker static bool hasSameExtUse(Instruction *Inst, const TargetLowering &TLI) {
4077*9880d681SAndroid Build Coastguard Worker   assert(!Inst->use_empty() && "Input must have at least one use");
4078*9880d681SAndroid Build Coastguard Worker   const Instruction *FirstUser = cast<Instruction>(*Inst->user_begin());
4079*9880d681SAndroid Build Coastguard Worker   bool IsSExt = isa<SExtInst>(FirstUser);
4080*9880d681SAndroid Build Coastguard Worker   Type *ExtTy = FirstUser->getType();
4081*9880d681SAndroid Build Coastguard Worker   for (const User *U : Inst->users()) {
4082*9880d681SAndroid Build Coastguard Worker     const Instruction *UI = cast<Instruction>(U);
4083*9880d681SAndroid Build Coastguard Worker     if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI)))
4084*9880d681SAndroid Build Coastguard Worker       return false;
4085*9880d681SAndroid Build Coastguard Worker     Type *CurTy = UI->getType();
4086*9880d681SAndroid Build Coastguard Worker     // Same input and output types: Same instruction after CSE.
4087*9880d681SAndroid Build Coastguard Worker     if (CurTy == ExtTy)
4088*9880d681SAndroid Build Coastguard Worker       continue;
4089*9880d681SAndroid Build Coastguard Worker 
4090*9880d681SAndroid Build Coastguard Worker     // If IsSExt is true, we are in this situation:
4091*9880d681SAndroid Build Coastguard Worker     // a = Inst
4092*9880d681SAndroid Build Coastguard Worker     // b = sext ty1 a to ty2
4093*9880d681SAndroid Build Coastguard Worker     // c = sext ty1 a to ty3
4094*9880d681SAndroid Build Coastguard Worker     // Assuming ty2 is shorter than ty3, this could be turned into:
4095*9880d681SAndroid Build Coastguard Worker     // a = Inst
4096*9880d681SAndroid Build Coastguard Worker     // b = sext ty1 a to ty2
4097*9880d681SAndroid Build Coastguard Worker     // c = sext ty2 b to ty3
4098*9880d681SAndroid Build Coastguard Worker     // However, the last sext is not free.
4099*9880d681SAndroid Build Coastguard Worker     if (IsSExt)
4100*9880d681SAndroid Build Coastguard Worker       return false;
4101*9880d681SAndroid Build Coastguard Worker 
4102*9880d681SAndroid Build Coastguard Worker     // This is a ZExt, maybe this is free to extend from one type to another.
4103*9880d681SAndroid Build Coastguard Worker     // In that case, we would not account for a different use.
4104*9880d681SAndroid Build Coastguard Worker     Type *NarrowTy;
4105*9880d681SAndroid Build Coastguard Worker     Type *LargeTy;
4106*9880d681SAndroid Build Coastguard Worker     if (ExtTy->getScalarType()->getIntegerBitWidth() >
4107*9880d681SAndroid Build Coastguard Worker         CurTy->getScalarType()->getIntegerBitWidth()) {
4108*9880d681SAndroid Build Coastguard Worker       NarrowTy = CurTy;
4109*9880d681SAndroid Build Coastguard Worker       LargeTy = ExtTy;
4110*9880d681SAndroid Build Coastguard Worker     } else {
4111*9880d681SAndroid Build Coastguard Worker       NarrowTy = ExtTy;
4112*9880d681SAndroid Build Coastguard Worker       LargeTy = CurTy;
4113*9880d681SAndroid Build Coastguard Worker     }
4114*9880d681SAndroid Build Coastguard Worker 
4115*9880d681SAndroid Build Coastguard Worker     if (!TLI.isZExtFree(NarrowTy, LargeTy))
4116*9880d681SAndroid Build Coastguard Worker       return false;
4117*9880d681SAndroid Build Coastguard Worker   }
4118*9880d681SAndroid Build Coastguard Worker   // All uses are the same or can be derived from one another for free.
4119*9880d681SAndroid Build Coastguard Worker   return true;
4120*9880d681SAndroid Build Coastguard Worker }
4121*9880d681SAndroid Build Coastguard Worker 
4122*9880d681SAndroid Build Coastguard Worker /// \brief Try to form ExtLd by promoting \p Exts until they reach a
4123*9880d681SAndroid Build Coastguard Worker /// load instruction.
4124*9880d681SAndroid Build Coastguard Worker /// If an ext(load) can be formed, it is returned via \p LI for the load
4125*9880d681SAndroid Build Coastguard Worker /// and \p Inst for the extension.
4126*9880d681SAndroid Build Coastguard Worker /// Otherwise LI == nullptr and Inst == nullptr.
4127*9880d681SAndroid Build Coastguard Worker /// When some promotion happened, \p TPT contains the proper state to
4128*9880d681SAndroid Build Coastguard Worker /// revert them.
4129*9880d681SAndroid Build Coastguard Worker ///
4130*9880d681SAndroid Build Coastguard Worker /// \return true when promoting was necessary to expose the ext(load)
4131*9880d681SAndroid Build Coastguard Worker /// opportunity, false otherwise.
4132*9880d681SAndroid Build Coastguard Worker ///
4133*9880d681SAndroid Build Coastguard Worker /// Example:
4134*9880d681SAndroid Build Coastguard Worker /// \code
4135*9880d681SAndroid Build Coastguard Worker /// %ld = load i32* %addr
4136*9880d681SAndroid Build Coastguard Worker /// %add = add nuw i32 %ld, 4
4137*9880d681SAndroid Build Coastguard Worker /// %zext = zext i32 %add to i64
4138*9880d681SAndroid Build Coastguard Worker /// \endcode
4139*9880d681SAndroid Build Coastguard Worker /// =>
4140*9880d681SAndroid Build Coastguard Worker /// \code
4141*9880d681SAndroid Build Coastguard Worker /// %ld = load i32* %addr
4142*9880d681SAndroid Build Coastguard Worker /// %zext = zext i32 %ld to i64
4143*9880d681SAndroid Build Coastguard Worker /// %add = add nuw i64 %zext, 4
4144*9880d681SAndroid Build Coastguard Worker /// \encode
4145*9880d681SAndroid Build Coastguard Worker /// Thanks to the promotion, we can match zext(load i32*) to i64.
extLdPromotion(TypePromotionTransaction & TPT,LoadInst * & LI,Instruction * & Inst,const SmallVectorImpl<Instruction * > & Exts,unsigned CreatedInstsCost=0)4146*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::extLdPromotion(TypePromotionTransaction &TPT,
4147*9880d681SAndroid Build Coastguard Worker                                     LoadInst *&LI, Instruction *&Inst,
4148*9880d681SAndroid Build Coastguard Worker                                     const SmallVectorImpl<Instruction *> &Exts,
4149*9880d681SAndroid Build Coastguard Worker                                     unsigned CreatedInstsCost = 0) {
4150*9880d681SAndroid Build Coastguard Worker   // Iterate over all the extensions to see if one form an ext(load).
4151*9880d681SAndroid Build Coastguard Worker   for (auto I : Exts) {
4152*9880d681SAndroid Build Coastguard Worker     // Check if we directly have ext(load).
4153*9880d681SAndroid Build Coastguard Worker     if ((LI = dyn_cast<LoadInst>(I->getOperand(0)))) {
4154*9880d681SAndroid Build Coastguard Worker       Inst = I;
4155*9880d681SAndroid Build Coastguard Worker       // No promotion happened here.
4156*9880d681SAndroid Build Coastguard Worker       return false;
4157*9880d681SAndroid Build Coastguard Worker     }
4158*9880d681SAndroid Build Coastguard Worker     // Check whether or not we want to do any promotion.
4159*9880d681SAndroid Build Coastguard Worker     if (!TLI || !TLI->enableExtLdPromotion() || DisableExtLdPromotion)
4160*9880d681SAndroid Build Coastguard Worker       continue;
4161*9880d681SAndroid Build Coastguard Worker     // Get the action to perform the promotion.
4162*9880d681SAndroid Build Coastguard Worker     TypePromotionHelper::Action TPH = TypePromotionHelper::getAction(
4163*9880d681SAndroid Build Coastguard Worker         I, InsertedInsts, *TLI, PromotedInsts);
4164*9880d681SAndroid Build Coastguard Worker     // Check if we can promote.
4165*9880d681SAndroid Build Coastguard Worker     if (!TPH)
4166*9880d681SAndroid Build Coastguard Worker       continue;
4167*9880d681SAndroid Build Coastguard Worker     // Save the current state.
4168*9880d681SAndroid Build Coastguard Worker     TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4169*9880d681SAndroid Build Coastguard Worker         TPT.getRestorationPoint();
4170*9880d681SAndroid Build Coastguard Worker     SmallVector<Instruction *, 4> NewExts;
4171*9880d681SAndroid Build Coastguard Worker     unsigned NewCreatedInstsCost = 0;
4172*9880d681SAndroid Build Coastguard Worker     unsigned ExtCost = !TLI->isExtFree(I);
4173*9880d681SAndroid Build Coastguard Worker     // Promote.
4174*9880d681SAndroid Build Coastguard Worker     Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost,
4175*9880d681SAndroid Build Coastguard Worker                              &NewExts, nullptr, *TLI);
4176*9880d681SAndroid Build Coastguard Worker     assert(PromotedVal &&
4177*9880d681SAndroid Build Coastguard Worker            "TypePromotionHelper should have filtered out those cases");
4178*9880d681SAndroid Build Coastguard Worker 
4179*9880d681SAndroid Build Coastguard Worker     // We would be able to merge only one extension in a load.
4180*9880d681SAndroid Build Coastguard Worker     // Therefore, if we have more than 1 new extension we heuristically
4181*9880d681SAndroid Build Coastguard Worker     // cut this search path, because it means we degrade the code quality.
4182*9880d681SAndroid Build Coastguard Worker     // With exactly 2, the transformation is neutral, because we will merge
4183*9880d681SAndroid Build Coastguard Worker     // one extension but leave one. However, we optimistically keep going,
4184*9880d681SAndroid Build Coastguard Worker     // because the new extension may be removed too.
4185*9880d681SAndroid Build Coastguard Worker     long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;
4186*9880d681SAndroid Build Coastguard Worker     TotalCreatedInstsCost -= ExtCost;
4187*9880d681SAndroid Build Coastguard Worker     if (!StressExtLdPromotion &&
4188*9880d681SAndroid Build Coastguard Worker         (TotalCreatedInstsCost > 1 ||
4189*9880d681SAndroid Build Coastguard Worker          !isPromotedInstructionLegal(*TLI, *DL, PromotedVal))) {
4190*9880d681SAndroid Build Coastguard Worker       // The promotion is not profitable, rollback to the previous state.
4191*9880d681SAndroid Build Coastguard Worker       TPT.rollback(LastKnownGood);
4192*9880d681SAndroid Build Coastguard Worker       continue;
4193*9880d681SAndroid Build Coastguard Worker     }
4194*9880d681SAndroid Build Coastguard Worker     // The promotion is profitable.
4195*9880d681SAndroid Build Coastguard Worker     // Check if it exposes an ext(load).
4196*9880d681SAndroid Build Coastguard Worker     (void)extLdPromotion(TPT, LI, Inst, NewExts, TotalCreatedInstsCost);
4197*9880d681SAndroid Build Coastguard Worker     if (LI && (StressExtLdPromotion || NewCreatedInstsCost <= ExtCost ||
4198*9880d681SAndroid Build Coastguard Worker                // If we have created a new extension, i.e., now we have two
4199*9880d681SAndroid Build Coastguard Worker                // extensions. We must make sure one of them is merged with
4200*9880d681SAndroid Build Coastguard Worker                // the load, otherwise we may degrade the code quality.
4201*9880d681SAndroid Build Coastguard Worker                (LI->hasOneUse() || hasSameExtUse(LI, *TLI))))
4202*9880d681SAndroid Build Coastguard Worker       // Promotion happened.
4203*9880d681SAndroid Build Coastguard Worker       return true;
4204*9880d681SAndroid Build Coastguard Worker     // If this does not help to expose an ext(load) then, rollback.
4205*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
4206*9880d681SAndroid Build Coastguard Worker   }
4207*9880d681SAndroid Build Coastguard Worker   // None of the extension can form an ext(load).
4208*9880d681SAndroid Build Coastguard Worker   LI = nullptr;
4209*9880d681SAndroid Build Coastguard Worker   Inst = nullptr;
4210*9880d681SAndroid Build Coastguard Worker   return false;
4211*9880d681SAndroid Build Coastguard Worker }
4212*9880d681SAndroid Build Coastguard Worker 
4213*9880d681SAndroid Build Coastguard Worker /// Move a zext or sext fed by a load into the same basic block as the load,
4214*9880d681SAndroid Build Coastguard Worker /// unless conditions are unfavorable. This allows SelectionDAG to fold the
4215*9880d681SAndroid Build Coastguard Worker /// extend into the load.
4216*9880d681SAndroid Build Coastguard Worker /// \p I[in/out] the extension may be modified during the process if some
4217*9880d681SAndroid Build Coastguard Worker /// promotions apply.
4218*9880d681SAndroid Build Coastguard Worker ///
moveExtToFormExtLoad(Instruction * & I)4219*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::moveExtToFormExtLoad(Instruction *&I) {
4220*9880d681SAndroid Build Coastguard Worker   // Try to promote a chain of computation if it allows to form
4221*9880d681SAndroid Build Coastguard Worker   // an extended load.
4222*9880d681SAndroid Build Coastguard Worker   TypePromotionTransaction TPT;
4223*9880d681SAndroid Build Coastguard Worker   TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4224*9880d681SAndroid Build Coastguard Worker     TPT.getRestorationPoint();
4225*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction *, 1> Exts;
4226*9880d681SAndroid Build Coastguard Worker   Exts.push_back(I);
4227*9880d681SAndroid Build Coastguard Worker   // Look for a load being extended.
4228*9880d681SAndroid Build Coastguard Worker   LoadInst *LI = nullptr;
4229*9880d681SAndroid Build Coastguard Worker   Instruction *OldExt = I;
4230*9880d681SAndroid Build Coastguard Worker   bool HasPromoted = extLdPromotion(TPT, LI, I, Exts);
4231*9880d681SAndroid Build Coastguard Worker   if (!LI || !I) {
4232*9880d681SAndroid Build Coastguard Worker     assert(!HasPromoted && !LI && "If we did not match any load instruction "
4233*9880d681SAndroid Build Coastguard Worker                                   "the code must remain the same");
4234*9880d681SAndroid Build Coastguard Worker     I = OldExt;
4235*9880d681SAndroid Build Coastguard Worker     return false;
4236*9880d681SAndroid Build Coastguard Worker   }
4237*9880d681SAndroid Build Coastguard Worker 
4238*9880d681SAndroid Build Coastguard Worker   // If they're already in the same block, there's nothing to do.
4239*9880d681SAndroid Build Coastguard Worker   // Make the cheap checks first if we did not promote.
4240*9880d681SAndroid Build Coastguard Worker   // If we promoted, we need to check if it is indeed profitable.
4241*9880d681SAndroid Build Coastguard Worker   if (!HasPromoted && LI->getParent() == I->getParent())
4242*9880d681SAndroid Build Coastguard Worker     return false;
4243*9880d681SAndroid Build Coastguard Worker 
4244*9880d681SAndroid Build Coastguard Worker   EVT VT = TLI->getValueType(*DL, I->getType());
4245*9880d681SAndroid Build Coastguard Worker   EVT LoadVT = TLI->getValueType(*DL, LI->getType());
4246*9880d681SAndroid Build Coastguard Worker 
4247*9880d681SAndroid Build Coastguard Worker   // If the load has other users and the truncate is not free, this probably
4248*9880d681SAndroid Build Coastguard Worker   // isn't worthwhile.
4249*9880d681SAndroid Build Coastguard Worker   if (!LI->hasOneUse() && TLI &&
4250*9880d681SAndroid Build Coastguard Worker       (TLI->isTypeLegal(LoadVT) || !TLI->isTypeLegal(VT)) &&
4251*9880d681SAndroid Build Coastguard Worker       !TLI->isTruncateFree(I->getType(), LI->getType())) {
4252*9880d681SAndroid Build Coastguard Worker     I = OldExt;
4253*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
4254*9880d681SAndroid Build Coastguard Worker     return false;
4255*9880d681SAndroid Build Coastguard Worker   }
4256*9880d681SAndroid Build Coastguard Worker 
4257*9880d681SAndroid Build Coastguard Worker   // Check whether the target supports casts folded into loads.
4258*9880d681SAndroid Build Coastguard Worker   unsigned LType;
4259*9880d681SAndroid Build Coastguard Worker   if (isa<ZExtInst>(I))
4260*9880d681SAndroid Build Coastguard Worker     LType = ISD::ZEXTLOAD;
4261*9880d681SAndroid Build Coastguard Worker   else {
4262*9880d681SAndroid Build Coastguard Worker     assert(isa<SExtInst>(I) && "Unexpected ext type!");
4263*9880d681SAndroid Build Coastguard Worker     LType = ISD::SEXTLOAD;
4264*9880d681SAndroid Build Coastguard Worker   }
4265*9880d681SAndroid Build Coastguard Worker   if (TLI && !TLI->isLoadExtLegal(LType, VT, LoadVT)) {
4266*9880d681SAndroid Build Coastguard Worker     I = OldExt;
4267*9880d681SAndroid Build Coastguard Worker     TPT.rollback(LastKnownGood);
4268*9880d681SAndroid Build Coastguard Worker     return false;
4269*9880d681SAndroid Build Coastguard Worker   }
4270*9880d681SAndroid Build Coastguard Worker 
4271*9880d681SAndroid Build Coastguard Worker   // Move the extend into the same block as the load, so that SelectionDAG
4272*9880d681SAndroid Build Coastguard Worker   // can fold it.
4273*9880d681SAndroid Build Coastguard Worker   TPT.commit();
4274*9880d681SAndroid Build Coastguard Worker   I->removeFromParent();
4275*9880d681SAndroid Build Coastguard Worker   I->insertAfter(LI);
4276*9880d681SAndroid Build Coastguard Worker   ++NumExtsMoved;
4277*9880d681SAndroid Build Coastguard Worker   return true;
4278*9880d681SAndroid Build Coastguard Worker }
4279*9880d681SAndroid Build Coastguard Worker 
optimizeExtUses(Instruction * I)4280*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeExtUses(Instruction *I) {
4281*9880d681SAndroid Build Coastguard Worker   BasicBlock *DefBB = I->getParent();
4282*9880d681SAndroid Build Coastguard Worker 
4283*9880d681SAndroid Build Coastguard Worker   // If the result of a {s|z}ext and its source are both live out, rewrite all
4284*9880d681SAndroid Build Coastguard Worker   // other uses of the source with result of extension.
4285*9880d681SAndroid Build Coastguard Worker   Value *Src = I->getOperand(0);
4286*9880d681SAndroid Build Coastguard Worker   if (Src->hasOneUse())
4287*9880d681SAndroid Build Coastguard Worker     return false;
4288*9880d681SAndroid Build Coastguard Worker 
4289*9880d681SAndroid Build Coastguard Worker   // Only do this xform if truncating is free.
4290*9880d681SAndroid Build Coastguard Worker   if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType()))
4291*9880d681SAndroid Build Coastguard Worker     return false;
4292*9880d681SAndroid Build Coastguard Worker 
4293*9880d681SAndroid Build Coastguard Worker   // Only safe to perform the optimization if the source is also defined in
4294*9880d681SAndroid Build Coastguard Worker   // this block.
4295*9880d681SAndroid Build Coastguard Worker   if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent())
4296*9880d681SAndroid Build Coastguard Worker     return false;
4297*9880d681SAndroid Build Coastguard Worker 
4298*9880d681SAndroid Build Coastguard Worker   bool DefIsLiveOut = false;
4299*9880d681SAndroid Build Coastguard Worker   for (User *U : I->users()) {
4300*9880d681SAndroid Build Coastguard Worker     Instruction *UI = cast<Instruction>(U);
4301*9880d681SAndroid Build Coastguard Worker 
4302*9880d681SAndroid Build Coastguard Worker     // Figure out which BB this ext is used in.
4303*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = UI->getParent();
4304*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) continue;
4305*9880d681SAndroid Build Coastguard Worker     DefIsLiveOut = true;
4306*9880d681SAndroid Build Coastguard Worker     break;
4307*9880d681SAndroid Build Coastguard Worker   }
4308*9880d681SAndroid Build Coastguard Worker   if (!DefIsLiveOut)
4309*9880d681SAndroid Build Coastguard Worker     return false;
4310*9880d681SAndroid Build Coastguard Worker 
4311*9880d681SAndroid Build Coastguard Worker   // Make sure none of the uses are PHI nodes.
4312*9880d681SAndroid Build Coastguard Worker   for (User *U : Src->users()) {
4313*9880d681SAndroid Build Coastguard Worker     Instruction *UI = cast<Instruction>(U);
4314*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = UI->getParent();
4315*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) continue;
4316*9880d681SAndroid Build Coastguard Worker     // Be conservative. We don't want this xform to end up introducing
4317*9880d681SAndroid Build Coastguard Worker     // reloads just before load / store instructions.
4318*9880d681SAndroid Build Coastguard Worker     if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI))
4319*9880d681SAndroid Build Coastguard Worker       return false;
4320*9880d681SAndroid Build Coastguard Worker   }
4321*9880d681SAndroid Build Coastguard Worker 
4322*9880d681SAndroid Build Coastguard Worker   // InsertedTruncs - Only insert one trunc in each block once.
4323*9880d681SAndroid Build Coastguard Worker   DenseMap<BasicBlock*, Instruction*> InsertedTruncs;
4324*9880d681SAndroid Build Coastguard Worker 
4325*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
4326*9880d681SAndroid Build Coastguard Worker   for (Use &U : Src->uses()) {
4327*9880d681SAndroid Build Coastguard Worker     Instruction *User = cast<Instruction>(U.getUser());
4328*9880d681SAndroid Build Coastguard Worker 
4329*9880d681SAndroid Build Coastguard Worker     // Figure out which BB this ext is used in.
4330*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = User->getParent();
4331*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) continue;
4332*9880d681SAndroid Build Coastguard Worker 
4333*9880d681SAndroid Build Coastguard Worker     // Both src and def are live in this block. Rewrite the use.
4334*9880d681SAndroid Build Coastguard Worker     Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
4335*9880d681SAndroid Build Coastguard Worker 
4336*9880d681SAndroid Build Coastguard Worker     if (!InsertedTrunc) {
4337*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
4338*9880d681SAndroid Build Coastguard Worker       assert(InsertPt != UserBB->end());
4339*9880d681SAndroid Build Coastguard Worker       InsertedTrunc = new TruncInst(I, Src->getType(), "", &*InsertPt);
4340*9880d681SAndroid Build Coastguard Worker       InsertedInsts.insert(InsertedTrunc);
4341*9880d681SAndroid Build Coastguard Worker     }
4342*9880d681SAndroid Build Coastguard Worker 
4343*9880d681SAndroid Build Coastguard Worker     // Replace a use of the {s|z}ext source with a use of the result.
4344*9880d681SAndroid Build Coastguard Worker     U = InsertedTrunc;
4345*9880d681SAndroid Build Coastguard Worker     ++NumExtUses;
4346*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
4347*9880d681SAndroid Build Coastguard Worker   }
4348*9880d681SAndroid Build Coastguard Worker 
4349*9880d681SAndroid Build Coastguard Worker   return MadeChange;
4350*9880d681SAndroid Build Coastguard Worker }
4351*9880d681SAndroid Build Coastguard Worker 
4352*9880d681SAndroid Build Coastguard Worker // Find loads whose uses only use some of the loaded value's bits.  Add an "and"
4353*9880d681SAndroid Build Coastguard Worker // just after the load if the target can fold this into one extload instruction,
4354*9880d681SAndroid Build Coastguard Worker // with the hope of eliminating some of the other later "and" instructions using
4355*9880d681SAndroid Build Coastguard Worker // the loaded value.  "and"s that are made trivially redundant by the insertion
4356*9880d681SAndroid Build Coastguard Worker // of the new "and" are removed by this function, while others (e.g. those whose
4357*9880d681SAndroid Build Coastguard Worker // path from the load goes through a phi) are left for isel to potentially
4358*9880d681SAndroid Build Coastguard Worker // remove.
4359*9880d681SAndroid Build Coastguard Worker //
4360*9880d681SAndroid Build Coastguard Worker // For example:
4361*9880d681SAndroid Build Coastguard Worker //
4362*9880d681SAndroid Build Coastguard Worker // b0:
4363*9880d681SAndroid Build Coastguard Worker //   x = load i32
4364*9880d681SAndroid Build Coastguard Worker //   ...
4365*9880d681SAndroid Build Coastguard Worker // b1:
4366*9880d681SAndroid Build Coastguard Worker //   y = and x, 0xff
4367*9880d681SAndroid Build Coastguard Worker //   z = use y
4368*9880d681SAndroid Build Coastguard Worker //
4369*9880d681SAndroid Build Coastguard Worker // becomes:
4370*9880d681SAndroid Build Coastguard Worker //
4371*9880d681SAndroid Build Coastguard Worker // b0:
4372*9880d681SAndroid Build Coastguard Worker //   x = load i32
4373*9880d681SAndroid Build Coastguard Worker //   x' = and x, 0xff
4374*9880d681SAndroid Build Coastguard Worker //   ...
4375*9880d681SAndroid Build Coastguard Worker // b1:
4376*9880d681SAndroid Build Coastguard Worker //   z = use x'
4377*9880d681SAndroid Build Coastguard Worker //
4378*9880d681SAndroid Build Coastguard Worker // whereas:
4379*9880d681SAndroid Build Coastguard Worker //
4380*9880d681SAndroid Build Coastguard Worker // b0:
4381*9880d681SAndroid Build Coastguard Worker //   x1 = load i32
4382*9880d681SAndroid Build Coastguard Worker //   ...
4383*9880d681SAndroid Build Coastguard Worker // b1:
4384*9880d681SAndroid Build Coastguard Worker //   x2 = load i32
4385*9880d681SAndroid Build Coastguard Worker //   ...
4386*9880d681SAndroid Build Coastguard Worker // b2:
4387*9880d681SAndroid Build Coastguard Worker //   x = phi x1, x2
4388*9880d681SAndroid Build Coastguard Worker //   y = and x, 0xff
4389*9880d681SAndroid Build Coastguard Worker //
4390*9880d681SAndroid Build Coastguard Worker // becomes (after a call to optimizeLoadExt for each load):
4391*9880d681SAndroid Build Coastguard Worker //
4392*9880d681SAndroid Build Coastguard Worker // b0:
4393*9880d681SAndroid Build Coastguard Worker //   x1 = load i32
4394*9880d681SAndroid Build Coastguard Worker //   x1' = and x1, 0xff
4395*9880d681SAndroid Build Coastguard Worker //   ...
4396*9880d681SAndroid Build Coastguard Worker // b1:
4397*9880d681SAndroid Build Coastguard Worker //   x2 = load i32
4398*9880d681SAndroid Build Coastguard Worker //   x2' = and x2, 0xff
4399*9880d681SAndroid Build Coastguard Worker //   ...
4400*9880d681SAndroid Build Coastguard Worker // b2:
4401*9880d681SAndroid Build Coastguard Worker //   x = phi x1', x2'
4402*9880d681SAndroid Build Coastguard Worker //   y = and x, 0xff
4403*9880d681SAndroid Build Coastguard Worker //
4404*9880d681SAndroid Build Coastguard Worker 
optimizeLoadExt(LoadInst * Load)4405*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
4406*9880d681SAndroid Build Coastguard Worker 
4407*9880d681SAndroid Build Coastguard Worker   if (!Load->isSimple() ||
4408*9880d681SAndroid Build Coastguard Worker       !(Load->getType()->isIntegerTy() || Load->getType()->isPointerTy()))
4409*9880d681SAndroid Build Coastguard Worker     return false;
4410*9880d681SAndroid Build Coastguard Worker 
4411*9880d681SAndroid Build Coastguard Worker   // Skip loads we've already transformed or have no reason to transform.
4412*9880d681SAndroid Build Coastguard Worker   if (Load->hasOneUse()) {
4413*9880d681SAndroid Build Coastguard Worker     User *LoadUser = *Load->user_begin();
4414*9880d681SAndroid Build Coastguard Worker     if (cast<Instruction>(LoadUser)->getParent() == Load->getParent() &&
4415*9880d681SAndroid Build Coastguard Worker         !dyn_cast<PHINode>(LoadUser))
4416*9880d681SAndroid Build Coastguard Worker       return false;
4417*9880d681SAndroid Build Coastguard Worker   }
4418*9880d681SAndroid Build Coastguard Worker 
4419*9880d681SAndroid Build Coastguard Worker   // Look at all uses of Load, looking through phis, to determine how many bits
4420*9880d681SAndroid Build Coastguard Worker   // of the loaded value are needed.
4421*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction *, 8> WorkList;
4422*9880d681SAndroid Build Coastguard Worker   SmallPtrSet<Instruction *, 16> Visited;
4423*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction *, 8> AndsToMaybeRemove;
4424*9880d681SAndroid Build Coastguard Worker   for (auto *U : Load->users())
4425*9880d681SAndroid Build Coastguard Worker     WorkList.push_back(cast<Instruction>(U));
4426*9880d681SAndroid Build Coastguard Worker 
4427*9880d681SAndroid Build Coastguard Worker   EVT LoadResultVT = TLI->getValueType(*DL, Load->getType());
4428*9880d681SAndroid Build Coastguard Worker   unsigned BitWidth = LoadResultVT.getSizeInBits();
4429*9880d681SAndroid Build Coastguard Worker   APInt DemandBits(BitWidth, 0);
4430*9880d681SAndroid Build Coastguard Worker   APInt WidestAndBits(BitWidth, 0);
4431*9880d681SAndroid Build Coastguard Worker 
4432*9880d681SAndroid Build Coastguard Worker   while (!WorkList.empty()) {
4433*9880d681SAndroid Build Coastguard Worker     Instruction *I = WorkList.back();
4434*9880d681SAndroid Build Coastguard Worker     WorkList.pop_back();
4435*9880d681SAndroid Build Coastguard Worker 
4436*9880d681SAndroid Build Coastguard Worker     // Break use-def graph loops.
4437*9880d681SAndroid Build Coastguard Worker     if (!Visited.insert(I).second)
4438*9880d681SAndroid Build Coastguard Worker       continue;
4439*9880d681SAndroid Build Coastguard Worker 
4440*9880d681SAndroid Build Coastguard Worker     // For a PHI node, push all of its users.
4441*9880d681SAndroid Build Coastguard Worker     if (auto *Phi = dyn_cast<PHINode>(I)) {
4442*9880d681SAndroid Build Coastguard Worker       for (auto *U : Phi->users())
4443*9880d681SAndroid Build Coastguard Worker         WorkList.push_back(cast<Instruction>(U));
4444*9880d681SAndroid Build Coastguard Worker       continue;
4445*9880d681SAndroid Build Coastguard Worker     }
4446*9880d681SAndroid Build Coastguard Worker 
4447*9880d681SAndroid Build Coastguard Worker     switch (I->getOpcode()) {
4448*9880d681SAndroid Build Coastguard Worker     case llvm::Instruction::And: {
4449*9880d681SAndroid Build Coastguard Worker       auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1));
4450*9880d681SAndroid Build Coastguard Worker       if (!AndC)
4451*9880d681SAndroid Build Coastguard Worker         return false;
4452*9880d681SAndroid Build Coastguard Worker       APInt AndBits = AndC->getValue();
4453*9880d681SAndroid Build Coastguard Worker       DemandBits |= AndBits;
4454*9880d681SAndroid Build Coastguard Worker       // Keep track of the widest and mask we see.
4455*9880d681SAndroid Build Coastguard Worker       if (AndBits.ugt(WidestAndBits))
4456*9880d681SAndroid Build Coastguard Worker         WidestAndBits = AndBits;
4457*9880d681SAndroid Build Coastguard Worker       if (AndBits == WidestAndBits && I->getOperand(0) == Load)
4458*9880d681SAndroid Build Coastguard Worker         AndsToMaybeRemove.push_back(I);
4459*9880d681SAndroid Build Coastguard Worker       break;
4460*9880d681SAndroid Build Coastguard Worker     }
4461*9880d681SAndroid Build Coastguard Worker 
4462*9880d681SAndroid Build Coastguard Worker     case llvm::Instruction::Shl: {
4463*9880d681SAndroid Build Coastguard Worker       auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1));
4464*9880d681SAndroid Build Coastguard Worker       if (!ShlC)
4465*9880d681SAndroid Build Coastguard Worker         return false;
4466*9880d681SAndroid Build Coastguard Worker       uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1);
4467*9880d681SAndroid Build Coastguard Worker       auto ShlDemandBits = APInt::getAllOnesValue(BitWidth).lshr(ShiftAmt);
4468*9880d681SAndroid Build Coastguard Worker       DemandBits |= ShlDemandBits;
4469*9880d681SAndroid Build Coastguard Worker       break;
4470*9880d681SAndroid Build Coastguard Worker     }
4471*9880d681SAndroid Build Coastguard Worker 
4472*9880d681SAndroid Build Coastguard Worker     case llvm::Instruction::Trunc: {
4473*9880d681SAndroid Build Coastguard Worker       EVT TruncVT = TLI->getValueType(*DL, I->getType());
4474*9880d681SAndroid Build Coastguard Worker       unsigned TruncBitWidth = TruncVT.getSizeInBits();
4475*9880d681SAndroid Build Coastguard Worker       auto TruncBits = APInt::getAllOnesValue(TruncBitWidth).zext(BitWidth);
4476*9880d681SAndroid Build Coastguard Worker       DemandBits |= TruncBits;
4477*9880d681SAndroid Build Coastguard Worker       break;
4478*9880d681SAndroid Build Coastguard Worker     }
4479*9880d681SAndroid Build Coastguard Worker 
4480*9880d681SAndroid Build Coastguard Worker     default:
4481*9880d681SAndroid Build Coastguard Worker       return false;
4482*9880d681SAndroid Build Coastguard Worker     }
4483*9880d681SAndroid Build Coastguard Worker   }
4484*9880d681SAndroid Build Coastguard Worker 
4485*9880d681SAndroid Build Coastguard Worker   uint32_t ActiveBits = DemandBits.getActiveBits();
4486*9880d681SAndroid Build Coastguard Worker   // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the
4487*9880d681SAndroid Build Coastguard Worker   // target even if isLoadExtLegal says an i1 EXTLOAD is valid.  For example,
4488*9880d681SAndroid Build Coastguard Worker   // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but
4489*9880d681SAndroid Build Coastguard Worker   // (and (load x) 1) is not matched as a single instruction, rather as a LDR
4490*9880d681SAndroid Build Coastguard Worker   // followed by an AND.
4491*9880d681SAndroid Build Coastguard Worker   // TODO: Look into removing this restriction by fixing backends to either
4492*9880d681SAndroid Build Coastguard Worker   // return false for isLoadExtLegal for i1 or have them select this pattern to
4493*9880d681SAndroid Build Coastguard Worker   // a single instruction.
4494*9880d681SAndroid Build Coastguard Worker   //
4495*9880d681SAndroid Build Coastguard Worker   // Also avoid hoisting if we didn't see any ands with the exact DemandBits
4496*9880d681SAndroid Build Coastguard Worker   // mask, since these are the only ands that will be removed by isel.
4497*9880d681SAndroid Build Coastguard Worker   if (ActiveBits <= 1 || !APIntOps::isMask(ActiveBits, DemandBits) ||
4498*9880d681SAndroid Build Coastguard Worker       WidestAndBits != DemandBits)
4499*9880d681SAndroid Build Coastguard Worker     return false;
4500*9880d681SAndroid Build Coastguard Worker 
4501*9880d681SAndroid Build Coastguard Worker   LLVMContext &Ctx = Load->getType()->getContext();
4502*9880d681SAndroid Build Coastguard Worker   Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits);
4503*9880d681SAndroid Build Coastguard Worker   EVT TruncVT = TLI->getValueType(*DL, TruncTy);
4504*9880d681SAndroid Build Coastguard Worker 
4505*9880d681SAndroid Build Coastguard Worker   // Reject cases that won't be matched as extloads.
4506*9880d681SAndroid Build Coastguard Worker   if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() ||
4507*9880d681SAndroid Build Coastguard Worker       !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT))
4508*9880d681SAndroid Build Coastguard Worker     return false;
4509*9880d681SAndroid Build Coastguard Worker 
4510*9880d681SAndroid Build Coastguard Worker   IRBuilder<> Builder(Load->getNextNode());
4511*9880d681SAndroid Build Coastguard Worker   auto *NewAnd = dyn_cast<Instruction>(
4512*9880d681SAndroid Build Coastguard Worker       Builder.CreateAnd(Load, ConstantInt::get(Ctx, DemandBits)));
4513*9880d681SAndroid Build Coastguard Worker 
4514*9880d681SAndroid Build Coastguard Worker   // Replace all uses of load with new and (except for the use of load in the
4515*9880d681SAndroid Build Coastguard Worker   // new and itself).
4516*9880d681SAndroid Build Coastguard Worker   Load->replaceAllUsesWith(NewAnd);
4517*9880d681SAndroid Build Coastguard Worker   NewAnd->setOperand(0, Load);
4518*9880d681SAndroid Build Coastguard Worker 
4519*9880d681SAndroid Build Coastguard Worker   // Remove any and instructions that are now redundant.
4520*9880d681SAndroid Build Coastguard Worker   for (auto *And : AndsToMaybeRemove)
4521*9880d681SAndroid Build Coastguard Worker     // Check that the and mask is the same as the one we decided to put on the
4522*9880d681SAndroid Build Coastguard Worker     // new and.
4523*9880d681SAndroid Build Coastguard Worker     if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) {
4524*9880d681SAndroid Build Coastguard Worker       And->replaceAllUsesWith(NewAnd);
4525*9880d681SAndroid Build Coastguard Worker       if (&*CurInstIterator == And)
4526*9880d681SAndroid Build Coastguard Worker         CurInstIterator = std::next(And->getIterator());
4527*9880d681SAndroid Build Coastguard Worker       And->eraseFromParent();
4528*9880d681SAndroid Build Coastguard Worker       ++NumAndUses;
4529*9880d681SAndroid Build Coastguard Worker     }
4530*9880d681SAndroid Build Coastguard Worker 
4531*9880d681SAndroid Build Coastguard Worker   ++NumAndsAdded;
4532*9880d681SAndroid Build Coastguard Worker   return true;
4533*9880d681SAndroid Build Coastguard Worker }
4534*9880d681SAndroid Build Coastguard Worker 
4535*9880d681SAndroid Build Coastguard Worker /// Check if V (an operand of a select instruction) is an expensive instruction
4536*9880d681SAndroid Build Coastguard Worker /// that is only used once.
sinkSelectOperand(const TargetTransformInfo * TTI,Value * V)4537*9880d681SAndroid Build Coastguard Worker static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V) {
4538*9880d681SAndroid Build Coastguard Worker   auto *I = dyn_cast<Instruction>(V);
4539*9880d681SAndroid Build Coastguard Worker   // If it's safe to speculatively execute, then it should not have side
4540*9880d681SAndroid Build Coastguard Worker   // effects; therefore, it's safe to sink and possibly *not* execute.
4541*9880d681SAndroid Build Coastguard Worker   return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) &&
4542*9880d681SAndroid Build Coastguard Worker          TTI->getUserCost(I) >= TargetTransformInfo::TCC_Expensive;
4543*9880d681SAndroid Build Coastguard Worker }
4544*9880d681SAndroid Build Coastguard Worker 
4545*9880d681SAndroid Build Coastguard Worker /// Returns true if a SelectInst should be turned into an explicit branch.
isFormingBranchFromSelectProfitable(const TargetTransformInfo * TTI,const TargetLowering * TLI,SelectInst * SI)4546*9880d681SAndroid Build Coastguard Worker static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
4547*9880d681SAndroid Build Coastguard Worker                                                 const TargetLowering *TLI,
4548*9880d681SAndroid Build Coastguard Worker                                                 SelectInst *SI) {
4549*9880d681SAndroid Build Coastguard Worker   // If even a predictable select is cheap, then a branch can't be cheaper.
4550*9880d681SAndroid Build Coastguard Worker   if (!TLI->isPredictableSelectExpensive())
4551*9880d681SAndroid Build Coastguard Worker     return false;
4552*9880d681SAndroid Build Coastguard Worker 
4553*9880d681SAndroid Build Coastguard Worker   // FIXME: This should use the same heuristics as IfConversion to determine
4554*9880d681SAndroid Build Coastguard Worker   // whether a select is better represented as a branch.
4555*9880d681SAndroid Build Coastguard Worker 
4556*9880d681SAndroid Build Coastguard Worker   // If metadata tells us that the select condition is obviously predictable,
4557*9880d681SAndroid Build Coastguard Worker   // then we want to replace the select with a branch.
4558*9880d681SAndroid Build Coastguard Worker   uint64_t TrueWeight, FalseWeight;
4559*9880d681SAndroid Build Coastguard Worker   if (SI->extractProfMetadata(TrueWeight, FalseWeight)) {
4560*9880d681SAndroid Build Coastguard Worker     uint64_t Max = std::max(TrueWeight, FalseWeight);
4561*9880d681SAndroid Build Coastguard Worker     uint64_t Sum = TrueWeight + FalseWeight;
4562*9880d681SAndroid Build Coastguard Worker     if (Sum != 0) {
4563*9880d681SAndroid Build Coastguard Worker       auto Probability = BranchProbability::getBranchProbability(Max, Sum);
4564*9880d681SAndroid Build Coastguard Worker       if (Probability > TLI->getPredictableBranchThreshold())
4565*9880d681SAndroid Build Coastguard Worker         return true;
4566*9880d681SAndroid Build Coastguard Worker     }
4567*9880d681SAndroid Build Coastguard Worker   }
4568*9880d681SAndroid Build Coastguard Worker 
4569*9880d681SAndroid Build Coastguard Worker   CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition());
4570*9880d681SAndroid Build Coastguard Worker 
4571*9880d681SAndroid Build Coastguard Worker   // If a branch is predictable, an out-of-order CPU can avoid blocking on its
4572*9880d681SAndroid Build Coastguard Worker   // comparison condition. If the compare has more than one use, there's
4573*9880d681SAndroid Build Coastguard Worker   // probably another cmov or setcc around, so it's not worth emitting a branch.
4574*9880d681SAndroid Build Coastguard Worker   if (!Cmp || !Cmp->hasOneUse())
4575*9880d681SAndroid Build Coastguard Worker     return false;
4576*9880d681SAndroid Build Coastguard Worker 
4577*9880d681SAndroid Build Coastguard Worker   // If either operand of the select is expensive and only needed on one side
4578*9880d681SAndroid Build Coastguard Worker   // of the select, we should form a branch.
4579*9880d681SAndroid Build Coastguard Worker   if (sinkSelectOperand(TTI, SI->getTrueValue()) ||
4580*9880d681SAndroid Build Coastguard Worker       sinkSelectOperand(TTI, SI->getFalseValue()))
4581*9880d681SAndroid Build Coastguard Worker     return true;
4582*9880d681SAndroid Build Coastguard Worker 
4583*9880d681SAndroid Build Coastguard Worker   return false;
4584*9880d681SAndroid Build Coastguard Worker }
4585*9880d681SAndroid Build Coastguard Worker 
4586*9880d681SAndroid Build Coastguard Worker 
4587*9880d681SAndroid Build Coastguard Worker /// If we have a SelectInst that will likely profit from branch prediction,
4588*9880d681SAndroid Build Coastguard Worker /// turn it into a branch.
optimizeSelectInst(SelectInst * SI)4589*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
4590*9880d681SAndroid Build Coastguard Worker   bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1);
4591*9880d681SAndroid Build Coastguard Worker 
4592*9880d681SAndroid Build Coastguard Worker   // Can we convert the 'select' to CF ?
4593*9880d681SAndroid Build Coastguard Worker   if (DisableSelectToBranch || OptSize || !TLI || VectorCond ||
4594*9880d681SAndroid Build Coastguard Worker       SI->getMetadata(LLVMContext::MD_unpredictable))
4595*9880d681SAndroid Build Coastguard Worker     return false;
4596*9880d681SAndroid Build Coastguard Worker 
4597*9880d681SAndroid Build Coastguard Worker   TargetLowering::SelectSupportKind SelectKind;
4598*9880d681SAndroid Build Coastguard Worker   if (VectorCond)
4599*9880d681SAndroid Build Coastguard Worker     SelectKind = TargetLowering::VectorMaskSelect;
4600*9880d681SAndroid Build Coastguard Worker   else if (SI->getType()->isVectorTy())
4601*9880d681SAndroid Build Coastguard Worker     SelectKind = TargetLowering::ScalarCondVectorVal;
4602*9880d681SAndroid Build Coastguard Worker   else
4603*9880d681SAndroid Build Coastguard Worker     SelectKind = TargetLowering::ScalarValSelect;
4604*9880d681SAndroid Build Coastguard Worker 
4605*9880d681SAndroid Build Coastguard Worker   if (TLI->isSelectSupported(SelectKind) &&
4606*9880d681SAndroid Build Coastguard Worker       !isFormingBranchFromSelectProfitable(TTI, TLI, SI))
4607*9880d681SAndroid Build Coastguard Worker     return false;
4608*9880d681SAndroid Build Coastguard Worker 
4609*9880d681SAndroid Build Coastguard Worker   ModifiedDT = true;
4610*9880d681SAndroid Build Coastguard Worker 
4611*9880d681SAndroid Build Coastguard Worker   // Transform a sequence like this:
4612*9880d681SAndroid Build Coastguard Worker   //    start:
4613*9880d681SAndroid Build Coastguard Worker   //       %cmp = cmp uge i32 %a, %b
4614*9880d681SAndroid Build Coastguard Worker   //       %sel = select i1 %cmp, i32 %c, i32 %d
4615*9880d681SAndroid Build Coastguard Worker   //
4616*9880d681SAndroid Build Coastguard Worker   // Into:
4617*9880d681SAndroid Build Coastguard Worker   //    start:
4618*9880d681SAndroid Build Coastguard Worker   //       %cmp = cmp uge i32 %a, %b
4619*9880d681SAndroid Build Coastguard Worker   //       br i1 %cmp, label %select.true, label %select.false
4620*9880d681SAndroid Build Coastguard Worker   //    select.true:
4621*9880d681SAndroid Build Coastguard Worker   //       br label %select.end
4622*9880d681SAndroid Build Coastguard Worker   //    select.false:
4623*9880d681SAndroid Build Coastguard Worker   //       br label %select.end
4624*9880d681SAndroid Build Coastguard Worker   //    select.end:
4625*9880d681SAndroid Build Coastguard Worker   //       %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ]
4626*9880d681SAndroid Build Coastguard Worker   //
4627*9880d681SAndroid Build Coastguard Worker   // In addition, we may sink instructions that produce %c or %d from
4628*9880d681SAndroid Build Coastguard Worker   // the entry block into the destination(s) of the new branch.
4629*9880d681SAndroid Build Coastguard Worker   // If the true or false blocks do not contain a sunken instruction, that
4630*9880d681SAndroid Build Coastguard Worker   // block and its branch may be optimized away. In that case, one side of the
4631*9880d681SAndroid Build Coastguard Worker   // first branch will point directly to select.end, and the corresponding PHI
4632*9880d681SAndroid Build Coastguard Worker   // predecessor block will be the start block.
4633*9880d681SAndroid Build Coastguard Worker 
4634*9880d681SAndroid Build Coastguard Worker   // First, we split the block containing the select into 2 blocks.
4635*9880d681SAndroid Build Coastguard Worker   BasicBlock *StartBlock = SI->getParent();
4636*9880d681SAndroid Build Coastguard Worker   BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(SI));
4637*9880d681SAndroid Build Coastguard Worker   BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end");
4638*9880d681SAndroid Build Coastguard Worker 
4639*9880d681SAndroid Build Coastguard Worker   // Delete the unconditional branch that was just created by the split.
4640*9880d681SAndroid Build Coastguard Worker   StartBlock->getTerminator()->eraseFromParent();
4641*9880d681SAndroid Build Coastguard Worker 
4642*9880d681SAndroid Build Coastguard Worker   // These are the new basic blocks for the conditional branch.
4643*9880d681SAndroid Build Coastguard Worker   // At least one will become an actual new basic block.
4644*9880d681SAndroid Build Coastguard Worker   BasicBlock *TrueBlock = nullptr;
4645*9880d681SAndroid Build Coastguard Worker   BasicBlock *FalseBlock = nullptr;
4646*9880d681SAndroid Build Coastguard Worker 
4647*9880d681SAndroid Build Coastguard Worker   // Sink expensive instructions into the conditional blocks to avoid executing
4648*9880d681SAndroid Build Coastguard Worker   // them speculatively.
4649*9880d681SAndroid Build Coastguard Worker   if (sinkSelectOperand(TTI, SI->getTrueValue())) {
4650*9880d681SAndroid Build Coastguard Worker     TrueBlock = BasicBlock::Create(SI->getContext(), "select.true.sink",
4651*9880d681SAndroid Build Coastguard Worker                                    EndBlock->getParent(), EndBlock);
4652*9880d681SAndroid Build Coastguard Worker     auto *TrueBranch = BranchInst::Create(EndBlock, TrueBlock);
4653*9880d681SAndroid Build Coastguard Worker     auto *TrueInst = cast<Instruction>(SI->getTrueValue());
4654*9880d681SAndroid Build Coastguard Worker     TrueInst->moveBefore(TrueBranch);
4655*9880d681SAndroid Build Coastguard Worker   }
4656*9880d681SAndroid Build Coastguard Worker   if (sinkSelectOperand(TTI, SI->getFalseValue())) {
4657*9880d681SAndroid Build Coastguard Worker     FalseBlock = BasicBlock::Create(SI->getContext(), "select.false.sink",
4658*9880d681SAndroid Build Coastguard Worker                                     EndBlock->getParent(), EndBlock);
4659*9880d681SAndroid Build Coastguard Worker     auto *FalseBranch = BranchInst::Create(EndBlock, FalseBlock);
4660*9880d681SAndroid Build Coastguard Worker     auto *FalseInst = cast<Instruction>(SI->getFalseValue());
4661*9880d681SAndroid Build Coastguard Worker     FalseInst->moveBefore(FalseBranch);
4662*9880d681SAndroid Build Coastguard Worker   }
4663*9880d681SAndroid Build Coastguard Worker 
4664*9880d681SAndroid Build Coastguard Worker   // If there was nothing to sink, then arbitrarily choose the 'false' side
4665*9880d681SAndroid Build Coastguard Worker   // for a new input value to the PHI.
4666*9880d681SAndroid Build Coastguard Worker   if (TrueBlock == FalseBlock) {
4667*9880d681SAndroid Build Coastguard Worker     assert(TrueBlock == nullptr &&
4668*9880d681SAndroid Build Coastguard Worker            "Unexpected basic block transform while optimizing select");
4669*9880d681SAndroid Build Coastguard Worker 
4670*9880d681SAndroid Build Coastguard Worker     FalseBlock = BasicBlock::Create(SI->getContext(), "select.false",
4671*9880d681SAndroid Build Coastguard Worker                                     EndBlock->getParent(), EndBlock);
4672*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(EndBlock, FalseBlock);
4673*9880d681SAndroid Build Coastguard Worker   }
4674*9880d681SAndroid Build Coastguard Worker 
4675*9880d681SAndroid Build Coastguard Worker   // Insert the real conditional branch based on the original condition.
4676*9880d681SAndroid Build Coastguard Worker   // If we did not create a new block for one of the 'true' or 'false' paths
4677*9880d681SAndroid Build Coastguard Worker   // of the condition, it means that side of the branch goes to the end block
4678*9880d681SAndroid Build Coastguard Worker   // directly and the path originates from the start block from the point of
4679*9880d681SAndroid Build Coastguard Worker   // view of the new PHI.
4680*9880d681SAndroid Build Coastguard Worker   if (TrueBlock == nullptr) {
4681*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(EndBlock, FalseBlock, SI->getCondition(), SI);
4682*9880d681SAndroid Build Coastguard Worker     TrueBlock = StartBlock;
4683*9880d681SAndroid Build Coastguard Worker   } else if (FalseBlock == nullptr) {
4684*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(TrueBlock, EndBlock, SI->getCondition(), SI);
4685*9880d681SAndroid Build Coastguard Worker     FalseBlock = StartBlock;
4686*9880d681SAndroid Build Coastguard Worker   } else {
4687*9880d681SAndroid Build Coastguard Worker     BranchInst::Create(TrueBlock, FalseBlock, SI->getCondition(), SI);
4688*9880d681SAndroid Build Coastguard Worker   }
4689*9880d681SAndroid Build Coastguard Worker 
4690*9880d681SAndroid Build Coastguard Worker   // The select itself is replaced with a PHI Node.
4691*9880d681SAndroid Build Coastguard Worker   PHINode *PN = PHINode::Create(SI->getType(), 2, "", &EndBlock->front());
4692*9880d681SAndroid Build Coastguard Worker   PN->takeName(SI);
4693*9880d681SAndroid Build Coastguard Worker   PN->addIncoming(SI->getTrueValue(), TrueBlock);
4694*9880d681SAndroid Build Coastguard Worker   PN->addIncoming(SI->getFalseValue(), FalseBlock);
4695*9880d681SAndroid Build Coastguard Worker 
4696*9880d681SAndroid Build Coastguard Worker   SI->replaceAllUsesWith(PN);
4697*9880d681SAndroid Build Coastguard Worker   SI->eraseFromParent();
4698*9880d681SAndroid Build Coastguard Worker 
4699*9880d681SAndroid Build Coastguard Worker   // Instruct OptimizeBlock to skip to the next block.
4700*9880d681SAndroid Build Coastguard Worker   CurInstIterator = StartBlock->end();
4701*9880d681SAndroid Build Coastguard Worker   ++NumSelectsExpanded;
4702*9880d681SAndroid Build Coastguard Worker   return true;
4703*9880d681SAndroid Build Coastguard Worker }
4704*9880d681SAndroid Build Coastguard Worker 
isBroadcastShuffle(ShuffleVectorInst * SVI)4705*9880d681SAndroid Build Coastguard Worker static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
4706*9880d681SAndroid Build Coastguard Worker   SmallVector<int, 16> Mask(SVI->getShuffleMask());
4707*9880d681SAndroid Build Coastguard Worker   int SplatElem = -1;
4708*9880d681SAndroid Build Coastguard Worker   for (unsigned i = 0; i < Mask.size(); ++i) {
4709*9880d681SAndroid Build Coastguard Worker     if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
4710*9880d681SAndroid Build Coastguard Worker       return false;
4711*9880d681SAndroid Build Coastguard Worker     SplatElem = Mask[i];
4712*9880d681SAndroid Build Coastguard Worker   }
4713*9880d681SAndroid Build Coastguard Worker 
4714*9880d681SAndroid Build Coastguard Worker   return true;
4715*9880d681SAndroid Build Coastguard Worker }
4716*9880d681SAndroid Build Coastguard Worker 
4717*9880d681SAndroid Build Coastguard Worker /// Some targets have expensive vector shifts if the lanes aren't all the same
4718*9880d681SAndroid Build Coastguard Worker /// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases
4719*9880d681SAndroid Build Coastguard Worker /// it's often worth sinking a shufflevector splat down to its use so that
4720*9880d681SAndroid Build Coastguard Worker /// codegen can spot all lanes are identical.
optimizeShuffleVectorInst(ShuffleVectorInst * SVI)4721*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
4722*9880d681SAndroid Build Coastguard Worker   BasicBlock *DefBB = SVI->getParent();
4723*9880d681SAndroid Build Coastguard Worker 
4724*9880d681SAndroid Build Coastguard Worker   // Only do this xform if variable vector shifts are particularly expensive.
4725*9880d681SAndroid Build Coastguard Worker   if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType()))
4726*9880d681SAndroid Build Coastguard Worker     return false;
4727*9880d681SAndroid Build Coastguard Worker 
4728*9880d681SAndroid Build Coastguard Worker   // We only expect better codegen by sinking a shuffle if we can recognise a
4729*9880d681SAndroid Build Coastguard Worker   // constant splat.
4730*9880d681SAndroid Build Coastguard Worker   if (!isBroadcastShuffle(SVI))
4731*9880d681SAndroid Build Coastguard Worker     return false;
4732*9880d681SAndroid Build Coastguard Worker 
4733*9880d681SAndroid Build Coastguard Worker   // InsertedShuffles - Only insert a shuffle in each block once.
4734*9880d681SAndroid Build Coastguard Worker   DenseMap<BasicBlock*, Instruction*> InsertedShuffles;
4735*9880d681SAndroid Build Coastguard Worker 
4736*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
4737*9880d681SAndroid Build Coastguard Worker   for (User *U : SVI->users()) {
4738*9880d681SAndroid Build Coastguard Worker     Instruction *UI = cast<Instruction>(U);
4739*9880d681SAndroid Build Coastguard Worker 
4740*9880d681SAndroid Build Coastguard Worker     // Figure out which BB this ext is used in.
4741*9880d681SAndroid Build Coastguard Worker     BasicBlock *UserBB = UI->getParent();
4742*9880d681SAndroid Build Coastguard Worker     if (UserBB == DefBB) continue;
4743*9880d681SAndroid Build Coastguard Worker 
4744*9880d681SAndroid Build Coastguard Worker     // For now only apply this when the splat is used by a shift instruction.
4745*9880d681SAndroid Build Coastguard Worker     if (!UI->isShift()) continue;
4746*9880d681SAndroid Build Coastguard Worker 
4747*9880d681SAndroid Build Coastguard Worker     // Everything checks out, sink the shuffle if the user's block doesn't
4748*9880d681SAndroid Build Coastguard Worker     // already have a copy.
4749*9880d681SAndroid Build Coastguard Worker     Instruction *&InsertedShuffle = InsertedShuffles[UserBB];
4750*9880d681SAndroid Build Coastguard Worker 
4751*9880d681SAndroid Build Coastguard Worker     if (!InsertedShuffle) {
4752*9880d681SAndroid Build Coastguard Worker       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
4753*9880d681SAndroid Build Coastguard Worker       assert(InsertPt != UserBB->end());
4754*9880d681SAndroid Build Coastguard Worker       InsertedShuffle =
4755*9880d681SAndroid Build Coastguard Worker           new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
4756*9880d681SAndroid Build Coastguard Worker                                 SVI->getOperand(2), "", &*InsertPt);
4757*9880d681SAndroid Build Coastguard Worker     }
4758*9880d681SAndroid Build Coastguard Worker 
4759*9880d681SAndroid Build Coastguard Worker     UI->replaceUsesOfWith(SVI, InsertedShuffle);
4760*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
4761*9880d681SAndroid Build Coastguard Worker   }
4762*9880d681SAndroid Build Coastguard Worker 
4763*9880d681SAndroid Build Coastguard Worker   // If we removed all uses, nuke the shuffle.
4764*9880d681SAndroid Build Coastguard Worker   if (SVI->use_empty()) {
4765*9880d681SAndroid Build Coastguard Worker     SVI->eraseFromParent();
4766*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
4767*9880d681SAndroid Build Coastguard Worker   }
4768*9880d681SAndroid Build Coastguard Worker 
4769*9880d681SAndroid Build Coastguard Worker   return MadeChange;
4770*9880d681SAndroid Build Coastguard Worker }
4771*9880d681SAndroid Build Coastguard Worker 
optimizeSwitchInst(SwitchInst * SI)4772*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {
4773*9880d681SAndroid Build Coastguard Worker   if (!TLI || !DL)
4774*9880d681SAndroid Build Coastguard Worker     return false;
4775*9880d681SAndroid Build Coastguard Worker 
4776*9880d681SAndroid Build Coastguard Worker   Value *Cond = SI->getCondition();
4777*9880d681SAndroid Build Coastguard Worker   Type *OldType = Cond->getType();
4778*9880d681SAndroid Build Coastguard Worker   LLVMContext &Context = Cond->getContext();
4779*9880d681SAndroid Build Coastguard Worker   MVT RegType = TLI->getRegisterType(Context, TLI->getValueType(*DL, OldType));
4780*9880d681SAndroid Build Coastguard Worker   unsigned RegWidth = RegType.getSizeInBits();
4781*9880d681SAndroid Build Coastguard Worker 
4782*9880d681SAndroid Build Coastguard Worker   if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth())
4783*9880d681SAndroid Build Coastguard Worker     return false;
4784*9880d681SAndroid Build Coastguard Worker 
4785*9880d681SAndroid Build Coastguard Worker   // If the register width is greater than the type width, expand the condition
4786*9880d681SAndroid Build Coastguard Worker   // of the switch instruction and each case constant to the width of the
4787*9880d681SAndroid Build Coastguard Worker   // register. By widening the type of the switch condition, subsequent
4788*9880d681SAndroid Build Coastguard Worker   // comparisons (for case comparisons) will not need to be extended to the
4789*9880d681SAndroid Build Coastguard Worker   // preferred register width, so we will potentially eliminate N-1 extends,
4790*9880d681SAndroid Build Coastguard Worker   // where N is the number of cases in the switch.
4791*9880d681SAndroid Build Coastguard Worker   auto *NewType = Type::getIntNTy(Context, RegWidth);
4792*9880d681SAndroid Build Coastguard Worker 
4793*9880d681SAndroid Build Coastguard Worker   // Zero-extend the switch condition and case constants unless the switch
4794*9880d681SAndroid Build Coastguard Worker   // condition is a function argument that is already being sign-extended.
4795*9880d681SAndroid Build Coastguard Worker   // In that case, we can avoid an unnecessary mask/extension by sign-extending
4796*9880d681SAndroid Build Coastguard Worker   // everything instead.
4797*9880d681SAndroid Build Coastguard Worker   Instruction::CastOps ExtType = Instruction::ZExt;
4798*9880d681SAndroid Build Coastguard Worker   if (auto *Arg = dyn_cast<Argument>(Cond))
4799*9880d681SAndroid Build Coastguard Worker     if (Arg->hasSExtAttr())
4800*9880d681SAndroid Build Coastguard Worker       ExtType = Instruction::SExt;
4801*9880d681SAndroid Build Coastguard Worker 
4802*9880d681SAndroid Build Coastguard Worker   auto *ExtInst = CastInst::Create(ExtType, Cond, NewType);
4803*9880d681SAndroid Build Coastguard Worker   ExtInst->insertBefore(SI);
4804*9880d681SAndroid Build Coastguard Worker   SI->setCondition(ExtInst);
4805*9880d681SAndroid Build Coastguard Worker   for (SwitchInst::CaseIt Case : SI->cases()) {
4806*9880d681SAndroid Build Coastguard Worker     APInt NarrowConst = Case.getCaseValue()->getValue();
4807*9880d681SAndroid Build Coastguard Worker     APInt WideConst = (ExtType == Instruction::ZExt) ?
4808*9880d681SAndroid Build Coastguard Worker                       NarrowConst.zext(RegWidth) : NarrowConst.sext(RegWidth);
4809*9880d681SAndroid Build Coastguard Worker     Case.setValue(ConstantInt::get(Context, WideConst));
4810*9880d681SAndroid Build Coastguard Worker   }
4811*9880d681SAndroid Build Coastguard Worker 
4812*9880d681SAndroid Build Coastguard Worker   return true;
4813*9880d681SAndroid Build Coastguard Worker }
4814*9880d681SAndroid Build Coastguard Worker 
4815*9880d681SAndroid Build Coastguard Worker namespace {
4816*9880d681SAndroid Build Coastguard Worker /// \brief Helper class to promote a scalar operation to a vector one.
4817*9880d681SAndroid Build Coastguard Worker /// This class is used to move downward extractelement transition.
4818*9880d681SAndroid Build Coastguard Worker /// E.g.,
4819*9880d681SAndroid Build Coastguard Worker /// a = vector_op <2 x i32>
4820*9880d681SAndroid Build Coastguard Worker /// b = extractelement <2 x i32> a, i32 0
4821*9880d681SAndroid Build Coastguard Worker /// c = scalar_op b
4822*9880d681SAndroid Build Coastguard Worker /// store c
4823*9880d681SAndroid Build Coastguard Worker ///
4824*9880d681SAndroid Build Coastguard Worker /// =>
4825*9880d681SAndroid Build Coastguard Worker /// a = vector_op <2 x i32>
4826*9880d681SAndroid Build Coastguard Worker /// c = vector_op a (equivalent to scalar_op on the related lane)
4827*9880d681SAndroid Build Coastguard Worker /// * d = extractelement <2 x i32> c, i32 0
4828*9880d681SAndroid Build Coastguard Worker /// * store d
4829*9880d681SAndroid Build Coastguard Worker /// Assuming both extractelement and store can be combine, we get rid of the
4830*9880d681SAndroid Build Coastguard Worker /// transition.
4831*9880d681SAndroid Build Coastguard Worker class VectorPromoteHelper {
4832*9880d681SAndroid Build Coastguard Worker   /// DataLayout associated with the current module.
4833*9880d681SAndroid Build Coastguard Worker   const DataLayout &DL;
4834*9880d681SAndroid Build Coastguard Worker 
4835*9880d681SAndroid Build Coastguard Worker   /// Used to perform some checks on the legality of vector operations.
4836*9880d681SAndroid Build Coastguard Worker   const TargetLowering &TLI;
4837*9880d681SAndroid Build Coastguard Worker 
4838*9880d681SAndroid Build Coastguard Worker   /// Used to estimated the cost of the promoted chain.
4839*9880d681SAndroid Build Coastguard Worker   const TargetTransformInfo &TTI;
4840*9880d681SAndroid Build Coastguard Worker 
4841*9880d681SAndroid Build Coastguard Worker   /// The transition being moved downwards.
4842*9880d681SAndroid Build Coastguard Worker   Instruction *Transition;
4843*9880d681SAndroid Build Coastguard Worker   /// The sequence of instructions to be promoted.
4844*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction *, 4> InstsToBePromoted;
4845*9880d681SAndroid Build Coastguard Worker   /// Cost of combining a store and an extract.
4846*9880d681SAndroid Build Coastguard Worker   unsigned StoreExtractCombineCost;
4847*9880d681SAndroid Build Coastguard Worker   /// Instruction that will be combined with the transition.
4848*9880d681SAndroid Build Coastguard Worker   Instruction *CombineInst;
4849*9880d681SAndroid Build Coastguard Worker 
4850*9880d681SAndroid Build Coastguard Worker   /// \brief The instruction that represents the current end of the transition.
4851*9880d681SAndroid Build Coastguard Worker   /// Since we are faking the promotion until we reach the end of the chain
4852*9880d681SAndroid Build Coastguard Worker   /// of computation, we need a way to get the current end of the transition.
getEndOfTransition() const4853*9880d681SAndroid Build Coastguard Worker   Instruction *getEndOfTransition() const {
4854*9880d681SAndroid Build Coastguard Worker     if (InstsToBePromoted.empty())
4855*9880d681SAndroid Build Coastguard Worker       return Transition;
4856*9880d681SAndroid Build Coastguard Worker     return InstsToBePromoted.back();
4857*9880d681SAndroid Build Coastguard Worker   }
4858*9880d681SAndroid Build Coastguard Worker 
4859*9880d681SAndroid Build Coastguard Worker   /// \brief Return the index of the original value in the transition.
4860*9880d681SAndroid Build Coastguard Worker   /// E.g., for "extractelement <2 x i32> c, i32 1" the original value,
4861*9880d681SAndroid Build Coastguard Worker   /// c, is at index 0.
getTransitionOriginalValueIdx() const4862*9880d681SAndroid Build Coastguard Worker   unsigned getTransitionOriginalValueIdx() const {
4863*9880d681SAndroid Build Coastguard Worker     assert(isa<ExtractElementInst>(Transition) &&
4864*9880d681SAndroid Build Coastguard Worker            "Other kind of transitions are not supported yet");
4865*9880d681SAndroid Build Coastguard Worker     return 0;
4866*9880d681SAndroid Build Coastguard Worker   }
4867*9880d681SAndroid Build Coastguard Worker 
4868*9880d681SAndroid Build Coastguard Worker   /// \brief Return the index of the index in the transition.
4869*9880d681SAndroid Build Coastguard Worker   /// E.g., for "extractelement <2 x i32> c, i32 0" the index
4870*9880d681SAndroid Build Coastguard Worker   /// is at index 1.
getTransitionIdx() const4871*9880d681SAndroid Build Coastguard Worker   unsigned getTransitionIdx() const {
4872*9880d681SAndroid Build Coastguard Worker     assert(isa<ExtractElementInst>(Transition) &&
4873*9880d681SAndroid Build Coastguard Worker            "Other kind of transitions are not supported yet");
4874*9880d681SAndroid Build Coastguard Worker     return 1;
4875*9880d681SAndroid Build Coastguard Worker   }
4876*9880d681SAndroid Build Coastguard Worker 
4877*9880d681SAndroid Build Coastguard Worker   /// \brief Get the type of the transition.
4878*9880d681SAndroid Build Coastguard Worker   /// This is the type of the original value.
4879*9880d681SAndroid Build Coastguard Worker   /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the
4880*9880d681SAndroid Build Coastguard Worker   /// transition is <2 x i32>.
getTransitionType() const4881*9880d681SAndroid Build Coastguard Worker   Type *getTransitionType() const {
4882*9880d681SAndroid Build Coastguard Worker     return Transition->getOperand(getTransitionOriginalValueIdx())->getType();
4883*9880d681SAndroid Build Coastguard Worker   }
4884*9880d681SAndroid Build Coastguard Worker 
4885*9880d681SAndroid Build Coastguard Worker   /// \brief Promote \p ToBePromoted by moving \p Def downward through.
4886*9880d681SAndroid Build Coastguard Worker   /// I.e., we have the following sequence:
4887*9880d681SAndroid Build Coastguard Worker   /// Def = Transition <ty1> a to <ty2>
4888*9880d681SAndroid Build Coastguard Worker   /// b = ToBePromoted <ty2> Def, ...
4889*9880d681SAndroid Build Coastguard Worker   /// =>
4890*9880d681SAndroid Build Coastguard Worker   /// b = ToBePromoted <ty1> a, ...
4891*9880d681SAndroid Build Coastguard Worker   /// Def = Transition <ty1> ToBePromoted to <ty2>
4892*9880d681SAndroid Build Coastguard Worker   void promoteImpl(Instruction *ToBePromoted);
4893*9880d681SAndroid Build Coastguard Worker 
4894*9880d681SAndroid Build Coastguard Worker   /// \brief Check whether or not it is profitable to promote all the
4895*9880d681SAndroid Build Coastguard Worker   /// instructions enqueued to be promoted.
isProfitableToPromote()4896*9880d681SAndroid Build Coastguard Worker   bool isProfitableToPromote() {
4897*9880d681SAndroid Build Coastguard Worker     Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx());
4898*9880d681SAndroid Build Coastguard Worker     unsigned Index = isa<ConstantInt>(ValIdx)
4899*9880d681SAndroid Build Coastguard Worker                          ? cast<ConstantInt>(ValIdx)->getZExtValue()
4900*9880d681SAndroid Build Coastguard Worker                          : -1;
4901*9880d681SAndroid Build Coastguard Worker     Type *PromotedType = getTransitionType();
4902*9880d681SAndroid Build Coastguard Worker 
4903*9880d681SAndroid Build Coastguard Worker     StoreInst *ST = cast<StoreInst>(CombineInst);
4904*9880d681SAndroid Build Coastguard Worker     unsigned AS = ST->getPointerAddressSpace();
4905*9880d681SAndroid Build Coastguard Worker     unsigned Align = ST->getAlignment();
4906*9880d681SAndroid Build Coastguard Worker     // Check if this store is supported.
4907*9880d681SAndroid Build Coastguard Worker     if (!TLI.allowsMisalignedMemoryAccesses(
4908*9880d681SAndroid Build Coastguard Worker             TLI.getValueType(DL, ST->getValueOperand()->getType()), AS,
4909*9880d681SAndroid Build Coastguard Worker             Align)) {
4910*9880d681SAndroid Build Coastguard Worker       // If this is not supported, there is no way we can combine
4911*9880d681SAndroid Build Coastguard Worker       // the extract with the store.
4912*9880d681SAndroid Build Coastguard Worker       return false;
4913*9880d681SAndroid Build Coastguard Worker     }
4914*9880d681SAndroid Build Coastguard Worker 
4915*9880d681SAndroid Build Coastguard Worker     // The scalar chain of computation has to pay for the transition
4916*9880d681SAndroid Build Coastguard Worker     // scalar to vector.
4917*9880d681SAndroid Build Coastguard Worker     // The vector chain has to account for the combining cost.
4918*9880d681SAndroid Build Coastguard Worker     uint64_t ScalarCost =
4919*9880d681SAndroid Build Coastguard Worker         TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index);
4920*9880d681SAndroid Build Coastguard Worker     uint64_t VectorCost = StoreExtractCombineCost;
4921*9880d681SAndroid Build Coastguard Worker     for (const auto &Inst : InstsToBePromoted) {
4922*9880d681SAndroid Build Coastguard Worker       // Compute the cost.
4923*9880d681SAndroid Build Coastguard Worker       // By construction, all instructions being promoted are arithmetic ones.
4924*9880d681SAndroid Build Coastguard Worker       // Moreover, one argument is a constant that can be viewed as a splat
4925*9880d681SAndroid Build Coastguard Worker       // constant.
4926*9880d681SAndroid Build Coastguard Worker       Value *Arg0 = Inst->getOperand(0);
4927*9880d681SAndroid Build Coastguard Worker       bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) ||
4928*9880d681SAndroid Build Coastguard Worker                             isa<ConstantFP>(Arg0);
4929*9880d681SAndroid Build Coastguard Worker       TargetTransformInfo::OperandValueKind Arg0OVK =
4930*9880d681SAndroid Build Coastguard Worker           IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
4931*9880d681SAndroid Build Coastguard Worker                          : TargetTransformInfo::OK_AnyValue;
4932*9880d681SAndroid Build Coastguard Worker       TargetTransformInfo::OperandValueKind Arg1OVK =
4933*9880d681SAndroid Build Coastguard Worker           !IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
4934*9880d681SAndroid Build Coastguard Worker                           : TargetTransformInfo::OK_AnyValue;
4935*9880d681SAndroid Build Coastguard Worker       ScalarCost += TTI.getArithmeticInstrCost(
4936*9880d681SAndroid Build Coastguard Worker           Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK);
4937*9880d681SAndroid Build Coastguard Worker       VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType,
4938*9880d681SAndroid Build Coastguard Worker                                                Arg0OVK, Arg1OVK);
4939*9880d681SAndroid Build Coastguard Worker     }
4940*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Estimated cost of computation to be promoted:\nScalar: "
4941*9880d681SAndroid Build Coastguard Worker                  << ScalarCost << "\nVector: " << VectorCost << '\n');
4942*9880d681SAndroid Build Coastguard Worker     return ScalarCost > VectorCost;
4943*9880d681SAndroid Build Coastguard Worker   }
4944*9880d681SAndroid Build Coastguard Worker 
4945*9880d681SAndroid Build Coastguard Worker   /// \brief Generate a constant vector with \p Val with the same
4946*9880d681SAndroid Build Coastguard Worker   /// number of elements as the transition.
4947*9880d681SAndroid Build Coastguard Worker   /// \p UseSplat defines whether or not \p Val should be replicated
4948*9880d681SAndroid Build Coastguard Worker   /// across the whole vector.
4949*9880d681SAndroid Build Coastguard Worker   /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>,
4950*9880d681SAndroid Build Coastguard Worker   /// otherwise we generate a vector with as many undef as possible:
4951*9880d681SAndroid Build Coastguard Worker   /// <undef, ..., undef, Val, undef, ..., undef> where \p Val is only
4952*9880d681SAndroid Build Coastguard Worker   /// used at the index of the extract.
getConstantVector(Constant * Val,bool UseSplat) const4953*9880d681SAndroid Build Coastguard Worker   Value *getConstantVector(Constant *Val, bool UseSplat) const {
4954*9880d681SAndroid Build Coastguard Worker     unsigned ExtractIdx = UINT_MAX;
4955*9880d681SAndroid Build Coastguard Worker     if (!UseSplat) {
4956*9880d681SAndroid Build Coastguard Worker       // If we cannot determine where the constant must be, we have to
4957*9880d681SAndroid Build Coastguard Worker       // use a splat constant.
4958*9880d681SAndroid Build Coastguard Worker       Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());
4959*9880d681SAndroid Build Coastguard Worker       if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))
4960*9880d681SAndroid Build Coastguard Worker         ExtractIdx = CstVal->getSExtValue();
4961*9880d681SAndroid Build Coastguard Worker       else
4962*9880d681SAndroid Build Coastguard Worker         UseSplat = true;
4963*9880d681SAndroid Build Coastguard Worker     }
4964*9880d681SAndroid Build Coastguard Worker 
4965*9880d681SAndroid Build Coastguard Worker     unsigned End = getTransitionType()->getVectorNumElements();
4966*9880d681SAndroid Build Coastguard Worker     if (UseSplat)
4967*9880d681SAndroid Build Coastguard Worker       return ConstantVector::getSplat(End, Val);
4968*9880d681SAndroid Build Coastguard Worker 
4969*9880d681SAndroid Build Coastguard Worker     SmallVector<Constant *, 4> ConstVec;
4970*9880d681SAndroid Build Coastguard Worker     UndefValue *UndefVal = UndefValue::get(Val->getType());
4971*9880d681SAndroid Build Coastguard Worker     for (unsigned Idx = 0; Idx != End; ++Idx) {
4972*9880d681SAndroid Build Coastguard Worker       if (Idx == ExtractIdx)
4973*9880d681SAndroid Build Coastguard Worker         ConstVec.push_back(Val);
4974*9880d681SAndroid Build Coastguard Worker       else
4975*9880d681SAndroid Build Coastguard Worker         ConstVec.push_back(UndefVal);
4976*9880d681SAndroid Build Coastguard Worker     }
4977*9880d681SAndroid Build Coastguard Worker     return ConstantVector::get(ConstVec);
4978*9880d681SAndroid Build Coastguard Worker   }
4979*9880d681SAndroid Build Coastguard Worker 
4980*9880d681SAndroid Build Coastguard Worker   /// \brief Check if promoting to a vector type an operand at \p OperandIdx
4981*9880d681SAndroid Build Coastguard Worker   /// in \p Use can trigger undefined behavior.
canCauseUndefinedBehavior(const Instruction * Use,unsigned OperandIdx)4982*9880d681SAndroid Build Coastguard Worker   static bool canCauseUndefinedBehavior(const Instruction *Use,
4983*9880d681SAndroid Build Coastguard Worker                                         unsigned OperandIdx) {
4984*9880d681SAndroid Build Coastguard Worker     // This is not safe to introduce undef when the operand is on
4985*9880d681SAndroid Build Coastguard Worker     // the right hand side of a division-like instruction.
4986*9880d681SAndroid Build Coastguard Worker     if (OperandIdx != 1)
4987*9880d681SAndroid Build Coastguard Worker       return false;
4988*9880d681SAndroid Build Coastguard Worker     switch (Use->getOpcode()) {
4989*9880d681SAndroid Build Coastguard Worker     default:
4990*9880d681SAndroid Build Coastguard Worker       return false;
4991*9880d681SAndroid Build Coastguard Worker     case Instruction::SDiv:
4992*9880d681SAndroid Build Coastguard Worker     case Instruction::UDiv:
4993*9880d681SAndroid Build Coastguard Worker     case Instruction::SRem:
4994*9880d681SAndroid Build Coastguard Worker     case Instruction::URem:
4995*9880d681SAndroid Build Coastguard Worker       return true;
4996*9880d681SAndroid Build Coastguard Worker     case Instruction::FDiv:
4997*9880d681SAndroid Build Coastguard Worker     case Instruction::FRem:
4998*9880d681SAndroid Build Coastguard Worker       return !Use->hasNoNaNs();
4999*9880d681SAndroid Build Coastguard Worker     }
5000*9880d681SAndroid Build Coastguard Worker     llvm_unreachable(nullptr);
5001*9880d681SAndroid Build Coastguard Worker   }
5002*9880d681SAndroid Build Coastguard Worker 
5003*9880d681SAndroid Build Coastguard Worker public:
VectorPromoteHelper(const DataLayout & DL,const TargetLowering & TLI,const TargetTransformInfo & TTI,Instruction * Transition,unsigned CombineCost)5004*9880d681SAndroid Build Coastguard Worker   VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI,
5005*9880d681SAndroid Build Coastguard Worker                       const TargetTransformInfo &TTI, Instruction *Transition,
5006*9880d681SAndroid Build Coastguard Worker                       unsigned CombineCost)
5007*9880d681SAndroid Build Coastguard Worker       : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),
5008*9880d681SAndroid Build Coastguard Worker         StoreExtractCombineCost(CombineCost), CombineInst(nullptr) {
5009*9880d681SAndroid Build Coastguard Worker     assert(Transition && "Do not know how to promote null");
5010*9880d681SAndroid Build Coastguard Worker   }
5011*9880d681SAndroid Build Coastguard Worker 
5012*9880d681SAndroid Build Coastguard Worker   /// \brief Check if we can promote \p ToBePromoted to \p Type.
canPromote(const Instruction * ToBePromoted) const5013*9880d681SAndroid Build Coastguard Worker   bool canPromote(const Instruction *ToBePromoted) const {
5014*9880d681SAndroid Build Coastguard Worker     // We could support CastInst too.
5015*9880d681SAndroid Build Coastguard Worker     return isa<BinaryOperator>(ToBePromoted);
5016*9880d681SAndroid Build Coastguard Worker   }
5017*9880d681SAndroid Build Coastguard Worker 
5018*9880d681SAndroid Build Coastguard Worker   /// \brief Check if it is profitable to promote \p ToBePromoted
5019*9880d681SAndroid Build Coastguard Worker   /// by moving downward the transition through.
shouldPromote(const Instruction * ToBePromoted) const5020*9880d681SAndroid Build Coastguard Worker   bool shouldPromote(const Instruction *ToBePromoted) const {
5021*9880d681SAndroid Build Coastguard Worker     // Promote only if all the operands can be statically expanded.
5022*9880d681SAndroid Build Coastguard Worker     // Indeed, we do not want to introduce any new kind of transitions.
5023*9880d681SAndroid Build Coastguard Worker     for (const Use &U : ToBePromoted->operands()) {
5024*9880d681SAndroid Build Coastguard Worker       const Value *Val = U.get();
5025*9880d681SAndroid Build Coastguard Worker       if (Val == getEndOfTransition()) {
5026*9880d681SAndroid Build Coastguard Worker         // If the use is a division and the transition is on the rhs,
5027*9880d681SAndroid Build Coastguard Worker         // we cannot promote the operation, otherwise we may create a
5028*9880d681SAndroid Build Coastguard Worker         // division by zero.
5029*9880d681SAndroid Build Coastguard Worker         if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()))
5030*9880d681SAndroid Build Coastguard Worker           return false;
5031*9880d681SAndroid Build Coastguard Worker         continue;
5032*9880d681SAndroid Build Coastguard Worker       }
5033*9880d681SAndroid Build Coastguard Worker       if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) &&
5034*9880d681SAndroid Build Coastguard Worker           !isa<ConstantFP>(Val))
5035*9880d681SAndroid Build Coastguard Worker         return false;
5036*9880d681SAndroid Build Coastguard Worker     }
5037*9880d681SAndroid Build Coastguard Worker     // Check that the resulting operation is legal.
5038*9880d681SAndroid Build Coastguard Worker     int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode());
5039*9880d681SAndroid Build Coastguard Worker     if (!ISDOpcode)
5040*9880d681SAndroid Build Coastguard Worker       return false;
5041*9880d681SAndroid Build Coastguard Worker     return StressStoreExtract ||
5042*9880d681SAndroid Build Coastguard Worker            TLI.isOperationLegalOrCustom(
5043*9880d681SAndroid Build Coastguard Worker                ISDOpcode, TLI.getValueType(DL, getTransitionType(), true));
5044*9880d681SAndroid Build Coastguard Worker   }
5045*9880d681SAndroid Build Coastguard Worker 
5046*9880d681SAndroid Build Coastguard Worker   /// \brief Check whether or not \p Use can be combined
5047*9880d681SAndroid Build Coastguard Worker   /// with the transition.
5048*9880d681SAndroid Build Coastguard Worker   /// I.e., is it possible to do Use(Transition) => AnotherUse?
canCombine(const Instruction * Use)5049*9880d681SAndroid Build Coastguard Worker   bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); }
5050*9880d681SAndroid Build Coastguard Worker 
5051*9880d681SAndroid Build Coastguard Worker   /// \brief Record \p ToBePromoted as part of the chain to be promoted.
enqueueForPromotion(Instruction * ToBePromoted)5052*9880d681SAndroid Build Coastguard Worker   void enqueueForPromotion(Instruction *ToBePromoted) {
5053*9880d681SAndroid Build Coastguard Worker     InstsToBePromoted.push_back(ToBePromoted);
5054*9880d681SAndroid Build Coastguard Worker   }
5055*9880d681SAndroid Build Coastguard Worker 
5056*9880d681SAndroid Build Coastguard Worker   /// \brief Set the instruction that will be combined with the transition.
recordCombineInstruction(Instruction * ToBeCombined)5057*9880d681SAndroid Build Coastguard Worker   void recordCombineInstruction(Instruction *ToBeCombined) {
5058*9880d681SAndroid Build Coastguard Worker     assert(canCombine(ToBeCombined) && "Unsupported instruction to combine");
5059*9880d681SAndroid Build Coastguard Worker     CombineInst = ToBeCombined;
5060*9880d681SAndroid Build Coastguard Worker   }
5061*9880d681SAndroid Build Coastguard Worker 
5062*9880d681SAndroid Build Coastguard Worker   /// \brief Promote all the instructions enqueued for promotion if it is
5063*9880d681SAndroid Build Coastguard Worker   /// is profitable.
5064*9880d681SAndroid Build Coastguard Worker   /// \return True if the promotion happened, false otherwise.
promote()5065*9880d681SAndroid Build Coastguard Worker   bool promote() {
5066*9880d681SAndroid Build Coastguard Worker     // Check if there is something to promote.
5067*9880d681SAndroid Build Coastguard Worker     // Right now, if we do not have anything to combine with,
5068*9880d681SAndroid Build Coastguard Worker     // we assume the promotion is not profitable.
5069*9880d681SAndroid Build Coastguard Worker     if (InstsToBePromoted.empty() || !CombineInst)
5070*9880d681SAndroid Build Coastguard Worker       return false;
5071*9880d681SAndroid Build Coastguard Worker 
5072*9880d681SAndroid Build Coastguard Worker     // Check cost.
5073*9880d681SAndroid Build Coastguard Worker     if (!StressStoreExtract && !isProfitableToPromote())
5074*9880d681SAndroid Build Coastguard Worker       return false;
5075*9880d681SAndroid Build Coastguard Worker 
5076*9880d681SAndroid Build Coastguard Worker     // Promote.
5077*9880d681SAndroid Build Coastguard Worker     for (auto &ToBePromoted : InstsToBePromoted)
5078*9880d681SAndroid Build Coastguard Worker       promoteImpl(ToBePromoted);
5079*9880d681SAndroid Build Coastguard Worker     InstsToBePromoted.clear();
5080*9880d681SAndroid Build Coastguard Worker     return true;
5081*9880d681SAndroid Build Coastguard Worker   }
5082*9880d681SAndroid Build Coastguard Worker };
5083*9880d681SAndroid Build Coastguard Worker } // End of anonymous namespace.
5084*9880d681SAndroid Build Coastguard Worker 
promoteImpl(Instruction * ToBePromoted)5085*9880d681SAndroid Build Coastguard Worker void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) {
5086*9880d681SAndroid Build Coastguard Worker   // At this point, we know that all the operands of ToBePromoted but Def
5087*9880d681SAndroid Build Coastguard Worker   // can be statically promoted.
5088*9880d681SAndroid Build Coastguard Worker   // For Def, we need to use its parameter in ToBePromoted:
5089*9880d681SAndroid Build Coastguard Worker   // b = ToBePromoted ty1 a
5090*9880d681SAndroid Build Coastguard Worker   // Def = Transition ty1 b to ty2
5091*9880d681SAndroid Build Coastguard Worker   // Move the transition down.
5092*9880d681SAndroid Build Coastguard Worker   // 1. Replace all uses of the promoted operation by the transition.
5093*9880d681SAndroid Build Coastguard Worker   // = ... b => = ... Def.
5094*9880d681SAndroid Build Coastguard Worker   assert(ToBePromoted->getType() == Transition->getType() &&
5095*9880d681SAndroid Build Coastguard Worker          "The type of the result of the transition does not match "
5096*9880d681SAndroid Build Coastguard Worker          "the final type");
5097*9880d681SAndroid Build Coastguard Worker   ToBePromoted->replaceAllUsesWith(Transition);
5098*9880d681SAndroid Build Coastguard Worker   // 2. Update the type of the uses.
5099*9880d681SAndroid Build Coastguard Worker   // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def.
5100*9880d681SAndroid Build Coastguard Worker   Type *TransitionTy = getTransitionType();
5101*9880d681SAndroid Build Coastguard Worker   ToBePromoted->mutateType(TransitionTy);
5102*9880d681SAndroid Build Coastguard Worker   // 3. Update all the operands of the promoted operation with promoted
5103*9880d681SAndroid Build Coastguard Worker   // operands.
5104*9880d681SAndroid Build Coastguard Worker   // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a.
5105*9880d681SAndroid Build Coastguard Worker   for (Use &U : ToBePromoted->operands()) {
5106*9880d681SAndroid Build Coastguard Worker     Value *Val = U.get();
5107*9880d681SAndroid Build Coastguard Worker     Value *NewVal = nullptr;
5108*9880d681SAndroid Build Coastguard Worker     if (Val == Transition)
5109*9880d681SAndroid Build Coastguard Worker       NewVal = Transition->getOperand(getTransitionOriginalValueIdx());
5110*9880d681SAndroid Build Coastguard Worker     else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) ||
5111*9880d681SAndroid Build Coastguard Worker              isa<ConstantFP>(Val)) {
5112*9880d681SAndroid Build Coastguard Worker       // Use a splat constant if it is not safe to use undef.
5113*9880d681SAndroid Build Coastguard Worker       NewVal = getConstantVector(
5114*9880d681SAndroid Build Coastguard Worker           cast<Constant>(Val),
5115*9880d681SAndroid Build Coastguard Worker           isa<UndefValue>(Val) ||
5116*9880d681SAndroid Build Coastguard Worker               canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()));
5117*9880d681SAndroid Build Coastguard Worker     } else
5118*9880d681SAndroid Build Coastguard Worker       llvm_unreachable("Did you modified shouldPromote and forgot to update "
5119*9880d681SAndroid Build Coastguard Worker                        "this?");
5120*9880d681SAndroid Build Coastguard Worker     ToBePromoted->setOperand(U.getOperandNo(), NewVal);
5121*9880d681SAndroid Build Coastguard Worker   }
5122*9880d681SAndroid Build Coastguard Worker   Transition->removeFromParent();
5123*9880d681SAndroid Build Coastguard Worker   Transition->insertAfter(ToBePromoted);
5124*9880d681SAndroid Build Coastguard Worker   Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted);
5125*9880d681SAndroid Build Coastguard Worker }
5126*9880d681SAndroid Build Coastguard Worker 
5127*9880d681SAndroid Build Coastguard Worker /// Some targets can do store(extractelement) with one instruction.
5128*9880d681SAndroid Build Coastguard Worker /// Try to push the extractelement towards the stores when the target
5129*9880d681SAndroid Build Coastguard Worker /// has this feature and this is profitable.
optimizeExtractElementInst(Instruction * Inst)5130*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) {
5131*9880d681SAndroid Build Coastguard Worker   unsigned CombineCost = UINT_MAX;
5132*9880d681SAndroid Build Coastguard Worker   if (DisableStoreExtract || !TLI ||
5133*9880d681SAndroid Build Coastguard Worker       (!StressStoreExtract &&
5134*9880d681SAndroid Build Coastguard Worker        !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(),
5135*9880d681SAndroid Build Coastguard Worker                                        Inst->getOperand(1), CombineCost)))
5136*9880d681SAndroid Build Coastguard Worker     return false;
5137*9880d681SAndroid Build Coastguard Worker 
5138*9880d681SAndroid Build Coastguard Worker   // At this point we know that Inst is a vector to scalar transition.
5139*9880d681SAndroid Build Coastguard Worker   // Try to move it down the def-use chain, until:
5140*9880d681SAndroid Build Coastguard Worker   // - We can combine the transition with its single use
5141*9880d681SAndroid Build Coastguard Worker   //   => we got rid of the transition.
5142*9880d681SAndroid Build Coastguard Worker   // - We escape the current basic block
5143*9880d681SAndroid Build Coastguard Worker   //   => we would need to check that we are moving it at a cheaper place and
5144*9880d681SAndroid Build Coastguard Worker   //      we do not do that for now.
5145*9880d681SAndroid Build Coastguard Worker   BasicBlock *Parent = Inst->getParent();
5146*9880d681SAndroid Build Coastguard Worker   DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n');
5147*9880d681SAndroid Build Coastguard Worker   VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost);
5148*9880d681SAndroid Build Coastguard Worker   // If the transition has more than one use, assume this is not going to be
5149*9880d681SAndroid Build Coastguard Worker   // beneficial.
5150*9880d681SAndroid Build Coastguard Worker   while (Inst->hasOneUse()) {
5151*9880d681SAndroid Build Coastguard Worker     Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin());
5152*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n');
5153*9880d681SAndroid Build Coastguard Worker 
5154*9880d681SAndroid Build Coastguard Worker     if (ToBePromoted->getParent() != Parent) {
5155*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Instruction to promote is in a different block ("
5156*9880d681SAndroid Build Coastguard Worker                    << ToBePromoted->getParent()->getName()
5157*9880d681SAndroid Build Coastguard Worker                    << ") than the transition (" << Parent->getName() << ").\n");
5158*9880d681SAndroid Build Coastguard Worker       return false;
5159*9880d681SAndroid Build Coastguard Worker     }
5160*9880d681SAndroid Build Coastguard Worker 
5161*9880d681SAndroid Build Coastguard Worker     if (VPH.canCombine(ToBePromoted)) {
5162*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "Assume " << *Inst << '\n'
5163*9880d681SAndroid Build Coastguard Worker                    << "will be combined with: " << *ToBePromoted << '\n');
5164*9880d681SAndroid Build Coastguard Worker       VPH.recordCombineInstruction(ToBePromoted);
5165*9880d681SAndroid Build Coastguard Worker       bool Changed = VPH.promote();
5166*9880d681SAndroid Build Coastguard Worker       NumStoreExtractExposed += Changed;
5167*9880d681SAndroid Build Coastguard Worker       return Changed;
5168*9880d681SAndroid Build Coastguard Worker     }
5169*9880d681SAndroid Build Coastguard Worker 
5170*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Try promoting.\n");
5171*9880d681SAndroid Build Coastguard Worker     if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))
5172*9880d681SAndroid Build Coastguard Worker       return false;
5173*9880d681SAndroid Build Coastguard Worker 
5174*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n");
5175*9880d681SAndroid Build Coastguard Worker 
5176*9880d681SAndroid Build Coastguard Worker     VPH.enqueueForPromotion(ToBePromoted);
5177*9880d681SAndroid Build Coastguard Worker     Inst = ToBePromoted;
5178*9880d681SAndroid Build Coastguard Worker   }
5179*9880d681SAndroid Build Coastguard Worker   return false;
5180*9880d681SAndroid Build Coastguard Worker }
5181*9880d681SAndroid Build Coastguard Worker 
optimizeInst(Instruction * I,bool & ModifiedDT)5182*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeInst(Instruction *I, bool& ModifiedDT) {
5183*9880d681SAndroid Build Coastguard Worker   // Bail out if we inserted the instruction to prevent optimizations from
5184*9880d681SAndroid Build Coastguard Worker   // stepping on each other's toes.
5185*9880d681SAndroid Build Coastguard Worker   if (InsertedInsts.count(I))
5186*9880d681SAndroid Build Coastguard Worker     return false;
5187*9880d681SAndroid Build Coastguard Worker 
5188*9880d681SAndroid Build Coastguard Worker   if (PHINode *P = dyn_cast<PHINode>(I)) {
5189*9880d681SAndroid Build Coastguard Worker     // It is possible for very late stage optimizations (such as SimplifyCFG)
5190*9880d681SAndroid Build Coastguard Worker     // to introduce PHI nodes too late to be cleaned up.  If we detect such a
5191*9880d681SAndroid Build Coastguard Worker     // trivial PHI, go ahead and zap it here.
5192*9880d681SAndroid Build Coastguard Worker     if (Value *V = SimplifyInstruction(P, *DL, TLInfo, nullptr)) {
5193*9880d681SAndroid Build Coastguard Worker       P->replaceAllUsesWith(V);
5194*9880d681SAndroid Build Coastguard Worker       P->eraseFromParent();
5195*9880d681SAndroid Build Coastguard Worker       ++NumPHIsElim;
5196*9880d681SAndroid Build Coastguard Worker       return true;
5197*9880d681SAndroid Build Coastguard Worker     }
5198*9880d681SAndroid Build Coastguard Worker     return false;
5199*9880d681SAndroid Build Coastguard Worker   }
5200*9880d681SAndroid Build Coastguard Worker 
5201*9880d681SAndroid Build Coastguard Worker   if (CastInst *CI = dyn_cast<CastInst>(I)) {
5202*9880d681SAndroid Build Coastguard Worker     // If the source of the cast is a constant, then this should have
5203*9880d681SAndroid Build Coastguard Worker     // already been constant folded.  The only reason NOT to constant fold
5204*9880d681SAndroid Build Coastguard Worker     // it is if something (e.g. LSR) was careful to place the constant
5205*9880d681SAndroid Build Coastguard Worker     // evaluation in a block other than then one that uses it (e.g. to hoist
5206*9880d681SAndroid Build Coastguard Worker     // the address of globals out of a loop).  If this is the case, we don't
5207*9880d681SAndroid Build Coastguard Worker     // want to forward-subst the cast.
5208*9880d681SAndroid Build Coastguard Worker     if (isa<Constant>(CI->getOperand(0)))
5209*9880d681SAndroid Build Coastguard Worker       return false;
5210*9880d681SAndroid Build Coastguard Worker 
5211*9880d681SAndroid Build Coastguard Worker     if (TLI && OptimizeNoopCopyExpression(CI, *TLI, *DL))
5212*9880d681SAndroid Build Coastguard Worker       return true;
5213*9880d681SAndroid Build Coastguard Worker 
5214*9880d681SAndroid Build Coastguard Worker     if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {
5215*9880d681SAndroid Build Coastguard Worker       /// Sink a zext or sext into its user blocks if the target type doesn't
5216*9880d681SAndroid Build Coastguard Worker       /// fit in one register
5217*9880d681SAndroid Build Coastguard Worker       if (TLI &&
5218*9880d681SAndroid Build Coastguard Worker           TLI->getTypeAction(CI->getContext(),
5219*9880d681SAndroid Build Coastguard Worker                              TLI->getValueType(*DL, CI->getType())) ==
5220*9880d681SAndroid Build Coastguard Worker               TargetLowering::TypeExpandInteger) {
5221*9880d681SAndroid Build Coastguard Worker         return SinkCast(CI);
5222*9880d681SAndroid Build Coastguard Worker       } else {
5223*9880d681SAndroid Build Coastguard Worker         bool MadeChange = moveExtToFormExtLoad(I);
5224*9880d681SAndroid Build Coastguard Worker         return MadeChange | optimizeExtUses(I);
5225*9880d681SAndroid Build Coastguard Worker       }
5226*9880d681SAndroid Build Coastguard Worker     }
5227*9880d681SAndroid Build Coastguard Worker     return false;
5228*9880d681SAndroid Build Coastguard Worker   }
5229*9880d681SAndroid Build Coastguard Worker 
5230*9880d681SAndroid Build Coastguard Worker   if (CmpInst *CI = dyn_cast<CmpInst>(I))
5231*9880d681SAndroid Build Coastguard Worker     if (!TLI || !TLI->hasMultipleConditionRegisters())
5232*9880d681SAndroid Build Coastguard Worker       return OptimizeCmpExpression(CI, TLI);
5233*9880d681SAndroid Build Coastguard Worker 
5234*9880d681SAndroid Build Coastguard Worker   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
5235*9880d681SAndroid Build Coastguard Worker     stripInvariantGroupMetadata(*LI);
5236*9880d681SAndroid Build Coastguard Worker     if (TLI) {
5237*9880d681SAndroid Build Coastguard Worker       bool Modified = optimizeLoadExt(LI);
5238*9880d681SAndroid Build Coastguard Worker       unsigned AS = LI->getPointerAddressSpace();
5239*9880d681SAndroid Build Coastguard Worker       Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS);
5240*9880d681SAndroid Build Coastguard Worker       return Modified;
5241*9880d681SAndroid Build Coastguard Worker     }
5242*9880d681SAndroid Build Coastguard Worker     return false;
5243*9880d681SAndroid Build Coastguard Worker   }
5244*9880d681SAndroid Build Coastguard Worker 
5245*9880d681SAndroid Build Coastguard Worker   if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
5246*9880d681SAndroid Build Coastguard Worker     stripInvariantGroupMetadata(*SI);
5247*9880d681SAndroid Build Coastguard Worker     if (TLI) {
5248*9880d681SAndroid Build Coastguard Worker       unsigned AS = SI->getPointerAddressSpace();
5249*9880d681SAndroid Build Coastguard Worker       return optimizeMemoryInst(I, SI->getOperand(1),
5250*9880d681SAndroid Build Coastguard Worker                                 SI->getOperand(0)->getType(), AS);
5251*9880d681SAndroid Build Coastguard Worker     }
5252*9880d681SAndroid Build Coastguard Worker     return false;
5253*9880d681SAndroid Build Coastguard Worker   }
5254*9880d681SAndroid Build Coastguard Worker 
5255*9880d681SAndroid Build Coastguard Worker   BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I);
5256*9880d681SAndroid Build Coastguard Worker 
5257*9880d681SAndroid Build Coastguard Worker   if (BinOp && (BinOp->getOpcode() == Instruction::AShr ||
5258*9880d681SAndroid Build Coastguard Worker                 BinOp->getOpcode() == Instruction::LShr)) {
5259*9880d681SAndroid Build Coastguard Worker     ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1));
5260*9880d681SAndroid Build Coastguard Worker     if (TLI && CI && TLI->hasExtractBitsInsn())
5261*9880d681SAndroid Build Coastguard Worker       return OptimizeExtractBits(BinOp, CI, *TLI, *DL);
5262*9880d681SAndroid Build Coastguard Worker 
5263*9880d681SAndroid Build Coastguard Worker     return false;
5264*9880d681SAndroid Build Coastguard Worker   }
5265*9880d681SAndroid Build Coastguard Worker 
5266*9880d681SAndroid Build Coastguard Worker   if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
5267*9880d681SAndroid Build Coastguard Worker     if (GEPI->hasAllZeroIndices()) {
5268*9880d681SAndroid Build Coastguard Worker       /// The GEP operand must be a pointer, so must its result -> BitCast
5269*9880d681SAndroid Build Coastguard Worker       Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(),
5270*9880d681SAndroid Build Coastguard Worker                                         GEPI->getName(), GEPI);
5271*9880d681SAndroid Build Coastguard Worker       GEPI->replaceAllUsesWith(NC);
5272*9880d681SAndroid Build Coastguard Worker       GEPI->eraseFromParent();
5273*9880d681SAndroid Build Coastguard Worker       ++NumGEPsElim;
5274*9880d681SAndroid Build Coastguard Worker       optimizeInst(NC, ModifiedDT);
5275*9880d681SAndroid Build Coastguard Worker       return true;
5276*9880d681SAndroid Build Coastguard Worker     }
5277*9880d681SAndroid Build Coastguard Worker     return false;
5278*9880d681SAndroid Build Coastguard Worker   }
5279*9880d681SAndroid Build Coastguard Worker 
5280*9880d681SAndroid Build Coastguard Worker   if (CallInst *CI = dyn_cast<CallInst>(I))
5281*9880d681SAndroid Build Coastguard Worker     return optimizeCallInst(CI, ModifiedDT);
5282*9880d681SAndroid Build Coastguard Worker 
5283*9880d681SAndroid Build Coastguard Worker   if (SelectInst *SI = dyn_cast<SelectInst>(I))
5284*9880d681SAndroid Build Coastguard Worker     return optimizeSelectInst(SI);
5285*9880d681SAndroid Build Coastguard Worker 
5286*9880d681SAndroid Build Coastguard Worker   if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
5287*9880d681SAndroid Build Coastguard Worker     return optimizeShuffleVectorInst(SVI);
5288*9880d681SAndroid Build Coastguard Worker 
5289*9880d681SAndroid Build Coastguard Worker   if (auto *Switch = dyn_cast<SwitchInst>(I))
5290*9880d681SAndroid Build Coastguard Worker     return optimizeSwitchInst(Switch);
5291*9880d681SAndroid Build Coastguard Worker 
5292*9880d681SAndroid Build Coastguard Worker   if (isa<ExtractElementInst>(I))
5293*9880d681SAndroid Build Coastguard Worker     return optimizeExtractElementInst(I);
5294*9880d681SAndroid Build Coastguard Worker 
5295*9880d681SAndroid Build Coastguard Worker   return false;
5296*9880d681SAndroid Build Coastguard Worker }
5297*9880d681SAndroid Build Coastguard Worker 
5298*9880d681SAndroid Build Coastguard Worker /// Given an OR instruction, check to see if this is a bitreverse
5299*9880d681SAndroid Build Coastguard Worker /// idiom. If so, insert the new intrinsic and return true.
makeBitReverse(Instruction & I,const DataLayout & DL,const TargetLowering & TLI)5300*9880d681SAndroid Build Coastguard Worker static bool makeBitReverse(Instruction &I, const DataLayout &DL,
5301*9880d681SAndroid Build Coastguard Worker                            const TargetLowering &TLI) {
5302*9880d681SAndroid Build Coastguard Worker   if (!I.getType()->isIntegerTy() ||
5303*9880d681SAndroid Build Coastguard Worker       !TLI.isOperationLegalOrCustom(ISD::BITREVERSE,
5304*9880d681SAndroid Build Coastguard Worker                                     TLI.getValueType(DL, I.getType(), true)))
5305*9880d681SAndroid Build Coastguard Worker     return false;
5306*9880d681SAndroid Build Coastguard Worker 
5307*9880d681SAndroid Build Coastguard Worker   SmallVector<Instruction*, 4> Insts;
5308*9880d681SAndroid Build Coastguard Worker   if (!recognizeBSwapOrBitReverseIdiom(&I, false, true, Insts))
5309*9880d681SAndroid Build Coastguard Worker     return false;
5310*9880d681SAndroid Build Coastguard Worker   Instruction *LastInst = Insts.back();
5311*9880d681SAndroid Build Coastguard Worker   I.replaceAllUsesWith(LastInst);
5312*9880d681SAndroid Build Coastguard Worker   RecursivelyDeleteTriviallyDeadInstructions(&I);
5313*9880d681SAndroid Build Coastguard Worker   return true;
5314*9880d681SAndroid Build Coastguard Worker }
5315*9880d681SAndroid Build Coastguard Worker 
5316*9880d681SAndroid Build Coastguard Worker // In this pass we look for GEP and cast instructions that are used
5317*9880d681SAndroid Build Coastguard Worker // across basic blocks and rewrite them to improve basic-block-at-a-time
5318*9880d681SAndroid Build Coastguard Worker // selection.
optimizeBlock(BasicBlock & BB,bool & ModifiedDT)5319*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, bool& ModifiedDT) {
5320*9880d681SAndroid Build Coastguard Worker   SunkAddrs.clear();
5321*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
5322*9880d681SAndroid Build Coastguard Worker 
5323*9880d681SAndroid Build Coastguard Worker   CurInstIterator = BB.begin();
5324*9880d681SAndroid Build Coastguard Worker   while (CurInstIterator != BB.end()) {
5325*9880d681SAndroid Build Coastguard Worker     MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);
5326*9880d681SAndroid Build Coastguard Worker     if (ModifiedDT)
5327*9880d681SAndroid Build Coastguard Worker       return true;
5328*9880d681SAndroid Build Coastguard Worker   }
5329*9880d681SAndroid Build Coastguard Worker 
5330*9880d681SAndroid Build Coastguard Worker   bool MadeBitReverse = true;
5331*9880d681SAndroid Build Coastguard Worker   while (TLI && MadeBitReverse) {
5332*9880d681SAndroid Build Coastguard Worker     MadeBitReverse = false;
5333*9880d681SAndroid Build Coastguard Worker     for (auto &I : reverse(BB)) {
5334*9880d681SAndroid Build Coastguard Worker       if (makeBitReverse(I, *DL, *TLI)) {
5335*9880d681SAndroid Build Coastguard Worker         MadeBitReverse = MadeChange = true;
5336*9880d681SAndroid Build Coastguard Worker         ModifiedDT = true;
5337*9880d681SAndroid Build Coastguard Worker         break;
5338*9880d681SAndroid Build Coastguard Worker       }
5339*9880d681SAndroid Build Coastguard Worker     }
5340*9880d681SAndroid Build Coastguard Worker   }
5341*9880d681SAndroid Build Coastguard Worker   MadeChange |= dupRetToEnableTailCallOpts(&BB);
5342*9880d681SAndroid Build Coastguard Worker 
5343*9880d681SAndroid Build Coastguard Worker   return MadeChange;
5344*9880d681SAndroid Build Coastguard Worker }
5345*9880d681SAndroid Build Coastguard Worker 
5346*9880d681SAndroid Build Coastguard Worker // llvm.dbg.value is far away from the value then iSel may not be able
5347*9880d681SAndroid Build Coastguard Worker // handle it properly. iSel will drop llvm.dbg.value if it can not
5348*9880d681SAndroid Build Coastguard Worker // find a node corresponding to the value.
placeDbgValues(Function & F)5349*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::placeDbgValues(Function &F) {
5350*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
5351*9880d681SAndroid Build Coastguard Worker   for (BasicBlock &BB : F) {
5352*9880d681SAndroid Build Coastguard Worker     Instruction *PrevNonDbgInst = nullptr;
5353*9880d681SAndroid Build Coastguard Worker     for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) {
5354*9880d681SAndroid Build Coastguard Worker       Instruction *Insn = &*BI++;
5355*9880d681SAndroid Build Coastguard Worker       DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn);
5356*9880d681SAndroid Build Coastguard Worker       // Leave dbg.values that refer to an alloca alone. These
5357*9880d681SAndroid Build Coastguard Worker       // instrinsics describe the address of a variable (= the alloca)
5358*9880d681SAndroid Build Coastguard Worker       // being taken.  They should not be moved next to the alloca
5359*9880d681SAndroid Build Coastguard Worker       // (and to the beginning of the scope), but rather stay close to
5360*9880d681SAndroid Build Coastguard Worker       // where said address is used.
5361*9880d681SAndroid Build Coastguard Worker       if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) {
5362*9880d681SAndroid Build Coastguard Worker         PrevNonDbgInst = Insn;
5363*9880d681SAndroid Build Coastguard Worker         continue;
5364*9880d681SAndroid Build Coastguard Worker       }
5365*9880d681SAndroid Build Coastguard Worker 
5366*9880d681SAndroid Build Coastguard Worker       Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue());
5367*9880d681SAndroid Build Coastguard Worker       if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) {
5368*9880d681SAndroid Build Coastguard Worker         // If VI is a phi in a block with an EHPad terminator, we can't insert
5369*9880d681SAndroid Build Coastguard Worker         // after it.
5370*9880d681SAndroid Build Coastguard Worker         if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad())
5371*9880d681SAndroid Build Coastguard Worker           continue;
5372*9880d681SAndroid Build Coastguard Worker         DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI);
5373*9880d681SAndroid Build Coastguard Worker         DVI->removeFromParent();
5374*9880d681SAndroid Build Coastguard Worker         if (isa<PHINode>(VI))
5375*9880d681SAndroid Build Coastguard Worker           DVI->insertBefore(&*VI->getParent()->getFirstInsertionPt());
5376*9880d681SAndroid Build Coastguard Worker         else
5377*9880d681SAndroid Build Coastguard Worker           DVI->insertAfter(VI);
5378*9880d681SAndroid Build Coastguard Worker         MadeChange = true;
5379*9880d681SAndroid Build Coastguard Worker         ++NumDbgValueMoved;
5380*9880d681SAndroid Build Coastguard Worker       }
5381*9880d681SAndroid Build Coastguard Worker     }
5382*9880d681SAndroid Build Coastguard Worker   }
5383*9880d681SAndroid Build Coastguard Worker   return MadeChange;
5384*9880d681SAndroid Build Coastguard Worker }
5385*9880d681SAndroid Build Coastguard Worker 
5386*9880d681SAndroid Build Coastguard Worker // If there is a sequence that branches based on comparing a single bit
5387*9880d681SAndroid Build Coastguard Worker // against zero that can be combined into a single instruction, and the
5388*9880d681SAndroid Build Coastguard Worker // target supports folding these into a single instruction, sink the
5389*9880d681SAndroid Build Coastguard Worker // mask and compare into the branch uses. Do this before OptimizeBlock ->
5390*9880d681SAndroid Build Coastguard Worker // OptimizeInst -> OptimizeCmpExpression, which perturbs the pattern being
5391*9880d681SAndroid Build Coastguard Worker // searched for.
sinkAndCmp(Function & F)5392*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::sinkAndCmp(Function &F) {
5393*9880d681SAndroid Build Coastguard Worker   if (!EnableAndCmpSinking)
5394*9880d681SAndroid Build Coastguard Worker     return false;
5395*9880d681SAndroid Build Coastguard Worker   if (!TLI || !TLI->isMaskAndBranchFoldingLegal())
5396*9880d681SAndroid Build Coastguard Worker     return false;
5397*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
5398*9880d681SAndroid Build Coastguard Worker   for (BasicBlock &BB : F) {
5399*9880d681SAndroid Build Coastguard Worker     // Does this BB end with the following?
5400*9880d681SAndroid Build Coastguard Worker     //   %andVal = and %val, #single-bit-set
5401*9880d681SAndroid Build Coastguard Worker     //   %icmpVal = icmp %andResult, 0
5402*9880d681SAndroid Build Coastguard Worker     //   br i1 %cmpVal label %dest1, label %dest2"
5403*9880d681SAndroid Build Coastguard Worker     BranchInst *Brcc = dyn_cast<BranchInst>(BB.getTerminator());
5404*9880d681SAndroid Build Coastguard Worker     if (!Brcc || !Brcc->isConditional())
5405*9880d681SAndroid Build Coastguard Worker       continue;
5406*9880d681SAndroid Build Coastguard Worker     ICmpInst *Cmp = dyn_cast<ICmpInst>(Brcc->getOperand(0));
5407*9880d681SAndroid Build Coastguard Worker     if (!Cmp || Cmp->getParent() != &BB)
5408*9880d681SAndroid Build Coastguard Worker       continue;
5409*9880d681SAndroid Build Coastguard Worker     ConstantInt *Zero = dyn_cast<ConstantInt>(Cmp->getOperand(1));
5410*9880d681SAndroid Build Coastguard Worker     if (!Zero || !Zero->isZero())
5411*9880d681SAndroid Build Coastguard Worker       continue;
5412*9880d681SAndroid Build Coastguard Worker     Instruction *And = dyn_cast<Instruction>(Cmp->getOperand(0));
5413*9880d681SAndroid Build Coastguard Worker     if (!And || And->getOpcode() != Instruction::And || And->getParent() != &BB)
5414*9880d681SAndroid Build Coastguard Worker       continue;
5415*9880d681SAndroid Build Coastguard Worker     ConstantInt* Mask = dyn_cast<ConstantInt>(And->getOperand(1));
5416*9880d681SAndroid Build Coastguard Worker     if (!Mask || !Mask->getUniqueInteger().isPowerOf2())
5417*9880d681SAndroid Build Coastguard Worker       continue;
5418*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "found and; icmp ?,0; brcc\n"); DEBUG(BB.dump());
5419*9880d681SAndroid Build Coastguard Worker 
5420*9880d681SAndroid Build Coastguard Worker     // Push the "and; icmp" for any users that are conditional branches.
5421*9880d681SAndroid Build Coastguard Worker     // Since there can only be one branch use per BB, we don't need to keep
5422*9880d681SAndroid Build Coastguard Worker     // track of which BBs we insert into.
5423*9880d681SAndroid Build Coastguard Worker     for (Use &TheUse : Cmp->uses()) {
5424*9880d681SAndroid Build Coastguard Worker       // Find brcc use.
5425*9880d681SAndroid Build Coastguard Worker       BranchInst *BrccUser = dyn_cast<BranchInst>(TheUse);
5426*9880d681SAndroid Build Coastguard Worker       if (!BrccUser || !BrccUser->isConditional())
5427*9880d681SAndroid Build Coastguard Worker         continue;
5428*9880d681SAndroid Build Coastguard Worker       BasicBlock *UserBB = BrccUser->getParent();
5429*9880d681SAndroid Build Coastguard Worker       if (UserBB == &BB) continue;
5430*9880d681SAndroid Build Coastguard Worker       DEBUG(dbgs() << "found Brcc use\n");
5431*9880d681SAndroid Build Coastguard Worker 
5432*9880d681SAndroid Build Coastguard Worker       // Sink the "and; icmp" to use.
5433*9880d681SAndroid Build Coastguard Worker       MadeChange = true;
5434*9880d681SAndroid Build Coastguard Worker       BinaryOperator *NewAnd =
5435*9880d681SAndroid Build Coastguard Worker         BinaryOperator::CreateAnd(And->getOperand(0), And->getOperand(1), "",
5436*9880d681SAndroid Build Coastguard Worker                                   BrccUser);
5437*9880d681SAndroid Build Coastguard Worker       CmpInst *NewCmp =
5438*9880d681SAndroid Build Coastguard Worker         CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), NewAnd, Zero,
5439*9880d681SAndroid Build Coastguard Worker                         "", BrccUser);
5440*9880d681SAndroid Build Coastguard Worker       TheUse = NewCmp;
5441*9880d681SAndroid Build Coastguard Worker       ++NumAndCmpsMoved;
5442*9880d681SAndroid Build Coastguard Worker       DEBUG(BrccUser->getParent()->dump());
5443*9880d681SAndroid Build Coastguard Worker     }
5444*9880d681SAndroid Build Coastguard Worker   }
5445*9880d681SAndroid Build Coastguard Worker   return MadeChange;
5446*9880d681SAndroid Build Coastguard Worker }
5447*9880d681SAndroid Build Coastguard Worker 
5448*9880d681SAndroid Build Coastguard Worker /// \brief Scale down both weights to fit into uint32_t.
scaleWeights(uint64_t & NewTrue,uint64_t & NewFalse)5449*9880d681SAndroid Build Coastguard Worker static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
5450*9880d681SAndroid Build Coastguard Worker   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
5451*9880d681SAndroid Build Coastguard Worker   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
5452*9880d681SAndroid Build Coastguard Worker   NewTrue = NewTrue / Scale;
5453*9880d681SAndroid Build Coastguard Worker   NewFalse = NewFalse / Scale;
5454*9880d681SAndroid Build Coastguard Worker }
5455*9880d681SAndroid Build Coastguard Worker 
5456*9880d681SAndroid Build Coastguard Worker /// \brief Some targets prefer to split a conditional branch like:
5457*9880d681SAndroid Build Coastguard Worker /// \code
5458*9880d681SAndroid Build Coastguard Worker ///   %0 = icmp ne i32 %a, 0
5459*9880d681SAndroid Build Coastguard Worker ///   %1 = icmp ne i32 %b, 0
5460*9880d681SAndroid Build Coastguard Worker ///   %or.cond = or i1 %0, %1
5461*9880d681SAndroid Build Coastguard Worker ///   br i1 %or.cond, label %TrueBB, label %FalseBB
5462*9880d681SAndroid Build Coastguard Worker /// \endcode
5463*9880d681SAndroid Build Coastguard Worker /// into multiple branch instructions like:
5464*9880d681SAndroid Build Coastguard Worker /// \code
5465*9880d681SAndroid Build Coastguard Worker ///   bb1:
5466*9880d681SAndroid Build Coastguard Worker ///     %0 = icmp ne i32 %a, 0
5467*9880d681SAndroid Build Coastguard Worker ///     br i1 %0, label %TrueBB, label %bb2
5468*9880d681SAndroid Build Coastguard Worker ///   bb2:
5469*9880d681SAndroid Build Coastguard Worker ///     %1 = icmp ne i32 %b, 0
5470*9880d681SAndroid Build Coastguard Worker ///     br i1 %1, label %TrueBB, label %FalseBB
5471*9880d681SAndroid Build Coastguard Worker /// \endcode
5472*9880d681SAndroid Build Coastguard Worker /// This usually allows instruction selection to do even further optimizations
5473*9880d681SAndroid Build Coastguard Worker /// and combine the compare with the branch instruction. Currently this is
5474*9880d681SAndroid Build Coastguard Worker /// applied for targets which have "cheap" jump instructions.
5475*9880d681SAndroid Build Coastguard Worker ///
5476*9880d681SAndroid Build Coastguard Worker /// FIXME: Remove the (equivalent?) implementation in SelectionDAG.
5477*9880d681SAndroid Build Coastguard Worker ///
splitBranchCondition(Function & F)5478*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::splitBranchCondition(Function &F) {
5479*9880d681SAndroid Build Coastguard Worker   if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive())
5480*9880d681SAndroid Build Coastguard Worker     return false;
5481*9880d681SAndroid Build Coastguard Worker 
5482*9880d681SAndroid Build Coastguard Worker   bool MadeChange = false;
5483*9880d681SAndroid Build Coastguard Worker   for (auto &BB : F) {
5484*9880d681SAndroid Build Coastguard Worker     // Does this BB end with the following?
5485*9880d681SAndroid Build Coastguard Worker     //   %cond1 = icmp|fcmp|binary instruction ...
5486*9880d681SAndroid Build Coastguard Worker     //   %cond2 = icmp|fcmp|binary instruction ...
5487*9880d681SAndroid Build Coastguard Worker     //   %cond.or = or|and i1 %cond1, cond2
5488*9880d681SAndroid Build Coastguard Worker     //   br i1 %cond.or label %dest1, label %dest2"
5489*9880d681SAndroid Build Coastguard Worker     BinaryOperator *LogicOp;
5490*9880d681SAndroid Build Coastguard Worker     BasicBlock *TBB, *FBB;
5491*9880d681SAndroid Build Coastguard Worker     if (!match(BB.getTerminator(), m_Br(m_OneUse(m_BinOp(LogicOp)), TBB, FBB)))
5492*9880d681SAndroid Build Coastguard Worker       continue;
5493*9880d681SAndroid Build Coastguard Worker 
5494*9880d681SAndroid Build Coastguard Worker     auto *Br1 = cast<BranchInst>(BB.getTerminator());
5495*9880d681SAndroid Build Coastguard Worker     if (Br1->getMetadata(LLVMContext::MD_unpredictable))
5496*9880d681SAndroid Build Coastguard Worker       continue;
5497*9880d681SAndroid Build Coastguard Worker 
5498*9880d681SAndroid Build Coastguard Worker     unsigned Opc;
5499*9880d681SAndroid Build Coastguard Worker     Value *Cond1, *Cond2;
5500*9880d681SAndroid Build Coastguard Worker     if (match(LogicOp, m_And(m_OneUse(m_Value(Cond1)),
5501*9880d681SAndroid Build Coastguard Worker                              m_OneUse(m_Value(Cond2)))))
5502*9880d681SAndroid Build Coastguard Worker       Opc = Instruction::And;
5503*9880d681SAndroid Build Coastguard Worker     else if (match(LogicOp, m_Or(m_OneUse(m_Value(Cond1)),
5504*9880d681SAndroid Build Coastguard Worker                                  m_OneUse(m_Value(Cond2)))))
5505*9880d681SAndroid Build Coastguard Worker       Opc = Instruction::Or;
5506*9880d681SAndroid Build Coastguard Worker     else
5507*9880d681SAndroid Build Coastguard Worker       continue;
5508*9880d681SAndroid Build Coastguard Worker 
5509*9880d681SAndroid Build Coastguard Worker     if (!match(Cond1, m_CombineOr(m_Cmp(), m_BinOp())) ||
5510*9880d681SAndroid Build Coastguard Worker         !match(Cond2, m_CombineOr(m_Cmp(), m_BinOp()))   )
5511*9880d681SAndroid Build Coastguard Worker       continue;
5512*9880d681SAndroid Build Coastguard Worker 
5513*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump());
5514*9880d681SAndroid Build Coastguard Worker 
5515*9880d681SAndroid Build Coastguard Worker     // Create a new BB.
5516*9880d681SAndroid Build Coastguard Worker     auto TmpBB =
5517*9880d681SAndroid Build Coastguard Worker         BasicBlock::Create(BB.getContext(), BB.getName() + ".cond.split",
5518*9880d681SAndroid Build Coastguard Worker                            BB.getParent(), BB.getNextNode());
5519*9880d681SAndroid Build Coastguard Worker 
5520*9880d681SAndroid Build Coastguard Worker     // Update original basic block by using the first condition directly by the
5521*9880d681SAndroid Build Coastguard Worker     // branch instruction and removing the no longer needed and/or instruction.
5522*9880d681SAndroid Build Coastguard Worker     Br1->setCondition(Cond1);
5523*9880d681SAndroid Build Coastguard Worker     LogicOp->eraseFromParent();
5524*9880d681SAndroid Build Coastguard Worker 
5525*9880d681SAndroid Build Coastguard Worker     // Depending on the conditon we have to either replace the true or the false
5526*9880d681SAndroid Build Coastguard Worker     // successor of the original branch instruction.
5527*9880d681SAndroid Build Coastguard Worker     if (Opc == Instruction::And)
5528*9880d681SAndroid Build Coastguard Worker       Br1->setSuccessor(0, TmpBB);
5529*9880d681SAndroid Build Coastguard Worker     else
5530*9880d681SAndroid Build Coastguard Worker       Br1->setSuccessor(1, TmpBB);
5531*9880d681SAndroid Build Coastguard Worker 
5532*9880d681SAndroid Build Coastguard Worker     // Fill in the new basic block.
5533*9880d681SAndroid Build Coastguard Worker     auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB);
5534*9880d681SAndroid Build Coastguard Worker     if (auto *I = dyn_cast<Instruction>(Cond2)) {
5535*9880d681SAndroid Build Coastguard Worker       I->removeFromParent();
5536*9880d681SAndroid Build Coastguard Worker       I->insertBefore(Br2);
5537*9880d681SAndroid Build Coastguard Worker     }
5538*9880d681SAndroid Build Coastguard Worker 
5539*9880d681SAndroid Build Coastguard Worker     // Update PHI nodes in both successors. The original BB needs to be
5540*9880d681SAndroid Build Coastguard Worker     // replaced in one succesor's PHI nodes, because the branch comes now from
5541*9880d681SAndroid Build Coastguard Worker     // the newly generated BB (NewBB). In the other successor we need to add one
5542*9880d681SAndroid Build Coastguard Worker     // incoming edge to the PHI nodes, because both branch instructions target
5543*9880d681SAndroid Build Coastguard Worker     // now the same successor. Depending on the original branch condition
5544*9880d681SAndroid Build Coastguard Worker     // (and/or) we have to swap the successors (TrueDest, FalseDest), so that
5545*9880d681SAndroid Build Coastguard Worker     // we perfrom the correct update for the PHI nodes.
5546*9880d681SAndroid Build Coastguard Worker     // This doesn't change the successor order of the just created branch
5547*9880d681SAndroid Build Coastguard Worker     // instruction (or any other instruction).
5548*9880d681SAndroid Build Coastguard Worker     if (Opc == Instruction::Or)
5549*9880d681SAndroid Build Coastguard Worker       std::swap(TBB, FBB);
5550*9880d681SAndroid Build Coastguard Worker 
5551*9880d681SAndroid Build Coastguard Worker     // Replace the old BB with the new BB.
5552*9880d681SAndroid Build Coastguard Worker     for (auto &I : *TBB) {
5553*9880d681SAndroid Build Coastguard Worker       PHINode *PN = dyn_cast<PHINode>(&I);
5554*9880d681SAndroid Build Coastguard Worker       if (!PN)
5555*9880d681SAndroid Build Coastguard Worker         break;
5556*9880d681SAndroid Build Coastguard Worker       int i;
5557*9880d681SAndroid Build Coastguard Worker       while ((i = PN->getBasicBlockIndex(&BB)) >= 0)
5558*9880d681SAndroid Build Coastguard Worker         PN->setIncomingBlock(i, TmpBB);
5559*9880d681SAndroid Build Coastguard Worker     }
5560*9880d681SAndroid Build Coastguard Worker 
5561*9880d681SAndroid Build Coastguard Worker     // Add another incoming edge form the new BB.
5562*9880d681SAndroid Build Coastguard Worker     for (auto &I : *FBB) {
5563*9880d681SAndroid Build Coastguard Worker       PHINode *PN = dyn_cast<PHINode>(&I);
5564*9880d681SAndroid Build Coastguard Worker       if (!PN)
5565*9880d681SAndroid Build Coastguard Worker         break;
5566*9880d681SAndroid Build Coastguard Worker       auto *Val = PN->getIncomingValueForBlock(&BB);
5567*9880d681SAndroid Build Coastguard Worker       PN->addIncoming(Val, TmpBB);
5568*9880d681SAndroid Build Coastguard Worker     }
5569*9880d681SAndroid Build Coastguard Worker 
5570*9880d681SAndroid Build Coastguard Worker     // Update the branch weights (from SelectionDAGBuilder::
5571*9880d681SAndroid Build Coastguard Worker     // FindMergedConditions).
5572*9880d681SAndroid Build Coastguard Worker     if (Opc == Instruction::Or) {
5573*9880d681SAndroid Build Coastguard Worker       // Codegen X | Y as:
5574*9880d681SAndroid Build Coastguard Worker       // BB1:
5575*9880d681SAndroid Build Coastguard Worker       //   jmp_if_X TBB
5576*9880d681SAndroid Build Coastguard Worker       //   jmp TmpBB
5577*9880d681SAndroid Build Coastguard Worker       // TmpBB:
5578*9880d681SAndroid Build Coastguard Worker       //   jmp_if_Y TBB
5579*9880d681SAndroid Build Coastguard Worker       //   jmp FBB
5580*9880d681SAndroid Build Coastguard Worker       //
5581*9880d681SAndroid Build Coastguard Worker 
5582*9880d681SAndroid Build Coastguard Worker       // We have flexibility in setting Prob for BB1 and Prob for NewBB.
5583*9880d681SAndroid Build Coastguard Worker       // The requirement is that
5584*9880d681SAndroid Build Coastguard Worker       //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
5585*9880d681SAndroid Build Coastguard Worker       //     = TrueProb for orignal BB.
5586*9880d681SAndroid Build Coastguard Worker       // Assuming the orignal weights are A and B, one choice is to set BB1's
5587*9880d681SAndroid Build Coastguard Worker       // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
5588*9880d681SAndroid Build Coastguard Worker       // assumes that
5589*9880d681SAndroid Build Coastguard Worker       //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
5590*9880d681SAndroid Build Coastguard Worker       // Another choice is to assume TrueProb for BB1 equals to TrueProb for
5591*9880d681SAndroid Build Coastguard Worker       // TmpBB, but the math is more complicated.
5592*9880d681SAndroid Build Coastguard Worker       uint64_t TrueWeight, FalseWeight;
5593*9880d681SAndroid Build Coastguard Worker       if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
5594*9880d681SAndroid Build Coastguard Worker         uint64_t NewTrueWeight = TrueWeight;
5595*9880d681SAndroid Build Coastguard Worker         uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;
5596*9880d681SAndroid Build Coastguard Worker         scaleWeights(NewTrueWeight, NewFalseWeight);
5597*9880d681SAndroid Build Coastguard Worker         Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext())
5598*9880d681SAndroid Build Coastguard Worker                          .createBranchWeights(TrueWeight, FalseWeight));
5599*9880d681SAndroid Build Coastguard Worker 
5600*9880d681SAndroid Build Coastguard Worker         NewTrueWeight = TrueWeight;
5601*9880d681SAndroid Build Coastguard Worker         NewFalseWeight = 2 * FalseWeight;
5602*9880d681SAndroid Build Coastguard Worker         scaleWeights(NewTrueWeight, NewFalseWeight);
5603*9880d681SAndroid Build Coastguard Worker         Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext())
5604*9880d681SAndroid Build Coastguard Worker                          .createBranchWeights(TrueWeight, FalseWeight));
5605*9880d681SAndroid Build Coastguard Worker       }
5606*9880d681SAndroid Build Coastguard Worker     } else {
5607*9880d681SAndroid Build Coastguard Worker       // Codegen X & Y as:
5608*9880d681SAndroid Build Coastguard Worker       // BB1:
5609*9880d681SAndroid Build Coastguard Worker       //   jmp_if_X TmpBB
5610*9880d681SAndroid Build Coastguard Worker       //   jmp FBB
5611*9880d681SAndroid Build Coastguard Worker       // TmpBB:
5612*9880d681SAndroid Build Coastguard Worker       //   jmp_if_Y TBB
5613*9880d681SAndroid Build Coastguard Worker       //   jmp FBB
5614*9880d681SAndroid Build Coastguard Worker       //
5615*9880d681SAndroid Build Coastguard Worker       //  This requires creation of TmpBB after CurBB.
5616*9880d681SAndroid Build Coastguard Worker 
5617*9880d681SAndroid Build Coastguard Worker       // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
5618*9880d681SAndroid Build Coastguard Worker       // The requirement is that
5619*9880d681SAndroid Build Coastguard Worker       //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
5620*9880d681SAndroid Build Coastguard Worker       //     = FalseProb for orignal BB.
5621*9880d681SAndroid Build Coastguard Worker       // Assuming the orignal weights are A and B, one choice is to set BB1's
5622*9880d681SAndroid Build Coastguard Worker       // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
5623*9880d681SAndroid Build Coastguard Worker       // assumes that
5624*9880d681SAndroid Build Coastguard Worker       //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
5625*9880d681SAndroid Build Coastguard Worker       uint64_t TrueWeight, FalseWeight;
5626*9880d681SAndroid Build Coastguard Worker       if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
5627*9880d681SAndroid Build Coastguard Worker         uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;
5628*9880d681SAndroid Build Coastguard Worker         uint64_t NewFalseWeight = FalseWeight;
5629*9880d681SAndroid Build Coastguard Worker         scaleWeights(NewTrueWeight, NewFalseWeight);
5630*9880d681SAndroid Build Coastguard Worker         Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext())
5631*9880d681SAndroid Build Coastguard Worker                          .createBranchWeights(TrueWeight, FalseWeight));
5632*9880d681SAndroid Build Coastguard Worker 
5633*9880d681SAndroid Build Coastguard Worker         NewTrueWeight = 2 * TrueWeight;
5634*9880d681SAndroid Build Coastguard Worker         NewFalseWeight = FalseWeight;
5635*9880d681SAndroid Build Coastguard Worker         scaleWeights(NewTrueWeight, NewFalseWeight);
5636*9880d681SAndroid Build Coastguard Worker         Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext())
5637*9880d681SAndroid Build Coastguard Worker                          .createBranchWeights(TrueWeight, FalseWeight));
5638*9880d681SAndroid Build Coastguard Worker       }
5639*9880d681SAndroid Build Coastguard Worker     }
5640*9880d681SAndroid Build Coastguard Worker 
5641*9880d681SAndroid Build Coastguard Worker     // Note: No point in getting fancy here, since the DT info is never
5642*9880d681SAndroid Build Coastguard Worker     // available to CodeGenPrepare.
5643*9880d681SAndroid Build Coastguard Worker     ModifiedDT = true;
5644*9880d681SAndroid Build Coastguard Worker 
5645*9880d681SAndroid Build Coastguard Worker     MadeChange = true;
5646*9880d681SAndroid Build Coastguard Worker 
5647*9880d681SAndroid Build Coastguard Worker     DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump();
5648*9880d681SAndroid Build Coastguard Worker           TmpBB->dump());
5649*9880d681SAndroid Build Coastguard Worker   }
5650*9880d681SAndroid Build Coastguard Worker   return MadeChange;
5651*9880d681SAndroid Build Coastguard Worker }
5652*9880d681SAndroid Build Coastguard Worker 
stripInvariantGroupMetadata(Instruction & I)5653*9880d681SAndroid Build Coastguard Worker void CodeGenPrepare::stripInvariantGroupMetadata(Instruction &I) {
5654*9880d681SAndroid Build Coastguard Worker   if (auto *InvariantMD = I.getMetadata(LLVMContext::MD_invariant_group))
5655*9880d681SAndroid Build Coastguard Worker     I.dropUnknownNonDebugMetadata(InvariantMD->getMetadataID());
5656*9880d681SAndroid Build Coastguard Worker }
5657