1*9880d681SAndroid Build Coastguard Worker //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
2*9880d681SAndroid Build Coastguard Worker //
3*9880d681SAndroid Build Coastguard Worker // The LLVM Compiler Infrastructure
4*9880d681SAndroid Build Coastguard Worker //
5*9880d681SAndroid Build Coastguard Worker // This file is distributed under the University of Illinois Open Source
6*9880d681SAndroid Build Coastguard Worker // License. See LICENSE.TXT for details.
7*9880d681SAndroid Build Coastguard Worker //
8*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
9*9880d681SAndroid Build Coastguard Worker //
10*9880d681SAndroid Build Coastguard Worker // This pass munges the code in the input function to better prepare it for
11*9880d681SAndroid Build Coastguard Worker // SelectionDAG-based code generation. This works around limitations in it's
12*9880d681SAndroid Build Coastguard Worker // basic-block-at-a-time approach. It should eventually be removed.
13*9880d681SAndroid Build Coastguard Worker //
14*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
15*9880d681SAndroid Build Coastguard Worker
16*9880d681SAndroid Build Coastguard Worker #include "llvm/CodeGen/Passes.h"
17*9880d681SAndroid Build Coastguard Worker #include "llvm/ADT/DenseMap.h"
18*9880d681SAndroid Build Coastguard Worker #include "llvm/ADT/SmallSet.h"
19*9880d681SAndroid Build Coastguard Worker #include "llvm/ADT/Statistic.h"
20*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/InstructionSimplify.h"
21*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/LoopInfo.h"
22*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/TargetLibraryInfo.h"
23*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/TargetTransformInfo.h"
24*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/ValueTracking.h"
25*9880d681SAndroid Build Coastguard Worker #include "llvm/Analysis/MemoryBuiltins.h"
26*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/CallSite.h"
27*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Constants.h"
28*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/DataLayout.h"
29*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/DerivedTypes.h"
30*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Dominators.h"
31*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Function.h"
32*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/GetElementPtrTypeIterator.h"
33*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/IRBuilder.h"
34*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/InlineAsm.h"
35*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Instructions.h"
36*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/IntrinsicInst.h"
37*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/MDBuilder.h"
38*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/PatternMatch.h"
39*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/Statepoint.h"
40*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/ValueHandle.h"
41*9880d681SAndroid Build Coastguard Worker #include "llvm/IR/ValueMap.h"
42*9880d681SAndroid Build Coastguard Worker #include "llvm/Pass.h"
43*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/BranchProbability.h"
44*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/CommandLine.h"
45*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/Debug.h"
46*9880d681SAndroid Build Coastguard Worker #include "llvm/Support/raw_ostream.h"
47*9880d681SAndroid Build Coastguard Worker #include "llvm/Target/TargetLowering.h"
48*9880d681SAndroid Build Coastguard Worker #include "llvm/Target/TargetSubtargetInfo.h"
49*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/BasicBlockUtils.h"
50*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/BuildLibCalls.h"
51*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/BypassSlowDivision.h"
52*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/Local.h"
53*9880d681SAndroid Build Coastguard Worker #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
54*9880d681SAndroid Build Coastguard Worker using namespace llvm;
55*9880d681SAndroid Build Coastguard Worker using namespace llvm::PatternMatch;
56*9880d681SAndroid Build Coastguard Worker
57*9880d681SAndroid Build Coastguard Worker #define DEBUG_TYPE "codegenprepare"
58*9880d681SAndroid Build Coastguard Worker
59*9880d681SAndroid Build Coastguard Worker STATISTIC(NumBlocksElim, "Number of blocks eliminated");
60*9880d681SAndroid Build Coastguard Worker STATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated");
61*9880d681SAndroid Build Coastguard Worker STATISTIC(NumGEPsElim, "Number of GEPs converted to casts");
62*9880d681SAndroid Build Coastguard Worker STATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of "
63*9880d681SAndroid Build Coastguard Worker "sunken Cmps");
64*9880d681SAndroid Build Coastguard Worker STATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses "
65*9880d681SAndroid Build Coastguard Worker "of sunken Casts");
66*9880d681SAndroid Build Coastguard Worker STATISTIC(NumMemoryInsts, "Number of memory instructions whose address "
67*9880d681SAndroid Build Coastguard Worker "computations were sunk");
68*9880d681SAndroid Build Coastguard Worker STATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads");
69*9880d681SAndroid Build Coastguard Worker STATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized");
70*9880d681SAndroid Build Coastguard Worker STATISTIC(NumAndsAdded,
71*9880d681SAndroid Build Coastguard Worker "Number of and mask instructions added to form ext loads");
72*9880d681SAndroid Build Coastguard Worker STATISTIC(NumAndUses, "Number of uses of and mask instructions optimized");
73*9880d681SAndroid Build Coastguard Worker STATISTIC(NumRetsDup, "Number of return instructions duplicated");
74*9880d681SAndroid Build Coastguard Worker STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
75*9880d681SAndroid Build Coastguard Worker STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");
76*9880d681SAndroid Build Coastguard Worker STATISTIC(NumAndCmpsMoved, "Number of and/cmp's pushed into branches");
77*9880d681SAndroid Build Coastguard Worker STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");
78*9880d681SAndroid Build Coastguard Worker
79*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableBranchOpts(
80*9880d681SAndroid Build Coastguard Worker "disable-cgp-branch-opts", cl::Hidden, cl::init(false),
81*9880d681SAndroid Build Coastguard Worker cl::desc("Disable branch optimizations in CodeGenPrepare"));
82*9880d681SAndroid Build Coastguard Worker
83*9880d681SAndroid Build Coastguard Worker static cl::opt<bool>
84*9880d681SAndroid Build Coastguard Worker DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false),
85*9880d681SAndroid Build Coastguard Worker cl::desc("Disable GC optimizations in CodeGenPrepare"));
86*9880d681SAndroid Build Coastguard Worker
87*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableSelectToBranch(
88*9880d681SAndroid Build Coastguard Worker "disable-cgp-select2branch", cl::Hidden, cl::init(false),
89*9880d681SAndroid Build Coastguard Worker cl::desc("Disable select to branch conversion."));
90*9880d681SAndroid Build Coastguard Worker
91*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> AddrSinkUsingGEPs(
92*9880d681SAndroid Build Coastguard Worker "addr-sink-using-gep", cl::Hidden, cl::init(false),
93*9880d681SAndroid Build Coastguard Worker cl::desc("Address sinking in CGP using GEPs."));
94*9880d681SAndroid Build Coastguard Worker
95*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> EnableAndCmpSinking(
96*9880d681SAndroid Build Coastguard Worker "enable-andcmp-sinking", cl::Hidden, cl::init(true),
97*9880d681SAndroid Build Coastguard Worker cl::desc("Enable sinkinig and/cmp into branches."));
98*9880d681SAndroid Build Coastguard Worker
99*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableStoreExtract(
100*9880d681SAndroid Build Coastguard Worker "disable-cgp-store-extract", cl::Hidden, cl::init(false),
101*9880d681SAndroid Build Coastguard Worker cl::desc("Disable store(extract) optimizations in CodeGenPrepare"));
102*9880d681SAndroid Build Coastguard Worker
103*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> StressStoreExtract(
104*9880d681SAndroid Build Coastguard Worker "stress-cgp-store-extract", cl::Hidden, cl::init(false),
105*9880d681SAndroid Build Coastguard Worker cl::desc("Stress test store(extract) optimizations in CodeGenPrepare"));
106*9880d681SAndroid Build Coastguard Worker
107*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisableExtLdPromotion(
108*9880d681SAndroid Build Coastguard Worker "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
109*9880d681SAndroid Build Coastguard Worker cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
110*9880d681SAndroid Build Coastguard Worker "CodeGenPrepare"));
111*9880d681SAndroid Build Coastguard Worker
112*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> StressExtLdPromotion(
113*9880d681SAndroid Build Coastguard Worker "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
114*9880d681SAndroid Build Coastguard Worker cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
115*9880d681SAndroid Build Coastguard Worker "optimization in CodeGenPrepare"));
116*9880d681SAndroid Build Coastguard Worker
117*9880d681SAndroid Build Coastguard Worker static cl::opt<bool> DisablePreheaderProtect(
118*9880d681SAndroid Build Coastguard Worker "disable-preheader-prot", cl::Hidden, cl::init(false),
119*9880d681SAndroid Build Coastguard Worker cl::desc("Disable protection against removing loop preheaders"));
120*9880d681SAndroid Build Coastguard Worker
121*9880d681SAndroid Build Coastguard Worker namespace {
122*9880d681SAndroid Build Coastguard Worker typedef SmallPtrSet<Instruction *, 16> SetOfInstrs;
123*9880d681SAndroid Build Coastguard Worker typedef PointerIntPair<Type *, 1, bool> TypeIsSExt;
124*9880d681SAndroid Build Coastguard Worker typedef DenseMap<Instruction *, TypeIsSExt> InstrToOrigTy;
125*9880d681SAndroid Build Coastguard Worker class TypePromotionTransaction;
126*9880d681SAndroid Build Coastguard Worker
127*9880d681SAndroid Build Coastguard Worker class CodeGenPrepare : public FunctionPass {
128*9880d681SAndroid Build Coastguard Worker const TargetMachine *TM;
129*9880d681SAndroid Build Coastguard Worker const TargetLowering *TLI;
130*9880d681SAndroid Build Coastguard Worker const TargetTransformInfo *TTI;
131*9880d681SAndroid Build Coastguard Worker const TargetLibraryInfo *TLInfo;
132*9880d681SAndroid Build Coastguard Worker const LoopInfo *LI;
133*9880d681SAndroid Build Coastguard Worker
134*9880d681SAndroid Build Coastguard Worker /// As we scan instructions optimizing them, this is the next instruction
135*9880d681SAndroid Build Coastguard Worker /// to optimize. Transforms that can invalidate this should update it.
136*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator CurInstIterator;
137*9880d681SAndroid Build Coastguard Worker
138*9880d681SAndroid Build Coastguard Worker /// Keeps track of non-local addresses that have been sunk into a block.
139*9880d681SAndroid Build Coastguard Worker /// This allows us to avoid inserting duplicate code for blocks with
140*9880d681SAndroid Build Coastguard Worker /// multiple load/stores of the same address.
141*9880d681SAndroid Build Coastguard Worker ValueMap<Value*, Value*> SunkAddrs;
142*9880d681SAndroid Build Coastguard Worker
143*9880d681SAndroid Build Coastguard Worker /// Keeps track of all instructions inserted for the current function.
144*9880d681SAndroid Build Coastguard Worker SetOfInstrs InsertedInsts;
145*9880d681SAndroid Build Coastguard Worker /// Keeps track of the type of the related instruction before their
146*9880d681SAndroid Build Coastguard Worker /// promotion for the current function.
147*9880d681SAndroid Build Coastguard Worker InstrToOrigTy PromotedInsts;
148*9880d681SAndroid Build Coastguard Worker
149*9880d681SAndroid Build Coastguard Worker /// True if CFG is modified in any way.
150*9880d681SAndroid Build Coastguard Worker bool ModifiedDT;
151*9880d681SAndroid Build Coastguard Worker
152*9880d681SAndroid Build Coastguard Worker /// True if optimizing for size.
153*9880d681SAndroid Build Coastguard Worker bool OptSize;
154*9880d681SAndroid Build Coastguard Worker
155*9880d681SAndroid Build Coastguard Worker /// DataLayout for the Function being processed.
156*9880d681SAndroid Build Coastguard Worker const DataLayout *DL;
157*9880d681SAndroid Build Coastguard Worker
158*9880d681SAndroid Build Coastguard Worker public:
159*9880d681SAndroid Build Coastguard Worker static char ID; // Pass identification, replacement for typeid
CodeGenPrepare(const TargetMachine * TM=nullptr)160*9880d681SAndroid Build Coastguard Worker explicit CodeGenPrepare(const TargetMachine *TM = nullptr)
161*9880d681SAndroid Build Coastguard Worker : FunctionPass(ID), TM(TM), TLI(nullptr), TTI(nullptr), DL(nullptr) {
162*9880d681SAndroid Build Coastguard Worker initializeCodeGenPreparePass(*PassRegistry::getPassRegistry());
163*9880d681SAndroid Build Coastguard Worker }
164*9880d681SAndroid Build Coastguard Worker bool runOnFunction(Function &F) override;
165*9880d681SAndroid Build Coastguard Worker
getPassName() const166*9880d681SAndroid Build Coastguard Worker const char *getPassName() const override { return "CodeGen Prepare"; }
167*9880d681SAndroid Build Coastguard Worker
getAnalysisUsage(AnalysisUsage & AU) const168*9880d681SAndroid Build Coastguard Worker void getAnalysisUsage(AnalysisUsage &AU) const override {
169*9880d681SAndroid Build Coastguard Worker // FIXME: When we can selectively preserve passes, preserve the domtree.
170*9880d681SAndroid Build Coastguard Worker AU.addRequired<TargetLibraryInfoWrapperPass>();
171*9880d681SAndroid Build Coastguard Worker AU.addRequired<TargetTransformInfoWrapperPass>();
172*9880d681SAndroid Build Coastguard Worker AU.addRequired<LoopInfoWrapperPass>();
173*9880d681SAndroid Build Coastguard Worker }
174*9880d681SAndroid Build Coastguard Worker
175*9880d681SAndroid Build Coastguard Worker private:
176*9880d681SAndroid Build Coastguard Worker bool eliminateFallThrough(Function &F);
177*9880d681SAndroid Build Coastguard Worker bool eliminateMostlyEmptyBlocks(Function &F);
178*9880d681SAndroid Build Coastguard Worker bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const;
179*9880d681SAndroid Build Coastguard Worker void eliminateMostlyEmptyBlock(BasicBlock *BB);
180*9880d681SAndroid Build Coastguard Worker bool optimizeBlock(BasicBlock &BB, bool& ModifiedDT);
181*9880d681SAndroid Build Coastguard Worker bool optimizeInst(Instruction *I, bool& ModifiedDT);
182*9880d681SAndroid Build Coastguard Worker bool optimizeMemoryInst(Instruction *I, Value *Addr,
183*9880d681SAndroid Build Coastguard Worker Type *AccessTy, unsigned AS);
184*9880d681SAndroid Build Coastguard Worker bool optimizeInlineAsmInst(CallInst *CS);
185*9880d681SAndroid Build Coastguard Worker bool optimizeCallInst(CallInst *CI, bool& ModifiedDT);
186*9880d681SAndroid Build Coastguard Worker bool moveExtToFormExtLoad(Instruction *&I);
187*9880d681SAndroid Build Coastguard Worker bool optimizeExtUses(Instruction *I);
188*9880d681SAndroid Build Coastguard Worker bool optimizeLoadExt(LoadInst *I);
189*9880d681SAndroid Build Coastguard Worker bool optimizeSelectInst(SelectInst *SI);
190*9880d681SAndroid Build Coastguard Worker bool optimizeShuffleVectorInst(ShuffleVectorInst *SI);
191*9880d681SAndroid Build Coastguard Worker bool optimizeSwitchInst(SwitchInst *CI);
192*9880d681SAndroid Build Coastguard Worker bool optimizeExtractElementInst(Instruction *Inst);
193*9880d681SAndroid Build Coastguard Worker bool dupRetToEnableTailCallOpts(BasicBlock *BB);
194*9880d681SAndroid Build Coastguard Worker bool placeDbgValues(Function &F);
195*9880d681SAndroid Build Coastguard Worker bool sinkAndCmp(Function &F);
196*9880d681SAndroid Build Coastguard Worker bool extLdPromotion(TypePromotionTransaction &TPT, LoadInst *&LI,
197*9880d681SAndroid Build Coastguard Worker Instruction *&Inst,
198*9880d681SAndroid Build Coastguard Worker const SmallVectorImpl<Instruction *> &Exts,
199*9880d681SAndroid Build Coastguard Worker unsigned CreatedInstCost);
200*9880d681SAndroid Build Coastguard Worker bool splitBranchCondition(Function &F);
201*9880d681SAndroid Build Coastguard Worker bool simplifyOffsetableRelocate(Instruction &I);
202*9880d681SAndroid Build Coastguard Worker void stripInvariantGroupMetadata(Instruction &I);
203*9880d681SAndroid Build Coastguard Worker };
204*9880d681SAndroid Build Coastguard Worker }
205*9880d681SAndroid Build Coastguard Worker
206*9880d681SAndroid Build Coastguard Worker char CodeGenPrepare::ID = 0;
207*9880d681SAndroid Build Coastguard Worker INITIALIZE_TM_PASS(CodeGenPrepare, "codegenprepare",
208*9880d681SAndroid Build Coastguard Worker "Optimize for code generation", false, false)
209*9880d681SAndroid Build Coastguard Worker
createCodeGenPreparePass(const TargetMachine * TM)210*9880d681SAndroid Build Coastguard Worker FunctionPass *llvm::createCodeGenPreparePass(const TargetMachine *TM) {
211*9880d681SAndroid Build Coastguard Worker return new CodeGenPrepare(TM);
212*9880d681SAndroid Build Coastguard Worker }
213*9880d681SAndroid Build Coastguard Worker
runOnFunction(Function & F)214*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::runOnFunction(Function &F) {
215*9880d681SAndroid Build Coastguard Worker if (skipFunction(F))
216*9880d681SAndroid Build Coastguard Worker return false;
217*9880d681SAndroid Build Coastguard Worker
218*9880d681SAndroid Build Coastguard Worker DL = &F.getParent()->getDataLayout();
219*9880d681SAndroid Build Coastguard Worker
220*9880d681SAndroid Build Coastguard Worker bool EverMadeChange = false;
221*9880d681SAndroid Build Coastguard Worker // Clear per function information.
222*9880d681SAndroid Build Coastguard Worker InsertedInsts.clear();
223*9880d681SAndroid Build Coastguard Worker PromotedInsts.clear();
224*9880d681SAndroid Build Coastguard Worker
225*9880d681SAndroid Build Coastguard Worker ModifiedDT = false;
226*9880d681SAndroid Build Coastguard Worker if (TM)
227*9880d681SAndroid Build Coastguard Worker TLI = TM->getSubtargetImpl(F)->getTargetLowering();
228*9880d681SAndroid Build Coastguard Worker TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
229*9880d681SAndroid Build Coastguard Worker TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
230*9880d681SAndroid Build Coastguard Worker LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
231*9880d681SAndroid Build Coastguard Worker OptSize = F.optForSize();
232*9880d681SAndroid Build Coastguard Worker
233*9880d681SAndroid Build Coastguard Worker /// This optimization identifies DIV instructions that can be
234*9880d681SAndroid Build Coastguard Worker /// profitably bypassed and carried out with a shorter, faster divide.
235*9880d681SAndroid Build Coastguard Worker if (!OptSize && TLI && TLI->isSlowDivBypassed()) {
236*9880d681SAndroid Build Coastguard Worker const DenseMap<unsigned int, unsigned int> &BypassWidths =
237*9880d681SAndroid Build Coastguard Worker TLI->getBypassSlowDivWidths();
238*9880d681SAndroid Build Coastguard Worker BasicBlock* BB = &*F.begin();
239*9880d681SAndroid Build Coastguard Worker while (BB != nullptr) {
240*9880d681SAndroid Build Coastguard Worker // bypassSlowDivision may create new BBs, but we don't want to reapply the
241*9880d681SAndroid Build Coastguard Worker // optimization to those blocks.
242*9880d681SAndroid Build Coastguard Worker BasicBlock* Next = BB->getNextNode();
243*9880d681SAndroid Build Coastguard Worker EverMadeChange |= bypassSlowDivision(BB, BypassWidths);
244*9880d681SAndroid Build Coastguard Worker BB = Next;
245*9880d681SAndroid Build Coastguard Worker }
246*9880d681SAndroid Build Coastguard Worker }
247*9880d681SAndroid Build Coastguard Worker
248*9880d681SAndroid Build Coastguard Worker // Eliminate blocks that contain only PHI nodes and an
249*9880d681SAndroid Build Coastguard Worker // unconditional branch.
250*9880d681SAndroid Build Coastguard Worker EverMadeChange |= eliminateMostlyEmptyBlocks(F);
251*9880d681SAndroid Build Coastguard Worker
252*9880d681SAndroid Build Coastguard Worker // llvm.dbg.value is far away from the value then iSel may not be able
253*9880d681SAndroid Build Coastguard Worker // handle it properly. iSel will drop llvm.dbg.value if it can not
254*9880d681SAndroid Build Coastguard Worker // find a node corresponding to the value.
255*9880d681SAndroid Build Coastguard Worker EverMadeChange |= placeDbgValues(F);
256*9880d681SAndroid Build Coastguard Worker
257*9880d681SAndroid Build Coastguard Worker // If there is a mask, compare against zero, and branch that can be combined
258*9880d681SAndroid Build Coastguard Worker // into a single target instruction, push the mask and compare into branch
259*9880d681SAndroid Build Coastguard Worker // users. Do this before OptimizeBlock -> OptimizeInst ->
260*9880d681SAndroid Build Coastguard Worker // OptimizeCmpExpression, which perturbs the pattern being searched for.
261*9880d681SAndroid Build Coastguard Worker if (!DisableBranchOpts) {
262*9880d681SAndroid Build Coastguard Worker EverMadeChange |= sinkAndCmp(F);
263*9880d681SAndroid Build Coastguard Worker EverMadeChange |= splitBranchCondition(F);
264*9880d681SAndroid Build Coastguard Worker }
265*9880d681SAndroid Build Coastguard Worker
266*9880d681SAndroid Build Coastguard Worker bool MadeChange = true;
267*9880d681SAndroid Build Coastguard Worker while (MadeChange) {
268*9880d681SAndroid Build Coastguard Worker MadeChange = false;
269*9880d681SAndroid Build Coastguard Worker for (Function::iterator I = F.begin(); I != F.end(); ) {
270*9880d681SAndroid Build Coastguard Worker BasicBlock *BB = &*I++;
271*9880d681SAndroid Build Coastguard Worker bool ModifiedDTOnIteration = false;
272*9880d681SAndroid Build Coastguard Worker MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration);
273*9880d681SAndroid Build Coastguard Worker
274*9880d681SAndroid Build Coastguard Worker // Restart BB iteration if the dominator tree of the Function was changed
275*9880d681SAndroid Build Coastguard Worker if (ModifiedDTOnIteration)
276*9880d681SAndroid Build Coastguard Worker break;
277*9880d681SAndroid Build Coastguard Worker }
278*9880d681SAndroid Build Coastguard Worker EverMadeChange |= MadeChange;
279*9880d681SAndroid Build Coastguard Worker }
280*9880d681SAndroid Build Coastguard Worker
281*9880d681SAndroid Build Coastguard Worker SunkAddrs.clear();
282*9880d681SAndroid Build Coastguard Worker
283*9880d681SAndroid Build Coastguard Worker if (!DisableBranchOpts) {
284*9880d681SAndroid Build Coastguard Worker MadeChange = false;
285*9880d681SAndroid Build Coastguard Worker SmallPtrSet<BasicBlock*, 8> WorkList;
286*9880d681SAndroid Build Coastguard Worker for (BasicBlock &BB : F) {
287*9880d681SAndroid Build Coastguard Worker SmallVector<BasicBlock *, 2> Successors(succ_begin(&BB), succ_end(&BB));
288*9880d681SAndroid Build Coastguard Worker MadeChange |= ConstantFoldTerminator(&BB, true);
289*9880d681SAndroid Build Coastguard Worker if (!MadeChange) continue;
290*9880d681SAndroid Build Coastguard Worker
291*9880d681SAndroid Build Coastguard Worker for (SmallVectorImpl<BasicBlock*>::iterator
292*9880d681SAndroid Build Coastguard Worker II = Successors.begin(), IE = Successors.end(); II != IE; ++II)
293*9880d681SAndroid Build Coastguard Worker if (pred_begin(*II) == pred_end(*II))
294*9880d681SAndroid Build Coastguard Worker WorkList.insert(*II);
295*9880d681SAndroid Build Coastguard Worker }
296*9880d681SAndroid Build Coastguard Worker
297*9880d681SAndroid Build Coastguard Worker // Delete the dead blocks and any of their dead successors.
298*9880d681SAndroid Build Coastguard Worker MadeChange |= !WorkList.empty();
299*9880d681SAndroid Build Coastguard Worker while (!WorkList.empty()) {
300*9880d681SAndroid Build Coastguard Worker BasicBlock *BB = *WorkList.begin();
301*9880d681SAndroid Build Coastguard Worker WorkList.erase(BB);
302*9880d681SAndroid Build Coastguard Worker SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB));
303*9880d681SAndroid Build Coastguard Worker
304*9880d681SAndroid Build Coastguard Worker DeleteDeadBlock(BB);
305*9880d681SAndroid Build Coastguard Worker
306*9880d681SAndroid Build Coastguard Worker for (SmallVectorImpl<BasicBlock*>::iterator
307*9880d681SAndroid Build Coastguard Worker II = Successors.begin(), IE = Successors.end(); II != IE; ++II)
308*9880d681SAndroid Build Coastguard Worker if (pred_begin(*II) == pred_end(*II))
309*9880d681SAndroid Build Coastguard Worker WorkList.insert(*II);
310*9880d681SAndroid Build Coastguard Worker }
311*9880d681SAndroid Build Coastguard Worker
312*9880d681SAndroid Build Coastguard Worker // Merge pairs of basic blocks with unconditional branches, connected by
313*9880d681SAndroid Build Coastguard Worker // a single edge.
314*9880d681SAndroid Build Coastguard Worker if (EverMadeChange || MadeChange)
315*9880d681SAndroid Build Coastguard Worker MadeChange |= eliminateFallThrough(F);
316*9880d681SAndroid Build Coastguard Worker
317*9880d681SAndroid Build Coastguard Worker EverMadeChange |= MadeChange;
318*9880d681SAndroid Build Coastguard Worker }
319*9880d681SAndroid Build Coastguard Worker
320*9880d681SAndroid Build Coastguard Worker if (!DisableGCOpts) {
321*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction *, 2> Statepoints;
322*9880d681SAndroid Build Coastguard Worker for (BasicBlock &BB : F)
323*9880d681SAndroid Build Coastguard Worker for (Instruction &I : BB)
324*9880d681SAndroid Build Coastguard Worker if (isStatepoint(I))
325*9880d681SAndroid Build Coastguard Worker Statepoints.push_back(&I);
326*9880d681SAndroid Build Coastguard Worker for (auto &I : Statepoints)
327*9880d681SAndroid Build Coastguard Worker EverMadeChange |= simplifyOffsetableRelocate(*I);
328*9880d681SAndroid Build Coastguard Worker }
329*9880d681SAndroid Build Coastguard Worker
330*9880d681SAndroid Build Coastguard Worker return EverMadeChange;
331*9880d681SAndroid Build Coastguard Worker }
332*9880d681SAndroid Build Coastguard Worker
333*9880d681SAndroid Build Coastguard Worker /// Merge basic blocks which are connected by a single edge, where one of the
334*9880d681SAndroid Build Coastguard Worker /// basic blocks has a single successor pointing to the other basic block,
335*9880d681SAndroid Build Coastguard Worker /// which has a single predecessor.
eliminateFallThrough(Function & F)336*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::eliminateFallThrough(Function &F) {
337*9880d681SAndroid Build Coastguard Worker bool Changed = false;
338*9880d681SAndroid Build Coastguard Worker // Scan all of the blocks in the function, except for the entry block.
339*9880d681SAndroid Build Coastguard Worker for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) {
340*9880d681SAndroid Build Coastguard Worker BasicBlock *BB = &*I++;
341*9880d681SAndroid Build Coastguard Worker // If the destination block has a single pred, then this is a trivial
342*9880d681SAndroid Build Coastguard Worker // edge, just collapse it.
343*9880d681SAndroid Build Coastguard Worker BasicBlock *SinglePred = BB->getSinglePredecessor();
344*9880d681SAndroid Build Coastguard Worker
345*9880d681SAndroid Build Coastguard Worker // Don't merge if BB's address is taken.
346*9880d681SAndroid Build Coastguard Worker if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue;
347*9880d681SAndroid Build Coastguard Worker
348*9880d681SAndroid Build Coastguard Worker BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator());
349*9880d681SAndroid Build Coastguard Worker if (Term && !Term->isConditional()) {
350*9880d681SAndroid Build Coastguard Worker Changed = true;
351*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "To merge:\n"<< *SinglePred << "\n\n\n");
352*9880d681SAndroid Build Coastguard Worker // Remember if SinglePred was the entry block of the function.
353*9880d681SAndroid Build Coastguard Worker // If so, we will need to move BB back to the entry position.
354*9880d681SAndroid Build Coastguard Worker bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
355*9880d681SAndroid Build Coastguard Worker MergeBasicBlockIntoOnlyPred(BB, nullptr);
356*9880d681SAndroid Build Coastguard Worker
357*9880d681SAndroid Build Coastguard Worker if (isEntry && BB != &BB->getParent()->getEntryBlock())
358*9880d681SAndroid Build Coastguard Worker BB->moveBefore(&BB->getParent()->getEntryBlock());
359*9880d681SAndroid Build Coastguard Worker
360*9880d681SAndroid Build Coastguard Worker // We have erased a block. Update the iterator.
361*9880d681SAndroid Build Coastguard Worker I = BB->getIterator();
362*9880d681SAndroid Build Coastguard Worker }
363*9880d681SAndroid Build Coastguard Worker }
364*9880d681SAndroid Build Coastguard Worker return Changed;
365*9880d681SAndroid Build Coastguard Worker }
366*9880d681SAndroid Build Coastguard Worker
367*9880d681SAndroid Build Coastguard Worker /// Eliminate blocks that contain only PHI nodes, debug info directives, and an
368*9880d681SAndroid Build Coastguard Worker /// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split
369*9880d681SAndroid Build Coastguard Worker /// edges in ways that are non-optimal for isel. Start by eliminating these
370*9880d681SAndroid Build Coastguard Worker /// blocks so we can split them the way we want them.
eliminateMostlyEmptyBlocks(Function & F)371*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F) {
372*9880d681SAndroid Build Coastguard Worker SmallPtrSet<BasicBlock *, 16> Preheaders;
373*9880d681SAndroid Build Coastguard Worker SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end());
374*9880d681SAndroid Build Coastguard Worker while (!LoopList.empty()) {
375*9880d681SAndroid Build Coastguard Worker Loop *L = LoopList.pop_back_val();
376*9880d681SAndroid Build Coastguard Worker LoopList.insert(LoopList.end(), L->begin(), L->end());
377*9880d681SAndroid Build Coastguard Worker if (BasicBlock *Preheader = L->getLoopPreheader())
378*9880d681SAndroid Build Coastguard Worker Preheaders.insert(Preheader);
379*9880d681SAndroid Build Coastguard Worker }
380*9880d681SAndroid Build Coastguard Worker
381*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
382*9880d681SAndroid Build Coastguard Worker // Note that this intentionally skips the entry block.
383*9880d681SAndroid Build Coastguard Worker for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) {
384*9880d681SAndroid Build Coastguard Worker BasicBlock *BB = &*I++;
385*9880d681SAndroid Build Coastguard Worker
386*9880d681SAndroid Build Coastguard Worker // If this block doesn't end with an uncond branch, ignore it.
387*9880d681SAndroid Build Coastguard Worker BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());
388*9880d681SAndroid Build Coastguard Worker if (!BI || !BI->isUnconditional())
389*9880d681SAndroid Build Coastguard Worker continue;
390*9880d681SAndroid Build Coastguard Worker
391*9880d681SAndroid Build Coastguard Worker // If the instruction before the branch (skipping debug info) isn't a phi
392*9880d681SAndroid Build Coastguard Worker // node, then other stuff is happening here.
393*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator BBI = BI->getIterator();
394*9880d681SAndroid Build Coastguard Worker if (BBI != BB->begin()) {
395*9880d681SAndroid Build Coastguard Worker --BBI;
396*9880d681SAndroid Build Coastguard Worker while (isa<DbgInfoIntrinsic>(BBI)) {
397*9880d681SAndroid Build Coastguard Worker if (BBI == BB->begin())
398*9880d681SAndroid Build Coastguard Worker break;
399*9880d681SAndroid Build Coastguard Worker --BBI;
400*9880d681SAndroid Build Coastguard Worker }
401*9880d681SAndroid Build Coastguard Worker if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI))
402*9880d681SAndroid Build Coastguard Worker continue;
403*9880d681SAndroid Build Coastguard Worker }
404*9880d681SAndroid Build Coastguard Worker
405*9880d681SAndroid Build Coastguard Worker // Do not break infinite loops.
406*9880d681SAndroid Build Coastguard Worker BasicBlock *DestBB = BI->getSuccessor(0);
407*9880d681SAndroid Build Coastguard Worker if (DestBB == BB)
408*9880d681SAndroid Build Coastguard Worker continue;
409*9880d681SAndroid Build Coastguard Worker
410*9880d681SAndroid Build Coastguard Worker if (!canMergeBlocks(BB, DestBB))
411*9880d681SAndroid Build Coastguard Worker continue;
412*9880d681SAndroid Build Coastguard Worker
413*9880d681SAndroid Build Coastguard Worker // Do not delete loop preheaders if doing so would create a critical edge.
414*9880d681SAndroid Build Coastguard Worker // Loop preheaders can be good locations to spill registers. If the
415*9880d681SAndroid Build Coastguard Worker // preheader is deleted and we create a critical edge, registers may be
416*9880d681SAndroid Build Coastguard Worker // spilled in the loop body instead.
417*9880d681SAndroid Build Coastguard Worker if (!DisablePreheaderProtect && Preheaders.count(BB) &&
418*9880d681SAndroid Build Coastguard Worker !(BB->getSinglePredecessor() && BB->getSinglePredecessor()->getSingleSuccessor()))
419*9880d681SAndroid Build Coastguard Worker continue;
420*9880d681SAndroid Build Coastguard Worker
421*9880d681SAndroid Build Coastguard Worker eliminateMostlyEmptyBlock(BB);
422*9880d681SAndroid Build Coastguard Worker MadeChange = true;
423*9880d681SAndroid Build Coastguard Worker }
424*9880d681SAndroid Build Coastguard Worker return MadeChange;
425*9880d681SAndroid Build Coastguard Worker }
426*9880d681SAndroid Build Coastguard Worker
427*9880d681SAndroid Build Coastguard Worker /// Return true if we can merge BB into DestBB if there is a single
428*9880d681SAndroid Build Coastguard Worker /// unconditional branch between them, and BB contains no other non-phi
429*9880d681SAndroid Build Coastguard Worker /// instructions.
canMergeBlocks(const BasicBlock * BB,const BasicBlock * DestBB) const430*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB,
431*9880d681SAndroid Build Coastguard Worker const BasicBlock *DestBB) const {
432*9880d681SAndroid Build Coastguard Worker // We only want to eliminate blocks whose phi nodes are used by phi nodes in
433*9880d681SAndroid Build Coastguard Worker // the successor. If there are more complex condition (e.g. preheaders),
434*9880d681SAndroid Build Coastguard Worker // don't mess around with them.
435*9880d681SAndroid Build Coastguard Worker BasicBlock::const_iterator BBI = BB->begin();
436*9880d681SAndroid Build Coastguard Worker while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
437*9880d681SAndroid Build Coastguard Worker for (const User *U : PN->users()) {
438*9880d681SAndroid Build Coastguard Worker const Instruction *UI = cast<Instruction>(U);
439*9880d681SAndroid Build Coastguard Worker if (UI->getParent() != DestBB || !isa<PHINode>(UI))
440*9880d681SAndroid Build Coastguard Worker return false;
441*9880d681SAndroid Build Coastguard Worker // If User is inside DestBB block and it is a PHINode then check
442*9880d681SAndroid Build Coastguard Worker // incoming value. If incoming value is not from BB then this is
443*9880d681SAndroid Build Coastguard Worker // a complex condition (e.g. preheaders) we want to avoid here.
444*9880d681SAndroid Build Coastguard Worker if (UI->getParent() == DestBB) {
445*9880d681SAndroid Build Coastguard Worker if (const PHINode *UPN = dyn_cast<PHINode>(UI))
446*9880d681SAndroid Build Coastguard Worker for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) {
447*9880d681SAndroid Build Coastguard Worker Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I));
448*9880d681SAndroid Build Coastguard Worker if (Insn && Insn->getParent() == BB &&
449*9880d681SAndroid Build Coastguard Worker Insn->getParent() != UPN->getIncomingBlock(I))
450*9880d681SAndroid Build Coastguard Worker return false;
451*9880d681SAndroid Build Coastguard Worker }
452*9880d681SAndroid Build Coastguard Worker }
453*9880d681SAndroid Build Coastguard Worker }
454*9880d681SAndroid Build Coastguard Worker }
455*9880d681SAndroid Build Coastguard Worker
456*9880d681SAndroid Build Coastguard Worker // If BB and DestBB contain any common predecessors, then the phi nodes in BB
457*9880d681SAndroid Build Coastguard Worker // and DestBB may have conflicting incoming values for the block. If so, we
458*9880d681SAndroid Build Coastguard Worker // can't merge the block.
459*9880d681SAndroid Build Coastguard Worker const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin());
460*9880d681SAndroid Build Coastguard Worker if (!DestBBPN) return true; // no conflict.
461*9880d681SAndroid Build Coastguard Worker
462*9880d681SAndroid Build Coastguard Worker // Collect the preds of BB.
463*9880d681SAndroid Build Coastguard Worker SmallPtrSet<const BasicBlock*, 16> BBPreds;
464*9880d681SAndroid Build Coastguard Worker if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
465*9880d681SAndroid Build Coastguard Worker // It is faster to get preds from a PHI than with pred_iterator.
466*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
467*9880d681SAndroid Build Coastguard Worker BBPreds.insert(BBPN->getIncomingBlock(i));
468*9880d681SAndroid Build Coastguard Worker } else {
469*9880d681SAndroid Build Coastguard Worker BBPreds.insert(pred_begin(BB), pred_end(BB));
470*9880d681SAndroid Build Coastguard Worker }
471*9880d681SAndroid Build Coastguard Worker
472*9880d681SAndroid Build Coastguard Worker // Walk the preds of DestBB.
473*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) {
474*9880d681SAndroid Build Coastguard Worker BasicBlock *Pred = DestBBPN->getIncomingBlock(i);
475*9880d681SAndroid Build Coastguard Worker if (BBPreds.count(Pred)) { // Common predecessor?
476*9880d681SAndroid Build Coastguard Worker BBI = DestBB->begin();
477*9880d681SAndroid Build Coastguard Worker while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
478*9880d681SAndroid Build Coastguard Worker const Value *V1 = PN->getIncomingValueForBlock(Pred);
479*9880d681SAndroid Build Coastguard Worker const Value *V2 = PN->getIncomingValueForBlock(BB);
480*9880d681SAndroid Build Coastguard Worker
481*9880d681SAndroid Build Coastguard Worker // If V2 is a phi node in BB, look up what the mapped value will be.
482*9880d681SAndroid Build Coastguard Worker if (const PHINode *V2PN = dyn_cast<PHINode>(V2))
483*9880d681SAndroid Build Coastguard Worker if (V2PN->getParent() == BB)
484*9880d681SAndroid Build Coastguard Worker V2 = V2PN->getIncomingValueForBlock(Pred);
485*9880d681SAndroid Build Coastguard Worker
486*9880d681SAndroid Build Coastguard Worker // If there is a conflict, bail out.
487*9880d681SAndroid Build Coastguard Worker if (V1 != V2) return false;
488*9880d681SAndroid Build Coastguard Worker }
489*9880d681SAndroid Build Coastguard Worker }
490*9880d681SAndroid Build Coastguard Worker }
491*9880d681SAndroid Build Coastguard Worker
492*9880d681SAndroid Build Coastguard Worker return true;
493*9880d681SAndroid Build Coastguard Worker }
494*9880d681SAndroid Build Coastguard Worker
495*9880d681SAndroid Build Coastguard Worker
496*9880d681SAndroid Build Coastguard Worker /// Eliminate a basic block that has only phi's and an unconditional branch in
497*9880d681SAndroid Build Coastguard Worker /// it.
eliminateMostlyEmptyBlock(BasicBlock * BB)498*9880d681SAndroid Build Coastguard Worker void CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
499*9880d681SAndroid Build Coastguard Worker BranchInst *BI = cast<BranchInst>(BB->getTerminator());
500*9880d681SAndroid Build Coastguard Worker BasicBlock *DestBB = BI->getSuccessor(0);
501*9880d681SAndroid Build Coastguard Worker
502*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB);
503*9880d681SAndroid Build Coastguard Worker
504*9880d681SAndroid Build Coastguard Worker // If the destination block has a single pred, then this is a trivial edge,
505*9880d681SAndroid Build Coastguard Worker // just collapse it.
506*9880d681SAndroid Build Coastguard Worker if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) {
507*9880d681SAndroid Build Coastguard Worker if (SinglePred != DestBB) {
508*9880d681SAndroid Build Coastguard Worker // Remember if SinglePred was the entry block of the function. If so, we
509*9880d681SAndroid Build Coastguard Worker // will need to move BB back to the entry position.
510*9880d681SAndroid Build Coastguard Worker bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
511*9880d681SAndroid Build Coastguard Worker MergeBasicBlockIntoOnlyPred(DestBB, nullptr);
512*9880d681SAndroid Build Coastguard Worker
513*9880d681SAndroid Build Coastguard Worker if (isEntry && BB != &BB->getParent()->getEntryBlock())
514*9880d681SAndroid Build Coastguard Worker BB->moveBefore(&BB->getParent()->getEntryBlock());
515*9880d681SAndroid Build Coastguard Worker
516*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
517*9880d681SAndroid Build Coastguard Worker return;
518*9880d681SAndroid Build Coastguard Worker }
519*9880d681SAndroid Build Coastguard Worker }
520*9880d681SAndroid Build Coastguard Worker
521*9880d681SAndroid Build Coastguard Worker // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB
522*9880d681SAndroid Build Coastguard Worker // to handle the new incoming edges it is about to have.
523*9880d681SAndroid Build Coastguard Worker PHINode *PN;
524*9880d681SAndroid Build Coastguard Worker for (BasicBlock::iterator BBI = DestBB->begin();
525*9880d681SAndroid Build Coastguard Worker (PN = dyn_cast<PHINode>(BBI)); ++BBI) {
526*9880d681SAndroid Build Coastguard Worker // Remove the incoming value for BB, and remember it.
527*9880d681SAndroid Build Coastguard Worker Value *InVal = PN->removeIncomingValue(BB, false);
528*9880d681SAndroid Build Coastguard Worker
529*9880d681SAndroid Build Coastguard Worker // Two options: either the InVal is a phi node defined in BB or it is some
530*9880d681SAndroid Build Coastguard Worker // value that dominates BB.
531*9880d681SAndroid Build Coastguard Worker PHINode *InValPhi = dyn_cast<PHINode>(InVal);
532*9880d681SAndroid Build Coastguard Worker if (InValPhi && InValPhi->getParent() == BB) {
533*9880d681SAndroid Build Coastguard Worker // Add all of the input values of the input PHI as inputs of this phi.
534*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i)
535*9880d681SAndroid Build Coastguard Worker PN->addIncoming(InValPhi->getIncomingValue(i),
536*9880d681SAndroid Build Coastguard Worker InValPhi->getIncomingBlock(i));
537*9880d681SAndroid Build Coastguard Worker } else {
538*9880d681SAndroid Build Coastguard Worker // Otherwise, add one instance of the dominating value for each edge that
539*9880d681SAndroid Build Coastguard Worker // we will be adding.
540*9880d681SAndroid Build Coastguard Worker if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
541*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
542*9880d681SAndroid Build Coastguard Worker PN->addIncoming(InVal, BBPN->getIncomingBlock(i));
543*9880d681SAndroid Build Coastguard Worker } else {
544*9880d681SAndroid Build Coastguard Worker for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
545*9880d681SAndroid Build Coastguard Worker PN->addIncoming(InVal, *PI);
546*9880d681SAndroid Build Coastguard Worker }
547*9880d681SAndroid Build Coastguard Worker }
548*9880d681SAndroid Build Coastguard Worker }
549*9880d681SAndroid Build Coastguard Worker
550*9880d681SAndroid Build Coastguard Worker // The PHIs are now updated, change everything that refers to BB to use
551*9880d681SAndroid Build Coastguard Worker // DestBB and remove BB.
552*9880d681SAndroid Build Coastguard Worker BB->replaceAllUsesWith(DestBB);
553*9880d681SAndroid Build Coastguard Worker BB->eraseFromParent();
554*9880d681SAndroid Build Coastguard Worker ++NumBlocksElim;
555*9880d681SAndroid Build Coastguard Worker
556*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
557*9880d681SAndroid Build Coastguard Worker }
558*9880d681SAndroid Build Coastguard Worker
559*9880d681SAndroid Build Coastguard Worker // Computes a map of base pointer relocation instructions to corresponding
560*9880d681SAndroid Build Coastguard Worker // derived pointer relocation instructions given a vector of all relocate calls
computeBaseDerivedRelocateMap(const SmallVectorImpl<GCRelocateInst * > & AllRelocateCalls,DenseMap<GCRelocateInst *,SmallVector<GCRelocateInst *,2>> & RelocateInstMap)561*9880d681SAndroid Build Coastguard Worker static void computeBaseDerivedRelocateMap(
562*9880d681SAndroid Build Coastguard Worker const SmallVectorImpl<GCRelocateInst *> &AllRelocateCalls,
563*9880d681SAndroid Build Coastguard Worker DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>>
564*9880d681SAndroid Build Coastguard Worker &RelocateInstMap) {
565*9880d681SAndroid Build Coastguard Worker // Collect information in two maps: one primarily for locating the base object
566*9880d681SAndroid Build Coastguard Worker // while filling the second map; the second map is the final structure holding
567*9880d681SAndroid Build Coastguard Worker // a mapping between Base and corresponding Derived relocate calls
568*9880d681SAndroid Build Coastguard Worker DenseMap<std::pair<unsigned, unsigned>, GCRelocateInst *> RelocateIdxMap;
569*9880d681SAndroid Build Coastguard Worker for (auto *ThisRelocate : AllRelocateCalls) {
570*9880d681SAndroid Build Coastguard Worker auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
571*9880d681SAndroid Build Coastguard Worker ThisRelocate->getDerivedPtrIndex());
572*9880d681SAndroid Build Coastguard Worker RelocateIdxMap.insert(std::make_pair(K, ThisRelocate));
573*9880d681SAndroid Build Coastguard Worker }
574*9880d681SAndroid Build Coastguard Worker for (auto &Item : RelocateIdxMap) {
575*9880d681SAndroid Build Coastguard Worker std::pair<unsigned, unsigned> Key = Item.first;
576*9880d681SAndroid Build Coastguard Worker if (Key.first == Key.second)
577*9880d681SAndroid Build Coastguard Worker // Base relocation: nothing to insert
578*9880d681SAndroid Build Coastguard Worker continue;
579*9880d681SAndroid Build Coastguard Worker
580*9880d681SAndroid Build Coastguard Worker GCRelocateInst *I = Item.second;
581*9880d681SAndroid Build Coastguard Worker auto BaseKey = std::make_pair(Key.first, Key.first);
582*9880d681SAndroid Build Coastguard Worker
583*9880d681SAndroid Build Coastguard Worker // We're iterating over RelocateIdxMap so we cannot modify it.
584*9880d681SAndroid Build Coastguard Worker auto MaybeBase = RelocateIdxMap.find(BaseKey);
585*9880d681SAndroid Build Coastguard Worker if (MaybeBase == RelocateIdxMap.end())
586*9880d681SAndroid Build Coastguard Worker // TODO: We might want to insert a new base object relocate and gep off
587*9880d681SAndroid Build Coastguard Worker // that, if there are enough derived object relocates.
588*9880d681SAndroid Build Coastguard Worker continue;
589*9880d681SAndroid Build Coastguard Worker
590*9880d681SAndroid Build Coastguard Worker RelocateInstMap[MaybeBase->second].push_back(I);
591*9880d681SAndroid Build Coastguard Worker }
592*9880d681SAndroid Build Coastguard Worker }
593*9880d681SAndroid Build Coastguard Worker
594*9880d681SAndroid Build Coastguard Worker // Accepts a GEP and extracts the operands into a vector provided they're all
595*9880d681SAndroid Build Coastguard Worker // small integer constants
getGEPSmallConstantIntOffsetV(GetElementPtrInst * GEP,SmallVectorImpl<Value * > & OffsetV)596*9880d681SAndroid Build Coastguard Worker static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP,
597*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Value *> &OffsetV) {
598*9880d681SAndroid Build Coastguard Worker for (unsigned i = 1; i < GEP->getNumOperands(); i++) {
599*9880d681SAndroid Build Coastguard Worker // Only accept small constant integer operands
600*9880d681SAndroid Build Coastguard Worker auto Op = dyn_cast<ConstantInt>(GEP->getOperand(i));
601*9880d681SAndroid Build Coastguard Worker if (!Op || Op->getZExtValue() > 20)
602*9880d681SAndroid Build Coastguard Worker return false;
603*9880d681SAndroid Build Coastguard Worker }
604*9880d681SAndroid Build Coastguard Worker
605*9880d681SAndroid Build Coastguard Worker for (unsigned i = 1; i < GEP->getNumOperands(); i++)
606*9880d681SAndroid Build Coastguard Worker OffsetV.push_back(GEP->getOperand(i));
607*9880d681SAndroid Build Coastguard Worker return true;
608*9880d681SAndroid Build Coastguard Worker }
609*9880d681SAndroid Build Coastguard Worker
610*9880d681SAndroid Build Coastguard Worker // Takes a RelocatedBase (base pointer relocation instruction) and Targets to
611*9880d681SAndroid Build Coastguard Worker // replace, computes a replacement, and affects it.
612*9880d681SAndroid Build Coastguard Worker static bool
simplifyRelocatesOffABase(GCRelocateInst * RelocatedBase,const SmallVectorImpl<GCRelocateInst * > & Targets)613*9880d681SAndroid Build Coastguard Worker simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase,
614*9880d681SAndroid Build Coastguard Worker const SmallVectorImpl<GCRelocateInst *> &Targets) {
615*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
616*9880d681SAndroid Build Coastguard Worker for (GCRelocateInst *ToReplace : Targets) {
617*9880d681SAndroid Build Coastguard Worker assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() &&
618*9880d681SAndroid Build Coastguard Worker "Not relocating a derived object of the original base object");
619*9880d681SAndroid Build Coastguard Worker if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
620*9880d681SAndroid Build Coastguard Worker // A duplicate relocate call. TODO: coalesce duplicates.
621*9880d681SAndroid Build Coastguard Worker continue;
622*9880d681SAndroid Build Coastguard Worker }
623*9880d681SAndroid Build Coastguard Worker
624*9880d681SAndroid Build Coastguard Worker if (RelocatedBase->getParent() != ToReplace->getParent()) {
625*9880d681SAndroid Build Coastguard Worker // Base and derived relocates are in different basic blocks.
626*9880d681SAndroid Build Coastguard Worker // In this case transform is only valid when base dominates derived
627*9880d681SAndroid Build Coastguard Worker // relocate. However it would be too expensive to check dominance
628*9880d681SAndroid Build Coastguard Worker // for each such relocate, so we skip the whole transformation.
629*9880d681SAndroid Build Coastguard Worker continue;
630*9880d681SAndroid Build Coastguard Worker }
631*9880d681SAndroid Build Coastguard Worker
632*9880d681SAndroid Build Coastguard Worker Value *Base = ToReplace->getBasePtr();
633*9880d681SAndroid Build Coastguard Worker auto Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr());
634*9880d681SAndroid Build Coastguard Worker if (!Derived || Derived->getPointerOperand() != Base)
635*9880d681SAndroid Build Coastguard Worker continue;
636*9880d681SAndroid Build Coastguard Worker
637*9880d681SAndroid Build Coastguard Worker SmallVector<Value *, 2> OffsetV;
638*9880d681SAndroid Build Coastguard Worker if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV))
639*9880d681SAndroid Build Coastguard Worker continue;
640*9880d681SAndroid Build Coastguard Worker
641*9880d681SAndroid Build Coastguard Worker // Create a Builder and replace the target callsite with a gep
642*9880d681SAndroid Build Coastguard Worker assert(RelocatedBase->getNextNode() &&
643*9880d681SAndroid Build Coastguard Worker "Should always have one since it's not a terminator");
644*9880d681SAndroid Build Coastguard Worker
645*9880d681SAndroid Build Coastguard Worker // Insert after RelocatedBase
646*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(RelocatedBase->getNextNode());
647*9880d681SAndroid Build Coastguard Worker Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
648*9880d681SAndroid Build Coastguard Worker
649*9880d681SAndroid Build Coastguard Worker // If gc_relocate does not match the actual type, cast it to the right type.
650*9880d681SAndroid Build Coastguard Worker // In theory, there must be a bitcast after gc_relocate if the type does not
651*9880d681SAndroid Build Coastguard Worker // match, and we should reuse it to get the derived pointer. But it could be
652*9880d681SAndroid Build Coastguard Worker // cases like this:
653*9880d681SAndroid Build Coastguard Worker // bb1:
654*9880d681SAndroid Build Coastguard Worker // ...
655*9880d681SAndroid Build Coastguard Worker // %g1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
656*9880d681SAndroid Build Coastguard Worker // br label %merge
657*9880d681SAndroid Build Coastguard Worker //
658*9880d681SAndroid Build Coastguard Worker // bb2:
659*9880d681SAndroid Build Coastguard Worker // ...
660*9880d681SAndroid Build Coastguard Worker // %g2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
661*9880d681SAndroid Build Coastguard Worker // br label %merge
662*9880d681SAndroid Build Coastguard Worker //
663*9880d681SAndroid Build Coastguard Worker // merge:
664*9880d681SAndroid Build Coastguard Worker // %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ]
665*9880d681SAndroid Build Coastguard Worker // %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)*
666*9880d681SAndroid Build Coastguard Worker //
667*9880d681SAndroid Build Coastguard Worker // In this case, we can not find the bitcast any more. So we insert a new bitcast
668*9880d681SAndroid Build Coastguard Worker // no matter there is already one or not. In this way, we can handle all cases, and
669*9880d681SAndroid Build Coastguard Worker // the extra bitcast should be optimized away in later passes.
670*9880d681SAndroid Build Coastguard Worker Value *ActualRelocatedBase = RelocatedBase;
671*9880d681SAndroid Build Coastguard Worker if (RelocatedBase->getType() != Base->getType()) {
672*9880d681SAndroid Build Coastguard Worker ActualRelocatedBase =
673*9880d681SAndroid Build Coastguard Worker Builder.CreateBitCast(RelocatedBase, Base->getType());
674*9880d681SAndroid Build Coastguard Worker }
675*9880d681SAndroid Build Coastguard Worker Value *Replacement = Builder.CreateGEP(
676*9880d681SAndroid Build Coastguard Worker Derived->getSourceElementType(), ActualRelocatedBase, makeArrayRef(OffsetV));
677*9880d681SAndroid Build Coastguard Worker Replacement->takeName(ToReplace);
678*9880d681SAndroid Build Coastguard Worker // If the newly generated derived pointer's type does not match the original derived
679*9880d681SAndroid Build Coastguard Worker // pointer's type, cast the new derived pointer to match it. Same reasoning as above.
680*9880d681SAndroid Build Coastguard Worker Value *ActualReplacement = Replacement;
681*9880d681SAndroid Build Coastguard Worker if (Replacement->getType() != ToReplace->getType()) {
682*9880d681SAndroid Build Coastguard Worker ActualReplacement =
683*9880d681SAndroid Build Coastguard Worker Builder.CreateBitCast(Replacement, ToReplace->getType());
684*9880d681SAndroid Build Coastguard Worker }
685*9880d681SAndroid Build Coastguard Worker ToReplace->replaceAllUsesWith(ActualReplacement);
686*9880d681SAndroid Build Coastguard Worker ToReplace->eraseFromParent();
687*9880d681SAndroid Build Coastguard Worker
688*9880d681SAndroid Build Coastguard Worker MadeChange = true;
689*9880d681SAndroid Build Coastguard Worker }
690*9880d681SAndroid Build Coastguard Worker return MadeChange;
691*9880d681SAndroid Build Coastguard Worker }
692*9880d681SAndroid Build Coastguard Worker
693*9880d681SAndroid Build Coastguard Worker // Turns this:
694*9880d681SAndroid Build Coastguard Worker //
695*9880d681SAndroid Build Coastguard Worker // %base = ...
696*9880d681SAndroid Build Coastguard Worker // %ptr = gep %base + 15
697*9880d681SAndroid Build Coastguard Worker // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
698*9880d681SAndroid Build Coastguard Worker // %base' = relocate(%tok, i32 4, i32 4)
699*9880d681SAndroid Build Coastguard Worker // %ptr' = relocate(%tok, i32 4, i32 5)
700*9880d681SAndroid Build Coastguard Worker // %val = load %ptr'
701*9880d681SAndroid Build Coastguard Worker //
702*9880d681SAndroid Build Coastguard Worker // into this:
703*9880d681SAndroid Build Coastguard Worker //
704*9880d681SAndroid Build Coastguard Worker // %base = ...
705*9880d681SAndroid Build Coastguard Worker // %ptr = gep %base + 15
706*9880d681SAndroid Build Coastguard Worker // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
707*9880d681SAndroid Build Coastguard Worker // %base' = gc.relocate(%tok, i32 4, i32 4)
708*9880d681SAndroid Build Coastguard Worker // %ptr' = gep %base' + 15
709*9880d681SAndroid Build Coastguard Worker // %val = load %ptr'
simplifyOffsetableRelocate(Instruction & I)710*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::simplifyOffsetableRelocate(Instruction &I) {
711*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
712*9880d681SAndroid Build Coastguard Worker SmallVector<GCRelocateInst *, 2> AllRelocateCalls;
713*9880d681SAndroid Build Coastguard Worker
714*9880d681SAndroid Build Coastguard Worker for (auto *U : I.users())
715*9880d681SAndroid Build Coastguard Worker if (GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U))
716*9880d681SAndroid Build Coastguard Worker // Collect all the relocate calls associated with a statepoint
717*9880d681SAndroid Build Coastguard Worker AllRelocateCalls.push_back(Relocate);
718*9880d681SAndroid Build Coastguard Worker
719*9880d681SAndroid Build Coastguard Worker // We need atleast one base pointer relocation + one derived pointer
720*9880d681SAndroid Build Coastguard Worker // relocation to mangle
721*9880d681SAndroid Build Coastguard Worker if (AllRelocateCalls.size() < 2)
722*9880d681SAndroid Build Coastguard Worker return false;
723*9880d681SAndroid Build Coastguard Worker
724*9880d681SAndroid Build Coastguard Worker // RelocateInstMap is a mapping from the base relocate instruction to the
725*9880d681SAndroid Build Coastguard Worker // corresponding derived relocate instructions
726*9880d681SAndroid Build Coastguard Worker DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>> RelocateInstMap;
727*9880d681SAndroid Build Coastguard Worker computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap);
728*9880d681SAndroid Build Coastguard Worker if (RelocateInstMap.empty())
729*9880d681SAndroid Build Coastguard Worker return false;
730*9880d681SAndroid Build Coastguard Worker
731*9880d681SAndroid Build Coastguard Worker for (auto &Item : RelocateInstMap)
732*9880d681SAndroid Build Coastguard Worker // Item.first is the RelocatedBase to offset against
733*9880d681SAndroid Build Coastguard Worker // Item.second is the vector of Targets to replace
734*9880d681SAndroid Build Coastguard Worker MadeChange = simplifyRelocatesOffABase(Item.first, Item.second);
735*9880d681SAndroid Build Coastguard Worker return MadeChange;
736*9880d681SAndroid Build Coastguard Worker }
737*9880d681SAndroid Build Coastguard Worker
738*9880d681SAndroid Build Coastguard Worker /// SinkCast - Sink the specified cast instruction into its user blocks
SinkCast(CastInst * CI)739*9880d681SAndroid Build Coastguard Worker static bool SinkCast(CastInst *CI) {
740*9880d681SAndroid Build Coastguard Worker BasicBlock *DefBB = CI->getParent();
741*9880d681SAndroid Build Coastguard Worker
742*9880d681SAndroid Build Coastguard Worker /// InsertedCasts - Only insert a cast in each block once.
743*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock*, CastInst*> InsertedCasts;
744*9880d681SAndroid Build Coastguard Worker
745*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
746*9880d681SAndroid Build Coastguard Worker for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
747*9880d681SAndroid Build Coastguard Worker UI != E; ) {
748*9880d681SAndroid Build Coastguard Worker Use &TheUse = UI.getUse();
749*9880d681SAndroid Build Coastguard Worker Instruction *User = cast<Instruction>(*UI);
750*9880d681SAndroid Build Coastguard Worker
751*9880d681SAndroid Build Coastguard Worker // Figure out which BB this cast is used in. For PHI's this is the
752*9880d681SAndroid Build Coastguard Worker // appropriate predecessor block.
753*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = User->getParent();
754*9880d681SAndroid Build Coastguard Worker if (PHINode *PN = dyn_cast<PHINode>(User)) {
755*9880d681SAndroid Build Coastguard Worker UserBB = PN->getIncomingBlock(TheUse);
756*9880d681SAndroid Build Coastguard Worker }
757*9880d681SAndroid Build Coastguard Worker
758*9880d681SAndroid Build Coastguard Worker // Preincrement use iterator so we don't invalidate it.
759*9880d681SAndroid Build Coastguard Worker ++UI;
760*9880d681SAndroid Build Coastguard Worker
761*9880d681SAndroid Build Coastguard Worker // The first insertion point of a block containing an EH pad is after the
762*9880d681SAndroid Build Coastguard Worker // pad. If the pad is the user, we cannot sink the cast past the pad.
763*9880d681SAndroid Build Coastguard Worker if (User->isEHPad())
764*9880d681SAndroid Build Coastguard Worker continue;
765*9880d681SAndroid Build Coastguard Worker
766*9880d681SAndroid Build Coastguard Worker // If the block selected to receive the cast is an EH pad that does not
767*9880d681SAndroid Build Coastguard Worker // allow non-PHI instructions before the terminator, we can't sink the
768*9880d681SAndroid Build Coastguard Worker // cast.
769*9880d681SAndroid Build Coastguard Worker if (UserBB->getTerminator()->isEHPad())
770*9880d681SAndroid Build Coastguard Worker continue;
771*9880d681SAndroid Build Coastguard Worker
772*9880d681SAndroid Build Coastguard Worker // If this user is in the same block as the cast, don't change the cast.
773*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) continue;
774*9880d681SAndroid Build Coastguard Worker
775*9880d681SAndroid Build Coastguard Worker // If we have already inserted a cast into this block, use it.
776*9880d681SAndroid Build Coastguard Worker CastInst *&InsertedCast = InsertedCasts[UserBB];
777*9880d681SAndroid Build Coastguard Worker
778*9880d681SAndroid Build Coastguard Worker if (!InsertedCast) {
779*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
780*9880d681SAndroid Build Coastguard Worker assert(InsertPt != UserBB->end());
781*9880d681SAndroid Build Coastguard Worker InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0),
782*9880d681SAndroid Build Coastguard Worker CI->getType(), "", &*InsertPt);
783*9880d681SAndroid Build Coastguard Worker }
784*9880d681SAndroid Build Coastguard Worker
785*9880d681SAndroid Build Coastguard Worker // Replace a use of the cast with a use of the new cast.
786*9880d681SAndroid Build Coastguard Worker TheUse = InsertedCast;
787*9880d681SAndroid Build Coastguard Worker MadeChange = true;
788*9880d681SAndroid Build Coastguard Worker ++NumCastUses;
789*9880d681SAndroid Build Coastguard Worker }
790*9880d681SAndroid Build Coastguard Worker
791*9880d681SAndroid Build Coastguard Worker // If we removed all uses, nuke the cast.
792*9880d681SAndroid Build Coastguard Worker if (CI->use_empty()) {
793*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
794*9880d681SAndroid Build Coastguard Worker MadeChange = true;
795*9880d681SAndroid Build Coastguard Worker }
796*9880d681SAndroid Build Coastguard Worker
797*9880d681SAndroid Build Coastguard Worker return MadeChange;
798*9880d681SAndroid Build Coastguard Worker }
799*9880d681SAndroid Build Coastguard Worker
800*9880d681SAndroid Build Coastguard Worker /// If the specified cast instruction is a noop copy (e.g. it's casting from
801*9880d681SAndroid Build Coastguard Worker /// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
802*9880d681SAndroid Build Coastguard Worker /// reduce the number of virtual registers that must be created and coalesced.
803*9880d681SAndroid Build Coastguard Worker ///
804*9880d681SAndroid Build Coastguard Worker /// Return true if any changes are made.
805*9880d681SAndroid Build Coastguard Worker ///
OptimizeNoopCopyExpression(CastInst * CI,const TargetLowering & TLI,const DataLayout & DL)806*9880d681SAndroid Build Coastguard Worker static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI,
807*9880d681SAndroid Build Coastguard Worker const DataLayout &DL) {
808*9880d681SAndroid Build Coastguard Worker // If this is a noop copy,
809*9880d681SAndroid Build Coastguard Worker EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType());
810*9880d681SAndroid Build Coastguard Worker EVT DstVT = TLI.getValueType(DL, CI->getType());
811*9880d681SAndroid Build Coastguard Worker
812*9880d681SAndroid Build Coastguard Worker // This is an fp<->int conversion?
813*9880d681SAndroid Build Coastguard Worker if (SrcVT.isInteger() != DstVT.isInteger())
814*9880d681SAndroid Build Coastguard Worker return false;
815*9880d681SAndroid Build Coastguard Worker
816*9880d681SAndroid Build Coastguard Worker // If this is an extension, it will be a zero or sign extension, which
817*9880d681SAndroid Build Coastguard Worker // isn't a noop.
818*9880d681SAndroid Build Coastguard Worker if (SrcVT.bitsLT(DstVT)) return false;
819*9880d681SAndroid Build Coastguard Worker
820*9880d681SAndroid Build Coastguard Worker // If these values will be promoted, find out what they will be promoted
821*9880d681SAndroid Build Coastguard Worker // to. This helps us consider truncates on PPC as noop copies when they
822*9880d681SAndroid Build Coastguard Worker // are.
823*9880d681SAndroid Build Coastguard Worker if (TLI.getTypeAction(CI->getContext(), SrcVT) ==
824*9880d681SAndroid Build Coastguard Worker TargetLowering::TypePromoteInteger)
825*9880d681SAndroid Build Coastguard Worker SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT);
826*9880d681SAndroid Build Coastguard Worker if (TLI.getTypeAction(CI->getContext(), DstVT) ==
827*9880d681SAndroid Build Coastguard Worker TargetLowering::TypePromoteInteger)
828*9880d681SAndroid Build Coastguard Worker DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT);
829*9880d681SAndroid Build Coastguard Worker
830*9880d681SAndroid Build Coastguard Worker // If, after promotion, these are the same types, this is a noop copy.
831*9880d681SAndroid Build Coastguard Worker if (SrcVT != DstVT)
832*9880d681SAndroid Build Coastguard Worker return false;
833*9880d681SAndroid Build Coastguard Worker
834*9880d681SAndroid Build Coastguard Worker return SinkCast(CI);
835*9880d681SAndroid Build Coastguard Worker }
836*9880d681SAndroid Build Coastguard Worker
837*9880d681SAndroid Build Coastguard Worker /// Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if
838*9880d681SAndroid Build Coastguard Worker /// possible.
839*9880d681SAndroid Build Coastguard Worker ///
840*9880d681SAndroid Build Coastguard Worker /// Return true if any changes were made.
CombineUAddWithOverflow(CmpInst * CI)841*9880d681SAndroid Build Coastguard Worker static bool CombineUAddWithOverflow(CmpInst *CI) {
842*9880d681SAndroid Build Coastguard Worker Value *A, *B;
843*9880d681SAndroid Build Coastguard Worker Instruction *AddI;
844*9880d681SAndroid Build Coastguard Worker if (!match(CI,
845*9880d681SAndroid Build Coastguard Worker m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI))))
846*9880d681SAndroid Build Coastguard Worker return false;
847*9880d681SAndroid Build Coastguard Worker
848*9880d681SAndroid Build Coastguard Worker Type *Ty = AddI->getType();
849*9880d681SAndroid Build Coastguard Worker if (!isa<IntegerType>(Ty))
850*9880d681SAndroid Build Coastguard Worker return false;
851*9880d681SAndroid Build Coastguard Worker
852*9880d681SAndroid Build Coastguard Worker // We don't want to move around uses of condition values this late, so we we
853*9880d681SAndroid Build Coastguard Worker // check if it is legal to create the call to the intrinsic in the basic
854*9880d681SAndroid Build Coastguard Worker // block containing the icmp:
855*9880d681SAndroid Build Coastguard Worker
856*9880d681SAndroid Build Coastguard Worker if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse())
857*9880d681SAndroid Build Coastguard Worker return false;
858*9880d681SAndroid Build Coastguard Worker
859*9880d681SAndroid Build Coastguard Worker #ifndef NDEBUG
860*9880d681SAndroid Build Coastguard Worker // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption
861*9880d681SAndroid Build Coastguard Worker // for now:
862*9880d681SAndroid Build Coastguard Worker if (AddI->hasOneUse())
863*9880d681SAndroid Build Coastguard Worker assert(*AddI->user_begin() == CI && "expected!");
864*9880d681SAndroid Build Coastguard Worker #endif
865*9880d681SAndroid Build Coastguard Worker
866*9880d681SAndroid Build Coastguard Worker Module *M = CI->getModule();
867*9880d681SAndroid Build Coastguard Worker Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty);
868*9880d681SAndroid Build Coastguard Worker
869*9880d681SAndroid Build Coastguard Worker auto *InsertPt = AddI->hasOneUse() ? CI : AddI;
870*9880d681SAndroid Build Coastguard Worker
871*9880d681SAndroid Build Coastguard Worker auto *UAddWithOverflow =
872*9880d681SAndroid Build Coastguard Worker CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt);
873*9880d681SAndroid Build Coastguard Worker auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt);
874*9880d681SAndroid Build Coastguard Worker auto *Overflow =
875*9880d681SAndroid Build Coastguard Worker ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt);
876*9880d681SAndroid Build Coastguard Worker
877*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(Overflow);
878*9880d681SAndroid Build Coastguard Worker AddI->replaceAllUsesWith(UAdd);
879*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
880*9880d681SAndroid Build Coastguard Worker AddI->eraseFromParent();
881*9880d681SAndroid Build Coastguard Worker return true;
882*9880d681SAndroid Build Coastguard Worker }
883*9880d681SAndroid Build Coastguard Worker
884*9880d681SAndroid Build Coastguard Worker /// Sink the given CmpInst into user blocks to reduce the number of virtual
885*9880d681SAndroid Build Coastguard Worker /// registers that must be created and coalesced. This is a clear win except on
886*9880d681SAndroid Build Coastguard Worker /// targets with multiple condition code registers (PowerPC), where it might
887*9880d681SAndroid Build Coastguard Worker /// lose; some adjustment may be wanted there.
888*9880d681SAndroid Build Coastguard Worker ///
889*9880d681SAndroid Build Coastguard Worker /// Return true if any changes are made.
SinkCmpExpression(CmpInst * CI,const TargetLowering * TLI)890*9880d681SAndroid Build Coastguard Worker static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
891*9880d681SAndroid Build Coastguard Worker BasicBlock *DefBB = CI->getParent();
892*9880d681SAndroid Build Coastguard Worker
893*9880d681SAndroid Build Coastguard Worker // Avoid sinking soft-FP comparisons, since this can move them into a loop.
894*9880d681SAndroid Build Coastguard Worker if (TLI && TLI->useSoftFloat() && isa<FCmpInst>(CI))
895*9880d681SAndroid Build Coastguard Worker return false;
896*9880d681SAndroid Build Coastguard Worker
897*9880d681SAndroid Build Coastguard Worker // Only insert a cmp in each block once.
898*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock*, CmpInst*> InsertedCmps;
899*9880d681SAndroid Build Coastguard Worker
900*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
901*9880d681SAndroid Build Coastguard Worker for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
902*9880d681SAndroid Build Coastguard Worker UI != E; ) {
903*9880d681SAndroid Build Coastguard Worker Use &TheUse = UI.getUse();
904*9880d681SAndroid Build Coastguard Worker Instruction *User = cast<Instruction>(*UI);
905*9880d681SAndroid Build Coastguard Worker
906*9880d681SAndroid Build Coastguard Worker // Preincrement use iterator so we don't invalidate it.
907*9880d681SAndroid Build Coastguard Worker ++UI;
908*9880d681SAndroid Build Coastguard Worker
909*9880d681SAndroid Build Coastguard Worker // Don't bother for PHI nodes.
910*9880d681SAndroid Build Coastguard Worker if (isa<PHINode>(User))
911*9880d681SAndroid Build Coastguard Worker continue;
912*9880d681SAndroid Build Coastguard Worker
913*9880d681SAndroid Build Coastguard Worker // Figure out which BB this cmp is used in.
914*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = User->getParent();
915*9880d681SAndroid Build Coastguard Worker
916*9880d681SAndroid Build Coastguard Worker // If this user is in the same block as the cmp, don't change the cmp.
917*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) continue;
918*9880d681SAndroid Build Coastguard Worker
919*9880d681SAndroid Build Coastguard Worker // If we have already inserted a cmp into this block, use it.
920*9880d681SAndroid Build Coastguard Worker CmpInst *&InsertedCmp = InsertedCmps[UserBB];
921*9880d681SAndroid Build Coastguard Worker
922*9880d681SAndroid Build Coastguard Worker if (!InsertedCmp) {
923*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
924*9880d681SAndroid Build Coastguard Worker assert(InsertPt != UserBB->end());
925*9880d681SAndroid Build Coastguard Worker InsertedCmp =
926*9880d681SAndroid Build Coastguard Worker CmpInst::Create(CI->getOpcode(), CI->getPredicate(),
927*9880d681SAndroid Build Coastguard Worker CI->getOperand(0), CI->getOperand(1), "", &*InsertPt);
928*9880d681SAndroid Build Coastguard Worker }
929*9880d681SAndroid Build Coastguard Worker
930*9880d681SAndroid Build Coastguard Worker // Replace a use of the cmp with a use of the new cmp.
931*9880d681SAndroid Build Coastguard Worker TheUse = InsertedCmp;
932*9880d681SAndroid Build Coastguard Worker MadeChange = true;
933*9880d681SAndroid Build Coastguard Worker ++NumCmpUses;
934*9880d681SAndroid Build Coastguard Worker }
935*9880d681SAndroid Build Coastguard Worker
936*9880d681SAndroid Build Coastguard Worker // If we removed all uses, nuke the cmp.
937*9880d681SAndroid Build Coastguard Worker if (CI->use_empty()) {
938*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
939*9880d681SAndroid Build Coastguard Worker MadeChange = true;
940*9880d681SAndroid Build Coastguard Worker }
941*9880d681SAndroid Build Coastguard Worker
942*9880d681SAndroid Build Coastguard Worker return MadeChange;
943*9880d681SAndroid Build Coastguard Worker }
944*9880d681SAndroid Build Coastguard Worker
OptimizeCmpExpression(CmpInst * CI,const TargetLowering * TLI)945*9880d681SAndroid Build Coastguard Worker static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
946*9880d681SAndroid Build Coastguard Worker if (SinkCmpExpression(CI, TLI))
947*9880d681SAndroid Build Coastguard Worker return true;
948*9880d681SAndroid Build Coastguard Worker
949*9880d681SAndroid Build Coastguard Worker if (CombineUAddWithOverflow(CI))
950*9880d681SAndroid Build Coastguard Worker return true;
951*9880d681SAndroid Build Coastguard Worker
952*9880d681SAndroid Build Coastguard Worker return false;
953*9880d681SAndroid Build Coastguard Worker }
954*9880d681SAndroid Build Coastguard Worker
955*9880d681SAndroid Build Coastguard Worker /// Check if the candidates could be combined with a shift instruction, which
956*9880d681SAndroid Build Coastguard Worker /// includes:
957*9880d681SAndroid Build Coastguard Worker /// 1. Truncate instruction
958*9880d681SAndroid Build Coastguard Worker /// 2. And instruction and the imm is a mask of the low bits:
959*9880d681SAndroid Build Coastguard Worker /// imm & (imm+1) == 0
isExtractBitsCandidateUse(Instruction * User)960*9880d681SAndroid Build Coastguard Worker static bool isExtractBitsCandidateUse(Instruction *User) {
961*9880d681SAndroid Build Coastguard Worker if (!isa<TruncInst>(User)) {
962*9880d681SAndroid Build Coastguard Worker if (User->getOpcode() != Instruction::And ||
963*9880d681SAndroid Build Coastguard Worker !isa<ConstantInt>(User->getOperand(1)))
964*9880d681SAndroid Build Coastguard Worker return false;
965*9880d681SAndroid Build Coastguard Worker
966*9880d681SAndroid Build Coastguard Worker const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue();
967*9880d681SAndroid Build Coastguard Worker
968*9880d681SAndroid Build Coastguard Worker if ((Cimm & (Cimm + 1)).getBoolValue())
969*9880d681SAndroid Build Coastguard Worker return false;
970*9880d681SAndroid Build Coastguard Worker }
971*9880d681SAndroid Build Coastguard Worker return true;
972*9880d681SAndroid Build Coastguard Worker }
973*9880d681SAndroid Build Coastguard Worker
974*9880d681SAndroid Build Coastguard Worker /// Sink both shift and truncate instruction to the use of truncate's BB.
975*9880d681SAndroid Build Coastguard Worker static bool
SinkShiftAndTruncate(BinaryOperator * ShiftI,Instruction * User,ConstantInt * CI,DenseMap<BasicBlock *,BinaryOperator * > & InsertedShifts,const TargetLowering & TLI,const DataLayout & DL)976*9880d681SAndroid Build Coastguard Worker SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
977*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts,
978*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI, const DataLayout &DL) {
979*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = User->getParent();
980*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock *, CastInst *> InsertedTruncs;
981*9880d681SAndroid Build Coastguard Worker TruncInst *TruncI = dyn_cast<TruncInst>(User);
982*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
983*9880d681SAndroid Build Coastguard Worker
984*9880d681SAndroid Build Coastguard Worker for (Value::user_iterator TruncUI = TruncI->user_begin(),
985*9880d681SAndroid Build Coastguard Worker TruncE = TruncI->user_end();
986*9880d681SAndroid Build Coastguard Worker TruncUI != TruncE;) {
987*9880d681SAndroid Build Coastguard Worker
988*9880d681SAndroid Build Coastguard Worker Use &TruncTheUse = TruncUI.getUse();
989*9880d681SAndroid Build Coastguard Worker Instruction *TruncUser = cast<Instruction>(*TruncUI);
990*9880d681SAndroid Build Coastguard Worker // Preincrement use iterator so we don't invalidate it.
991*9880d681SAndroid Build Coastguard Worker
992*9880d681SAndroid Build Coastguard Worker ++TruncUI;
993*9880d681SAndroid Build Coastguard Worker
994*9880d681SAndroid Build Coastguard Worker int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode());
995*9880d681SAndroid Build Coastguard Worker if (!ISDOpcode)
996*9880d681SAndroid Build Coastguard Worker continue;
997*9880d681SAndroid Build Coastguard Worker
998*9880d681SAndroid Build Coastguard Worker // If the use is actually a legal node, there will not be an
999*9880d681SAndroid Build Coastguard Worker // implicit truncate.
1000*9880d681SAndroid Build Coastguard Worker // FIXME: always querying the result type is just an
1001*9880d681SAndroid Build Coastguard Worker // approximation; some nodes' legality is determined by the
1002*9880d681SAndroid Build Coastguard Worker // operand or other means. There's no good way to find out though.
1003*9880d681SAndroid Build Coastguard Worker if (TLI.isOperationLegalOrCustom(
1004*9880d681SAndroid Build Coastguard Worker ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true)))
1005*9880d681SAndroid Build Coastguard Worker continue;
1006*9880d681SAndroid Build Coastguard Worker
1007*9880d681SAndroid Build Coastguard Worker // Don't bother for PHI nodes.
1008*9880d681SAndroid Build Coastguard Worker if (isa<PHINode>(TruncUser))
1009*9880d681SAndroid Build Coastguard Worker continue;
1010*9880d681SAndroid Build Coastguard Worker
1011*9880d681SAndroid Build Coastguard Worker BasicBlock *TruncUserBB = TruncUser->getParent();
1012*9880d681SAndroid Build Coastguard Worker
1013*9880d681SAndroid Build Coastguard Worker if (UserBB == TruncUserBB)
1014*9880d681SAndroid Build Coastguard Worker continue;
1015*9880d681SAndroid Build Coastguard Worker
1016*9880d681SAndroid Build Coastguard Worker BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB];
1017*9880d681SAndroid Build Coastguard Worker CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
1018*9880d681SAndroid Build Coastguard Worker
1019*9880d681SAndroid Build Coastguard Worker if (!InsertedShift && !InsertedTrunc) {
1020*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt();
1021*9880d681SAndroid Build Coastguard Worker assert(InsertPt != TruncUserBB->end());
1022*9880d681SAndroid Build Coastguard Worker // Sink the shift
1023*9880d681SAndroid Build Coastguard Worker if (ShiftI->getOpcode() == Instruction::AShr)
1024*9880d681SAndroid Build Coastguard Worker InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
1025*9880d681SAndroid Build Coastguard Worker "", &*InsertPt);
1026*9880d681SAndroid Build Coastguard Worker else
1027*9880d681SAndroid Build Coastguard Worker InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
1028*9880d681SAndroid Build Coastguard Worker "", &*InsertPt);
1029*9880d681SAndroid Build Coastguard Worker
1030*9880d681SAndroid Build Coastguard Worker // Sink the trunc
1031*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt();
1032*9880d681SAndroid Build Coastguard Worker TruncInsertPt++;
1033*9880d681SAndroid Build Coastguard Worker assert(TruncInsertPt != TruncUserBB->end());
1034*9880d681SAndroid Build Coastguard Worker
1035*9880d681SAndroid Build Coastguard Worker InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift,
1036*9880d681SAndroid Build Coastguard Worker TruncI->getType(), "", &*TruncInsertPt);
1037*9880d681SAndroid Build Coastguard Worker
1038*9880d681SAndroid Build Coastguard Worker MadeChange = true;
1039*9880d681SAndroid Build Coastguard Worker
1040*9880d681SAndroid Build Coastguard Worker TruncTheUse = InsertedTrunc;
1041*9880d681SAndroid Build Coastguard Worker }
1042*9880d681SAndroid Build Coastguard Worker }
1043*9880d681SAndroid Build Coastguard Worker return MadeChange;
1044*9880d681SAndroid Build Coastguard Worker }
1045*9880d681SAndroid Build Coastguard Worker
1046*9880d681SAndroid Build Coastguard Worker /// Sink the shift *right* instruction into user blocks if the uses could
1047*9880d681SAndroid Build Coastguard Worker /// potentially be combined with this shift instruction and generate BitExtract
1048*9880d681SAndroid Build Coastguard Worker /// instruction. It will only be applied if the architecture supports BitExtract
1049*9880d681SAndroid Build Coastguard Worker /// instruction. Here is an example:
1050*9880d681SAndroid Build Coastguard Worker /// BB1:
1051*9880d681SAndroid Build Coastguard Worker /// %x.extract.shift = lshr i64 %arg1, 32
1052*9880d681SAndroid Build Coastguard Worker /// BB2:
1053*9880d681SAndroid Build Coastguard Worker /// %x.extract.trunc = trunc i64 %x.extract.shift to i16
1054*9880d681SAndroid Build Coastguard Worker /// ==>
1055*9880d681SAndroid Build Coastguard Worker ///
1056*9880d681SAndroid Build Coastguard Worker /// BB2:
1057*9880d681SAndroid Build Coastguard Worker /// %x.extract.shift.1 = lshr i64 %arg1, 32
1058*9880d681SAndroid Build Coastguard Worker /// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16
1059*9880d681SAndroid Build Coastguard Worker ///
1060*9880d681SAndroid Build Coastguard Worker /// CodeGen will recoginze the pattern in BB2 and generate BitExtract
1061*9880d681SAndroid Build Coastguard Worker /// instruction.
1062*9880d681SAndroid Build Coastguard Worker /// Return true if any changes are made.
OptimizeExtractBits(BinaryOperator * ShiftI,ConstantInt * CI,const TargetLowering & TLI,const DataLayout & DL)1063*9880d681SAndroid Build Coastguard Worker static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
1064*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI,
1065*9880d681SAndroid Build Coastguard Worker const DataLayout &DL) {
1066*9880d681SAndroid Build Coastguard Worker BasicBlock *DefBB = ShiftI->getParent();
1067*9880d681SAndroid Build Coastguard Worker
1068*9880d681SAndroid Build Coastguard Worker /// Only insert instructions in each block once.
1069*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts;
1070*9880d681SAndroid Build Coastguard Worker
1071*9880d681SAndroid Build Coastguard Worker bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType()));
1072*9880d681SAndroid Build Coastguard Worker
1073*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
1074*9880d681SAndroid Build Coastguard Worker for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end();
1075*9880d681SAndroid Build Coastguard Worker UI != E;) {
1076*9880d681SAndroid Build Coastguard Worker Use &TheUse = UI.getUse();
1077*9880d681SAndroid Build Coastguard Worker Instruction *User = cast<Instruction>(*UI);
1078*9880d681SAndroid Build Coastguard Worker // Preincrement use iterator so we don't invalidate it.
1079*9880d681SAndroid Build Coastguard Worker ++UI;
1080*9880d681SAndroid Build Coastguard Worker
1081*9880d681SAndroid Build Coastguard Worker // Don't bother for PHI nodes.
1082*9880d681SAndroid Build Coastguard Worker if (isa<PHINode>(User))
1083*9880d681SAndroid Build Coastguard Worker continue;
1084*9880d681SAndroid Build Coastguard Worker
1085*9880d681SAndroid Build Coastguard Worker if (!isExtractBitsCandidateUse(User))
1086*9880d681SAndroid Build Coastguard Worker continue;
1087*9880d681SAndroid Build Coastguard Worker
1088*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = User->getParent();
1089*9880d681SAndroid Build Coastguard Worker
1090*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) {
1091*9880d681SAndroid Build Coastguard Worker // If the shift and truncate instruction are in the same BB. The use of
1092*9880d681SAndroid Build Coastguard Worker // the truncate(TruncUse) may still introduce another truncate if not
1093*9880d681SAndroid Build Coastguard Worker // legal. In this case, we would like to sink both shift and truncate
1094*9880d681SAndroid Build Coastguard Worker // instruction to the BB of TruncUse.
1095*9880d681SAndroid Build Coastguard Worker // for example:
1096*9880d681SAndroid Build Coastguard Worker // BB1:
1097*9880d681SAndroid Build Coastguard Worker // i64 shift.result = lshr i64 opnd, imm
1098*9880d681SAndroid Build Coastguard Worker // trunc.result = trunc shift.result to i16
1099*9880d681SAndroid Build Coastguard Worker //
1100*9880d681SAndroid Build Coastguard Worker // BB2:
1101*9880d681SAndroid Build Coastguard Worker // ----> We will have an implicit truncate here if the architecture does
1102*9880d681SAndroid Build Coastguard Worker // not have i16 compare.
1103*9880d681SAndroid Build Coastguard Worker // cmp i16 trunc.result, opnd2
1104*9880d681SAndroid Build Coastguard Worker //
1105*9880d681SAndroid Build Coastguard Worker if (isa<TruncInst>(User) && shiftIsLegal
1106*9880d681SAndroid Build Coastguard Worker // If the type of the truncate is legal, no trucate will be
1107*9880d681SAndroid Build Coastguard Worker // introduced in other basic blocks.
1108*9880d681SAndroid Build Coastguard Worker &&
1109*9880d681SAndroid Build Coastguard Worker (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType()))))
1110*9880d681SAndroid Build Coastguard Worker MadeChange =
1111*9880d681SAndroid Build Coastguard Worker SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL);
1112*9880d681SAndroid Build Coastguard Worker
1113*9880d681SAndroid Build Coastguard Worker continue;
1114*9880d681SAndroid Build Coastguard Worker }
1115*9880d681SAndroid Build Coastguard Worker // If we have already inserted a shift into this block, use it.
1116*9880d681SAndroid Build Coastguard Worker BinaryOperator *&InsertedShift = InsertedShifts[UserBB];
1117*9880d681SAndroid Build Coastguard Worker
1118*9880d681SAndroid Build Coastguard Worker if (!InsertedShift) {
1119*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
1120*9880d681SAndroid Build Coastguard Worker assert(InsertPt != UserBB->end());
1121*9880d681SAndroid Build Coastguard Worker
1122*9880d681SAndroid Build Coastguard Worker if (ShiftI->getOpcode() == Instruction::AShr)
1123*9880d681SAndroid Build Coastguard Worker InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
1124*9880d681SAndroid Build Coastguard Worker "", &*InsertPt);
1125*9880d681SAndroid Build Coastguard Worker else
1126*9880d681SAndroid Build Coastguard Worker InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
1127*9880d681SAndroid Build Coastguard Worker "", &*InsertPt);
1128*9880d681SAndroid Build Coastguard Worker
1129*9880d681SAndroid Build Coastguard Worker MadeChange = true;
1130*9880d681SAndroid Build Coastguard Worker }
1131*9880d681SAndroid Build Coastguard Worker
1132*9880d681SAndroid Build Coastguard Worker // Replace a use of the shift with a use of the new shift.
1133*9880d681SAndroid Build Coastguard Worker TheUse = InsertedShift;
1134*9880d681SAndroid Build Coastguard Worker }
1135*9880d681SAndroid Build Coastguard Worker
1136*9880d681SAndroid Build Coastguard Worker // If we removed all uses, nuke the shift.
1137*9880d681SAndroid Build Coastguard Worker if (ShiftI->use_empty())
1138*9880d681SAndroid Build Coastguard Worker ShiftI->eraseFromParent();
1139*9880d681SAndroid Build Coastguard Worker
1140*9880d681SAndroid Build Coastguard Worker return MadeChange;
1141*9880d681SAndroid Build Coastguard Worker }
1142*9880d681SAndroid Build Coastguard Worker
1143*9880d681SAndroid Build Coastguard Worker // Translate a masked load intrinsic like
1144*9880d681SAndroid Build Coastguard Worker // <16 x i32 > @llvm.masked.load( <16 x i32>* %addr, i32 align,
1145*9880d681SAndroid Build Coastguard Worker // <16 x i1> %mask, <16 x i32> %passthru)
1146*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, with loading element one-by-one if
1147*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set
1148*9880d681SAndroid Build Coastguard Worker //
1149*9880d681SAndroid Build Coastguard Worker // %1 = bitcast i8* %addr to i32*
1150*9880d681SAndroid Build Coastguard Worker // %2 = extractelement <16 x i1> %mask, i32 0
1151*9880d681SAndroid Build Coastguard Worker // %3 = icmp eq i1 %2, true
1152*9880d681SAndroid Build Coastguard Worker // br i1 %3, label %cond.load, label %else
1153*9880d681SAndroid Build Coastguard Worker //
1154*9880d681SAndroid Build Coastguard Worker //cond.load: ; preds = %0
1155*9880d681SAndroid Build Coastguard Worker // %4 = getelementptr i32* %1, i32 0
1156*9880d681SAndroid Build Coastguard Worker // %5 = load i32* %4
1157*9880d681SAndroid Build Coastguard Worker // %6 = insertelement <16 x i32> undef, i32 %5, i32 0
1158*9880d681SAndroid Build Coastguard Worker // br label %else
1159*9880d681SAndroid Build Coastguard Worker //
1160*9880d681SAndroid Build Coastguard Worker //else: ; preds = %0, %cond.load
1161*9880d681SAndroid Build Coastguard Worker // %res.phi.else = phi <16 x i32> [ %6, %cond.load ], [ undef, %0 ]
1162*9880d681SAndroid Build Coastguard Worker // %7 = extractelement <16 x i1> %mask, i32 1
1163*9880d681SAndroid Build Coastguard Worker // %8 = icmp eq i1 %7, true
1164*9880d681SAndroid Build Coastguard Worker // br i1 %8, label %cond.load1, label %else2
1165*9880d681SAndroid Build Coastguard Worker //
1166*9880d681SAndroid Build Coastguard Worker //cond.load1: ; preds = %else
1167*9880d681SAndroid Build Coastguard Worker // %9 = getelementptr i32* %1, i32 1
1168*9880d681SAndroid Build Coastguard Worker // %10 = load i32* %9
1169*9880d681SAndroid Build Coastguard Worker // %11 = insertelement <16 x i32> %res.phi.else, i32 %10, i32 1
1170*9880d681SAndroid Build Coastguard Worker // br label %else2
1171*9880d681SAndroid Build Coastguard Worker //
1172*9880d681SAndroid Build Coastguard Worker //else2: ; preds = %else, %cond.load1
1173*9880d681SAndroid Build Coastguard Worker // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
1174*9880d681SAndroid Build Coastguard Worker // %12 = extractelement <16 x i1> %mask, i32 2
1175*9880d681SAndroid Build Coastguard Worker // %13 = icmp eq i1 %12, true
1176*9880d681SAndroid Build Coastguard Worker // br i1 %13, label %cond.load4, label %else5
1177*9880d681SAndroid Build Coastguard Worker //
scalarizeMaskedLoad(CallInst * CI)1178*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedLoad(CallInst *CI) {
1179*9880d681SAndroid Build Coastguard Worker Value *Ptr = CI->getArgOperand(0);
1180*9880d681SAndroid Build Coastguard Worker Value *Alignment = CI->getArgOperand(1);
1181*9880d681SAndroid Build Coastguard Worker Value *Mask = CI->getArgOperand(2);
1182*9880d681SAndroid Build Coastguard Worker Value *Src0 = CI->getArgOperand(3);
1183*9880d681SAndroid Build Coastguard Worker
1184*9880d681SAndroid Build Coastguard Worker unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1185*9880d681SAndroid Build Coastguard Worker VectorType *VecType = dyn_cast<VectorType>(CI->getType());
1186*9880d681SAndroid Build Coastguard Worker assert(VecType && "Unexpected return type of masked load intrinsic");
1187*9880d681SAndroid Build Coastguard Worker
1188*9880d681SAndroid Build Coastguard Worker Type *EltTy = CI->getType()->getVectorElementType();
1189*9880d681SAndroid Build Coastguard Worker
1190*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(CI->getContext());
1191*9880d681SAndroid Build Coastguard Worker Instruction *InsertPt = CI;
1192*9880d681SAndroid Build Coastguard Worker BasicBlock *IfBlock = CI->getParent();
1193*9880d681SAndroid Build Coastguard Worker BasicBlock *CondBlock = nullptr;
1194*9880d681SAndroid Build Coastguard Worker BasicBlock *PrevIfBlock = CI->getParent();
1195*9880d681SAndroid Build Coastguard Worker
1196*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1197*9880d681SAndroid Build Coastguard Worker Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1198*9880d681SAndroid Build Coastguard Worker
1199*9880d681SAndroid Build Coastguard Worker // Short-cut if the mask is all-true.
1200*9880d681SAndroid Build Coastguard Worker bool IsAllOnesMask = isa<Constant>(Mask) &&
1201*9880d681SAndroid Build Coastguard Worker cast<Constant>(Mask)->isAllOnesValue();
1202*9880d681SAndroid Build Coastguard Worker
1203*9880d681SAndroid Build Coastguard Worker if (IsAllOnesMask) {
1204*9880d681SAndroid Build Coastguard Worker Value *NewI = Builder.CreateAlignedLoad(Ptr, AlignVal);
1205*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(NewI);
1206*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1207*9880d681SAndroid Build Coastguard Worker return;
1208*9880d681SAndroid Build Coastguard Worker }
1209*9880d681SAndroid Build Coastguard Worker
1210*9880d681SAndroid Build Coastguard Worker // Adjust alignment for the scalar instruction.
1211*9880d681SAndroid Build Coastguard Worker AlignVal = std::min(AlignVal, VecType->getScalarSizeInBits()/8);
1212*9880d681SAndroid Build Coastguard Worker // Bitcast %addr fron i8* to EltTy*
1213*9880d681SAndroid Build Coastguard Worker Type *NewPtrType =
1214*9880d681SAndroid Build Coastguard Worker EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace());
1215*9880d681SAndroid Build Coastguard Worker Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
1216*9880d681SAndroid Build Coastguard Worker unsigned VectorWidth = VecType->getNumElements();
1217*9880d681SAndroid Build Coastguard Worker
1218*9880d681SAndroid Build Coastguard Worker Value *UndefVal = UndefValue::get(VecType);
1219*9880d681SAndroid Build Coastguard Worker
1220*9880d681SAndroid Build Coastguard Worker // The result vector
1221*9880d681SAndroid Build Coastguard Worker Value *VResult = UndefVal;
1222*9880d681SAndroid Build Coastguard Worker
1223*9880d681SAndroid Build Coastguard Worker if (isa<ConstantVector>(Mask)) {
1224*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1225*9880d681SAndroid Build Coastguard Worker if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1226*9880d681SAndroid Build Coastguard Worker continue;
1227*9880d681SAndroid Build Coastguard Worker Value *Gep =
1228*9880d681SAndroid Build Coastguard Worker Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1229*9880d681SAndroid Build Coastguard Worker LoadInst* Load = Builder.CreateAlignedLoad(Gep, AlignVal);
1230*9880d681SAndroid Build Coastguard Worker VResult = Builder.CreateInsertElement(VResult, Load,
1231*9880d681SAndroid Build Coastguard Worker Builder.getInt32(Idx));
1232*9880d681SAndroid Build Coastguard Worker }
1233*9880d681SAndroid Build Coastguard Worker Value *NewI = Builder.CreateSelect(Mask, VResult, Src0);
1234*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(NewI);
1235*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1236*9880d681SAndroid Build Coastguard Worker return;
1237*9880d681SAndroid Build Coastguard Worker }
1238*9880d681SAndroid Build Coastguard Worker
1239*9880d681SAndroid Build Coastguard Worker PHINode *Phi = nullptr;
1240*9880d681SAndroid Build Coastguard Worker Value *PrevPhi = UndefVal;
1241*9880d681SAndroid Build Coastguard Worker
1242*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1243*9880d681SAndroid Build Coastguard Worker
1244*9880d681SAndroid Build Coastguard Worker // Fill the "else" block, created in the previous iteration
1245*9880d681SAndroid Build Coastguard Worker //
1246*9880d681SAndroid Build Coastguard Worker // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
1247*9880d681SAndroid Build Coastguard Worker // %mask_1 = extractelement <16 x i1> %mask, i32 Idx
1248*9880d681SAndroid Build Coastguard Worker // %to_load = icmp eq i1 %mask_1, true
1249*9880d681SAndroid Build Coastguard Worker // br i1 %to_load, label %cond.load, label %else
1250*9880d681SAndroid Build Coastguard Worker //
1251*9880d681SAndroid Build Coastguard Worker if (Idx > 0) {
1252*9880d681SAndroid Build Coastguard Worker Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
1253*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(VResult, CondBlock);
1254*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(PrevPhi, PrevIfBlock);
1255*9880d681SAndroid Build Coastguard Worker PrevPhi = Phi;
1256*9880d681SAndroid Build Coastguard Worker VResult = Phi;
1257*9880d681SAndroid Build Coastguard Worker }
1258*9880d681SAndroid Build Coastguard Worker
1259*9880d681SAndroid Build Coastguard Worker Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx));
1260*9880d681SAndroid Build Coastguard Worker Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1261*9880d681SAndroid Build Coastguard Worker ConstantInt::get(Predicate->getType(), 1));
1262*9880d681SAndroid Build Coastguard Worker
1263*9880d681SAndroid Build Coastguard Worker // Create "cond" block
1264*9880d681SAndroid Build Coastguard Worker //
1265*9880d681SAndroid Build Coastguard Worker // %EltAddr = getelementptr i32* %1, i32 0
1266*9880d681SAndroid Build Coastguard Worker // %Elt = load i32* %EltAddr
1267*9880d681SAndroid Build Coastguard Worker // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
1268*9880d681SAndroid Build Coastguard Worker //
1269*9880d681SAndroid Build Coastguard Worker CondBlock = IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.load");
1270*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1271*9880d681SAndroid Build Coastguard Worker
1272*9880d681SAndroid Build Coastguard Worker Value *Gep =
1273*9880d681SAndroid Build Coastguard Worker Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1274*9880d681SAndroid Build Coastguard Worker LoadInst *Load = Builder.CreateAlignedLoad(Gep, AlignVal);
1275*9880d681SAndroid Build Coastguard Worker VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx));
1276*9880d681SAndroid Build Coastguard Worker
1277*9880d681SAndroid Build Coastguard Worker // Create "else" block, fill it in the next iteration
1278*9880d681SAndroid Build Coastguard Worker BasicBlock *NewIfBlock =
1279*9880d681SAndroid Build Coastguard Worker CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
1280*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1281*9880d681SAndroid Build Coastguard Worker Instruction *OldBr = IfBlock->getTerminator();
1282*9880d681SAndroid Build Coastguard Worker BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1283*9880d681SAndroid Build Coastguard Worker OldBr->eraseFromParent();
1284*9880d681SAndroid Build Coastguard Worker PrevIfBlock = IfBlock;
1285*9880d681SAndroid Build Coastguard Worker IfBlock = NewIfBlock;
1286*9880d681SAndroid Build Coastguard Worker }
1287*9880d681SAndroid Build Coastguard Worker
1288*9880d681SAndroid Build Coastguard Worker Phi = Builder.CreatePHI(VecType, 2, "res.phi.select");
1289*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(VResult, CondBlock);
1290*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(PrevPhi, PrevIfBlock);
1291*9880d681SAndroid Build Coastguard Worker Value *NewI = Builder.CreateSelect(Mask, Phi, Src0);
1292*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(NewI);
1293*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1294*9880d681SAndroid Build Coastguard Worker }
1295*9880d681SAndroid Build Coastguard Worker
1296*9880d681SAndroid Build Coastguard Worker // Translate a masked store intrinsic, like
1297*9880d681SAndroid Build Coastguard Worker // void @llvm.masked.store(<16 x i32> %src, <16 x i32>* %addr, i32 align,
1298*9880d681SAndroid Build Coastguard Worker // <16 x i1> %mask)
1299*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, that stores element one-by-one if
1300*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set
1301*9880d681SAndroid Build Coastguard Worker //
1302*9880d681SAndroid Build Coastguard Worker // %1 = bitcast i8* %addr to i32*
1303*9880d681SAndroid Build Coastguard Worker // %2 = extractelement <16 x i1> %mask, i32 0
1304*9880d681SAndroid Build Coastguard Worker // %3 = icmp eq i1 %2, true
1305*9880d681SAndroid Build Coastguard Worker // br i1 %3, label %cond.store, label %else
1306*9880d681SAndroid Build Coastguard Worker //
1307*9880d681SAndroid Build Coastguard Worker // cond.store: ; preds = %0
1308*9880d681SAndroid Build Coastguard Worker // %4 = extractelement <16 x i32> %val, i32 0
1309*9880d681SAndroid Build Coastguard Worker // %5 = getelementptr i32* %1, i32 0
1310*9880d681SAndroid Build Coastguard Worker // store i32 %4, i32* %5
1311*9880d681SAndroid Build Coastguard Worker // br label %else
1312*9880d681SAndroid Build Coastguard Worker //
1313*9880d681SAndroid Build Coastguard Worker // else: ; preds = %0, %cond.store
1314*9880d681SAndroid Build Coastguard Worker // %6 = extractelement <16 x i1> %mask, i32 1
1315*9880d681SAndroid Build Coastguard Worker // %7 = icmp eq i1 %6, true
1316*9880d681SAndroid Build Coastguard Worker // br i1 %7, label %cond.store1, label %else2
1317*9880d681SAndroid Build Coastguard Worker //
1318*9880d681SAndroid Build Coastguard Worker // cond.store1: ; preds = %else
1319*9880d681SAndroid Build Coastguard Worker // %8 = extractelement <16 x i32> %val, i32 1
1320*9880d681SAndroid Build Coastguard Worker // %9 = getelementptr i32* %1, i32 1
1321*9880d681SAndroid Build Coastguard Worker // store i32 %8, i32* %9
1322*9880d681SAndroid Build Coastguard Worker // br label %else2
1323*9880d681SAndroid Build Coastguard Worker // . . .
scalarizeMaskedStore(CallInst * CI)1324*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedStore(CallInst *CI) {
1325*9880d681SAndroid Build Coastguard Worker Value *Src = CI->getArgOperand(0);
1326*9880d681SAndroid Build Coastguard Worker Value *Ptr = CI->getArgOperand(1);
1327*9880d681SAndroid Build Coastguard Worker Value *Alignment = CI->getArgOperand(2);
1328*9880d681SAndroid Build Coastguard Worker Value *Mask = CI->getArgOperand(3);
1329*9880d681SAndroid Build Coastguard Worker
1330*9880d681SAndroid Build Coastguard Worker unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1331*9880d681SAndroid Build Coastguard Worker VectorType *VecType = dyn_cast<VectorType>(Src->getType());
1332*9880d681SAndroid Build Coastguard Worker assert(VecType && "Unexpected data type in masked store intrinsic");
1333*9880d681SAndroid Build Coastguard Worker
1334*9880d681SAndroid Build Coastguard Worker Type *EltTy = VecType->getElementType();
1335*9880d681SAndroid Build Coastguard Worker
1336*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(CI->getContext());
1337*9880d681SAndroid Build Coastguard Worker Instruction *InsertPt = CI;
1338*9880d681SAndroid Build Coastguard Worker BasicBlock *IfBlock = CI->getParent();
1339*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1340*9880d681SAndroid Build Coastguard Worker Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1341*9880d681SAndroid Build Coastguard Worker
1342*9880d681SAndroid Build Coastguard Worker // Short-cut if the mask is all-true.
1343*9880d681SAndroid Build Coastguard Worker bool IsAllOnesMask = isa<Constant>(Mask) &&
1344*9880d681SAndroid Build Coastguard Worker cast<Constant>(Mask)->isAllOnesValue();
1345*9880d681SAndroid Build Coastguard Worker
1346*9880d681SAndroid Build Coastguard Worker if (IsAllOnesMask) {
1347*9880d681SAndroid Build Coastguard Worker Builder.CreateAlignedStore(Src, Ptr, AlignVal);
1348*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1349*9880d681SAndroid Build Coastguard Worker return;
1350*9880d681SAndroid Build Coastguard Worker }
1351*9880d681SAndroid Build Coastguard Worker
1352*9880d681SAndroid Build Coastguard Worker // Adjust alignment for the scalar instruction.
1353*9880d681SAndroid Build Coastguard Worker AlignVal = std::max(AlignVal, VecType->getScalarSizeInBits()/8);
1354*9880d681SAndroid Build Coastguard Worker // Bitcast %addr fron i8* to EltTy*
1355*9880d681SAndroid Build Coastguard Worker Type *NewPtrType =
1356*9880d681SAndroid Build Coastguard Worker EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace());
1357*9880d681SAndroid Build Coastguard Worker Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
1358*9880d681SAndroid Build Coastguard Worker unsigned VectorWidth = VecType->getNumElements();
1359*9880d681SAndroid Build Coastguard Worker
1360*9880d681SAndroid Build Coastguard Worker if (isa<ConstantVector>(Mask)) {
1361*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1362*9880d681SAndroid Build Coastguard Worker if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1363*9880d681SAndroid Build Coastguard Worker continue;
1364*9880d681SAndroid Build Coastguard Worker Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx));
1365*9880d681SAndroid Build Coastguard Worker Value *Gep =
1366*9880d681SAndroid Build Coastguard Worker Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1367*9880d681SAndroid Build Coastguard Worker Builder.CreateAlignedStore(OneElt, Gep, AlignVal);
1368*9880d681SAndroid Build Coastguard Worker }
1369*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1370*9880d681SAndroid Build Coastguard Worker return;
1371*9880d681SAndroid Build Coastguard Worker }
1372*9880d681SAndroid Build Coastguard Worker
1373*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1374*9880d681SAndroid Build Coastguard Worker
1375*9880d681SAndroid Build Coastguard Worker // Fill the "else" block, created in the previous iteration
1376*9880d681SAndroid Build Coastguard Worker //
1377*9880d681SAndroid Build Coastguard Worker // %mask_1 = extractelement <16 x i1> %mask, i32 Idx
1378*9880d681SAndroid Build Coastguard Worker // %to_store = icmp eq i1 %mask_1, true
1379*9880d681SAndroid Build Coastguard Worker // br i1 %to_store, label %cond.store, label %else
1380*9880d681SAndroid Build Coastguard Worker //
1381*9880d681SAndroid Build Coastguard Worker Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx));
1382*9880d681SAndroid Build Coastguard Worker Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1383*9880d681SAndroid Build Coastguard Worker ConstantInt::get(Predicate->getType(), 1));
1384*9880d681SAndroid Build Coastguard Worker
1385*9880d681SAndroid Build Coastguard Worker // Create "cond" block
1386*9880d681SAndroid Build Coastguard Worker //
1387*9880d681SAndroid Build Coastguard Worker // %OneElt = extractelement <16 x i32> %Src, i32 Idx
1388*9880d681SAndroid Build Coastguard Worker // %EltAddr = getelementptr i32* %1, i32 0
1389*9880d681SAndroid Build Coastguard Worker // %store i32 %OneElt, i32* %EltAddr
1390*9880d681SAndroid Build Coastguard Worker //
1391*9880d681SAndroid Build Coastguard Worker BasicBlock *CondBlock =
1392*9880d681SAndroid Build Coastguard Worker IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store");
1393*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1394*9880d681SAndroid Build Coastguard Worker
1395*9880d681SAndroid Build Coastguard Worker Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx));
1396*9880d681SAndroid Build Coastguard Worker Value *Gep =
1397*9880d681SAndroid Build Coastguard Worker Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1398*9880d681SAndroid Build Coastguard Worker Builder.CreateAlignedStore(OneElt, Gep, AlignVal);
1399*9880d681SAndroid Build Coastguard Worker
1400*9880d681SAndroid Build Coastguard Worker // Create "else" block, fill it in the next iteration
1401*9880d681SAndroid Build Coastguard Worker BasicBlock *NewIfBlock =
1402*9880d681SAndroid Build Coastguard Worker CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
1403*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1404*9880d681SAndroid Build Coastguard Worker Instruction *OldBr = IfBlock->getTerminator();
1405*9880d681SAndroid Build Coastguard Worker BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1406*9880d681SAndroid Build Coastguard Worker OldBr->eraseFromParent();
1407*9880d681SAndroid Build Coastguard Worker IfBlock = NewIfBlock;
1408*9880d681SAndroid Build Coastguard Worker }
1409*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1410*9880d681SAndroid Build Coastguard Worker }
1411*9880d681SAndroid Build Coastguard Worker
1412*9880d681SAndroid Build Coastguard Worker // Translate a masked gather intrinsic like
1413*9880d681SAndroid Build Coastguard Worker // <16 x i32 > @llvm.masked.gather.v16i32( <16 x i32*> %Ptrs, i32 4,
1414*9880d681SAndroid Build Coastguard Worker // <16 x i1> %Mask, <16 x i32> %Src)
1415*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, with loading element one-by-one if
1416*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set
1417*9880d681SAndroid Build Coastguard Worker //
1418*9880d681SAndroid Build Coastguard Worker // % Ptrs = getelementptr i32, i32* %base, <16 x i64> %ind
1419*9880d681SAndroid Build Coastguard Worker // % Mask0 = extractelement <16 x i1> %Mask, i32 0
1420*9880d681SAndroid Build Coastguard Worker // % ToLoad0 = icmp eq i1 % Mask0, true
1421*9880d681SAndroid Build Coastguard Worker // br i1 % ToLoad0, label %cond.load, label %else
1422*9880d681SAndroid Build Coastguard Worker //
1423*9880d681SAndroid Build Coastguard Worker // cond.load:
1424*9880d681SAndroid Build Coastguard Worker // % Ptr0 = extractelement <16 x i32*> %Ptrs, i32 0
1425*9880d681SAndroid Build Coastguard Worker // % Load0 = load i32, i32* % Ptr0, align 4
1426*9880d681SAndroid Build Coastguard Worker // % Res0 = insertelement <16 x i32> undef, i32 % Load0, i32 0
1427*9880d681SAndroid Build Coastguard Worker // br label %else
1428*9880d681SAndroid Build Coastguard Worker //
1429*9880d681SAndroid Build Coastguard Worker // else:
1430*9880d681SAndroid Build Coastguard Worker // %res.phi.else = phi <16 x i32>[% Res0, %cond.load], [undef, % 0]
1431*9880d681SAndroid Build Coastguard Worker // % Mask1 = extractelement <16 x i1> %Mask, i32 1
1432*9880d681SAndroid Build Coastguard Worker // % ToLoad1 = icmp eq i1 % Mask1, true
1433*9880d681SAndroid Build Coastguard Worker // br i1 % ToLoad1, label %cond.load1, label %else2
1434*9880d681SAndroid Build Coastguard Worker //
1435*9880d681SAndroid Build Coastguard Worker // cond.load1:
1436*9880d681SAndroid Build Coastguard Worker // % Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
1437*9880d681SAndroid Build Coastguard Worker // % Load1 = load i32, i32* % Ptr1, align 4
1438*9880d681SAndroid Build Coastguard Worker // % Res1 = insertelement <16 x i32> %res.phi.else, i32 % Load1, i32 1
1439*9880d681SAndroid Build Coastguard Worker // br label %else2
1440*9880d681SAndroid Build Coastguard Worker // . . .
1441*9880d681SAndroid Build Coastguard Worker // % Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
1442*9880d681SAndroid Build Coastguard Worker // ret <16 x i32> %Result
scalarizeMaskedGather(CallInst * CI)1443*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedGather(CallInst *CI) {
1444*9880d681SAndroid Build Coastguard Worker Value *Ptrs = CI->getArgOperand(0);
1445*9880d681SAndroid Build Coastguard Worker Value *Alignment = CI->getArgOperand(1);
1446*9880d681SAndroid Build Coastguard Worker Value *Mask = CI->getArgOperand(2);
1447*9880d681SAndroid Build Coastguard Worker Value *Src0 = CI->getArgOperand(3);
1448*9880d681SAndroid Build Coastguard Worker
1449*9880d681SAndroid Build Coastguard Worker VectorType *VecType = dyn_cast<VectorType>(CI->getType());
1450*9880d681SAndroid Build Coastguard Worker
1451*9880d681SAndroid Build Coastguard Worker assert(VecType && "Unexpected return type of masked load intrinsic");
1452*9880d681SAndroid Build Coastguard Worker
1453*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(CI->getContext());
1454*9880d681SAndroid Build Coastguard Worker Instruction *InsertPt = CI;
1455*9880d681SAndroid Build Coastguard Worker BasicBlock *IfBlock = CI->getParent();
1456*9880d681SAndroid Build Coastguard Worker BasicBlock *CondBlock = nullptr;
1457*9880d681SAndroid Build Coastguard Worker BasicBlock *PrevIfBlock = CI->getParent();
1458*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1459*9880d681SAndroid Build Coastguard Worker unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1460*9880d681SAndroid Build Coastguard Worker
1461*9880d681SAndroid Build Coastguard Worker Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1462*9880d681SAndroid Build Coastguard Worker
1463*9880d681SAndroid Build Coastguard Worker Value *UndefVal = UndefValue::get(VecType);
1464*9880d681SAndroid Build Coastguard Worker
1465*9880d681SAndroid Build Coastguard Worker // The result vector
1466*9880d681SAndroid Build Coastguard Worker Value *VResult = UndefVal;
1467*9880d681SAndroid Build Coastguard Worker unsigned VectorWidth = VecType->getNumElements();
1468*9880d681SAndroid Build Coastguard Worker
1469*9880d681SAndroid Build Coastguard Worker // Shorten the way if the mask is a vector of constants.
1470*9880d681SAndroid Build Coastguard Worker bool IsConstMask = isa<ConstantVector>(Mask);
1471*9880d681SAndroid Build Coastguard Worker
1472*9880d681SAndroid Build Coastguard Worker if (IsConstMask) {
1473*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1474*9880d681SAndroid Build Coastguard Worker if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1475*9880d681SAndroid Build Coastguard Worker continue;
1476*9880d681SAndroid Build Coastguard Worker Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1477*9880d681SAndroid Build Coastguard Worker "Ptr" + Twine(Idx));
1478*9880d681SAndroid Build Coastguard Worker LoadInst *Load = Builder.CreateAlignedLoad(Ptr, AlignVal,
1479*9880d681SAndroid Build Coastguard Worker "Load" + Twine(Idx));
1480*9880d681SAndroid Build Coastguard Worker VResult = Builder.CreateInsertElement(VResult, Load,
1481*9880d681SAndroid Build Coastguard Worker Builder.getInt32(Idx),
1482*9880d681SAndroid Build Coastguard Worker "Res" + Twine(Idx));
1483*9880d681SAndroid Build Coastguard Worker }
1484*9880d681SAndroid Build Coastguard Worker Value *NewI = Builder.CreateSelect(Mask, VResult, Src0);
1485*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(NewI);
1486*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1487*9880d681SAndroid Build Coastguard Worker return;
1488*9880d681SAndroid Build Coastguard Worker }
1489*9880d681SAndroid Build Coastguard Worker
1490*9880d681SAndroid Build Coastguard Worker PHINode *Phi = nullptr;
1491*9880d681SAndroid Build Coastguard Worker Value *PrevPhi = UndefVal;
1492*9880d681SAndroid Build Coastguard Worker
1493*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1494*9880d681SAndroid Build Coastguard Worker
1495*9880d681SAndroid Build Coastguard Worker // Fill the "else" block, created in the previous iteration
1496*9880d681SAndroid Build Coastguard Worker //
1497*9880d681SAndroid Build Coastguard Worker // %Mask1 = extractelement <16 x i1> %Mask, i32 1
1498*9880d681SAndroid Build Coastguard Worker // %ToLoad1 = icmp eq i1 %Mask1, true
1499*9880d681SAndroid Build Coastguard Worker // br i1 %ToLoad1, label %cond.load, label %else
1500*9880d681SAndroid Build Coastguard Worker //
1501*9880d681SAndroid Build Coastguard Worker if (Idx > 0) {
1502*9880d681SAndroid Build Coastguard Worker Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
1503*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(VResult, CondBlock);
1504*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(PrevPhi, PrevIfBlock);
1505*9880d681SAndroid Build Coastguard Worker PrevPhi = Phi;
1506*9880d681SAndroid Build Coastguard Worker VResult = Phi;
1507*9880d681SAndroid Build Coastguard Worker }
1508*9880d681SAndroid Build Coastguard Worker
1509*9880d681SAndroid Build Coastguard Worker Value *Predicate = Builder.CreateExtractElement(Mask,
1510*9880d681SAndroid Build Coastguard Worker Builder.getInt32(Idx),
1511*9880d681SAndroid Build Coastguard Worker "Mask" + Twine(Idx));
1512*9880d681SAndroid Build Coastguard Worker Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1513*9880d681SAndroid Build Coastguard Worker ConstantInt::get(Predicate->getType(), 1),
1514*9880d681SAndroid Build Coastguard Worker "ToLoad" + Twine(Idx));
1515*9880d681SAndroid Build Coastguard Worker
1516*9880d681SAndroid Build Coastguard Worker // Create "cond" block
1517*9880d681SAndroid Build Coastguard Worker //
1518*9880d681SAndroid Build Coastguard Worker // %EltAddr = getelementptr i32* %1, i32 0
1519*9880d681SAndroid Build Coastguard Worker // %Elt = load i32* %EltAddr
1520*9880d681SAndroid Build Coastguard Worker // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
1521*9880d681SAndroid Build Coastguard Worker //
1522*9880d681SAndroid Build Coastguard Worker CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load");
1523*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1524*9880d681SAndroid Build Coastguard Worker
1525*9880d681SAndroid Build Coastguard Worker Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1526*9880d681SAndroid Build Coastguard Worker "Ptr" + Twine(Idx));
1527*9880d681SAndroid Build Coastguard Worker LoadInst *Load = Builder.CreateAlignedLoad(Ptr, AlignVal,
1528*9880d681SAndroid Build Coastguard Worker "Load" + Twine(Idx));
1529*9880d681SAndroid Build Coastguard Worker VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx),
1530*9880d681SAndroid Build Coastguard Worker "Res" + Twine(Idx));
1531*9880d681SAndroid Build Coastguard Worker
1532*9880d681SAndroid Build Coastguard Worker // Create "else" block, fill it in the next iteration
1533*9880d681SAndroid Build Coastguard Worker BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else");
1534*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1535*9880d681SAndroid Build Coastguard Worker Instruction *OldBr = IfBlock->getTerminator();
1536*9880d681SAndroid Build Coastguard Worker BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1537*9880d681SAndroid Build Coastguard Worker OldBr->eraseFromParent();
1538*9880d681SAndroid Build Coastguard Worker PrevIfBlock = IfBlock;
1539*9880d681SAndroid Build Coastguard Worker IfBlock = NewIfBlock;
1540*9880d681SAndroid Build Coastguard Worker }
1541*9880d681SAndroid Build Coastguard Worker
1542*9880d681SAndroid Build Coastguard Worker Phi = Builder.CreatePHI(VecType, 2, "res.phi.select");
1543*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(VResult, CondBlock);
1544*9880d681SAndroid Build Coastguard Worker Phi->addIncoming(PrevPhi, PrevIfBlock);
1545*9880d681SAndroid Build Coastguard Worker Value *NewI = Builder.CreateSelect(Mask, Phi, Src0);
1546*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(NewI);
1547*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1548*9880d681SAndroid Build Coastguard Worker }
1549*9880d681SAndroid Build Coastguard Worker
1550*9880d681SAndroid Build Coastguard Worker // Translate a masked scatter intrinsic, like
1551*9880d681SAndroid Build Coastguard Worker // void @llvm.masked.scatter.v16i32(<16 x i32> %Src, <16 x i32*>* %Ptrs, i32 4,
1552*9880d681SAndroid Build Coastguard Worker // <16 x i1> %Mask)
1553*9880d681SAndroid Build Coastguard Worker // to a chain of basic blocks, that stores element one-by-one if
1554*9880d681SAndroid Build Coastguard Worker // the appropriate mask bit is set.
1555*9880d681SAndroid Build Coastguard Worker //
1556*9880d681SAndroid Build Coastguard Worker // % Ptrs = getelementptr i32, i32* %ptr, <16 x i64> %ind
1557*9880d681SAndroid Build Coastguard Worker // % Mask0 = extractelement <16 x i1> % Mask, i32 0
1558*9880d681SAndroid Build Coastguard Worker // % ToStore0 = icmp eq i1 % Mask0, true
1559*9880d681SAndroid Build Coastguard Worker // br i1 %ToStore0, label %cond.store, label %else
1560*9880d681SAndroid Build Coastguard Worker //
1561*9880d681SAndroid Build Coastguard Worker // cond.store:
1562*9880d681SAndroid Build Coastguard Worker // % Elt0 = extractelement <16 x i32> %Src, i32 0
1563*9880d681SAndroid Build Coastguard Worker // % Ptr0 = extractelement <16 x i32*> %Ptrs, i32 0
1564*9880d681SAndroid Build Coastguard Worker // store i32 %Elt0, i32* % Ptr0, align 4
1565*9880d681SAndroid Build Coastguard Worker // br label %else
1566*9880d681SAndroid Build Coastguard Worker //
1567*9880d681SAndroid Build Coastguard Worker // else:
1568*9880d681SAndroid Build Coastguard Worker // % Mask1 = extractelement <16 x i1> % Mask, i32 1
1569*9880d681SAndroid Build Coastguard Worker // % ToStore1 = icmp eq i1 % Mask1, true
1570*9880d681SAndroid Build Coastguard Worker // br i1 % ToStore1, label %cond.store1, label %else2
1571*9880d681SAndroid Build Coastguard Worker //
1572*9880d681SAndroid Build Coastguard Worker // cond.store1:
1573*9880d681SAndroid Build Coastguard Worker // % Elt1 = extractelement <16 x i32> %Src, i32 1
1574*9880d681SAndroid Build Coastguard Worker // % Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
1575*9880d681SAndroid Build Coastguard Worker // store i32 % Elt1, i32* % Ptr1, align 4
1576*9880d681SAndroid Build Coastguard Worker // br label %else2
1577*9880d681SAndroid Build Coastguard Worker // . . .
scalarizeMaskedScatter(CallInst * CI)1578*9880d681SAndroid Build Coastguard Worker static void scalarizeMaskedScatter(CallInst *CI) {
1579*9880d681SAndroid Build Coastguard Worker Value *Src = CI->getArgOperand(0);
1580*9880d681SAndroid Build Coastguard Worker Value *Ptrs = CI->getArgOperand(1);
1581*9880d681SAndroid Build Coastguard Worker Value *Alignment = CI->getArgOperand(2);
1582*9880d681SAndroid Build Coastguard Worker Value *Mask = CI->getArgOperand(3);
1583*9880d681SAndroid Build Coastguard Worker
1584*9880d681SAndroid Build Coastguard Worker assert(isa<VectorType>(Src->getType()) &&
1585*9880d681SAndroid Build Coastguard Worker "Unexpected data type in masked scatter intrinsic");
1586*9880d681SAndroid Build Coastguard Worker assert(isa<VectorType>(Ptrs->getType()) &&
1587*9880d681SAndroid Build Coastguard Worker isa<PointerType>(Ptrs->getType()->getVectorElementType()) &&
1588*9880d681SAndroid Build Coastguard Worker "Vector of pointers is expected in masked scatter intrinsic");
1589*9880d681SAndroid Build Coastguard Worker
1590*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(CI->getContext());
1591*9880d681SAndroid Build Coastguard Worker Instruction *InsertPt = CI;
1592*9880d681SAndroid Build Coastguard Worker BasicBlock *IfBlock = CI->getParent();
1593*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1594*9880d681SAndroid Build Coastguard Worker Builder.SetCurrentDebugLocation(CI->getDebugLoc());
1595*9880d681SAndroid Build Coastguard Worker
1596*9880d681SAndroid Build Coastguard Worker unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1597*9880d681SAndroid Build Coastguard Worker unsigned VectorWidth = Src->getType()->getVectorNumElements();
1598*9880d681SAndroid Build Coastguard Worker
1599*9880d681SAndroid Build Coastguard Worker // Shorten the way if the mask is a vector of constants.
1600*9880d681SAndroid Build Coastguard Worker bool IsConstMask = isa<ConstantVector>(Mask);
1601*9880d681SAndroid Build Coastguard Worker
1602*9880d681SAndroid Build Coastguard Worker if (IsConstMask) {
1603*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1604*9880d681SAndroid Build Coastguard Worker if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1605*9880d681SAndroid Build Coastguard Worker continue;
1606*9880d681SAndroid Build Coastguard Worker Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx),
1607*9880d681SAndroid Build Coastguard Worker "Elt" + Twine(Idx));
1608*9880d681SAndroid Build Coastguard Worker Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1609*9880d681SAndroid Build Coastguard Worker "Ptr" + Twine(Idx));
1610*9880d681SAndroid Build Coastguard Worker Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
1611*9880d681SAndroid Build Coastguard Worker }
1612*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1613*9880d681SAndroid Build Coastguard Worker return;
1614*9880d681SAndroid Build Coastguard Worker }
1615*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1616*9880d681SAndroid Build Coastguard Worker // Fill the "else" block, created in the previous iteration
1617*9880d681SAndroid Build Coastguard Worker //
1618*9880d681SAndroid Build Coastguard Worker // % Mask1 = extractelement <16 x i1> % Mask, i32 Idx
1619*9880d681SAndroid Build Coastguard Worker // % ToStore = icmp eq i1 % Mask1, true
1620*9880d681SAndroid Build Coastguard Worker // br i1 % ToStore, label %cond.store, label %else
1621*9880d681SAndroid Build Coastguard Worker //
1622*9880d681SAndroid Build Coastguard Worker Value *Predicate = Builder.CreateExtractElement(Mask,
1623*9880d681SAndroid Build Coastguard Worker Builder.getInt32(Idx),
1624*9880d681SAndroid Build Coastguard Worker "Mask" + Twine(Idx));
1625*9880d681SAndroid Build Coastguard Worker Value *Cmp =
1626*9880d681SAndroid Build Coastguard Worker Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate,
1627*9880d681SAndroid Build Coastguard Worker ConstantInt::get(Predicate->getType(), 1),
1628*9880d681SAndroid Build Coastguard Worker "ToStore" + Twine(Idx));
1629*9880d681SAndroid Build Coastguard Worker
1630*9880d681SAndroid Build Coastguard Worker // Create "cond" block
1631*9880d681SAndroid Build Coastguard Worker //
1632*9880d681SAndroid Build Coastguard Worker // % Elt1 = extractelement <16 x i32> %Src, i32 1
1633*9880d681SAndroid Build Coastguard Worker // % Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
1634*9880d681SAndroid Build Coastguard Worker // %store i32 % Elt1, i32* % Ptr1
1635*9880d681SAndroid Build Coastguard Worker //
1636*9880d681SAndroid Build Coastguard Worker BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.store");
1637*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1638*9880d681SAndroid Build Coastguard Worker
1639*9880d681SAndroid Build Coastguard Worker Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx),
1640*9880d681SAndroid Build Coastguard Worker "Elt" + Twine(Idx));
1641*9880d681SAndroid Build Coastguard Worker Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1642*9880d681SAndroid Build Coastguard Worker "Ptr" + Twine(Idx));
1643*9880d681SAndroid Build Coastguard Worker Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
1644*9880d681SAndroid Build Coastguard Worker
1645*9880d681SAndroid Build Coastguard Worker // Create "else" block, fill it in the next iteration
1646*9880d681SAndroid Build Coastguard Worker BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else");
1647*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(InsertPt);
1648*9880d681SAndroid Build Coastguard Worker Instruction *OldBr = IfBlock->getTerminator();
1649*9880d681SAndroid Build Coastguard Worker BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
1650*9880d681SAndroid Build Coastguard Worker OldBr->eraseFromParent();
1651*9880d681SAndroid Build Coastguard Worker IfBlock = NewIfBlock;
1652*9880d681SAndroid Build Coastguard Worker }
1653*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1654*9880d681SAndroid Build Coastguard Worker }
1655*9880d681SAndroid Build Coastguard Worker
1656*9880d681SAndroid Build Coastguard Worker /// If counting leading or trailing zeros is an expensive operation and a zero
1657*9880d681SAndroid Build Coastguard Worker /// input is defined, add a check for zero to avoid calling the intrinsic.
1658*9880d681SAndroid Build Coastguard Worker ///
1659*9880d681SAndroid Build Coastguard Worker /// We want to transform:
1660*9880d681SAndroid Build Coastguard Worker /// %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
1661*9880d681SAndroid Build Coastguard Worker ///
1662*9880d681SAndroid Build Coastguard Worker /// into:
1663*9880d681SAndroid Build Coastguard Worker /// entry:
1664*9880d681SAndroid Build Coastguard Worker /// %cmpz = icmp eq i64 %A, 0
1665*9880d681SAndroid Build Coastguard Worker /// br i1 %cmpz, label %cond.end, label %cond.false
1666*9880d681SAndroid Build Coastguard Worker /// cond.false:
1667*9880d681SAndroid Build Coastguard Worker /// %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)
1668*9880d681SAndroid Build Coastguard Worker /// br label %cond.end
1669*9880d681SAndroid Build Coastguard Worker /// cond.end:
1670*9880d681SAndroid Build Coastguard Worker /// %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
1671*9880d681SAndroid Build Coastguard Worker ///
1672*9880d681SAndroid Build Coastguard Worker /// If the transform is performed, return true and set ModifiedDT to true.
despeculateCountZeros(IntrinsicInst * CountZeros,const TargetLowering * TLI,const DataLayout * DL,bool & ModifiedDT)1673*9880d681SAndroid Build Coastguard Worker static bool despeculateCountZeros(IntrinsicInst *CountZeros,
1674*9880d681SAndroid Build Coastguard Worker const TargetLowering *TLI,
1675*9880d681SAndroid Build Coastguard Worker const DataLayout *DL,
1676*9880d681SAndroid Build Coastguard Worker bool &ModifiedDT) {
1677*9880d681SAndroid Build Coastguard Worker if (!TLI || !DL)
1678*9880d681SAndroid Build Coastguard Worker return false;
1679*9880d681SAndroid Build Coastguard Worker
1680*9880d681SAndroid Build Coastguard Worker // If a zero input is undefined, it doesn't make sense to despeculate that.
1681*9880d681SAndroid Build Coastguard Worker if (match(CountZeros->getOperand(1), m_One()))
1682*9880d681SAndroid Build Coastguard Worker return false;
1683*9880d681SAndroid Build Coastguard Worker
1684*9880d681SAndroid Build Coastguard Worker // If it's cheap to speculate, there's nothing to do.
1685*9880d681SAndroid Build Coastguard Worker auto IntrinsicID = CountZeros->getIntrinsicID();
1686*9880d681SAndroid Build Coastguard Worker if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz()) ||
1687*9880d681SAndroid Build Coastguard Worker (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz()))
1688*9880d681SAndroid Build Coastguard Worker return false;
1689*9880d681SAndroid Build Coastguard Worker
1690*9880d681SAndroid Build Coastguard Worker // Only handle legal scalar cases. Anything else requires too much work.
1691*9880d681SAndroid Build Coastguard Worker Type *Ty = CountZeros->getType();
1692*9880d681SAndroid Build Coastguard Worker unsigned SizeInBits = Ty->getPrimitiveSizeInBits();
1693*9880d681SAndroid Build Coastguard Worker if (Ty->isVectorTy() || SizeInBits > DL->getLargestLegalIntTypeSizeInBits())
1694*9880d681SAndroid Build Coastguard Worker return false;
1695*9880d681SAndroid Build Coastguard Worker
1696*9880d681SAndroid Build Coastguard Worker // The intrinsic will be sunk behind a compare against zero and branch.
1697*9880d681SAndroid Build Coastguard Worker BasicBlock *StartBlock = CountZeros->getParent();
1698*9880d681SAndroid Build Coastguard Worker BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false");
1699*9880d681SAndroid Build Coastguard Worker
1700*9880d681SAndroid Build Coastguard Worker // Create another block after the count zero intrinsic. A PHI will be added
1701*9880d681SAndroid Build Coastguard Worker // in this block to select the result of the intrinsic or the bit-width
1702*9880d681SAndroid Build Coastguard Worker // constant if the input to the intrinsic is zero.
1703*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(CountZeros));
1704*9880d681SAndroid Build Coastguard Worker BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end");
1705*9880d681SAndroid Build Coastguard Worker
1706*9880d681SAndroid Build Coastguard Worker // Set up a builder to create a compare, conditional branch, and PHI.
1707*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(CountZeros->getContext());
1708*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(StartBlock->getTerminator());
1709*9880d681SAndroid Build Coastguard Worker Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc());
1710*9880d681SAndroid Build Coastguard Worker
1711*9880d681SAndroid Build Coastguard Worker // Replace the unconditional branch that was created by the first split with
1712*9880d681SAndroid Build Coastguard Worker // a compare against zero and a conditional branch.
1713*9880d681SAndroid Build Coastguard Worker Value *Zero = Constant::getNullValue(Ty);
1714*9880d681SAndroid Build Coastguard Worker Value *Cmp = Builder.CreateICmpEQ(CountZeros->getOperand(0), Zero, "cmpz");
1715*9880d681SAndroid Build Coastguard Worker Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
1716*9880d681SAndroid Build Coastguard Worker StartBlock->getTerminator()->eraseFromParent();
1717*9880d681SAndroid Build Coastguard Worker
1718*9880d681SAndroid Build Coastguard Worker // Create a PHI in the end block to select either the output of the intrinsic
1719*9880d681SAndroid Build Coastguard Worker // or the bit width of the operand.
1720*9880d681SAndroid Build Coastguard Worker Builder.SetInsertPoint(&EndBlock->front());
1721*9880d681SAndroid Build Coastguard Worker PHINode *PN = Builder.CreatePHI(Ty, 2, "ctz");
1722*9880d681SAndroid Build Coastguard Worker CountZeros->replaceAllUsesWith(PN);
1723*9880d681SAndroid Build Coastguard Worker Value *BitWidth = Builder.getInt(APInt(SizeInBits, SizeInBits));
1724*9880d681SAndroid Build Coastguard Worker PN->addIncoming(BitWidth, StartBlock);
1725*9880d681SAndroid Build Coastguard Worker PN->addIncoming(CountZeros, CallBlock);
1726*9880d681SAndroid Build Coastguard Worker
1727*9880d681SAndroid Build Coastguard Worker // We are explicitly handling the zero case, so we can set the intrinsic's
1728*9880d681SAndroid Build Coastguard Worker // undefined zero argument to 'true'. This will also prevent reprocessing the
1729*9880d681SAndroid Build Coastguard Worker // intrinsic; we only despeculate when a zero input is defined.
1730*9880d681SAndroid Build Coastguard Worker CountZeros->setArgOperand(1, Builder.getTrue());
1731*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
1732*9880d681SAndroid Build Coastguard Worker return true;
1733*9880d681SAndroid Build Coastguard Worker }
1734*9880d681SAndroid Build Coastguard Worker
optimizeCallInst(CallInst * CI,bool & ModifiedDT)1735*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool& ModifiedDT) {
1736*9880d681SAndroid Build Coastguard Worker BasicBlock *BB = CI->getParent();
1737*9880d681SAndroid Build Coastguard Worker
1738*9880d681SAndroid Build Coastguard Worker // Lower inline assembly if we can.
1739*9880d681SAndroid Build Coastguard Worker // If we found an inline asm expession, and if the target knows how to
1740*9880d681SAndroid Build Coastguard Worker // lower it to normal LLVM code, do so now.
1741*9880d681SAndroid Build Coastguard Worker if (TLI && isa<InlineAsm>(CI->getCalledValue())) {
1742*9880d681SAndroid Build Coastguard Worker if (TLI->ExpandInlineAsm(CI)) {
1743*9880d681SAndroid Build Coastguard Worker // Avoid invalidating the iterator.
1744*9880d681SAndroid Build Coastguard Worker CurInstIterator = BB->begin();
1745*9880d681SAndroid Build Coastguard Worker // Avoid processing instructions out of order, which could cause
1746*9880d681SAndroid Build Coastguard Worker // reuse before a value is defined.
1747*9880d681SAndroid Build Coastguard Worker SunkAddrs.clear();
1748*9880d681SAndroid Build Coastguard Worker return true;
1749*9880d681SAndroid Build Coastguard Worker }
1750*9880d681SAndroid Build Coastguard Worker // Sink address computing for memory operands into the block.
1751*9880d681SAndroid Build Coastguard Worker if (optimizeInlineAsmInst(CI))
1752*9880d681SAndroid Build Coastguard Worker return true;
1753*9880d681SAndroid Build Coastguard Worker }
1754*9880d681SAndroid Build Coastguard Worker
1755*9880d681SAndroid Build Coastguard Worker // Align the pointer arguments to this call if the target thinks it's a good
1756*9880d681SAndroid Build Coastguard Worker // idea
1757*9880d681SAndroid Build Coastguard Worker unsigned MinSize, PrefAlign;
1758*9880d681SAndroid Build Coastguard Worker if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
1759*9880d681SAndroid Build Coastguard Worker for (auto &Arg : CI->arg_operands()) {
1760*9880d681SAndroid Build Coastguard Worker // We want to align both objects whose address is used directly and
1761*9880d681SAndroid Build Coastguard Worker // objects whose address is used in casts and GEPs, though it only makes
1762*9880d681SAndroid Build Coastguard Worker // sense for GEPs if the offset is a multiple of the desired alignment and
1763*9880d681SAndroid Build Coastguard Worker // if size - offset meets the size threshold.
1764*9880d681SAndroid Build Coastguard Worker if (!Arg->getType()->isPointerTy())
1765*9880d681SAndroid Build Coastguard Worker continue;
1766*9880d681SAndroid Build Coastguard Worker APInt Offset(DL->getPointerSizeInBits(
1767*9880d681SAndroid Build Coastguard Worker cast<PointerType>(Arg->getType())->getAddressSpace()),
1768*9880d681SAndroid Build Coastguard Worker 0);
1769*9880d681SAndroid Build Coastguard Worker Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset);
1770*9880d681SAndroid Build Coastguard Worker uint64_t Offset2 = Offset.getLimitedValue();
1771*9880d681SAndroid Build Coastguard Worker if ((Offset2 & (PrefAlign-1)) != 0)
1772*9880d681SAndroid Build Coastguard Worker continue;
1773*9880d681SAndroid Build Coastguard Worker AllocaInst *AI;
1774*9880d681SAndroid Build Coastguard Worker if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign &&
1775*9880d681SAndroid Build Coastguard Worker DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)
1776*9880d681SAndroid Build Coastguard Worker AI->setAlignment(PrefAlign);
1777*9880d681SAndroid Build Coastguard Worker // Global variables can only be aligned if they are defined in this
1778*9880d681SAndroid Build Coastguard Worker // object (i.e. they are uniquely initialized in this object), and
1779*9880d681SAndroid Build Coastguard Worker // over-aligning global variables that have an explicit section is
1780*9880d681SAndroid Build Coastguard Worker // forbidden.
1781*9880d681SAndroid Build Coastguard Worker GlobalVariable *GV;
1782*9880d681SAndroid Build Coastguard Worker if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() &&
1783*9880d681SAndroid Build Coastguard Worker GV->getAlignment() < PrefAlign &&
1784*9880d681SAndroid Build Coastguard Worker DL->getTypeAllocSize(GV->getValueType()) >=
1785*9880d681SAndroid Build Coastguard Worker MinSize + Offset2)
1786*9880d681SAndroid Build Coastguard Worker GV->setAlignment(PrefAlign);
1787*9880d681SAndroid Build Coastguard Worker }
1788*9880d681SAndroid Build Coastguard Worker // If this is a memcpy (or similar) then we may be able to improve the
1789*9880d681SAndroid Build Coastguard Worker // alignment
1790*9880d681SAndroid Build Coastguard Worker if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) {
1791*9880d681SAndroid Build Coastguard Worker unsigned Align = getKnownAlignment(MI->getDest(), *DL);
1792*9880d681SAndroid Build Coastguard Worker if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
1793*9880d681SAndroid Build Coastguard Worker Align = std::min(Align, getKnownAlignment(MTI->getSource(), *DL));
1794*9880d681SAndroid Build Coastguard Worker if (Align > MI->getAlignment())
1795*9880d681SAndroid Build Coastguard Worker MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align));
1796*9880d681SAndroid Build Coastguard Worker }
1797*9880d681SAndroid Build Coastguard Worker }
1798*9880d681SAndroid Build Coastguard Worker
1799*9880d681SAndroid Build Coastguard Worker // If we have a cold call site, try to sink addressing computation into the
1800*9880d681SAndroid Build Coastguard Worker // cold block. This interacts with our handling for loads and stores to
1801*9880d681SAndroid Build Coastguard Worker // ensure that we can fold all uses of a potential addressing computation
1802*9880d681SAndroid Build Coastguard Worker // into their uses. TODO: generalize this to work over profiling data
1803*9880d681SAndroid Build Coastguard Worker if (!OptSize && CI->hasFnAttr(Attribute::Cold))
1804*9880d681SAndroid Build Coastguard Worker for (auto &Arg : CI->arg_operands()) {
1805*9880d681SAndroid Build Coastguard Worker if (!Arg->getType()->isPointerTy())
1806*9880d681SAndroid Build Coastguard Worker continue;
1807*9880d681SAndroid Build Coastguard Worker unsigned AS = Arg->getType()->getPointerAddressSpace();
1808*9880d681SAndroid Build Coastguard Worker return optimizeMemoryInst(CI, Arg, Arg->getType(), AS);
1809*9880d681SAndroid Build Coastguard Worker }
1810*9880d681SAndroid Build Coastguard Worker
1811*9880d681SAndroid Build Coastguard Worker IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
1812*9880d681SAndroid Build Coastguard Worker if (II) {
1813*9880d681SAndroid Build Coastguard Worker switch (II->getIntrinsicID()) {
1814*9880d681SAndroid Build Coastguard Worker default: break;
1815*9880d681SAndroid Build Coastguard Worker case Intrinsic::objectsize: {
1816*9880d681SAndroid Build Coastguard Worker // Lower all uses of llvm.objectsize.*
1817*9880d681SAndroid Build Coastguard Worker uint64_t Size;
1818*9880d681SAndroid Build Coastguard Worker Type *ReturnTy = CI->getType();
1819*9880d681SAndroid Build Coastguard Worker Constant *RetVal = nullptr;
1820*9880d681SAndroid Build Coastguard Worker ConstantInt *Op1 = cast<ConstantInt>(II->getArgOperand(1));
1821*9880d681SAndroid Build Coastguard Worker ObjSizeMode Mode = Op1->isZero() ? ObjSizeMode::Max : ObjSizeMode::Min;
1822*9880d681SAndroid Build Coastguard Worker if (getObjectSize(II->getArgOperand(0),
1823*9880d681SAndroid Build Coastguard Worker Size, *DL, TLInfo, false, Mode)) {
1824*9880d681SAndroid Build Coastguard Worker RetVal = ConstantInt::get(ReturnTy, Size);
1825*9880d681SAndroid Build Coastguard Worker } else {
1826*9880d681SAndroid Build Coastguard Worker RetVal = ConstantInt::get(ReturnTy,
1827*9880d681SAndroid Build Coastguard Worker Mode == ObjSizeMode::Min ? 0 : -1ULL);
1828*9880d681SAndroid Build Coastguard Worker }
1829*9880d681SAndroid Build Coastguard Worker // Substituting this can cause recursive simplifications, which can
1830*9880d681SAndroid Build Coastguard Worker // invalidate our iterator. Use a WeakVH to hold onto it in case this
1831*9880d681SAndroid Build Coastguard Worker // happens.
1832*9880d681SAndroid Build Coastguard Worker Value *CurValue = &*CurInstIterator;
1833*9880d681SAndroid Build Coastguard Worker WeakVH IterHandle(CurValue);
1834*9880d681SAndroid Build Coastguard Worker
1835*9880d681SAndroid Build Coastguard Worker replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr);
1836*9880d681SAndroid Build Coastguard Worker
1837*9880d681SAndroid Build Coastguard Worker // If the iterator instruction was recursively deleted, start over at the
1838*9880d681SAndroid Build Coastguard Worker // start of the block.
1839*9880d681SAndroid Build Coastguard Worker if (IterHandle != CurValue) {
1840*9880d681SAndroid Build Coastguard Worker CurInstIterator = BB->begin();
1841*9880d681SAndroid Build Coastguard Worker SunkAddrs.clear();
1842*9880d681SAndroid Build Coastguard Worker }
1843*9880d681SAndroid Build Coastguard Worker return true;
1844*9880d681SAndroid Build Coastguard Worker }
1845*9880d681SAndroid Build Coastguard Worker case Intrinsic::masked_load: {
1846*9880d681SAndroid Build Coastguard Worker // Scalarize unsupported vector masked load
1847*9880d681SAndroid Build Coastguard Worker if (!TTI->isLegalMaskedLoad(CI->getType())) {
1848*9880d681SAndroid Build Coastguard Worker scalarizeMaskedLoad(CI);
1849*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
1850*9880d681SAndroid Build Coastguard Worker return true;
1851*9880d681SAndroid Build Coastguard Worker }
1852*9880d681SAndroid Build Coastguard Worker return false;
1853*9880d681SAndroid Build Coastguard Worker }
1854*9880d681SAndroid Build Coastguard Worker case Intrinsic::masked_store: {
1855*9880d681SAndroid Build Coastguard Worker if (!TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType())) {
1856*9880d681SAndroid Build Coastguard Worker scalarizeMaskedStore(CI);
1857*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
1858*9880d681SAndroid Build Coastguard Worker return true;
1859*9880d681SAndroid Build Coastguard Worker }
1860*9880d681SAndroid Build Coastguard Worker return false;
1861*9880d681SAndroid Build Coastguard Worker }
1862*9880d681SAndroid Build Coastguard Worker case Intrinsic::masked_gather: {
1863*9880d681SAndroid Build Coastguard Worker if (!TTI->isLegalMaskedGather(CI->getType())) {
1864*9880d681SAndroid Build Coastguard Worker scalarizeMaskedGather(CI);
1865*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
1866*9880d681SAndroid Build Coastguard Worker return true;
1867*9880d681SAndroid Build Coastguard Worker }
1868*9880d681SAndroid Build Coastguard Worker return false;
1869*9880d681SAndroid Build Coastguard Worker }
1870*9880d681SAndroid Build Coastguard Worker case Intrinsic::masked_scatter: {
1871*9880d681SAndroid Build Coastguard Worker if (!TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType())) {
1872*9880d681SAndroid Build Coastguard Worker scalarizeMaskedScatter(CI);
1873*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
1874*9880d681SAndroid Build Coastguard Worker return true;
1875*9880d681SAndroid Build Coastguard Worker }
1876*9880d681SAndroid Build Coastguard Worker return false;
1877*9880d681SAndroid Build Coastguard Worker }
1878*9880d681SAndroid Build Coastguard Worker case Intrinsic::aarch64_stlxr:
1879*9880d681SAndroid Build Coastguard Worker case Intrinsic::aarch64_stxr: {
1880*9880d681SAndroid Build Coastguard Worker ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0));
1881*9880d681SAndroid Build Coastguard Worker if (!ExtVal || !ExtVal->hasOneUse() ||
1882*9880d681SAndroid Build Coastguard Worker ExtVal->getParent() == CI->getParent())
1883*9880d681SAndroid Build Coastguard Worker return false;
1884*9880d681SAndroid Build Coastguard Worker // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.
1885*9880d681SAndroid Build Coastguard Worker ExtVal->moveBefore(CI);
1886*9880d681SAndroid Build Coastguard Worker // Mark this instruction as "inserted by CGP", so that other
1887*9880d681SAndroid Build Coastguard Worker // optimizations don't touch it.
1888*9880d681SAndroid Build Coastguard Worker InsertedInsts.insert(ExtVal);
1889*9880d681SAndroid Build Coastguard Worker return true;
1890*9880d681SAndroid Build Coastguard Worker }
1891*9880d681SAndroid Build Coastguard Worker case Intrinsic::invariant_group_barrier:
1892*9880d681SAndroid Build Coastguard Worker II->replaceAllUsesWith(II->getArgOperand(0));
1893*9880d681SAndroid Build Coastguard Worker II->eraseFromParent();
1894*9880d681SAndroid Build Coastguard Worker return true;
1895*9880d681SAndroid Build Coastguard Worker
1896*9880d681SAndroid Build Coastguard Worker case Intrinsic::cttz:
1897*9880d681SAndroid Build Coastguard Worker case Intrinsic::ctlz:
1898*9880d681SAndroid Build Coastguard Worker // If counting zeros is expensive, try to avoid it.
1899*9880d681SAndroid Build Coastguard Worker return despeculateCountZeros(II, TLI, DL, ModifiedDT);
1900*9880d681SAndroid Build Coastguard Worker }
1901*9880d681SAndroid Build Coastguard Worker
1902*9880d681SAndroid Build Coastguard Worker if (TLI) {
1903*9880d681SAndroid Build Coastguard Worker // Unknown address space.
1904*9880d681SAndroid Build Coastguard Worker // TODO: Target hook to pick which address space the intrinsic cares
1905*9880d681SAndroid Build Coastguard Worker // about?
1906*9880d681SAndroid Build Coastguard Worker unsigned AddrSpace = ~0u;
1907*9880d681SAndroid Build Coastguard Worker SmallVector<Value*, 2> PtrOps;
1908*9880d681SAndroid Build Coastguard Worker Type *AccessTy;
1909*9880d681SAndroid Build Coastguard Worker if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy, AddrSpace))
1910*9880d681SAndroid Build Coastguard Worker while (!PtrOps.empty())
1911*9880d681SAndroid Build Coastguard Worker if (optimizeMemoryInst(II, PtrOps.pop_back_val(), AccessTy, AddrSpace))
1912*9880d681SAndroid Build Coastguard Worker return true;
1913*9880d681SAndroid Build Coastguard Worker }
1914*9880d681SAndroid Build Coastguard Worker }
1915*9880d681SAndroid Build Coastguard Worker
1916*9880d681SAndroid Build Coastguard Worker // From here on out we're working with named functions.
1917*9880d681SAndroid Build Coastguard Worker if (!CI->getCalledFunction()) return false;
1918*9880d681SAndroid Build Coastguard Worker
1919*9880d681SAndroid Build Coastguard Worker // Lower all default uses of _chk calls. This is very similar
1920*9880d681SAndroid Build Coastguard Worker // to what InstCombineCalls does, but here we are only lowering calls
1921*9880d681SAndroid Build Coastguard Worker // to fortified library functions (e.g. __memcpy_chk) that have the default
1922*9880d681SAndroid Build Coastguard Worker // "don't know" as the objectsize. Anything else should be left alone.
1923*9880d681SAndroid Build Coastguard Worker FortifiedLibCallSimplifier Simplifier(TLInfo, true);
1924*9880d681SAndroid Build Coastguard Worker if (Value *V = Simplifier.optimizeCall(CI)) {
1925*9880d681SAndroid Build Coastguard Worker CI->replaceAllUsesWith(V);
1926*9880d681SAndroid Build Coastguard Worker CI->eraseFromParent();
1927*9880d681SAndroid Build Coastguard Worker return true;
1928*9880d681SAndroid Build Coastguard Worker }
1929*9880d681SAndroid Build Coastguard Worker return false;
1930*9880d681SAndroid Build Coastguard Worker }
1931*9880d681SAndroid Build Coastguard Worker
1932*9880d681SAndroid Build Coastguard Worker /// Look for opportunities to duplicate return instructions to the predecessor
1933*9880d681SAndroid Build Coastguard Worker /// to enable tail call optimizations. The case it is currently looking for is:
1934*9880d681SAndroid Build Coastguard Worker /// @code
1935*9880d681SAndroid Build Coastguard Worker /// bb0:
1936*9880d681SAndroid Build Coastguard Worker /// %tmp0 = tail call i32 @f0()
1937*9880d681SAndroid Build Coastguard Worker /// br label %return
1938*9880d681SAndroid Build Coastguard Worker /// bb1:
1939*9880d681SAndroid Build Coastguard Worker /// %tmp1 = tail call i32 @f1()
1940*9880d681SAndroid Build Coastguard Worker /// br label %return
1941*9880d681SAndroid Build Coastguard Worker /// bb2:
1942*9880d681SAndroid Build Coastguard Worker /// %tmp2 = tail call i32 @f2()
1943*9880d681SAndroid Build Coastguard Worker /// br label %return
1944*9880d681SAndroid Build Coastguard Worker /// return:
1945*9880d681SAndroid Build Coastguard Worker /// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
1946*9880d681SAndroid Build Coastguard Worker /// ret i32 %retval
1947*9880d681SAndroid Build Coastguard Worker /// @endcode
1948*9880d681SAndroid Build Coastguard Worker ///
1949*9880d681SAndroid Build Coastguard Worker /// =>
1950*9880d681SAndroid Build Coastguard Worker ///
1951*9880d681SAndroid Build Coastguard Worker /// @code
1952*9880d681SAndroid Build Coastguard Worker /// bb0:
1953*9880d681SAndroid Build Coastguard Worker /// %tmp0 = tail call i32 @f0()
1954*9880d681SAndroid Build Coastguard Worker /// ret i32 %tmp0
1955*9880d681SAndroid Build Coastguard Worker /// bb1:
1956*9880d681SAndroid Build Coastguard Worker /// %tmp1 = tail call i32 @f1()
1957*9880d681SAndroid Build Coastguard Worker /// ret i32 %tmp1
1958*9880d681SAndroid Build Coastguard Worker /// bb2:
1959*9880d681SAndroid Build Coastguard Worker /// %tmp2 = tail call i32 @f2()
1960*9880d681SAndroid Build Coastguard Worker /// ret i32 %tmp2
1961*9880d681SAndroid Build Coastguard Worker /// @endcode
dupRetToEnableTailCallOpts(BasicBlock * BB)1962*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) {
1963*9880d681SAndroid Build Coastguard Worker if (!TLI)
1964*9880d681SAndroid Build Coastguard Worker return false;
1965*9880d681SAndroid Build Coastguard Worker
1966*9880d681SAndroid Build Coastguard Worker ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator());
1967*9880d681SAndroid Build Coastguard Worker if (!RI)
1968*9880d681SAndroid Build Coastguard Worker return false;
1969*9880d681SAndroid Build Coastguard Worker
1970*9880d681SAndroid Build Coastguard Worker PHINode *PN = nullptr;
1971*9880d681SAndroid Build Coastguard Worker BitCastInst *BCI = nullptr;
1972*9880d681SAndroid Build Coastguard Worker Value *V = RI->getReturnValue();
1973*9880d681SAndroid Build Coastguard Worker if (V) {
1974*9880d681SAndroid Build Coastguard Worker BCI = dyn_cast<BitCastInst>(V);
1975*9880d681SAndroid Build Coastguard Worker if (BCI)
1976*9880d681SAndroid Build Coastguard Worker V = BCI->getOperand(0);
1977*9880d681SAndroid Build Coastguard Worker
1978*9880d681SAndroid Build Coastguard Worker PN = dyn_cast<PHINode>(V);
1979*9880d681SAndroid Build Coastguard Worker if (!PN)
1980*9880d681SAndroid Build Coastguard Worker return false;
1981*9880d681SAndroid Build Coastguard Worker }
1982*9880d681SAndroid Build Coastguard Worker
1983*9880d681SAndroid Build Coastguard Worker if (PN && PN->getParent() != BB)
1984*9880d681SAndroid Build Coastguard Worker return false;
1985*9880d681SAndroid Build Coastguard Worker
1986*9880d681SAndroid Build Coastguard Worker // It's not safe to eliminate the sign / zero extension of the return value.
1987*9880d681SAndroid Build Coastguard Worker // See llvm::isInTailCallPosition().
1988*9880d681SAndroid Build Coastguard Worker const Function *F = BB->getParent();
1989*9880d681SAndroid Build Coastguard Worker AttributeSet CallerAttrs = F->getAttributes();
1990*9880d681SAndroid Build Coastguard Worker if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) ||
1991*9880d681SAndroid Build Coastguard Worker CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt))
1992*9880d681SAndroid Build Coastguard Worker return false;
1993*9880d681SAndroid Build Coastguard Worker
1994*9880d681SAndroid Build Coastguard Worker // Make sure there are no instructions between the PHI and return, or that the
1995*9880d681SAndroid Build Coastguard Worker // return is the first instruction in the block.
1996*9880d681SAndroid Build Coastguard Worker if (PN) {
1997*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator BI = BB->begin();
1998*9880d681SAndroid Build Coastguard Worker do { ++BI; } while (isa<DbgInfoIntrinsic>(BI));
1999*9880d681SAndroid Build Coastguard Worker if (&*BI == BCI)
2000*9880d681SAndroid Build Coastguard Worker // Also skip over the bitcast.
2001*9880d681SAndroid Build Coastguard Worker ++BI;
2002*9880d681SAndroid Build Coastguard Worker if (&*BI != RI)
2003*9880d681SAndroid Build Coastguard Worker return false;
2004*9880d681SAndroid Build Coastguard Worker } else {
2005*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator BI = BB->begin();
2006*9880d681SAndroid Build Coastguard Worker while (isa<DbgInfoIntrinsic>(BI)) ++BI;
2007*9880d681SAndroid Build Coastguard Worker if (&*BI != RI)
2008*9880d681SAndroid Build Coastguard Worker return false;
2009*9880d681SAndroid Build Coastguard Worker }
2010*9880d681SAndroid Build Coastguard Worker
2011*9880d681SAndroid Build Coastguard Worker /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
2012*9880d681SAndroid Build Coastguard Worker /// call.
2013*9880d681SAndroid Build Coastguard Worker SmallVector<CallInst*, 4> TailCalls;
2014*9880d681SAndroid Build Coastguard Worker if (PN) {
2015*9880d681SAndroid Build Coastguard Worker for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) {
2016*9880d681SAndroid Build Coastguard Worker CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I));
2017*9880d681SAndroid Build Coastguard Worker // Make sure the phi value is indeed produced by the tail call.
2018*9880d681SAndroid Build Coastguard Worker if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) &&
2019*9880d681SAndroid Build Coastguard Worker TLI->mayBeEmittedAsTailCall(CI))
2020*9880d681SAndroid Build Coastguard Worker TailCalls.push_back(CI);
2021*9880d681SAndroid Build Coastguard Worker }
2022*9880d681SAndroid Build Coastguard Worker } else {
2023*9880d681SAndroid Build Coastguard Worker SmallPtrSet<BasicBlock*, 4> VisitedBBs;
2024*9880d681SAndroid Build Coastguard Worker for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) {
2025*9880d681SAndroid Build Coastguard Worker if (!VisitedBBs.insert(*PI).second)
2026*9880d681SAndroid Build Coastguard Worker continue;
2027*9880d681SAndroid Build Coastguard Worker
2028*9880d681SAndroid Build Coastguard Worker BasicBlock::InstListType &InstList = (*PI)->getInstList();
2029*9880d681SAndroid Build Coastguard Worker BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin();
2030*9880d681SAndroid Build Coastguard Worker BasicBlock::InstListType::reverse_iterator RE = InstList.rend();
2031*9880d681SAndroid Build Coastguard Worker do { ++RI; } while (RI != RE && isa<DbgInfoIntrinsic>(&*RI));
2032*9880d681SAndroid Build Coastguard Worker if (RI == RE)
2033*9880d681SAndroid Build Coastguard Worker continue;
2034*9880d681SAndroid Build Coastguard Worker
2035*9880d681SAndroid Build Coastguard Worker CallInst *CI = dyn_cast<CallInst>(&*RI);
2036*9880d681SAndroid Build Coastguard Worker if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI))
2037*9880d681SAndroid Build Coastguard Worker TailCalls.push_back(CI);
2038*9880d681SAndroid Build Coastguard Worker }
2039*9880d681SAndroid Build Coastguard Worker }
2040*9880d681SAndroid Build Coastguard Worker
2041*9880d681SAndroid Build Coastguard Worker bool Changed = false;
2042*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) {
2043*9880d681SAndroid Build Coastguard Worker CallInst *CI = TailCalls[i];
2044*9880d681SAndroid Build Coastguard Worker CallSite CS(CI);
2045*9880d681SAndroid Build Coastguard Worker
2046*9880d681SAndroid Build Coastguard Worker // Conservatively require the attributes of the call to match those of the
2047*9880d681SAndroid Build Coastguard Worker // return. Ignore noalias because it doesn't affect the call sequence.
2048*9880d681SAndroid Build Coastguard Worker AttributeSet CalleeAttrs = CS.getAttributes();
2049*9880d681SAndroid Build Coastguard Worker if (AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex).
2050*9880d681SAndroid Build Coastguard Worker removeAttribute(Attribute::NoAlias) !=
2051*9880d681SAndroid Build Coastguard Worker AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex).
2052*9880d681SAndroid Build Coastguard Worker removeAttribute(Attribute::NoAlias))
2053*9880d681SAndroid Build Coastguard Worker continue;
2054*9880d681SAndroid Build Coastguard Worker
2055*9880d681SAndroid Build Coastguard Worker // Make sure the call instruction is followed by an unconditional branch to
2056*9880d681SAndroid Build Coastguard Worker // the return block.
2057*9880d681SAndroid Build Coastguard Worker BasicBlock *CallBB = CI->getParent();
2058*9880d681SAndroid Build Coastguard Worker BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator());
2059*9880d681SAndroid Build Coastguard Worker if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB)
2060*9880d681SAndroid Build Coastguard Worker continue;
2061*9880d681SAndroid Build Coastguard Worker
2062*9880d681SAndroid Build Coastguard Worker // Duplicate the return into CallBB.
2063*9880d681SAndroid Build Coastguard Worker (void)FoldReturnIntoUncondBranch(RI, BB, CallBB);
2064*9880d681SAndroid Build Coastguard Worker ModifiedDT = Changed = true;
2065*9880d681SAndroid Build Coastguard Worker ++NumRetsDup;
2066*9880d681SAndroid Build Coastguard Worker }
2067*9880d681SAndroid Build Coastguard Worker
2068*9880d681SAndroid Build Coastguard Worker // If we eliminated all predecessors of the block, delete the block now.
2069*9880d681SAndroid Build Coastguard Worker if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB))
2070*9880d681SAndroid Build Coastguard Worker BB->eraseFromParent();
2071*9880d681SAndroid Build Coastguard Worker
2072*9880d681SAndroid Build Coastguard Worker return Changed;
2073*9880d681SAndroid Build Coastguard Worker }
2074*9880d681SAndroid Build Coastguard Worker
2075*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
2076*9880d681SAndroid Build Coastguard Worker // Memory Optimization
2077*9880d681SAndroid Build Coastguard Worker //===----------------------------------------------------------------------===//
2078*9880d681SAndroid Build Coastguard Worker
2079*9880d681SAndroid Build Coastguard Worker namespace {
2080*9880d681SAndroid Build Coastguard Worker
2081*9880d681SAndroid Build Coastguard Worker /// This is an extended version of TargetLowering::AddrMode
2082*9880d681SAndroid Build Coastguard Worker /// which holds actual Value*'s for register values.
2083*9880d681SAndroid Build Coastguard Worker struct ExtAddrMode : public TargetLowering::AddrMode {
2084*9880d681SAndroid Build Coastguard Worker Value *BaseReg;
2085*9880d681SAndroid Build Coastguard Worker Value *ScaledReg;
ExtAddrMode__anon034c73f00211::ExtAddrMode2086*9880d681SAndroid Build Coastguard Worker ExtAddrMode() : BaseReg(nullptr), ScaledReg(nullptr) {}
2087*9880d681SAndroid Build Coastguard Worker void print(raw_ostream &OS) const;
2088*9880d681SAndroid Build Coastguard Worker void dump() const;
2089*9880d681SAndroid Build Coastguard Worker
operator ==__anon034c73f00211::ExtAddrMode2090*9880d681SAndroid Build Coastguard Worker bool operator==(const ExtAddrMode& O) const {
2091*9880d681SAndroid Build Coastguard Worker return (BaseReg == O.BaseReg) && (ScaledReg == O.ScaledReg) &&
2092*9880d681SAndroid Build Coastguard Worker (BaseGV == O.BaseGV) && (BaseOffs == O.BaseOffs) &&
2093*9880d681SAndroid Build Coastguard Worker (HasBaseReg == O.HasBaseReg) && (Scale == O.Scale);
2094*9880d681SAndroid Build Coastguard Worker }
2095*9880d681SAndroid Build Coastguard Worker };
2096*9880d681SAndroid Build Coastguard Worker
2097*9880d681SAndroid Build Coastguard Worker #ifndef NDEBUG
operator <<(raw_ostream & OS,const ExtAddrMode & AM)2098*9880d681SAndroid Build Coastguard Worker static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) {
2099*9880d681SAndroid Build Coastguard Worker AM.print(OS);
2100*9880d681SAndroid Build Coastguard Worker return OS;
2101*9880d681SAndroid Build Coastguard Worker }
2102*9880d681SAndroid Build Coastguard Worker #endif
2103*9880d681SAndroid Build Coastguard Worker
print(raw_ostream & OS) const2104*9880d681SAndroid Build Coastguard Worker void ExtAddrMode::print(raw_ostream &OS) const {
2105*9880d681SAndroid Build Coastguard Worker bool NeedPlus = false;
2106*9880d681SAndroid Build Coastguard Worker OS << "[";
2107*9880d681SAndroid Build Coastguard Worker if (BaseGV) {
2108*9880d681SAndroid Build Coastguard Worker OS << (NeedPlus ? " + " : "")
2109*9880d681SAndroid Build Coastguard Worker << "GV:";
2110*9880d681SAndroid Build Coastguard Worker BaseGV->printAsOperand(OS, /*PrintType=*/false);
2111*9880d681SAndroid Build Coastguard Worker NeedPlus = true;
2112*9880d681SAndroid Build Coastguard Worker }
2113*9880d681SAndroid Build Coastguard Worker
2114*9880d681SAndroid Build Coastguard Worker if (BaseOffs) {
2115*9880d681SAndroid Build Coastguard Worker OS << (NeedPlus ? " + " : "")
2116*9880d681SAndroid Build Coastguard Worker << BaseOffs;
2117*9880d681SAndroid Build Coastguard Worker NeedPlus = true;
2118*9880d681SAndroid Build Coastguard Worker }
2119*9880d681SAndroid Build Coastguard Worker
2120*9880d681SAndroid Build Coastguard Worker if (BaseReg) {
2121*9880d681SAndroid Build Coastguard Worker OS << (NeedPlus ? " + " : "")
2122*9880d681SAndroid Build Coastguard Worker << "Base:";
2123*9880d681SAndroid Build Coastguard Worker BaseReg->printAsOperand(OS, /*PrintType=*/false);
2124*9880d681SAndroid Build Coastguard Worker NeedPlus = true;
2125*9880d681SAndroid Build Coastguard Worker }
2126*9880d681SAndroid Build Coastguard Worker if (Scale) {
2127*9880d681SAndroid Build Coastguard Worker OS << (NeedPlus ? " + " : "")
2128*9880d681SAndroid Build Coastguard Worker << Scale << "*";
2129*9880d681SAndroid Build Coastguard Worker ScaledReg->printAsOperand(OS, /*PrintType=*/false);
2130*9880d681SAndroid Build Coastguard Worker }
2131*9880d681SAndroid Build Coastguard Worker
2132*9880d681SAndroid Build Coastguard Worker OS << ']';
2133*9880d681SAndroid Build Coastguard Worker }
2134*9880d681SAndroid Build Coastguard Worker
2135*9880d681SAndroid Build Coastguard Worker #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const2136*9880d681SAndroid Build Coastguard Worker LLVM_DUMP_METHOD void ExtAddrMode::dump() const {
2137*9880d681SAndroid Build Coastguard Worker print(dbgs());
2138*9880d681SAndroid Build Coastguard Worker dbgs() << '\n';
2139*9880d681SAndroid Build Coastguard Worker }
2140*9880d681SAndroid Build Coastguard Worker #endif
2141*9880d681SAndroid Build Coastguard Worker
2142*9880d681SAndroid Build Coastguard Worker /// \brief This class provides transaction based operation on the IR.
2143*9880d681SAndroid Build Coastguard Worker /// Every change made through this class is recorded in the internal state and
2144*9880d681SAndroid Build Coastguard Worker /// can be undone (rollback) until commit is called.
2145*9880d681SAndroid Build Coastguard Worker class TypePromotionTransaction {
2146*9880d681SAndroid Build Coastguard Worker
2147*9880d681SAndroid Build Coastguard Worker /// \brief This represents the common interface of the individual transaction.
2148*9880d681SAndroid Build Coastguard Worker /// Each class implements the logic for doing one specific modification on
2149*9880d681SAndroid Build Coastguard Worker /// the IR via the TypePromotionTransaction.
2150*9880d681SAndroid Build Coastguard Worker class TypePromotionAction {
2151*9880d681SAndroid Build Coastguard Worker protected:
2152*9880d681SAndroid Build Coastguard Worker /// The Instruction modified.
2153*9880d681SAndroid Build Coastguard Worker Instruction *Inst;
2154*9880d681SAndroid Build Coastguard Worker
2155*9880d681SAndroid Build Coastguard Worker public:
2156*9880d681SAndroid Build Coastguard Worker /// \brief Constructor of the action.
2157*9880d681SAndroid Build Coastguard Worker /// The constructor performs the related action on the IR.
TypePromotionAction(Instruction * Inst)2158*9880d681SAndroid Build Coastguard Worker TypePromotionAction(Instruction *Inst) : Inst(Inst) {}
2159*9880d681SAndroid Build Coastguard Worker
~TypePromotionAction()2160*9880d681SAndroid Build Coastguard Worker virtual ~TypePromotionAction() {}
2161*9880d681SAndroid Build Coastguard Worker
2162*9880d681SAndroid Build Coastguard Worker /// \brief Undo the modification done by this action.
2163*9880d681SAndroid Build Coastguard Worker /// When this method is called, the IR must be in the same state as it was
2164*9880d681SAndroid Build Coastguard Worker /// before this action was applied.
2165*9880d681SAndroid Build Coastguard Worker /// \pre Undoing the action works if and only if the IR is in the exact same
2166*9880d681SAndroid Build Coastguard Worker /// state as it was directly after this action was applied.
2167*9880d681SAndroid Build Coastguard Worker virtual void undo() = 0;
2168*9880d681SAndroid Build Coastguard Worker
2169*9880d681SAndroid Build Coastguard Worker /// \brief Advocate every change made by this action.
2170*9880d681SAndroid Build Coastguard Worker /// When the results on the IR of the action are to be kept, it is important
2171*9880d681SAndroid Build Coastguard Worker /// to call this function, otherwise hidden information may be kept forever.
commit()2172*9880d681SAndroid Build Coastguard Worker virtual void commit() {
2173*9880d681SAndroid Build Coastguard Worker // Nothing to be done, this action is not doing anything.
2174*9880d681SAndroid Build Coastguard Worker }
2175*9880d681SAndroid Build Coastguard Worker };
2176*9880d681SAndroid Build Coastguard Worker
2177*9880d681SAndroid Build Coastguard Worker /// \brief Utility to remember the position of an instruction.
2178*9880d681SAndroid Build Coastguard Worker class InsertionHandler {
2179*9880d681SAndroid Build Coastguard Worker /// Position of an instruction.
2180*9880d681SAndroid Build Coastguard Worker /// Either an instruction:
2181*9880d681SAndroid Build Coastguard Worker /// - Is the first in a basic block: BB is used.
2182*9880d681SAndroid Build Coastguard Worker /// - Has a previous instructon: PrevInst is used.
2183*9880d681SAndroid Build Coastguard Worker union {
2184*9880d681SAndroid Build Coastguard Worker Instruction *PrevInst;
2185*9880d681SAndroid Build Coastguard Worker BasicBlock *BB;
2186*9880d681SAndroid Build Coastguard Worker } Point;
2187*9880d681SAndroid Build Coastguard Worker /// Remember whether or not the instruction had a previous instruction.
2188*9880d681SAndroid Build Coastguard Worker bool HasPrevInstruction;
2189*9880d681SAndroid Build Coastguard Worker
2190*9880d681SAndroid Build Coastguard Worker public:
2191*9880d681SAndroid Build Coastguard Worker /// \brief Record the position of \p Inst.
InsertionHandler(Instruction * Inst)2192*9880d681SAndroid Build Coastguard Worker InsertionHandler(Instruction *Inst) {
2193*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator It = Inst->getIterator();
2194*9880d681SAndroid Build Coastguard Worker HasPrevInstruction = (It != (Inst->getParent()->begin()));
2195*9880d681SAndroid Build Coastguard Worker if (HasPrevInstruction)
2196*9880d681SAndroid Build Coastguard Worker Point.PrevInst = &*--It;
2197*9880d681SAndroid Build Coastguard Worker else
2198*9880d681SAndroid Build Coastguard Worker Point.BB = Inst->getParent();
2199*9880d681SAndroid Build Coastguard Worker }
2200*9880d681SAndroid Build Coastguard Worker
2201*9880d681SAndroid Build Coastguard Worker /// \brief Insert \p Inst at the recorded position.
insert(Instruction * Inst)2202*9880d681SAndroid Build Coastguard Worker void insert(Instruction *Inst) {
2203*9880d681SAndroid Build Coastguard Worker if (HasPrevInstruction) {
2204*9880d681SAndroid Build Coastguard Worker if (Inst->getParent())
2205*9880d681SAndroid Build Coastguard Worker Inst->removeFromParent();
2206*9880d681SAndroid Build Coastguard Worker Inst->insertAfter(Point.PrevInst);
2207*9880d681SAndroid Build Coastguard Worker } else {
2208*9880d681SAndroid Build Coastguard Worker Instruction *Position = &*Point.BB->getFirstInsertionPt();
2209*9880d681SAndroid Build Coastguard Worker if (Inst->getParent())
2210*9880d681SAndroid Build Coastguard Worker Inst->moveBefore(Position);
2211*9880d681SAndroid Build Coastguard Worker else
2212*9880d681SAndroid Build Coastguard Worker Inst->insertBefore(Position);
2213*9880d681SAndroid Build Coastguard Worker }
2214*9880d681SAndroid Build Coastguard Worker }
2215*9880d681SAndroid Build Coastguard Worker };
2216*9880d681SAndroid Build Coastguard Worker
2217*9880d681SAndroid Build Coastguard Worker /// \brief Move an instruction before another.
2218*9880d681SAndroid Build Coastguard Worker class InstructionMoveBefore : public TypePromotionAction {
2219*9880d681SAndroid Build Coastguard Worker /// Original position of the instruction.
2220*9880d681SAndroid Build Coastguard Worker InsertionHandler Position;
2221*9880d681SAndroid Build Coastguard Worker
2222*9880d681SAndroid Build Coastguard Worker public:
2223*9880d681SAndroid Build Coastguard Worker /// \brief Move \p Inst before \p Before.
InstructionMoveBefore(Instruction * Inst,Instruction * Before)2224*9880d681SAndroid Build Coastguard Worker InstructionMoveBefore(Instruction *Inst, Instruction *Before)
2225*9880d681SAndroid Build Coastguard Worker : TypePromotionAction(Inst), Position(Inst) {
2226*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before << "\n");
2227*9880d681SAndroid Build Coastguard Worker Inst->moveBefore(Before);
2228*9880d681SAndroid Build Coastguard Worker }
2229*9880d681SAndroid Build Coastguard Worker
2230*9880d681SAndroid Build Coastguard Worker /// \brief Move the instruction back to its original position.
undo()2231*9880d681SAndroid Build Coastguard Worker void undo() override {
2232*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n");
2233*9880d681SAndroid Build Coastguard Worker Position.insert(Inst);
2234*9880d681SAndroid Build Coastguard Worker }
2235*9880d681SAndroid Build Coastguard Worker };
2236*9880d681SAndroid Build Coastguard Worker
2237*9880d681SAndroid Build Coastguard Worker /// \brief Set the operand of an instruction with a new value.
2238*9880d681SAndroid Build Coastguard Worker class OperandSetter : public TypePromotionAction {
2239*9880d681SAndroid Build Coastguard Worker /// Original operand of the instruction.
2240*9880d681SAndroid Build Coastguard Worker Value *Origin;
2241*9880d681SAndroid Build Coastguard Worker /// Index of the modified instruction.
2242*9880d681SAndroid Build Coastguard Worker unsigned Idx;
2243*9880d681SAndroid Build Coastguard Worker
2244*9880d681SAndroid Build Coastguard Worker public:
2245*9880d681SAndroid Build Coastguard Worker /// \brief Set \p Idx operand of \p Inst with \p NewVal.
OperandSetter(Instruction * Inst,unsigned Idx,Value * NewVal)2246*9880d681SAndroid Build Coastguard Worker OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal)
2247*9880d681SAndroid Build Coastguard Worker : TypePromotionAction(Inst), Idx(Idx) {
2248*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n"
2249*9880d681SAndroid Build Coastguard Worker << "for:" << *Inst << "\n"
2250*9880d681SAndroid Build Coastguard Worker << "with:" << *NewVal << "\n");
2251*9880d681SAndroid Build Coastguard Worker Origin = Inst->getOperand(Idx);
2252*9880d681SAndroid Build Coastguard Worker Inst->setOperand(Idx, NewVal);
2253*9880d681SAndroid Build Coastguard Worker }
2254*9880d681SAndroid Build Coastguard Worker
2255*9880d681SAndroid Build Coastguard Worker /// \brief Restore the original value of the instruction.
undo()2256*9880d681SAndroid Build Coastguard Worker void undo() override {
2257*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n"
2258*9880d681SAndroid Build Coastguard Worker << "for: " << *Inst << "\n"
2259*9880d681SAndroid Build Coastguard Worker << "with: " << *Origin << "\n");
2260*9880d681SAndroid Build Coastguard Worker Inst->setOperand(Idx, Origin);
2261*9880d681SAndroid Build Coastguard Worker }
2262*9880d681SAndroid Build Coastguard Worker };
2263*9880d681SAndroid Build Coastguard Worker
2264*9880d681SAndroid Build Coastguard Worker /// \brief Hide the operands of an instruction.
2265*9880d681SAndroid Build Coastguard Worker /// Do as if this instruction was not using any of its operands.
2266*9880d681SAndroid Build Coastguard Worker class OperandsHider : public TypePromotionAction {
2267*9880d681SAndroid Build Coastguard Worker /// The list of original operands.
2268*9880d681SAndroid Build Coastguard Worker SmallVector<Value *, 4> OriginalValues;
2269*9880d681SAndroid Build Coastguard Worker
2270*9880d681SAndroid Build Coastguard Worker public:
2271*9880d681SAndroid Build Coastguard Worker /// \brief Remove \p Inst from the uses of the operands of \p Inst.
OperandsHider(Instruction * Inst)2272*9880d681SAndroid Build Coastguard Worker OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) {
2273*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n");
2274*9880d681SAndroid Build Coastguard Worker unsigned NumOpnds = Inst->getNumOperands();
2275*9880d681SAndroid Build Coastguard Worker OriginalValues.reserve(NumOpnds);
2276*9880d681SAndroid Build Coastguard Worker for (unsigned It = 0; It < NumOpnds; ++It) {
2277*9880d681SAndroid Build Coastguard Worker // Save the current operand.
2278*9880d681SAndroid Build Coastguard Worker Value *Val = Inst->getOperand(It);
2279*9880d681SAndroid Build Coastguard Worker OriginalValues.push_back(Val);
2280*9880d681SAndroid Build Coastguard Worker // Set a dummy one.
2281*9880d681SAndroid Build Coastguard Worker // We could use OperandSetter here, but that would imply an overhead
2282*9880d681SAndroid Build Coastguard Worker // that we are not willing to pay.
2283*9880d681SAndroid Build Coastguard Worker Inst->setOperand(It, UndefValue::get(Val->getType()));
2284*9880d681SAndroid Build Coastguard Worker }
2285*9880d681SAndroid Build Coastguard Worker }
2286*9880d681SAndroid Build Coastguard Worker
2287*9880d681SAndroid Build Coastguard Worker /// \brief Restore the original list of uses.
undo()2288*9880d681SAndroid Build Coastguard Worker void undo() override {
2289*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n");
2290*9880d681SAndroid Build Coastguard Worker for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)
2291*9880d681SAndroid Build Coastguard Worker Inst->setOperand(It, OriginalValues[It]);
2292*9880d681SAndroid Build Coastguard Worker }
2293*9880d681SAndroid Build Coastguard Worker };
2294*9880d681SAndroid Build Coastguard Worker
2295*9880d681SAndroid Build Coastguard Worker /// \brief Build a truncate instruction.
2296*9880d681SAndroid Build Coastguard Worker class TruncBuilder : public TypePromotionAction {
2297*9880d681SAndroid Build Coastguard Worker Value *Val;
2298*9880d681SAndroid Build Coastguard Worker public:
2299*9880d681SAndroid Build Coastguard Worker /// \brief Build a truncate instruction of \p Opnd producing a \p Ty
2300*9880d681SAndroid Build Coastguard Worker /// result.
2301*9880d681SAndroid Build Coastguard Worker /// trunc Opnd to Ty.
TruncBuilder(Instruction * Opnd,Type * Ty)2302*9880d681SAndroid Build Coastguard Worker TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) {
2303*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(Opnd);
2304*9880d681SAndroid Build Coastguard Worker Val = Builder.CreateTrunc(Opnd, Ty, "promoted");
2305*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n");
2306*9880d681SAndroid Build Coastguard Worker }
2307*9880d681SAndroid Build Coastguard Worker
2308*9880d681SAndroid Build Coastguard Worker /// \brief Get the built value.
getBuiltValue()2309*9880d681SAndroid Build Coastguard Worker Value *getBuiltValue() { return Val; }
2310*9880d681SAndroid Build Coastguard Worker
2311*9880d681SAndroid Build Coastguard Worker /// \brief Remove the built instruction.
undo()2312*9880d681SAndroid Build Coastguard Worker void undo() override {
2313*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n");
2314*9880d681SAndroid Build Coastguard Worker if (Instruction *IVal = dyn_cast<Instruction>(Val))
2315*9880d681SAndroid Build Coastguard Worker IVal->eraseFromParent();
2316*9880d681SAndroid Build Coastguard Worker }
2317*9880d681SAndroid Build Coastguard Worker };
2318*9880d681SAndroid Build Coastguard Worker
2319*9880d681SAndroid Build Coastguard Worker /// \brief Build a sign extension instruction.
2320*9880d681SAndroid Build Coastguard Worker class SExtBuilder : public TypePromotionAction {
2321*9880d681SAndroid Build Coastguard Worker Value *Val;
2322*9880d681SAndroid Build Coastguard Worker public:
2323*9880d681SAndroid Build Coastguard Worker /// \brief Build a sign extension instruction of \p Opnd producing a \p Ty
2324*9880d681SAndroid Build Coastguard Worker /// result.
2325*9880d681SAndroid Build Coastguard Worker /// sext Opnd to Ty.
SExtBuilder(Instruction * InsertPt,Value * Opnd,Type * Ty)2326*9880d681SAndroid Build Coastguard Worker SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
2327*9880d681SAndroid Build Coastguard Worker : TypePromotionAction(InsertPt) {
2328*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(InsertPt);
2329*9880d681SAndroid Build Coastguard Worker Val = Builder.CreateSExt(Opnd, Ty, "promoted");
2330*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n");
2331*9880d681SAndroid Build Coastguard Worker }
2332*9880d681SAndroid Build Coastguard Worker
2333*9880d681SAndroid Build Coastguard Worker /// \brief Get the built value.
getBuiltValue()2334*9880d681SAndroid Build Coastguard Worker Value *getBuiltValue() { return Val; }
2335*9880d681SAndroid Build Coastguard Worker
2336*9880d681SAndroid Build Coastguard Worker /// \brief Remove the built instruction.
undo()2337*9880d681SAndroid Build Coastguard Worker void undo() override {
2338*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n");
2339*9880d681SAndroid Build Coastguard Worker if (Instruction *IVal = dyn_cast<Instruction>(Val))
2340*9880d681SAndroid Build Coastguard Worker IVal->eraseFromParent();
2341*9880d681SAndroid Build Coastguard Worker }
2342*9880d681SAndroid Build Coastguard Worker };
2343*9880d681SAndroid Build Coastguard Worker
2344*9880d681SAndroid Build Coastguard Worker /// \brief Build a zero extension instruction.
2345*9880d681SAndroid Build Coastguard Worker class ZExtBuilder : public TypePromotionAction {
2346*9880d681SAndroid Build Coastguard Worker Value *Val;
2347*9880d681SAndroid Build Coastguard Worker public:
2348*9880d681SAndroid Build Coastguard Worker /// \brief Build a zero extension instruction of \p Opnd producing a \p Ty
2349*9880d681SAndroid Build Coastguard Worker /// result.
2350*9880d681SAndroid Build Coastguard Worker /// zext Opnd to Ty.
ZExtBuilder(Instruction * InsertPt,Value * Opnd,Type * Ty)2351*9880d681SAndroid Build Coastguard Worker ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
2352*9880d681SAndroid Build Coastguard Worker : TypePromotionAction(InsertPt) {
2353*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(InsertPt);
2354*9880d681SAndroid Build Coastguard Worker Val = Builder.CreateZExt(Opnd, Ty, "promoted");
2355*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n");
2356*9880d681SAndroid Build Coastguard Worker }
2357*9880d681SAndroid Build Coastguard Worker
2358*9880d681SAndroid Build Coastguard Worker /// \brief Get the built value.
getBuiltValue()2359*9880d681SAndroid Build Coastguard Worker Value *getBuiltValue() { return Val; }
2360*9880d681SAndroid Build Coastguard Worker
2361*9880d681SAndroid Build Coastguard Worker /// \brief Remove the built instruction.
undo()2362*9880d681SAndroid Build Coastguard Worker void undo() override {
2363*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n");
2364*9880d681SAndroid Build Coastguard Worker if (Instruction *IVal = dyn_cast<Instruction>(Val))
2365*9880d681SAndroid Build Coastguard Worker IVal->eraseFromParent();
2366*9880d681SAndroid Build Coastguard Worker }
2367*9880d681SAndroid Build Coastguard Worker };
2368*9880d681SAndroid Build Coastguard Worker
2369*9880d681SAndroid Build Coastguard Worker /// \brief Mutate an instruction to another type.
2370*9880d681SAndroid Build Coastguard Worker class TypeMutator : public TypePromotionAction {
2371*9880d681SAndroid Build Coastguard Worker /// Record the original type.
2372*9880d681SAndroid Build Coastguard Worker Type *OrigTy;
2373*9880d681SAndroid Build Coastguard Worker
2374*9880d681SAndroid Build Coastguard Worker public:
2375*9880d681SAndroid Build Coastguard Worker /// \brief Mutate the type of \p Inst into \p NewTy.
TypeMutator(Instruction * Inst,Type * NewTy)2376*9880d681SAndroid Build Coastguard Worker TypeMutator(Instruction *Inst, Type *NewTy)
2377*9880d681SAndroid Build Coastguard Worker : TypePromotionAction(Inst), OrigTy(Inst->getType()) {
2378*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy
2379*9880d681SAndroid Build Coastguard Worker << "\n");
2380*9880d681SAndroid Build Coastguard Worker Inst->mutateType(NewTy);
2381*9880d681SAndroid Build Coastguard Worker }
2382*9880d681SAndroid Build Coastguard Worker
2383*9880d681SAndroid Build Coastguard Worker /// \brief Mutate the instruction back to its original type.
undo()2384*9880d681SAndroid Build Coastguard Worker void undo() override {
2385*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy
2386*9880d681SAndroid Build Coastguard Worker << "\n");
2387*9880d681SAndroid Build Coastguard Worker Inst->mutateType(OrigTy);
2388*9880d681SAndroid Build Coastguard Worker }
2389*9880d681SAndroid Build Coastguard Worker };
2390*9880d681SAndroid Build Coastguard Worker
2391*9880d681SAndroid Build Coastguard Worker /// \brief Replace the uses of an instruction by another instruction.
2392*9880d681SAndroid Build Coastguard Worker class UsesReplacer : public TypePromotionAction {
2393*9880d681SAndroid Build Coastguard Worker /// Helper structure to keep track of the replaced uses.
2394*9880d681SAndroid Build Coastguard Worker struct InstructionAndIdx {
2395*9880d681SAndroid Build Coastguard Worker /// The instruction using the instruction.
2396*9880d681SAndroid Build Coastguard Worker Instruction *Inst;
2397*9880d681SAndroid Build Coastguard Worker /// The index where this instruction is used for Inst.
2398*9880d681SAndroid Build Coastguard Worker unsigned Idx;
InstructionAndIdx__anon034c73f00211::TypePromotionTransaction::UsesReplacer::InstructionAndIdx2399*9880d681SAndroid Build Coastguard Worker InstructionAndIdx(Instruction *Inst, unsigned Idx)
2400*9880d681SAndroid Build Coastguard Worker : Inst(Inst), Idx(Idx) {}
2401*9880d681SAndroid Build Coastguard Worker };
2402*9880d681SAndroid Build Coastguard Worker
2403*9880d681SAndroid Build Coastguard Worker /// Keep track of the original uses (pair Instruction, Index).
2404*9880d681SAndroid Build Coastguard Worker SmallVector<InstructionAndIdx, 4> OriginalUses;
2405*9880d681SAndroid Build Coastguard Worker typedef SmallVectorImpl<InstructionAndIdx>::iterator use_iterator;
2406*9880d681SAndroid Build Coastguard Worker
2407*9880d681SAndroid Build Coastguard Worker public:
2408*9880d681SAndroid Build Coastguard Worker /// \brief Replace all the use of \p Inst by \p New.
UsesReplacer(Instruction * Inst,Value * New)2409*9880d681SAndroid Build Coastguard Worker UsesReplacer(Instruction *Inst, Value *New) : TypePromotionAction(Inst) {
2410*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New
2411*9880d681SAndroid Build Coastguard Worker << "\n");
2412*9880d681SAndroid Build Coastguard Worker // Record the original uses.
2413*9880d681SAndroid Build Coastguard Worker for (Use &U : Inst->uses()) {
2414*9880d681SAndroid Build Coastguard Worker Instruction *UserI = cast<Instruction>(U.getUser());
2415*9880d681SAndroid Build Coastguard Worker OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));
2416*9880d681SAndroid Build Coastguard Worker }
2417*9880d681SAndroid Build Coastguard Worker // Now, we can replace the uses.
2418*9880d681SAndroid Build Coastguard Worker Inst->replaceAllUsesWith(New);
2419*9880d681SAndroid Build Coastguard Worker }
2420*9880d681SAndroid Build Coastguard Worker
2421*9880d681SAndroid Build Coastguard Worker /// \brief Reassign the original uses of Inst to Inst.
undo()2422*9880d681SAndroid Build Coastguard Worker void undo() override {
2423*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n");
2424*9880d681SAndroid Build Coastguard Worker for (use_iterator UseIt = OriginalUses.begin(),
2425*9880d681SAndroid Build Coastguard Worker EndIt = OriginalUses.end();
2426*9880d681SAndroid Build Coastguard Worker UseIt != EndIt; ++UseIt) {
2427*9880d681SAndroid Build Coastguard Worker UseIt->Inst->setOperand(UseIt->Idx, Inst);
2428*9880d681SAndroid Build Coastguard Worker }
2429*9880d681SAndroid Build Coastguard Worker }
2430*9880d681SAndroid Build Coastguard Worker };
2431*9880d681SAndroid Build Coastguard Worker
2432*9880d681SAndroid Build Coastguard Worker /// \brief Remove an instruction from the IR.
2433*9880d681SAndroid Build Coastguard Worker class InstructionRemover : public TypePromotionAction {
2434*9880d681SAndroid Build Coastguard Worker /// Original position of the instruction.
2435*9880d681SAndroid Build Coastguard Worker InsertionHandler Inserter;
2436*9880d681SAndroid Build Coastguard Worker /// Helper structure to hide all the link to the instruction. In other
2437*9880d681SAndroid Build Coastguard Worker /// words, this helps to do as if the instruction was removed.
2438*9880d681SAndroid Build Coastguard Worker OperandsHider Hider;
2439*9880d681SAndroid Build Coastguard Worker /// Keep track of the uses replaced, if any.
2440*9880d681SAndroid Build Coastguard Worker UsesReplacer *Replacer;
2441*9880d681SAndroid Build Coastguard Worker
2442*9880d681SAndroid Build Coastguard Worker public:
2443*9880d681SAndroid Build Coastguard Worker /// \brief Remove all reference of \p Inst and optinally replace all its
2444*9880d681SAndroid Build Coastguard Worker /// uses with New.
2445*9880d681SAndroid Build Coastguard Worker /// \pre If !Inst->use_empty(), then New != nullptr
InstructionRemover(Instruction * Inst,Value * New=nullptr)2446*9880d681SAndroid Build Coastguard Worker InstructionRemover(Instruction *Inst, Value *New = nullptr)
2447*9880d681SAndroid Build Coastguard Worker : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),
2448*9880d681SAndroid Build Coastguard Worker Replacer(nullptr) {
2449*9880d681SAndroid Build Coastguard Worker if (New)
2450*9880d681SAndroid Build Coastguard Worker Replacer = new UsesReplacer(Inst, New);
2451*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n");
2452*9880d681SAndroid Build Coastguard Worker Inst->removeFromParent();
2453*9880d681SAndroid Build Coastguard Worker }
2454*9880d681SAndroid Build Coastguard Worker
~InstructionRemover()2455*9880d681SAndroid Build Coastguard Worker ~InstructionRemover() override { delete Replacer; }
2456*9880d681SAndroid Build Coastguard Worker
2457*9880d681SAndroid Build Coastguard Worker /// \brief Really remove the instruction.
commit()2458*9880d681SAndroid Build Coastguard Worker void commit() override { delete Inst; }
2459*9880d681SAndroid Build Coastguard Worker
2460*9880d681SAndroid Build Coastguard Worker /// \brief Resurrect the instruction and reassign it to the proper uses if
2461*9880d681SAndroid Build Coastguard Worker /// new value was provided when build this action.
undo()2462*9880d681SAndroid Build Coastguard Worker void undo() override {
2463*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n");
2464*9880d681SAndroid Build Coastguard Worker Inserter.insert(Inst);
2465*9880d681SAndroid Build Coastguard Worker if (Replacer)
2466*9880d681SAndroid Build Coastguard Worker Replacer->undo();
2467*9880d681SAndroid Build Coastguard Worker Hider.undo();
2468*9880d681SAndroid Build Coastguard Worker }
2469*9880d681SAndroid Build Coastguard Worker };
2470*9880d681SAndroid Build Coastguard Worker
2471*9880d681SAndroid Build Coastguard Worker public:
2472*9880d681SAndroid Build Coastguard Worker /// Restoration point.
2473*9880d681SAndroid Build Coastguard Worker /// The restoration point is a pointer to an action instead of an iterator
2474*9880d681SAndroid Build Coastguard Worker /// because the iterator may be invalidated but not the pointer.
2475*9880d681SAndroid Build Coastguard Worker typedef const TypePromotionAction *ConstRestorationPt;
2476*9880d681SAndroid Build Coastguard Worker /// Advocate every changes made in that transaction.
2477*9880d681SAndroid Build Coastguard Worker void commit();
2478*9880d681SAndroid Build Coastguard Worker /// Undo all the changes made after the given point.
2479*9880d681SAndroid Build Coastguard Worker void rollback(ConstRestorationPt Point);
2480*9880d681SAndroid Build Coastguard Worker /// Get the current restoration point.
2481*9880d681SAndroid Build Coastguard Worker ConstRestorationPt getRestorationPoint() const;
2482*9880d681SAndroid Build Coastguard Worker
2483*9880d681SAndroid Build Coastguard Worker /// \name API for IR modification with state keeping to support rollback.
2484*9880d681SAndroid Build Coastguard Worker /// @{
2485*9880d681SAndroid Build Coastguard Worker /// Same as Instruction::setOperand.
2486*9880d681SAndroid Build Coastguard Worker void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal);
2487*9880d681SAndroid Build Coastguard Worker /// Same as Instruction::eraseFromParent.
2488*9880d681SAndroid Build Coastguard Worker void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr);
2489*9880d681SAndroid Build Coastguard Worker /// Same as Value::replaceAllUsesWith.
2490*9880d681SAndroid Build Coastguard Worker void replaceAllUsesWith(Instruction *Inst, Value *New);
2491*9880d681SAndroid Build Coastguard Worker /// Same as Value::mutateType.
2492*9880d681SAndroid Build Coastguard Worker void mutateType(Instruction *Inst, Type *NewTy);
2493*9880d681SAndroid Build Coastguard Worker /// Same as IRBuilder::createTrunc.
2494*9880d681SAndroid Build Coastguard Worker Value *createTrunc(Instruction *Opnd, Type *Ty);
2495*9880d681SAndroid Build Coastguard Worker /// Same as IRBuilder::createSExt.
2496*9880d681SAndroid Build Coastguard Worker Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty);
2497*9880d681SAndroid Build Coastguard Worker /// Same as IRBuilder::createZExt.
2498*9880d681SAndroid Build Coastguard Worker Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty);
2499*9880d681SAndroid Build Coastguard Worker /// Same as Instruction::moveBefore.
2500*9880d681SAndroid Build Coastguard Worker void moveBefore(Instruction *Inst, Instruction *Before);
2501*9880d681SAndroid Build Coastguard Worker /// @}
2502*9880d681SAndroid Build Coastguard Worker
2503*9880d681SAndroid Build Coastguard Worker private:
2504*9880d681SAndroid Build Coastguard Worker /// The ordered list of actions made so far.
2505*9880d681SAndroid Build Coastguard Worker SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions;
2506*9880d681SAndroid Build Coastguard Worker typedef SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator CommitPt;
2507*9880d681SAndroid Build Coastguard Worker };
2508*9880d681SAndroid Build Coastguard Worker
setOperand(Instruction * Inst,unsigned Idx,Value * NewVal)2509*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx,
2510*9880d681SAndroid Build Coastguard Worker Value *NewVal) {
2511*9880d681SAndroid Build Coastguard Worker Actions.push_back(
2512*9880d681SAndroid Build Coastguard Worker make_unique<TypePromotionTransaction::OperandSetter>(Inst, Idx, NewVal));
2513*9880d681SAndroid Build Coastguard Worker }
2514*9880d681SAndroid Build Coastguard Worker
eraseInstruction(Instruction * Inst,Value * NewVal)2515*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::eraseInstruction(Instruction *Inst,
2516*9880d681SAndroid Build Coastguard Worker Value *NewVal) {
2517*9880d681SAndroid Build Coastguard Worker Actions.push_back(
2518*9880d681SAndroid Build Coastguard Worker make_unique<TypePromotionTransaction::InstructionRemover>(Inst, NewVal));
2519*9880d681SAndroid Build Coastguard Worker }
2520*9880d681SAndroid Build Coastguard Worker
replaceAllUsesWith(Instruction * Inst,Value * New)2521*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst,
2522*9880d681SAndroid Build Coastguard Worker Value *New) {
2523*9880d681SAndroid Build Coastguard Worker Actions.push_back(make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
2524*9880d681SAndroid Build Coastguard Worker }
2525*9880d681SAndroid Build Coastguard Worker
mutateType(Instruction * Inst,Type * NewTy)2526*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) {
2527*9880d681SAndroid Build Coastguard Worker Actions.push_back(make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
2528*9880d681SAndroid Build Coastguard Worker }
2529*9880d681SAndroid Build Coastguard Worker
createTrunc(Instruction * Opnd,Type * Ty)2530*9880d681SAndroid Build Coastguard Worker Value *TypePromotionTransaction::createTrunc(Instruction *Opnd,
2531*9880d681SAndroid Build Coastguard Worker Type *Ty) {
2532*9880d681SAndroid Build Coastguard Worker std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty));
2533*9880d681SAndroid Build Coastguard Worker Value *Val = Ptr->getBuiltValue();
2534*9880d681SAndroid Build Coastguard Worker Actions.push_back(std::move(Ptr));
2535*9880d681SAndroid Build Coastguard Worker return Val;
2536*9880d681SAndroid Build Coastguard Worker }
2537*9880d681SAndroid Build Coastguard Worker
createSExt(Instruction * Inst,Value * Opnd,Type * Ty)2538*9880d681SAndroid Build Coastguard Worker Value *TypePromotionTransaction::createSExt(Instruction *Inst,
2539*9880d681SAndroid Build Coastguard Worker Value *Opnd, Type *Ty) {
2540*9880d681SAndroid Build Coastguard Worker std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty));
2541*9880d681SAndroid Build Coastguard Worker Value *Val = Ptr->getBuiltValue();
2542*9880d681SAndroid Build Coastguard Worker Actions.push_back(std::move(Ptr));
2543*9880d681SAndroid Build Coastguard Worker return Val;
2544*9880d681SAndroid Build Coastguard Worker }
2545*9880d681SAndroid Build Coastguard Worker
createZExt(Instruction * Inst,Value * Opnd,Type * Ty)2546*9880d681SAndroid Build Coastguard Worker Value *TypePromotionTransaction::createZExt(Instruction *Inst,
2547*9880d681SAndroid Build Coastguard Worker Value *Opnd, Type *Ty) {
2548*9880d681SAndroid Build Coastguard Worker std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty));
2549*9880d681SAndroid Build Coastguard Worker Value *Val = Ptr->getBuiltValue();
2550*9880d681SAndroid Build Coastguard Worker Actions.push_back(std::move(Ptr));
2551*9880d681SAndroid Build Coastguard Worker return Val;
2552*9880d681SAndroid Build Coastguard Worker }
2553*9880d681SAndroid Build Coastguard Worker
moveBefore(Instruction * Inst,Instruction * Before)2554*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::moveBefore(Instruction *Inst,
2555*9880d681SAndroid Build Coastguard Worker Instruction *Before) {
2556*9880d681SAndroid Build Coastguard Worker Actions.push_back(
2557*9880d681SAndroid Build Coastguard Worker make_unique<TypePromotionTransaction::InstructionMoveBefore>(Inst, Before));
2558*9880d681SAndroid Build Coastguard Worker }
2559*9880d681SAndroid Build Coastguard Worker
2560*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt
getRestorationPoint() const2561*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::getRestorationPoint() const {
2562*9880d681SAndroid Build Coastguard Worker return !Actions.empty() ? Actions.back().get() : nullptr;
2563*9880d681SAndroid Build Coastguard Worker }
2564*9880d681SAndroid Build Coastguard Worker
commit()2565*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::commit() {
2566*9880d681SAndroid Build Coastguard Worker for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt;
2567*9880d681SAndroid Build Coastguard Worker ++It)
2568*9880d681SAndroid Build Coastguard Worker (*It)->commit();
2569*9880d681SAndroid Build Coastguard Worker Actions.clear();
2570*9880d681SAndroid Build Coastguard Worker }
2571*9880d681SAndroid Build Coastguard Worker
rollback(TypePromotionTransaction::ConstRestorationPt Point)2572*9880d681SAndroid Build Coastguard Worker void TypePromotionTransaction::rollback(
2573*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt Point) {
2574*9880d681SAndroid Build Coastguard Worker while (!Actions.empty() && Point != Actions.back().get()) {
2575*9880d681SAndroid Build Coastguard Worker std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
2576*9880d681SAndroid Build Coastguard Worker Curr->undo();
2577*9880d681SAndroid Build Coastguard Worker }
2578*9880d681SAndroid Build Coastguard Worker }
2579*9880d681SAndroid Build Coastguard Worker
2580*9880d681SAndroid Build Coastguard Worker /// \brief A helper class for matching addressing modes.
2581*9880d681SAndroid Build Coastguard Worker ///
2582*9880d681SAndroid Build Coastguard Worker /// This encapsulates the logic for matching the target-legal addressing modes.
2583*9880d681SAndroid Build Coastguard Worker class AddressingModeMatcher {
2584*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction*> &AddrModeInsts;
2585*9880d681SAndroid Build Coastguard Worker const TargetMachine &TM;
2586*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI;
2587*9880d681SAndroid Build Coastguard Worker const DataLayout &DL;
2588*9880d681SAndroid Build Coastguard Worker
2589*9880d681SAndroid Build Coastguard Worker /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
2590*9880d681SAndroid Build Coastguard Worker /// the memory instruction that we're computing this address for.
2591*9880d681SAndroid Build Coastguard Worker Type *AccessTy;
2592*9880d681SAndroid Build Coastguard Worker unsigned AddrSpace;
2593*9880d681SAndroid Build Coastguard Worker Instruction *MemoryInst;
2594*9880d681SAndroid Build Coastguard Worker
2595*9880d681SAndroid Build Coastguard Worker /// This is the addressing mode that we're building up. This is
2596*9880d681SAndroid Build Coastguard Worker /// part of the return value of this addressing mode matching stuff.
2597*9880d681SAndroid Build Coastguard Worker ExtAddrMode &AddrMode;
2598*9880d681SAndroid Build Coastguard Worker
2599*9880d681SAndroid Build Coastguard Worker /// The instructions inserted by other CodeGenPrepare optimizations.
2600*9880d681SAndroid Build Coastguard Worker const SetOfInstrs &InsertedInsts;
2601*9880d681SAndroid Build Coastguard Worker /// A map from the instructions to their type before promotion.
2602*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts;
2603*9880d681SAndroid Build Coastguard Worker /// The ongoing transaction where every action should be registered.
2604*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction &TPT;
2605*9880d681SAndroid Build Coastguard Worker
2606*9880d681SAndroid Build Coastguard Worker /// This is set to true when we should not do profitability checks.
2607*9880d681SAndroid Build Coastguard Worker /// When true, IsProfitableToFoldIntoAddressingMode always returns true.
2608*9880d681SAndroid Build Coastguard Worker bool IgnoreProfitability;
2609*9880d681SAndroid Build Coastguard Worker
AddressingModeMatcher(SmallVectorImpl<Instruction * > & AMI,const TargetMachine & TM,Type * AT,unsigned AS,Instruction * MI,ExtAddrMode & AM,const SetOfInstrs & InsertedInsts,InstrToOrigTy & PromotedInsts,TypePromotionTransaction & TPT)2610*9880d681SAndroid Build Coastguard Worker AddressingModeMatcher(SmallVectorImpl<Instruction *> &AMI,
2611*9880d681SAndroid Build Coastguard Worker const TargetMachine &TM, Type *AT, unsigned AS,
2612*9880d681SAndroid Build Coastguard Worker Instruction *MI, ExtAddrMode &AM,
2613*9880d681SAndroid Build Coastguard Worker const SetOfInstrs &InsertedInsts,
2614*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts,
2615*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction &TPT)
2616*9880d681SAndroid Build Coastguard Worker : AddrModeInsts(AMI), TM(TM),
2617*9880d681SAndroid Build Coastguard Worker TLI(*TM.getSubtargetImpl(*MI->getParent()->getParent())
2618*9880d681SAndroid Build Coastguard Worker ->getTargetLowering()),
2619*9880d681SAndroid Build Coastguard Worker DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS),
2620*9880d681SAndroid Build Coastguard Worker MemoryInst(MI), AddrMode(AM), InsertedInsts(InsertedInsts),
2621*9880d681SAndroid Build Coastguard Worker PromotedInsts(PromotedInsts), TPT(TPT) {
2622*9880d681SAndroid Build Coastguard Worker IgnoreProfitability = false;
2623*9880d681SAndroid Build Coastguard Worker }
2624*9880d681SAndroid Build Coastguard Worker public:
2625*9880d681SAndroid Build Coastguard Worker
2626*9880d681SAndroid Build Coastguard Worker /// Find the maximal addressing mode that a load/store of V can fold,
2627*9880d681SAndroid Build Coastguard Worker /// give an access type of AccessTy. This returns a list of involved
2628*9880d681SAndroid Build Coastguard Worker /// instructions in AddrModeInsts.
2629*9880d681SAndroid Build Coastguard Worker /// \p InsertedInsts The instructions inserted by other CodeGenPrepare
2630*9880d681SAndroid Build Coastguard Worker /// optimizations.
2631*9880d681SAndroid Build Coastguard Worker /// \p PromotedInsts maps the instructions to their type before promotion.
2632*9880d681SAndroid Build Coastguard Worker /// \p The ongoing transaction where every action should be registered.
Match(Value * V,Type * AccessTy,unsigned AS,Instruction * MemoryInst,SmallVectorImpl<Instruction * > & AddrModeInsts,const TargetMachine & TM,const SetOfInstrs & InsertedInsts,InstrToOrigTy & PromotedInsts,TypePromotionTransaction & TPT)2633*9880d681SAndroid Build Coastguard Worker static ExtAddrMode Match(Value *V, Type *AccessTy, unsigned AS,
2634*9880d681SAndroid Build Coastguard Worker Instruction *MemoryInst,
2635*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction*> &AddrModeInsts,
2636*9880d681SAndroid Build Coastguard Worker const TargetMachine &TM,
2637*9880d681SAndroid Build Coastguard Worker const SetOfInstrs &InsertedInsts,
2638*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts,
2639*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction &TPT) {
2640*9880d681SAndroid Build Coastguard Worker ExtAddrMode Result;
2641*9880d681SAndroid Build Coastguard Worker
2642*9880d681SAndroid Build Coastguard Worker bool Success = AddressingModeMatcher(AddrModeInsts, TM, AccessTy, AS,
2643*9880d681SAndroid Build Coastguard Worker MemoryInst, Result, InsertedInsts,
2644*9880d681SAndroid Build Coastguard Worker PromotedInsts, TPT).matchAddr(V, 0);
2645*9880d681SAndroid Build Coastguard Worker (void)Success; assert(Success && "Couldn't select *anything*?");
2646*9880d681SAndroid Build Coastguard Worker return Result;
2647*9880d681SAndroid Build Coastguard Worker }
2648*9880d681SAndroid Build Coastguard Worker private:
2649*9880d681SAndroid Build Coastguard Worker bool matchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth);
2650*9880d681SAndroid Build Coastguard Worker bool matchAddr(Value *V, unsigned Depth);
2651*9880d681SAndroid Build Coastguard Worker bool matchOperationAddr(User *Operation, unsigned Opcode, unsigned Depth,
2652*9880d681SAndroid Build Coastguard Worker bool *MovedAway = nullptr);
2653*9880d681SAndroid Build Coastguard Worker bool isProfitableToFoldIntoAddressingMode(Instruction *I,
2654*9880d681SAndroid Build Coastguard Worker ExtAddrMode &AMBefore,
2655*9880d681SAndroid Build Coastguard Worker ExtAddrMode &AMAfter);
2656*9880d681SAndroid Build Coastguard Worker bool valueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2);
2657*9880d681SAndroid Build Coastguard Worker bool isPromotionProfitable(unsigned NewCost, unsigned OldCost,
2658*9880d681SAndroid Build Coastguard Worker Value *PromotedOperand) const;
2659*9880d681SAndroid Build Coastguard Worker };
2660*9880d681SAndroid Build Coastguard Worker
2661*9880d681SAndroid Build Coastguard Worker /// Try adding ScaleReg*Scale to the current addressing mode.
2662*9880d681SAndroid Build Coastguard Worker /// Return true and update AddrMode if this addr mode is legal for the target,
2663*9880d681SAndroid Build Coastguard Worker /// false if not.
matchScaledValue(Value * ScaleReg,int64_t Scale,unsigned Depth)2664*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale,
2665*9880d681SAndroid Build Coastguard Worker unsigned Depth) {
2666*9880d681SAndroid Build Coastguard Worker // If Scale is 1, then this is the same as adding ScaleReg to the addressing
2667*9880d681SAndroid Build Coastguard Worker // mode. Just process that directly.
2668*9880d681SAndroid Build Coastguard Worker if (Scale == 1)
2669*9880d681SAndroid Build Coastguard Worker return matchAddr(ScaleReg, Depth);
2670*9880d681SAndroid Build Coastguard Worker
2671*9880d681SAndroid Build Coastguard Worker // If the scale is 0, it takes nothing to add this.
2672*9880d681SAndroid Build Coastguard Worker if (Scale == 0)
2673*9880d681SAndroid Build Coastguard Worker return true;
2674*9880d681SAndroid Build Coastguard Worker
2675*9880d681SAndroid Build Coastguard Worker // If we already have a scale of this value, we can add to it, otherwise, we
2676*9880d681SAndroid Build Coastguard Worker // need an available scale field.
2677*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg)
2678*9880d681SAndroid Build Coastguard Worker return false;
2679*9880d681SAndroid Build Coastguard Worker
2680*9880d681SAndroid Build Coastguard Worker ExtAddrMode TestAddrMode = AddrMode;
2681*9880d681SAndroid Build Coastguard Worker
2682*9880d681SAndroid Build Coastguard Worker // Add scale to turn X*4+X*3 -> X*7. This could also do things like
2683*9880d681SAndroid Build Coastguard Worker // [A+B + A*7] -> [B+A*8].
2684*9880d681SAndroid Build Coastguard Worker TestAddrMode.Scale += Scale;
2685*9880d681SAndroid Build Coastguard Worker TestAddrMode.ScaledReg = ScaleReg;
2686*9880d681SAndroid Build Coastguard Worker
2687*9880d681SAndroid Build Coastguard Worker // If the new address isn't legal, bail out.
2688*9880d681SAndroid Build Coastguard Worker if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))
2689*9880d681SAndroid Build Coastguard Worker return false;
2690*9880d681SAndroid Build Coastguard Worker
2691*9880d681SAndroid Build Coastguard Worker // It was legal, so commit it.
2692*9880d681SAndroid Build Coastguard Worker AddrMode = TestAddrMode;
2693*9880d681SAndroid Build Coastguard Worker
2694*9880d681SAndroid Build Coastguard Worker // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now
2695*9880d681SAndroid Build Coastguard Worker // to see if ScaleReg is actually X+C. If so, we can turn this into adding
2696*9880d681SAndroid Build Coastguard Worker // X*Scale + C*Scale to addr mode.
2697*9880d681SAndroid Build Coastguard Worker ConstantInt *CI = nullptr; Value *AddLHS = nullptr;
2698*9880d681SAndroid Build Coastguard Worker if (isa<Instruction>(ScaleReg) && // not a constant expr.
2699*9880d681SAndroid Build Coastguard Worker match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) {
2700*9880d681SAndroid Build Coastguard Worker TestAddrMode.ScaledReg = AddLHS;
2701*9880d681SAndroid Build Coastguard Worker TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale;
2702*9880d681SAndroid Build Coastguard Worker
2703*9880d681SAndroid Build Coastguard Worker // If this addressing mode is legal, commit it and remember that we folded
2704*9880d681SAndroid Build Coastguard Worker // this instruction.
2705*9880d681SAndroid Build Coastguard Worker if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {
2706*9880d681SAndroid Build Coastguard Worker AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
2707*9880d681SAndroid Build Coastguard Worker AddrMode = TestAddrMode;
2708*9880d681SAndroid Build Coastguard Worker return true;
2709*9880d681SAndroid Build Coastguard Worker }
2710*9880d681SAndroid Build Coastguard Worker }
2711*9880d681SAndroid Build Coastguard Worker
2712*9880d681SAndroid Build Coastguard Worker // Otherwise, not (x+c)*scale, just return what we have.
2713*9880d681SAndroid Build Coastguard Worker return true;
2714*9880d681SAndroid Build Coastguard Worker }
2715*9880d681SAndroid Build Coastguard Worker
2716*9880d681SAndroid Build Coastguard Worker /// This is a little filter, which returns true if an addressing computation
2717*9880d681SAndroid Build Coastguard Worker /// involving I might be folded into a load/store accessing it.
2718*9880d681SAndroid Build Coastguard Worker /// This doesn't need to be perfect, but needs to accept at least
2719*9880d681SAndroid Build Coastguard Worker /// the set of instructions that MatchOperationAddr can.
MightBeFoldableInst(Instruction * I)2720*9880d681SAndroid Build Coastguard Worker static bool MightBeFoldableInst(Instruction *I) {
2721*9880d681SAndroid Build Coastguard Worker switch (I->getOpcode()) {
2722*9880d681SAndroid Build Coastguard Worker case Instruction::BitCast:
2723*9880d681SAndroid Build Coastguard Worker case Instruction::AddrSpaceCast:
2724*9880d681SAndroid Build Coastguard Worker // Don't touch identity bitcasts.
2725*9880d681SAndroid Build Coastguard Worker if (I->getType() == I->getOperand(0)->getType())
2726*9880d681SAndroid Build Coastguard Worker return false;
2727*9880d681SAndroid Build Coastguard Worker return I->getType()->isPointerTy() || I->getType()->isIntegerTy();
2728*9880d681SAndroid Build Coastguard Worker case Instruction::PtrToInt:
2729*9880d681SAndroid Build Coastguard Worker // PtrToInt is always a noop, as we know that the int type is pointer sized.
2730*9880d681SAndroid Build Coastguard Worker return true;
2731*9880d681SAndroid Build Coastguard Worker case Instruction::IntToPtr:
2732*9880d681SAndroid Build Coastguard Worker // We know the input is intptr_t, so this is foldable.
2733*9880d681SAndroid Build Coastguard Worker return true;
2734*9880d681SAndroid Build Coastguard Worker case Instruction::Add:
2735*9880d681SAndroid Build Coastguard Worker return true;
2736*9880d681SAndroid Build Coastguard Worker case Instruction::Mul:
2737*9880d681SAndroid Build Coastguard Worker case Instruction::Shl:
2738*9880d681SAndroid Build Coastguard Worker // Can only handle X*C and X << C.
2739*9880d681SAndroid Build Coastguard Worker return isa<ConstantInt>(I->getOperand(1));
2740*9880d681SAndroid Build Coastguard Worker case Instruction::GetElementPtr:
2741*9880d681SAndroid Build Coastguard Worker return true;
2742*9880d681SAndroid Build Coastguard Worker default:
2743*9880d681SAndroid Build Coastguard Worker return false;
2744*9880d681SAndroid Build Coastguard Worker }
2745*9880d681SAndroid Build Coastguard Worker }
2746*9880d681SAndroid Build Coastguard Worker
2747*9880d681SAndroid Build Coastguard Worker /// \brief Check whether or not \p Val is a legal instruction for \p TLI.
2748*9880d681SAndroid Build Coastguard Worker /// \note \p Val is assumed to be the product of some type promotion.
2749*9880d681SAndroid Build Coastguard Worker /// Therefore if \p Val has an undefined state in \p TLI, this is assumed
2750*9880d681SAndroid Build Coastguard Worker /// to be legal, as the non-promoted value would have had the same state.
isPromotedInstructionLegal(const TargetLowering & TLI,const DataLayout & DL,Value * Val)2751*9880d681SAndroid Build Coastguard Worker static bool isPromotedInstructionLegal(const TargetLowering &TLI,
2752*9880d681SAndroid Build Coastguard Worker const DataLayout &DL, Value *Val) {
2753*9880d681SAndroid Build Coastguard Worker Instruction *PromotedInst = dyn_cast<Instruction>(Val);
2754*9880d681SAndroid Build Coastguard Worker if (!PromotedInst)
2755*9880d681SAndroid Build Coastguard Worker return false;
2756*9880d681SAndroid Build Coastguard Worker int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode());
2757*9880d681SAndroid Build Coastguard Worker // If the ISDOpcode is undefined, it was undefined before the promotion.
2758*9880d681SAndroid Build Coastguard Worker if (!ISDOpcode)
2759*9880d681SAndroid Build Coastguard Worker return true;
2760*9880d681SAndroid Build Coastguard Worker // Otherwise, check if the promoted instruction is legal or not.
2761*9880d681SAndroid Build Coastguard Worker return TLI.isOperationLegalOrCustom(
2762*9880d681SAndroid Build Coastguard Worker ISDOpcode, TLI.getValueType(DL, PromotedInst->getType()));
2763*9880d681SAndroid Build Coastguard Worker }
2764*9880d681SAndroid Build Coastguard Worker
2765*9880d681SAndroid Build Coastguard Worker /// \brief Hepler class to perform type promotion.
2766*9880d681SAndroid Build Coastguard Worker class TypePromotionHelper {
2767*9880d681SAndroid Build Coastguard Worker /// \brief Utility function to check whether or not a sign or zero extension
2768*9880d681SAndroid Build Coastguard Worker /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by
2769*9880d681SAndroid Build Coastguard Worker /// either using the operands of \p Inst or promoting \p Inst.
2770*9880d681SAndroid Build Coastguard Worker /// The type of the extension is defined by \p IsSExt.
2771*9880d681SAndroid Build Coastguard Worker /// In other words, check if:
2772*9880d681SAndroid Build Coastguard Worker /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType.
2773*9880d681SAndroid Build Coastguard Worker /// #1 Promotion applies:
2774*9880d681SAndroid Build Coastguard Worker /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...).
2775*9880d681SAndroid Build Coastguard Worker /// #2 Operand reuses:
2776*9880d681SAndroid Build Coastguard Worker /// ext opnd1 to ConsideredExtType.
2777*9880d681SAndroid Build Coastguard Worker /// \p PromotedInsts maps the instructions to their type before promotion.
2778*9880d681SAndroid Build Coastguard Worker static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType,
2779*9880d681SAndroid Build Coastguard Worker const InstrToOrigTy &PromotedInsts, bool IsSExt);
2780*9880d681SAndroid Build Coastguard Worker
2781*9880d681SAndroid Build Coastguard Worker /// \brief Utility function to determine if \p OpIdx should be promoted when
2782*9880d681SAndroid Build Coastguard Worker /// promoting \p Inst.
shouldExtOperand(const Instruction * Inst,int OpIdx)2783*9880d681SAndroid Build Coastguard Worker static bool shouldExtOperand(const Instruction *Inst, int OpIdx) {
2784*9880d681SAndroid Build Coastguard Worker return !(isa<SelectInst>(Inst) && OpIdx == 0);
2785*9880d681SAndroid Build Coastguard Worker }
2786*9880d681SAndroid Build Coastguard Worker
2787*9880d681SAndroid Build Coastguard Worker /// \brief Utility function to promote the operand of \p Ext when this
2788*9880d681SAndroid Build Coastguard Worker /// operand is a promotable trunc or sext or zext.
2789*9880d681SAndroid Build Coastguard Worker /// \p PromotedInsts maps the instructions to their type before promotion.
2790*9880d681SAndroid Build Coastguard Worker /// \p CreatedInstsCost[out] contains the cost of all instructions
2791*9880d681SAndroid Build Coastguard Worker /// created to promote the operand of Ext.
2792*9880d681SAndroid Build Coastguard Worker /// Newly added extensions are inserted in \p Exts.
2793*9880d681SAndroid Build Coastguard Worker /// Newly added truncates are inserted in \p Truncs.
2794*9880d681SAndroid Build Coastguard Worker /// Should never be called directly.
2795*9880d681SAndroid Build Coastguard Worker /// \return The promoted value which is used instead of Ext.
2796*9880d681SAndroid Build Coastguard Worker static Value *promoteOperandForTruncAndAnyExt(
2797*9880d681SAndroid Build Coastguard Worker Instruction *Ext, TypePromotionTransaction &TPT,
2798*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2799*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
2800*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI);
2801*9880d681SAndroid Build Coastguard Worker
2802*9880d681SAndroid Build Coastguard Worker /// \brief Utility function to promote the operand of \p Ext when this
2803*9880d681SAndroid Build Coastguard Worker /// operand is promotable and is not a supported trunc or sext.
2804*9880d681SAndroid Build Coastguard Worker /// \p PromotedInsts maps the instructions to their type before promotion.
2805*9880d681SAndroid Build Coastguard Worker /// \p CreatedInstsCost[out] contains the cost of all the instructions
2806*9880d681SAndroid Build Coastguard Worker /// created to promote the operand of Ext.
2807*9880d681SAndroid Build Coastguard Worker /// Newly added extensions are inserted in \p Exts.
2808*9880d681SAndroid Build Coastguard Worker /// Newly added truncates are inserted in \p Truncs.
2809*9880d681SAndroid Build Coastguard Worker /// Should never be called directly.
2810*9880d681SAndroid Build Coastguard Worker /// \return The promoted value which is used instead of Ext.
2811*9880d681SAndroid Build Coastguard Worker static Value *promoteOperandForOther(Instruction *Ext,
2812*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction &TPT,
2813*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts,
2814*9880d681SAndroid Build Coastguard Worker unsigned &CreatedInstsCost,
2815*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
2816*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs,
2817*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI, bool IsSExt);
2818*9880d681SAndroid Build Coastguard Worker
2819*9880d681SAndroid Build Coastguard Worker /// \see promoteOperandForOther.
signExtendOperandForOther(Instruction * Ext,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI)2820*9880d681SAndroid Build Coastguard Worker static Value *signExtendOperandForOther(
2821*9880d681SAndroid Build Coastguard Worker Instruction *Ext, TypePromotionTransaction &TPT,
2822*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2823*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
2824*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
2825*9880d681SAndroid Build Coastguard Worker return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
2826*9880d681SAndroid Build Coastguard Worker Exts, Truncs, TLI, true);
2827*9880d681SAndroid Build Coastguard Worker }
2828*9880d681SAndroid Build Coastguard Worker
2829*9880d681SAndroid Build Coastguard Worker /// \see promoteOperandForOther.
zeroExtendOperandForOther(Instruction * Ext,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI)2830*9880d681SAndroid Build Coastguard Worker static Value *zeroExtendOperandForOther(
2831*9880d681SAndroid Build Coastguard Worker Instruction *Ext, TypePromotionTransaction &TPT,
2832*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2833*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
2834*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
2835*9880d681SAndroid Build Coastguard Worker return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
2836*9880d681SAndroid Build Coastguard Worker Exts, Truncs, TLI, false);
2837*9880d681SAndroid Build Coastguard Worker }
2838*9880d681SAndroid Build Coastguard Worker
2839*9880d681SAndroid Build Coastguard Worker public:
2840*9880d681SAndroid Build Coastguard Worker /// Type for the utility function that promotes the operand of Ext.
2841*9880d681SAndroid Build Coastguard Worker typedef Value *(*Action)(Instruction *Ext, TypePromotionTransaction &TPT,
2842*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts,
2843*9880d681SAndroid Build Coastguard Worker unsigned &CreatedInstsCost,
2844*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
2845*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs,
2846*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI);
2847*9880d681SAndroid Build Coastguard Worker /// \brief Given a sign/zero extend instruction \p Ext, return the approriate
2848*9880d681SAndroid Build Coastguard Worker /// action to promote the operand of \p Ext instead of using Ext.
2849*9880d681SAndroid Build Coastguard Worker /// \return NULL if no promotable action is possible with the current
2850*9880d681SAndroid Build Coastguard Worker /// sign extension.
2851*9880d681SAndroid Build Coastguard Worker /// \p InsertedInsts keeps track of all the instructions inserted by the
2852*9880d681SAndroid Build Coastguard Worker /// other CodeGenPrepare optimizations. This information is important
2853*9880d681SAndroid Build Coastguard Worker /// because we do not want to promote these instructions as CodeGenPrepare
2854*9880d681SAndroid Build Coastguard Worker /// will reinsert them later. Thus creating an infinite loop: create/remove.
2855*9880d681SAndroid Build Coastguard Worker /// \p PromotedInsts maps the instructions to their type before promotion.
2856*9880d681SAndroid Build Coastguard Worker static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts,
2857*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI,
2858*9880d681SAndroid Build Coastguard Worker const InstrToOrigTy &PromotedInsts);
2859*9880d681SAndroid Build Coastguard Worker };
2860*9880d681SAndroid Build Coastguard Worker
canGetThrough(const Instruction * Inst,Type * ConsideredExtType,const InstrToOrigTy & PromotedInsts,bool IsSExt)2861*9880d681SAndroid Build Coastguard Worker bool TypePromotionHelper::canGetThrough(const Instruction *Inst,
2862*9880d681SAndroid Build Coastguard Worker Type *ConsideredExtType,
2863*9880d681SAndroid Build Coastguard Worker const InstrToOrigTy &PromotedInsts,
2864*9880d681SAndroid Build Coastguard Worker bool IsSExt) {
2865*9880d681SAndroid Build Coastguard Worker // The promotion helper does not know how to deal with vector types yet.
2866*9880d681SAndroid Build Coastguard Worker // To be able to fix that, we would need to fix the places where we
2867*9880d681SAndroid Build Coastguard Worker // statically extend, e.g., constants and such.
2868*9880d681SAndroid Build Coastguard Worker if (Inst->getType()->isVectorTy())
2869*9880d681SAndroid Build Coastguard Worker return false;
2870*9880d681SAndroid Build Coastguard Worker
2871*9880d681SAndroid Build Coastguard Worker // We can always get through zext.
2872*9880d681SAndroid Build Coastguard Worker if (isa<ZExtInst>(Inst))
2873*9880d681SAndroid Build Coastguard Worker return true;
2874*9880d681SAndroid Build Coastguard Worker
2875*9880d681SAndroid Build Coastguard Worker // sext(sext) is ok too.
2876*9880d681SAndroid Build Coastguard Worker if (IsSExt && isa<SExtInst>(Inst))
2877*9880d681SAndroid Build Coastguard Worker return true;
2878*9880d681SAndroid Build Coastguard Worker
2879*9880d681SAndroid Build Coastguard Worker // We can get through binary operator, if it is legal. In other words, the
2880*9880d681SAndroid Build Coastguard Worker // binary operator must have a nuw or nsw flag.
2881*9880d681SAndroid Build Coastguard Worker const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst);
2882*9880d681SAndroid Build Coastguard Worker if (BinOp && isa<OverflowingBinaryOperator>(BinOp) &&
2883*9880d681SAndroid Build Coastguard Worker ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||
2884*9880d681SAndroid Build Coastguard Worker (IsSExt && BinOp->hasNoSignedWrap())))
2885*9880d681SAndroid Build Coastguard Worker return true;
2886*9880d681SAndroid Build Coastguard Worker
2887*9880d681SAndroid Build Coastguard Worker // Check if we can do the following simplification.
2888*9880d681SAndroid Build Coastguard Worker // ext(trunc(opnd)) --> ext(opnd)
2889*9880d681SAndroid Build Coastguard Worker if (!isa<TruncInst>(Inst))
2890*9880d681SAndroid Build Coastguard Worker return false;
2891*9880d681SAndroid Build Coastguard Worker
2892*9880d681SAndroid Build Coastguard Worker Value *OpndVal = Inst->getOperand(0);
2893*9880d681SAndroid Build Coastguard Worker // Check if we can use this operand in the extension.
2894*9880d681SAndroid Build Coastguard Worker // If the type is larger than the result type of the extension, we cannot.
2895*9880d681SAndroid Build Coastguard Worker if (!OpndVal->getType()->isIntegerTy() ||
2896*9880d681SAndroid Build Coastguard Worker OpndVal->getType()->getIntegerBitWidth() >
2897*9880d681SAndroid Build Coastguard Worker ConsideredExtType->getIntegerBitWidth())
2898*9880d681SAndroid Build Coastguard Worker return false;
2899*9880d681SAndroid Build Coastguard Worker
2900*9880d681SAndroid Build Coastguard Worker // If the operand of the truncate is not an instruction, we will not have
2901*9880d681SAndroid Build Coastguard Worker // any information on the dropped bits.
2902*9880d681SAndroid Build Coastguard Worker // (Actually we could for constant but it is not worth the extra logic).
2903*9880d681SAndroid Build Coastguard Worker Instruction *Opnd = dyn_cast<Instruction>(OpndVal);
2904*9880d681SAndroid Build Coastguard Worker if (!Opnd)
2905*9880d681SAndroid Build Coastguard Worker return false;
2906*9880d681SAndroid Build Coastguard Worker
2907*9880d681SAndroid Build Coastguard Worker // Check if the source of the type is narrow enough.
2908*9880d681SAndroid Build Coastguard Worker // I.e., check that trunc just drops extended bits of the same kind of
2909*9880d681SAndroid Build Coastguard Worker // the extension.
2910*9880d681SAndroid Build Coastguard Worker // #1 get the type of the operand and check the kind of the extended bits.
2911*9880d681SAndroid Build Coastguard Worker const Type *OpndType;
2912*9880d681SAndroid Build Coastguard Worker InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
2913*9880d681SAndroid Build Coastguard Worker if (It != PromotedInsts.end() && It->second.getInt() == IsSExt)
2914*9880d681SAndroid Build Coastguard Worker OpndType = It->second.getPointer();
2915*9880d681SAndroid Build Coastguard Worker else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
2916*9880d681SAndroid Build Coastguard Worker OpndType = Opnd->getOperand(0)->getType();
2917*9880d681SAndroid Build Coastguard Worker else
2918*9880d681SAndroid Build Coastguard Worker return false;
2919*9880d681SAndroid Build Coastguard Worker
2920*9880d681SAndroid Build Coastguard Worker // #2 check that the truncate just drops extended bits.
2921*9880d681SAndroid Build Coastguard Worker return Inst->getType()->getIntegerBitWidth() >=
2922*9880d681SAndroid Build Coastguard Worker OpndType->getIntegerBitWidth();
2923*9880d681SAndroid Build Coastguard Worker }
2924*9880d681SAndroid Build Coastguard Worker
getAction(Instruction * Ext,const SetOfInstrs & InsertedInsts,const TargetLowering & TLI,const InstrToOrigTy & PromotedInsts)2925*9880d681SAndroid Build Coastguard Worker TypePromotionHelper::Action TypePromotionHelper::getAction(
2926*9880d681SAndroid Build Coastguard Worker Instruction *Ext, const SetOfInstrs &InsertedInsts,
2927*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) {
2928*9880d681SAndroid Build Coastguard Worker assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2929*9880d681SAndroid Build Coastguard Worker "Unexpected instruction type");
2930*9880d681SAndroid Build Coastguard Worker Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0));
2931*9880d681SAndroid Build Coastguard Worker Type *ExtTy = Ext->getType();
2932*9880d681SAndroid Build Coastguard Worker bool IsSExt = isa<SExtInst>(Ext);
2933*9880d681SAndroid Build Coastguard Worker // If the operand of the extension is not an instruction, we cannot
2934*9880d681SAndroid Build Coastguard Worker // get through.
2935*9880d681SAndroid Build Coastguard Worker // If it, check we can get through.
2936*9880d681SAndroid Build Coastguard Worker if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
2937*9880d681SAndroid Build Coastguard Worker return nullptr;
2938*9880d681SAndroid Build Coastguard Worker
2939*9880d681SAndroid Build Coastguard Worker // Do not promote if the operand has been added by codegenprepare.
2940*9880d681SAndroid Build Coastguard Worker // Otherwise, it means we are undoing an optimization that is likely to be
2941*9880d681SAndroid Build Coastguard Worker // redone, thus causing potential infinite loop.
2942*9880d681SAndroid Build Coastguard Worker if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
2943*9880d681SAndroid Build Coastguard Worker return nullptr;
2944*9880d681SAndroid Build Coastguard Worker
2945*9880d681SAndroid Build Coastguard Worker // SExt or Trunc instructions.
2946*9880d681SAndroid Build Coastguard Worker // Return the related handler.
2947*9880d681SAndroid Build Coastguard Worker if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||
2948*9880d681SAndroid Build Coastguard Worker isa<ZExtInst>(ExtOpnd))
2949*9880d681SAndroid Build Coastguard Worker return promoteOperandForTruncAndAnyExt;
2950*9880d681SAndroid Build Coastguard Worker
2951*9880d681SAndroid Build Coastguard Worker // Regular instruction.
2952*9880d681SAndroid Build Coastguard Worker // Abort early if we will have to insert non-free instructions.
2953*9880d681SAndroid Build Coastguard Worker if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType()))
2954*9880d681SAndroid Build Coastguard Worker return nullptr;
2955*9880d681SAndroid Build Coastguard Worker return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
2956*9880d681SAndroid Build Coastguard Worker }
2957*9880d681SAndroid Build Coastguard Worker
promoteOperandForTruncAndAnyExt(llvm::Instruction * SExt,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI)2958*9880d681SAndroid Build Coastguard Worker Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
2959*9880d681SAndroid Build Coastguard Worker llvm::Instruction *SExt, TypePromotionTransaction &TPT,
2960*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
2961*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
2962*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
2963*9880d681SAndroid Build Coastguard Worker // By construction, the operand of SExt is an instruction. Otherwise we cannot
2964*9880d681SAndroid Build Coastguard Worker // get through it and this method should not be called.
2965*9880d681SAndroid Build Coastguard Worker Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0));
2966*9880d681SAndroid Build Coastguard Worker Value *ExtVal = SExt;
2967*9880d681SAndroid Build Coastguard Worker bool HasMergedNonFreeExt = false;
2968*9880d681SAndroid Build Coastguard Worker if (isa<ZExtInst>(SExtOpnd)) {
2969*9880d681SAndroid Build Coastguard Worker // Replace s|zext(zext(opnd))
2970*9880d681SAndroid Build Coastguard Worker // => zext(opnd).
2971*9880d681SAndroid Build Coastguard Worker HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd);
2972*9880d681SAndroid Build Coastguard Worker Value *ZExt =
2973*9880d681SAndroid Build Coastguard Worker TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType());
2974*9880d681SAndroid Build Coastguard Worker TPT.replaceAllUsesWith(SExt, ZExt);
2975*9880d681SAndroid Build Coastguard Worker TPT.eraseInstruction(SExt);
2976*9880d681SAndroid Build Coastguard Worker ExtVal = ZExt;
2977*9880d681SAndroid Build Coastguard Worker } else {
2978*9880d681SAndroid Build Coastguard Worker // Replace z|sext(trunc(opnd)) or sext(sext(opnd))
2979*9880d681SAndroid Build Coastguard Worker // => z|sext(opnd).
2980*9880d681SAndroid Build Coastguard Worker TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0));
2981*9880d681SAndroid Build Coastguard Worker }
2982*9880d681SAndroid Build Coastguard Worker CreatedInstsCost = 0;
2983*9880d681SAndroid Build Coastguard Worker
2984*9880d681SAndroid Build Coastguard Worker // Remove dead code.
2985*9880d681SAndroid Build Coastguard Worker if (SExtOpnd->use_empty())
2986*9880d681SAndroid Build Coastguard Worker TPT.eraseInstruction(SExtOpnd);
2987*9880d681SAndroid Build Coastguard Worker
2988*9880d681SAndroid Build Coastguard Worker // Check if the extension is still needed.
2989*9880d681SAndroid Build Coastguard Worker Instruction *ExtInst = dyn_cast<Instruction>(ExtVal);
2990*9880d681SAndroid Build Coastguard Worker if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) {
2991*9880d681SAndroid Build Coastguard Worker if (ExtInst) {
2992*9880d681SAndroid Build Coastguard Worker if (Exts)
2993*9880d681SAndroid Build Coastguard Worker Exts->push_back(ExtInst);
2994*9880d681SAndroid Build Coastguard Worker CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt;
2995*9880d681SAndroid Build Coastguard Worker }
2996*9880d681SAndroid Build Coastguard Worker return ExtVal;
2997*9880d681SAndroid Build Coastguard Worker }
2998*9880d681SAndroid Build Coastguard Worker
2999*9880d681SAndroid Build Coastguard Worker // At this point we have: ext ty opnd to ty.
3000*9880d681SAndroid Build Coastguard Worker // Reassign the uses of ExtInst to the opnd and remove ExtInst.
3001*9880d681SAndroid Build Coastguard Worker Value *NextVal = ExtInst->getOperand(0);
3002*9880d681SAndroid Build Coastguard Worker TPT.eraseInstruction(ExtInst, NextVal);
3003*9880d681SAndroid Build Coastguard Worker return NextVal;
3004*9880d681SAndroid Build Coastguard Worker }
3005*9880d681SAndroid Build Coastguard Worker
promoteOperandForOther(Instruction * Ext,TypePromotionTransaction & TPT,InstrToOrigTy & PromotedInsts,unsigned & CreatedInstsCost,SmallVectorImpl<Instruction * > * Exts,SmallVectorImpl<Instruction * > * Truncs,const TargetLowering & TLI,bool IsSExt)3006*9880d681SAndroid Build Coastguard Worker Value *TypePromotionHelper::promoteOperandForOther(
3007*9880d681SAndroid Build Coastguard Worker Instruction *Ext, TypePromotionTransaction &TPT,
3008*9880d681SAndroid Build Coastguard Worker InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
3009*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Exts,
3010*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI,
3011*9880d681SAndroid Build Coastguard Worker bool IsSExt) {
3012*9880d681SAndroid Build Coastguard Worker // By construction, the operand of Ext is an instruction. Otherwise we cannot
3013*9880d681SAndroid Build Coastguard Worker // get through it and this method should not be called.
3014*9880d681SAndroid Build Coastguard Worker Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0));
3015*9880d681SAndroid Build Coastguard Worker CreatedInstsCost = 0;
3016*9880d681SAndroid Build Coastguard Worker if (!ExtOpnd->hasOneUse()) {
3017*9880d681SAndroid Build Coastguard Worker // ExtOpnd will be promoted.
3018*9880d681SAndroid Build Coastguard Worker // All its uses, but Ext, will need to use a truncated value of the
3019*9880d681SAndroid Build Coastguard Worker // promoted version.
3020*9880d681SAndroid Build Coastguard Worker // Create the truncate now.
3021*9880d681SAndroid Build Coastguard Worker Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType());
3022*9880d681SAndroid Build Coastguard Worker if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {
3023*9880d681SAndroid Build Coastguard Worker ITrunc->removeFromParent();
3024*9880d681SAndroid Build Coastguard Worker // Insert it just after the definition.
3025*9880d681SAndroid Build Coastguard Worker ITrunc->insertAfter(ExtOpnd);
3026*9880d681SAndroid Build Coastguard Worker if (Truncs)
3027*9880d681SAndroid Build Coastguard Worker Truncs->push_back(ITrunc);
3028*9880d681SAndroid Build Coastguard Worker }
3029*9880d681SAndroid Build Coastguard Worker
3030*9880d681SAndroid Build Coastguard Worker TPT.replaceAllUsesWith(ExtOpnd, Trunc);
3031*9880d681SAndroid Build Coastguard Worker // Restore the operand of Ext (which has been replaced by the previous call
3032*9880d681SAndroid Build Coastguard Worker // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext.
3033*9880d681SAndroid Build Coastguard Worker TPT.setOperand(Ext, 0, ExtOpnd);
3034*9880d681SAndroid Build Coastguard Worker }
3035*9880d681SAndroid Build Coastguard Worker
3036*9880d681SAndroid Build Coastguard Worker // Get through the Instruction:
3037*9880d681SAndroid Build Coastguard Worker // 1. Update its type.
3038*9880d681SAndroid Build Coastguard Worker // 2. Replace the uses of Ext by Inst.
3039*9880d681SAndroid Build Coastguard Worker // 3. Extend each operand that needs to be extended.
3040*9880d681SAndroid Build Coastguard Worker
3041*9880d681SAndroid Build Coastguard Worker // Remember the original type of the instruction before promotion.
3042*9880d681SAndroid Build Coastguard Worker // This is useful to know that the high bits are sign extended bits.
3043*9880d681SAndroid Build Coastguard Worker PromotedInsts.insert(std::pair<Instruction *, TypeIsSExt>(
3044*9880d681SAndroid Build Coastguard Worker ExtOpnd, TypeIsSExt(ExtOpnd->getType(), IsSExt)));
3045*9880d681SAndroid Build Coastguard Worker // Step #1.
3046*9880d681SAndroid Build Coastguard Worker TPT.mutateType(ExtOpnd, Ext->getType());
3047*9880d681SAndroid Build Coastguard Worker // Step #2.
3048*9880d681SAndroid Build Coastguard Worker TPT.replaceAllUsesWith(Ext, ExtOpnd);
3049*9880d681SAndroid Build Coastguard Worker // Step #3.
3050*9880d681SAndroid Build Coastguard Worker Instruction *ExtForOpnd = Ext;
3051*9880d681SAndroid Build Coastguard Worker
3052*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Propagate Ext to operands\n");
3053*9880d681SAndroid Build Coastguard Worker for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx;
3054*9880d681SAndroid Build Coastguard Worker ++OpIdx) {
3055*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n');
3056*9880d681SAndroid Build Coastguard Worker if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() ||
3057*9880d681SAndroid Build Coastguard Worker !shouldExtOperand(ExtOpnd, OpIdx)) {
3058*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "No need to propagate\n");
3059*9880d681SAndroid Build Coastguard Worker continue;
3060*9880d681SAndroid Build Coastguard Worker }
3061*9880d681SAndroid Build Coastguard Worker // Check if we can statically extend the operand.
3062*9880d681SAndroid Build Coastguard Worker Value *Opnd = ExtOpnd->getOperand(OpIdx);
3063*9880d681SAndroid Build Coastguard Worker if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {
3064*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Statically extend\n");
3065*9880d681SAndroid Build Coastguard Worker unsigned BitWidth = Ext->getType()->getIntegerBitWidth();
3066*9880d681SAndroid Build Coastguard Worker APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth)
3067*9880d681SAndroid Build Coastguard Worker : Cst->getValue().zext(BitWidth);
3068*9880d681SAndroid Build Coastguard Worker TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal));
3069*9880d681SAndroid Build Coastguard Worker continue;
3070*9880d681SAndroid Build Coastguard Worker }
3071*9880d681SAndroid Build Coastguard Worker // UndefValue are typed, so we have to statically sign extend them.
3072*9880d681SAndroid Build Coastguard Worker if (isa<UndefValue>(Opnd)) {
3073*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Statically extend\n");
3074*9880d681SAndroid Build Coastguard Worker TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType()));
3075*9880d681SAndroid Build Coastguard Worker continue;
3076*9880d681SAndroid Build Coastguard Worker }
3077*9880d681SAndroid Build Coastguard Worker
3078*9880d681SAndroid Build Coastguard Worker // Otherwise we have to explicity sign extend the operand.
3079*9880d681SAndroid Build Coastguard Worker // Check if Ext was reused to extend an operand.
3080*9880d681SAndroid Build Coastguard Worker if (!ExtForOpnd) {
3081*9880d681SAndroid Build Coastguard Worker // If yes, create a new one.
3082*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "More operands to ext\n");
3083*9880d681SAndroid Build Coastguard Worker Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->getType())
3084*9880d681SAndroid Build Coastguard Worker : TPT.createZExt(Ext, Opnd, Ext->getType());
3085*9880d681SAndroid Build Coastguard Worker if (!isa<Instruction>(ValForExtOpnd)) {
3086*9880d681SAndroid Build Coastguard Worker TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
3087*9880d681SAndroid Build Coastguard Worker continue;
3088*9880d681SAndroid Build Coastguard Worker }
3089*9880d681SAndroid Build Coastguard Worker ExtForOpnd = cast<Instruction>(ValForExtOpnd);
3090*9880d681SAndroid Build Coastguard Worker }
3091*9880d681SAndroid Build Coastguard Worker if (Exts)
3092*9880d681SAndroid Build Coastguard Worker Exts->push_back(ExtForOpnd);
3093*9880d681SAndroid Build Coastguard Worker TPT.setOperand(ExtForOpnd, 0, Opnd);
3094*9880d681SAndroid Build Coastguard Worker
3095*9880d681SAndroid Build Coastguard Worker // Move the sign extension before the insertion point.
3096*9880d681SAndroid Build Coastguard Worker TPT.moveBefore(ExtForOpnd, ExtOpnd);
3097*9880d681SAndroid Build Coastguard Worker TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd);
3098*9880d681SAndroid Build Coastguard Worker CreatedInstsCost += !TLI.isExtFree(ExtForOpnd);
3099*9880d681SAndroid Build Coastguard Worker // If more sext are required, new instructions will have to be created.
3100*9880d681SAndroid Build Coastguard Worker ExtForOpnd = nullptr;
3101*9880d681SAndroid Build Coastguard Worker }
3102*9880d681SAndroid Build Coastguard Worker if (ExtForOpnd == Ext) {
3103*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Extension is useless now\n");
3104*9880d681SAndroid Build Coastguard Worker TPT.eraseInstruction(Ext);
3105*9880d681SAndroid Build Coastguard Worker }
3106*9880d681SAndroid Build Coastguard Worker return ExtOpnd;
3107*9880d681SAndroid Build Coastguard Worker }
3108*9880d681SAndroid Build Coastguard Worker
3109*9880d681SAndroid Build Coastguard Worker /// Check whether or not promoting an instruction to a wider type is profitable.
3110*9880d681SAndroid Build Coastguard Worker /// \p NewCost gives the cost of extension instructions created by the
3111*9880d681SAndroid Build Coastguard Worker /// promotion.
3112*9880d681SAndroid Build Coastguard Worker /// \p OldCost gives the cost of extension instructions before the promotion
3113*9880d681SAndroid Build Coastguard Worker /// plus the number of instructions that have been
3114*9880d681SAndroid Build Coastguard Worker /// matched in the addressing mode the promotion.
3115*9880d681SAndroid Build Coastguard Worker /// \p PromotedOperand is the value that has been promoted.
3116*9880d681SAndroid Build Coastguard Worker /// \return True if the promotion is profitable, false otherwise.
isPromotionProfitable(unsigned NewCost,unsigned OldCost,Value * PromotedOperand) const3117*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::isPromotionProfitable(
3118*9880d681SAndroid Build Coastguard Worker unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const {
3119*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost << '\n');
3120*9880d681SAndroid Build Coastguard Worker // The cost of the new extensions is greater than the cost of the
3121*9880d681SAndroid Build Coastguard Worker // old extension plus what we folded.
3122*9880d681SAndroid Build Coastguard Worker // This is not profitable.
3123*9880d681SAndroid Build Coastguard Worker if (NewCost > OldCost)
3124*9880d681SAndroid Build Coastguard Worker return false;
3125*9880d681SAndroid Build Coastguard Worker if (NewCost < OldCost)
3126*9880d681SAndroid Build Coastguard Worker return true;
3127*9880d681SAndroid Build Coastguard Worker // The promotion is neutral but it may help folding the sign extension in
3128*9880d681SAndroid Build Coastguard Worker // loads for instance.
3129*9880d681SAndroid Build Coastguard Worker // Check that we did not create an illegal instruction.
3130*9880d681SAndroid Build Coastguard Worker return isPromotedInstructionLegal(TLI, DL, PromotedOperand);
3131*9880d681SAndroid Build Coastguard Worker }
3132*9880d681SAndroid Build Coastguard Worker
3133*9880d681SAndroid Build Coastguard Worker /// Given an instruction or constant expr, see if we can fold the operation
3134*9880d681SAndroid Build Coastguard Worker /// into the addressing mode. If so, update the addressing mode and return
3135*9880d681SAndroid Build Coastguard Worker /// true, otherwise return false without modifying AddrMode.
3136*9880d681SAndroid Build Coastguard Worker /// If \p MovedAway is not NULL, it contains the information of whether or
3137*9880d681SAndroid Build Coastguard Worker /// not AddrInst has to be folded into the addressing mode on success.
3138*9880d681SAndroid Build Coastguard Worker /// If \p MovedAway == true, \p AddrInst will not be part of the addressing
3139*9880d681SAndroid Build Coastguard Worker /// because it has been moved away.
3140*9880d681SAndroid Build Coastguard Worker /// Thus AddrInst must not be added in the matched instructions.
3141*9880d681SAndroid Build Coastguard Worker /// This state can happen when AddrInst is a sext, since it may be moved away.
3142*9880d681SAndroid Build Coastguard Worker /// Therefore, AddrInst may not be valid when MovedAway is true and it must
3143*9880d681SAndroid Build Coastguard Worker /// not be referenced anymore.
matchOperationAddr(User * AddrInst,unsigned Opcode,unsigned Depth,bool * MovedAway)3144*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
3145*9880d681SAndroid Build Coastguard Worker unsigned Depth,
3146*9880d681SAndroid Build Coastguard Worker bool *MovedAway) {
3147*9880d681SAndroid Build Coastguard Worker // Avoid exponential behavior on extremely deep expression trees.
3148*9880d681SAndroid Build Coastguard Worker if (Depth >= 5) return false;
3149*9880d681SAndroid Build Coastguard Worker
3150*9880d681SAndroid Build Coastguard Worker // By default, all matched instructions stay in place.
3151*9880d681SAndroid Build Coastguard Worker if (MovedAway)
3152*9880d681SAndroid Build Coastguard Worker *MovedAway = false;
3153*9880d681SAndroid Build Coastguard Worker
3154*9880d681SAndroid Build Coastguard Worker switch (Opcode) {
3155*9880d681SAndroid Build Coastguard Worker case Instruction::PtrToInt:
3156*9880d681SAndroid Build Coastguard Worker // PtrToInt is always a noop, as we know that the int type is pointer sized.
3157*9880d681SAndroid Build Coastguard Worker return matchAddr(AddrInst->getOperand(0), Depth);
3158*9880d681SAndroid Build Coastguard Worker case Instruction::IntToPtr: {
3159*9880d681SAndroid Build Coastguard Worker auto AS = AddrInst->getType()->getPointerAddressSpace();
3160*9880d681SAndroid Build Coastguard Worker auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
3161*9880d681SAndroid Build Coastguard Worker // This inttoptr is a no-op if the integer type is pointer sized.
3162*9880d681SAndroid Build Coastguard Worker if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy)
3163*9880d681SAndroid Build Coastguard Worker return matchAddr(AddrInst->getOperand(0), Depth);
3164*9880d681SAndroid Build Coastguard Worker return false;
3165*9880d681SAndroid Build Coastguard Worker }
3166*9880d681SAndroid Build Coastguard Worker case Instruction::BitCast:
3167*9880d681SAndroid Build Coastguard Worker // BitCast is always a noop, and we can handle it as long as it is
3168*9880d681SAndroid Build Coastguard Worker // int->int or pointer->pointer (we don't want int<->fp or something).
3169*9880d681SAndroid Build Coastguard Worker if ((AddrInst->getOperand(0)->getType()->isPointerTy() ||
3170*9880d681SAndroid Build Coastguard Worker AddrInst->getOperand(0)->getType()->isIntegerTy()) &&
3171*9880d681SAndroid Build Coastguard Worker // Don't touch identity bitcasts. These were probably put here by LSR,
3172*9880d681SAndroid Build Coastguard Worker // and we don't want to mess around with them. Assume it knows what it
3173*9880d681SAndroid Build Coastguard Worker // is doing.
3174*9880d681SAndroid Build Coastguard Worker AddrInst->getOperand(0)->getType() != AddrInst->getType())
3175*9880d681SAndroid Build Coastguard Worker return matchAddr(AddrInst->getOperand(0), Depth);
3176*9880d681SAndroid Build Coastguard Worker return false;
3177*9880d681SAndroid Build Coastguard Worker case Instruction::AddrSpaceCast: {
3178*9880d681SAndroid Build Coastguard Worker unsigned SrcAS
3179*9880d681SAndroid Build Coastguard Worker = AddrInst->getOperand(0)->getType()->getPointerAddressSpace();
3180*9880d681SAndroid Build Coastguard Worker unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();
3181*9880d681SAndroid Build Coastguard Worker if (TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3182*9880d681SAndroid Build Coastguard Worker return matchAddr(AddrInst->getOperand(0), Depth);
3183*9880d681SAndroid Build Coastguard Worker return false;
3184*9880d681SAndroid Build Coastguard Worker }
3185*9880d681SAndroid Build Coastguard Worker case Instruction::Add: {
3186*9880d681SAndroid Build Coastguard Worker // Check to see if we can merge in the RHS then the LHS. If so, we win.
3187*9880d681SAndroid Build Coastguard Worker ExtAddrMode BackupAddrMode = AddrMode;
3188*9880d681SAndroid Build Coastguard Worker unsigned OldSize = AddrModeInsts.size();
3189*9880d681SAndroid Build Coastguard Worker // Start a transaction at this point.
3190*9880d681SAndroid Build Coastguard Worker // The LHS may match but not the RHS.
3191*9880d681SAndroid Build Coastguard Worker // Therefore, we need a higher level restoration point to undo partially
3192*9880d681SAndroid Build Coastguard Worker // matched operation.
3193*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3194*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
3195*9880d681SAndroid Build Coastguard Worker
3196*9880d681SAndroid Build Coastguard Worker if (matchAddr(AddrInst->getOperand(1), Depth+1) &&
3197*9880d681SAndroid Build Coastguard Worker matchAddr(AddrInst->getOperand(0), Depth+1))
3198*9880d681SAndroid Build Coastguard Worker return true;
3199*9880d681SAndroid Build Coastguard Worker
3200*9880d681SAndroid Build Coastguard Worker // Restore the old addr mode info.
3201*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3202*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3203*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3204*9880d681SAndroid Build Coastguard Worker
3205*9880d681SAndroid Build Coastguard Worker // Otherwise this was over-aggressive. Try merging in the LHS then the RHS.
3206*9880d681SAndroid Build Coastguard Worker if (matchAddr(AddrInst->getOperand(0), Depth+1) &&
3207*9880d681SAndroid Build Coastguard Worker matchAddr(AddrInst->getOperand(1), Depth+1))
3208*9880d681SAndroid Build Coastguard Worker return true;
3209*9880d681SAndroid Build Coastguard Worker
3210*9880d681SAndroid Build Coastguard Worker // Otherwise we definitely can't merge the ADD in.
3211*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3212*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3213*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3214*9880d681SAndroid Build Coastguard Worker break;
3215*9880d681SAndroid Build Coastguard Worker }
3216*9880d681SAndroid Build Coastguard Worker //case Instruction::Or:
3217*9880d681SAndroid Build Coastguard Worker // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD.
3218*9880d681SAndroid Build Coastguard Worker //break;
3219*9880d681SAndroid Build Coastguard Worker case Instruction::Mul:
3220*9880d681SAndroid Build Coastguard Worker case Instruction::Shl: {
3221*9880d681SAndroid Build Coastguard Worker // Can only handle X*C and X << C.
3222*9880d681SAndroid Build Coastguard Worker ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1));
3223*9880d681SAndroid Build Coastguard Worker if (!RHS)
3224*9880d681SAndroid Build Coastguard Worker return false;
3225*9880d681SAndroid Build Coastguard Worker int64_t Scale = RHS->getSExtValue();
3226*9880d681SAndroid Build Coastguard Worker if (Opcode == Instruction::Shl)
3227*9880d681SAndroid Build Coastguard Worker Scale = 1LL << Scale;
3228*9880d681SAndroid Build Coastguard Worker
3229*9880d681SAndroid Build Coastguard Worker return matchScaledValue(AddrInst->getOperand(0), Scale, Depth);
3230*9880d681SAndroid Build Coastguard Worker }
3231*9880d681SAndroid Build Coastguard Worker case Instruction::GetElementPtr: {
3232*9880d681SAndroid Build Coastguard Worker // Scan the GEP. We check it if it contains constant offsets and at most
3233*9880d681SAndroid Build Coastguard Worker // one variable offset.
3234*9880d681SAndroid Build Coastguard Worker int VariableOperand = -1;
3235*9880d681SAndroid Build Coastguard Worker unsigned VariableScale = 0;
3236*9880d681SAndroid Build Coastguard Worker
3237*9880d681SAndroid Build Coastguard Worker int64_t ConstantOffset = 0;
3238*9880d681SAndroid Build Coastguard Worker gep_type_iterator GTI = gep_type_begin(AddrInst);
3239*9880d681SAndroid Build Coastguard Worker for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
3240*9880d681SAndroid Build Coastguard Worker if (StructType *STy = dyn_cast<StructType>(*GTI)) {
3241*9880d681SAndroid Build Coastguard Worker const StructLayout *SL = DL.getStructLayout(STy);
3242*9880d681SAndroid Build Coastguard Worker unsigned Idx =
3243*9880d681SAndroid Build Coastguard Worker cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
3244*9880d681SAndroid Build Coastguard Worker ConstantOffset += SL->getElementOffset(Idx);
3245*9880d681SAndroid Build Coastguard Worker } else {
3246*9880d681SAndroid Build Coastguard Worker uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
3247*9880d681SAndroid Build Coastguard Worker if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
3248*9880d681SAndroid Build Coastguard Worker ConstantOffset += CI->getSExtValue()*TypeSize;
3249*9880d681SAndroid Build Coastguard Worker } else if (TypeSize) { // Scales of zero don't do anything.
3250*9880d681SAndroid Build Coastguard Worker // We only allow one variable index at the moment.
3251*9880d681SAndroid Build Coastguard Worker if (VariableOperand != -1)
3252*9880d681SAndroid Build Coastguard Worker return false;
3253*9880d681SAndroid Build Coastguard Worker
3254*9880d681SAndroid Build Coastguard Worker // Remember the variable index.
3255*9880d681SAndroid Build Coastguard Worker VariableOperand = i;
3256*9880d681SAndroid Build Coastguard Worker VariableScale = TypeSize;
3257*9880d681SAndroid Build Coastguard Worker }
3258*9880d681SAndroid Build Coastguard Worker }
3259*9880d681SAndroid Build Coastguard Worker }
3260*9880d681SAndroid Build Coastguard Worker
3261*9880d681SAndroid Build Coastguard Worker // A common case is for the GEP to only do a constant offset. In this case,
3262*9880d681SAndroid Build Coastguard Worker // just add it to the disp field and check validity.
3263*9880d681SAndroid Build Coastguard Worker if (VariableOperand == -1) {
3264*9880d681SAndroid Build Coastguard Worker AddrMode.BaseOffs += ConstantOffset;
3265*9880d681SAndroid Build Coastguard Worker if (ConstantOffset == 0 ||
3266*9880d681SAndroid Build Coastguard Worker TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) {
3267*9880d681SAndroid Build Coastguard Worker // Check to see if we can fold the base pointer in too.
3268*9880d681SAndroid Build Coastguard Worker if (matchAddr(AddrInst->getOperand(0), Depth+1))
3269*9880d681SAndroid Build Coastguard Worker return true;
3270*9880d681SAndroid Build Coastguard Worker }
3271*9880d681SAndroid Build Coastguard Worker AddrMode.BaseOffs -= ConstantOffset;
3272*9880d681SAndroid Build Coastguard Worker return false;
3273*9880d681SAndroid Build Coastguard Worker }
3274*9880d681SAndroid Build Coastguard Worker
3275*9880d681SAndroid Build Coastguard Worker // Save the valid addressing mode in case we can't match.
3276*9880d681SAndroid Build Coastguard Worker ExtAddrMode BackupAddrMode = AddrMode;
3277*9880d681SAndroid Build Coastguard Worker unsigned OldSize = AddrModeInsts.size();
3278*9880d681SAndroid Build Coastguard Worker
3279*9880d681SAndroid Build Coastguard Worker // See if the scale and offset amount is valid for this target.
3280*9880d681SAndroid Build Coastguard Worker AddrMode.BaseOffs += ConstantOffset;
3281*9880d681SAndroid Build Coastguard Worker
3282*9880d681SAndroid Build Coastguard Worker // Match the base operand of the GEP.
3283*9880d681SAndroid Build Coastguard Worker if (!matchAddr(AddrInst->getOperand(0), Depth+1)) {
3284*9880d681SAndroid Build Coastguard Worker // If it couldn't be matched, just stuff the value in a register.
3285*9880d681SAndroid Build Coastguard Worker if (AddrMode.HasBaseReg) {
3286*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3287*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3288*9880d681SAndroid Build Coastguard Worker return false;
3289*9880d681SAndroid Build Coastguard Worker }
3290*9880d681SAndroid Build Coastguard Worker AddrMode.HasBaseReg = true;
3291*9880d681SAndroid Build Coastguard Worker AddrMode.BaseReg = AddrInst->getOperand(0);
3292*9880d681SAndroid Build Coastguard Worker }
3293*9880d681SAndroid Build Coastguard Worker
3294*9880d681SAndroid Build Coastguard Worker // Match the remaining variable portion of the GEP.
3295*9880d681SAndroid Build Coastguard Worker if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale,
3296*9880d681SAndroid Build Coastguard Worker Depth)) {
3297*9880d681SAndroid Build Coastguard Worker // If it couldn't be matched, try stuffing the base into a register
3298*9880d681SAndroid Build Coastguard Worker // instead of matching it, and retrying the match of the scale.
3299*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3300*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3301*9880d681SAndroid Build Coastguard Worker if (AddrMode.HasBaseReg)
3302*9880d681SAndroid Build Coastguard Worker return false;
3303*9880d681SAndroid Build Coastguard Worker AddrMode.HasBaseReg = true;
3304*9880d681SAndroid Build Coastguard Worker AddrMode.BaseReg = AddrInst->getOperand(0);
3305*9880d681SAndroid Build Coastguard Worker AddrMode.BaseOffs += ConstantOffset;
3306*9880d681SAndroid Build Coastguard Worker if (!matchScaledValue(AddrInst->getOperand(VariableOperand),
3307*9880d681SAndroid Build Coastguard Worker VariableScale, Depth)) {
3308*9880d681SAndroid Build Coastguard Worker // If even that didn't work, bail.
3309*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3310*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3311*9880d681SAndroid Build Coastguard Worker return false;
3312*9880d681SAndroid Build Coastguard Worker }
3313*9880d681SAndroid Build Coastguard Worker }
3314*9880d681SAndroid Build Coastguard Worker
3315*9880d681SAndroid Build Coastguard Worker return true;
3316*9880d681SAndroid Build Coastguard Worker }
3317*9880d681SAndroid Build Coastguard Worker case Instruction::SExt:
3318*9880d681SAndroid Build Coastguard Worker case Instruction::ZExt: {
3319*9880d681SAndroid Build Coastguard Worker Instruction *Ext = dyn_cast<Instruction>(AddrInst);
3320*9880d681SAndroid Build Coastguard Worker if (!Ext)
3321*9880d681SAndroid Build Coastguard Worker return false;
3322*9880d681SAndroid Build Coastguard Worker
3323*9880d681SAndroid Build Coastguard Worker // Try to move this ext out of the way of the addressing mode.
3324*9880d681SAndroid Build Coastguard Worker // Ask for a method for doing so.
3325*9880d681SAndroid Build Coastguard Worker TypePromotionHelper::Action TPH =
3326*9880d681SAndroid Build Coastguard Worker TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);
3327*9880d681SAndroid Build Coastguard Worker if (!TPH)
3328*9880d681SAndroid Build Coastguard Worker return false;
3329*9880d681SAndroid Build Coastguard Worker
3330*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3331*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
3332*9880d681SAndroid Build Coastguard Worker unsigned CreatedInstsCost = 0;
3333*9880d681SAndroid Build Coastguard Worker unsigned ExtCost = !TLI.isExtFree(Ext);
3334*9880d681SAndroid Build Coastguard Worker Value *PromotedOperand =
3335*9880d681SAndroid Build Coastguard Worker TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI);
3336*9880d681SAndroid Build Coastguard Worker // SExt has been moved away.
3337*9880d681SAndroid Build Coastguard Worker // Thus either it will be rematched later in the recursive calls or it is
3338*9880d681SAndroid Build Coastguard Worker // gone. Anyway, we must not fold it into the addressing mode at this point.
3339*9880d681SAndroid Build Coastguard Worker // E.g.,
3340*9880d681SAndroid Build Coastguard Worker // op = add opnd, 1
3341*9880d681SAndroid Build Coastguard Worker // idx = ext op
3342*9880d681SAndroid Build Coastguard Worker // addr = gep base, idx
3343*9880d681SAndroid Build Coastguard Worker // is now:
3344*9880d681SAndroid Build Coastguard Worker // promotedOpnd = ext opnd <- no match here
3345*9880d681SAndroid Build Coastguard Worker // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls)
3346*9880d681SAndroid Build Coastguard Worker // addr = gep base, op <- match
3347*9880d681SAndroid Build Coastguard Worker if (MovedAway)
3348*9880d681SAndroid Build Coastguard Worker *MovedAway = true;
3349*9880d681SAndroid Build Coastguard Worker
3350*9880d681SAndroid Build Coastguard Worker assert(PromotedOperand &&
3351*9880d681SAndroid Build Coastguard Worker "TypePromotionHelper should have filtered out those cases");
3352*9880d681SAndroid Build Coastguard Worker
3353*9880d681SAndroid Build Coastguard Worker ExtAddrMode BackupAddrMode = AddrMode;
3354*9880d681SAndroid Build Coastguard Worker unsigned OldSize = AddrModeInsts.size();
3355*9880d681SAndroid Build Coastguard Worker
3356*9880d681SAndroid Build Coastguard Worker if (!matchAddr(PromotedOperand, Depth) ||
3357*9880d681SAndroid Build Coastguard Worker // The total of the new cost is equal to the cost of the created
3358*9880d681SAndroid Build Coastguard Worker // instructions.
3359*9880d681SAndroid Build Coastguard Worker // The total of the old cost is equal to the cost of the extension plus
3360*9880d681SAndroid Build Coastguard Worker // what we have saved in the addressing mode.
3361*9880d681SAndroid Build Coastguard Worker !isPromotionProfitable(CreatedInstsCost,
3362*9880d681SAndroid Build Coastguard Worker ExtCost + (AddrModeInsts.size() - OldSize),
3363*9880d681SAndroid Build Coastguard Worker PromotedOperand)) {
3364*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3365*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3366*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Sign extension does not pay off: rollback\n");
3367*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3368*9880d681SAndroid Build Coastguard Worker return false;
3369*9880d681SAndroid Build Coastguard Worker }
3370*9880d681SAndroid Build Coastguard Worker return true;
3371*9880d681SAndroid Build Coastguard Worker }
3372*9880d681SAndroid Build Coastguard Worker }
3373*9880d681SAndroid Build Coastguard Worker return false;
3374*9880d681SAndroid Build Coastguard Worker }
3375*9880d681SAndroid Build Coastguard Worker
3376*9880d681SAndroid Build Coastguard Worker /// If we can, try to add the value of 'Addr' into the current addressing mode.
3377*9880d681SAndroid Build Coastguard Worker /// If Addr can't be added to AddrMode this returns false and leaves AddrMode
3378*9880d681SAndroid Build Coastguard Worker /// unmodified. This assumes that Addr is either a pointer type or intptr_t
3379*9880d681SAndroid Build Coastguard Worker /// for the target.
3380*9880d681SAndroid Build Coastguard Worker ///
matchAddr(Value * Addr,unsigned Depth)3381*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) {
3382*9880d681SAndroid Build Coastguard Worker // Start a transaction at this point that we will rollback if the matching
3383*9880d681SAndroid Build Coastguard Worker // fails.
3384*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3385*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
3386*9880d681SAndroid Build Coastguard Worker if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
3387*9880d681SAndroid Build Coastguard Worker // Fold in immediates if legal for the target.
3388*9880d681SAndroid Build Coastguard Worker AddrMode.BaseOffs += CI->getSExtValue();
3389*9880d681SAndroid Build Coastguard Worker if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3390*9880d681SAndroid Build Coastguard Worker return true;
3391*9880d681SAndroid Build Coastguard Worker AddrMode.BaseOffs -= CI->getSExtValue();
3392*9880d681SAndroid Build Coastguard Worker } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
3393*9880d681SAndroid Build Coastguard Worker // If this is a global variable, try to fold it into the addressing mode.
3394*9880d681SAndroid Build Coastguard Worker if (!AddrMode.BaseGV) {
3395*9880d681SAndroid Build Coastguard Worker AddrMode.BaseGV = GV;
3396*9880d681SAndroid Build Coastguard Worker if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3397*9880d681SAndroid Build Coastguard Worker return true;
3398*9880d681SAndroid Build Coastguard Worker AddrMode.BaseGV = nullptr;
3399*9880d681SAndroid Build Coastguard Worker }
3400*9880d681SAndroid Build Coastguard Worker } else if (Instruction *I = dyn_cast<Instruction>(Addr)) {
3401*9880d681SAndroid Build Coastguard Worker ExtAddrMode BackupAddrMode = AddrMode;
3402*9880d681SAndroid Build Coastguard Worker unsigned OldSize = AddrModeInsts.size();
3403*9880d681SAndroid Build Coastguard Worker
3404*9880d681SAndroid Build Coastguard Worker // Check to see if it is possible to fold this operation.
3405*9880d681SAndroid Build Coastguard Worker bool MovedAway = false;
3406*9880d681SAndroid Build Coastguard Worker if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) {
3407*9880d681SAndroid Build Coastguard Worker // This instruction may have been moved away. If so, there is nothing
3408*9880d681SAndroid Build Coastguard Worker // to check here.
3409*9880d681SAndroid Build Coastguard Worker if (MovedAway)
3410*9880d681SAndroid Build Coastguard Worker return true;
3411*9880d681SAndroid Build Coastguard Worker // Okay, it's possible to fold this. Check to see if it is actually
3412*9880d681SAndroid Build Coastguard Worker // *profitable* to do so. We use a simple cost model to avoid increasing
3413*9880d681SAndroid Build Coastguard Worker // register pressure too much.
3414*9880d681SAndroid Build Coastguard Worker if (I->hasOneUse() ||
3415*9880d681SAndroid Build Coastguard Worker isProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) {
3416*9880d681SAndroid Build Coastguard Worker AddrModeInsts.push_back(I);
3417*9880d681SAndroid Build Coastguard Worker return true;
3418*9880d681SAndroid Build Coastguard Worker }
3419*9880d681SAndroid Build Coastguard Worker
3420*9880d681SAndroid Build Coastguard Worker // It isn't profitable to do this, roll back.
3421*9880d681SAndroid Build Coastguard Worker //cerr << "NOT FOLDING: " << *I;
3422*9880d681SAndroid Build Coastguard Worker AddrMode = BackupAddrMode;
3423*9880d681SAndroid Build Coastguard Worker AddrModeInsts.resize(OldSize);
3424*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3425*9880d681SAndroid Build Coastguard Worker }
3426*9880d681SAndroid Build Coastguard Worker } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) {
3427*9880d681SAndroid Build Coastguard Worker if (matchOperationAddr(CE, CE->getOpcode(), Depth))
3428*9880d681SAndroid Build Coastguard Worker return true;
3429*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3430*9880d681SAndroid Build Coastguard Worker } else if (isa<ConstantPointerNull>(Addr)) {
3431*9880d681SAndroid Build Coastguard Worker // Null pointer gets folded without affecting the addressing mode.
3432*9880d681SAndroid Build Coastguard Worker return true;
3433*9880d681SAndroid Build Coastguard Worker }
3434*9880d681SAndroid Build Coastguard Worker
3435*9880d681SAndroid Build Coastguard Worker // Worse case, the target should support [reg] addressing modes. :)
3436*9880d681SAndroid Build Coastguard Worker if (!AddrMode.HasBaseReg) {
3437*9880d681SAndroid Build Coastguard Worker AddrMode.HasBaseReg = true;
3438*9880d681SAndroid Build Coastguard Worker AddrMode.BaseReg = Addr;
3439*9880d681SAndroid Build Coastguard Worker // Still check for legality in case the target supports [imm] but not [i+r].
3440*9880d681SAndroid Build Coastguard Worker if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3441*9880d681SAndroid Build Coastguard Worker return true;
3442*9880d681SAndroid Build Coastguard Worker AddrMode.HasBaseReg = false;
3443*9880d681SAndroid Build Coastguard Worker AddrMode.BaseReg = nullptr;
3444*9880d681SAndroid Build Coastguard Worker }
3445*9880d681SAndroid Build Coastguard Worker
3446*9880d681SAndroid Build Coastguard Worker // If the base register is already taken, see if we can do [r+r].
3447*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale == 0) {
3448*9880d681SAndroid Build Coastguard Worker AddrMode.Scale = 1;
3449*9880d681SAndroid Build Coastguard Worker AddrMode.ScaledReg = Addr;
3450*9880d681SAndroid Build Coastguard Worker if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
3451*9880d681SAndroid Build Coastguard Worker return true;
3452*9880d681SAndroid Build Coastguard Worker AddrMode.Scale = 0;
3453*9880d681SAndroid Build Coastguard Worker AddrMode.ScaledReg = nullptr;
3454*9880d681SAndroid Build Coastguard Worker }
3455*9880d681SAndroid Build Coastguard Worker // Couldn't match.
3456*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3457*9880d681SAndroid Build Coastguard Worker return false;
3458*9880d681SAndroid Build Coastguard Worker }
3459*9880d681SAndroid Build Coastguard Worker
3460*9880d681SAndroid Build Coastguard Worker /// Check to see if all uses of OpVal by the specified inline asm call are due
3461*9880d681SAndroid Build Coastguard Worker /// to memory operands. If so, return true, otherwise return false.
IsOperandAMemoryOperand(CallInst * CI,InlineAsm * IA,Value * OpVal,const TargetMachine & TM)3462*9880d681SAndroid Build Coastguard Worker static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal,
3463*9880d681SAndroid Build Coastguard Worker const TargetMachine &TM) {
3464*9880d681SAndroid Build Coastguard Worker const Function *F = CI->getParent()->getParent();
3465*9880d681SAndroid Build Coastguard Worker const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
3466*9880d681SAndroid Build Coastguard Worker const TargetRegisterInfo *TRI = TM.getSubtargetImpl(*F)->getRegisterInfo();
3467*9880d681SAndroid Build Coastguard Worker TargetLowering::AsmOperandInfoVector TargetConstraints =
3468*9880d681SAndroid Build Coastguard Worker TLI->ParseConstraints(F->getParent()->getDataLayout(), TRI,
3469*9880d681SAndroid Build Coastguard Worker ImmutableCallSite(CI));
3470*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
3471*9880d681SAndroid Build Coastguard Worker TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
3472*9880d681SAndroid Build Coastguard Worker
3473*9880d681SAndroid Build Coastguard Worker // Compute the constraint code and ConstraintType to use.
3474*9880d681SAndroid Build Coastguard Worker TLI->ComputeConstraintToUse(OpInfo, SDValue());
3475*9880d681SAndroid Build Coastguard Worker
3476*9880d681SAndroid Build Coastguard Worker // If this asm operand is our Value*, and if it isn't an indirect memory
3477*9880d681SAndroid Build Coastguard Worker // operand, we can't fold it!
3478*9880d681SAndroid Build Coastguard Worker if (OpInfo.CallOperandVal == OpVal &&
3479*9880d681SAndroid Build Coastguard Worker (OpInfo.ConstraintType != TargetLowering::C_Memory ||
3480*9880d681SAndroid Build Coastguard Worker !OpInfo.isIndirect))
3481*9880d681SAndroid Build Coastguard Worker return false;
3482*9880d681SAndroid Build Coastguard Worker }
3483*9880d681SAndroid Build Coastguard Worker
3484*9880d681SAndroid Build Coastguard Worker return true;
3485*9880d681SAndroid Build Coastguard Worker }
3486*9880d681SAndroid Build Coastguard Worker
3487*9880d681SAndroid Build Coastguard Worker /// Recursively walk all the uses of I until we find a memory use.
3488*9880d681SAndroid Build Coastguard Worker /// If we find an obviously non-foldable instruction, return true.
3489*9880d681SAndroid Build Coastguard Worker /// Add the ultimately found memory instructions to MemoryUses.
FindAllMemoryUses(Instruction * I,SmallVectorImpl<std::pair<Instruction *,unsigned>> & MemoryUses,SmallPtrSetImpl<Instruction * > & ConsideredInsts,const TargetMachine & TM)3490*9880d681SAndroid Build Coastguard Worker static bool FindAllMemoryUses(
3491*9880d681SAndroid Build Coastguard Worker Instruction *I,
3492*9880d681SAndroid Build Coastguard Worker SmallVectorImpl<std::pair<Instruction *, unsigned>> &MemoryUses,
3493*9880d681SAndroid Build Coastguard Worker SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetMachine &TM) {
3494*9880d681SAndroid Build Coastguard Worker // If we already considered this instruction, we're done.
3495*9880d681SAndroid Build Coastguard Worker if (!ConsideredInsts.insert(I).second)
3496*9880d681SAndroid Build Coastguard Worker return false;
3497*9880d681SAndroid Build Coastguard Worker
3498*9880d681SAndroid Build Coastguard Worker // If this is an obviously unfoldable instruction, bail out.
3499*9880d681SAndroid Build Coastguard Worker if (!MightBeFoldableInst(I))
3500*9880d681SAndroid Build Coastguard Worker return true;
3501*9880d681SAndroid Build Coastguard Worker
3502*9880d681SAndroid Build Coastguard Worker const bool OptSize = I->getFunction()->optForSize();
3503*9880d681SAndroid Build Coastguard Worker
3504*9880d681SAndroid Build Coastguard Worker // Loop over all the uses, recursively processing them.
3505*9880d681SAndroid Build Coastguard Worker for (Use &U : I->uses()) {
3506*9880d681SAndroid Build Coastguard Worker Instruction *UserI = cast<Instruction>(U.getUser());
3507*9880d681SAndroid Build Coastguard Worker
3508*9880d681SAndroid Build Coastguard Worker if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) {
3509*9880d681SAndroid Build Coastguard Worker MemoryUses.push_back(std::make_pair(LI, U.getOperandNo()));
3510*9880d681SAndroid Build Coastguard Worker continue;
3511*9880d681SAndroid Build Coastguard Worker }
3512*9880d681SAndroid Build Coastguard Worker
3513*9880d681SAndroid Build Coastguard Worker if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) {
3514*9880d681SAndroid Build Coastguard Worker unsigned opNo = U.getOperandNo();
3515*9880d681SAndroid Build Coastguard Worker if (opNo == 0) return true; // Storing addr, not into addr.
3516*9880d681SAndroid Build Coastguard Worker MemoryUses.push_back(std::make_pair(SI, opNo));
3517*9880d681SAndroid Build Coastguard Worker continue;
3518*9880d681SAndroid Build Coastguard Worker }
3519*9880d681SAndroid Build Coastguard Worker
3520*9880d681SAndroid Build Coastguard Worker if (CallInst *CI = dyn_cast<CallInst>(UserI)) {
3521*9880d681SAndroid Build Coastguard Worker // If this is a cold call, we can sink the addressing calculation into
3522*9880d681SAndroid Build Coastguard Worker // the cold path. See optimizeCallInst
3523*9880d681SAndroid Build Coastguard Worker if (!OptSize && CI->hasFnAttr(Attribute::Cold))
3524*9880d681SAndroid Build Coastguard Worker continue;
3525*9880d681SAndroid Build Coastguard Worker
3526*9880d681SAndroid Build Coastguard Worker InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue());
3527*9880d681SAndroid Build Coastguard Worker if (!IA) return true;
3528*9880d681SAndroid Build Coastguard Worker
3529*9880d681SAndroid Build Coastguard Worker // If this is a memory operand, we're cool, otherwise bail out.
3530*9880d681SAndroid Build Coastguard Worker if (!IsOperandAMemoryOperand(CI, IA, I, TM))
3531*9880d681SAndroid Build Coastguard Worker return true;
3532*9880d681SAndroid Build Coastguard Worker continue;
3533*9880d681SAndroid Build Coastguard Worker }
3534*9880d681SAndroid Build Coastguard Worker
3535*9880d681SAndroid Build Coastguard Worker if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TM))
3536*9880d681SAndroid Build Coastguard Worker return true;
3537*9880d681SAndroid Build Coastguard Worker }
3538*9880d681SAndroid Build Coastguard Worker
3539*9880d681SAndroid Build Coastguard Worker return false;
3540*9880d681SAndroid Build Coastguard Worker }
3541*9880d681SAndroid Build Coastguard Worker
3542*9880d681SAndroid Build Coastguard Worker /// Return true if Val is already known to be live at the use site that we're
3543*9880d681SAndroid Build Coastguard Worker /// folding it into. If so, there is no cost to include it in the addressing
3544*9880d681SAndroid Build Coastguard Worker /// mode. KnownLive1 and KnownLive2 are two values that we know are live at the
3545*9880d681SAndroid Build Coastguard Worker /// instruction already.
valueAlreadyLiveAtInst(Value * Val,Value * KnownLive1,Value * KnownLive2)3546*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,Value *KnownLive1,
3547*9880d681SAndroid Build Coastguard Worker Value *KnownLive2) {
3548*9880d681SAndroid Build Coastguard Worker // If Val is either of the known-live values, we know it is live!
3549*9880d681SAndroid Build Coastguard Worker if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2)
3550*9880d681SAndroid Build Coastguard Worker return true;
3551*9880d681SAndroid Build Coastguard Worker
3552*9880d681SAndroid Build Coastguard Worker // All values other than instructions and arguments (e.g. constants) are live.
3553*9880d681SAndroid Build Coastguard Worker if (!isa<Instruction>(Val) && !isa<Argument>(Val)) return true;
3554*9880d681SAndroid Build Coastguard Worker
3555*9880d681SAndroid Build Coastguard Worker // If Val is a constant sized alloca in the entry block, it is live, this is
3556*9880d681SAndroid Build Coastguard Worker // true because it is just a reference to the stack/frame pointer, which is
3557*9880d681SAndroid Build Coastguard Worker // live for the whole function.
3558*9880d681SAndroid Build Coastguard Worker if (AllocaInst *AI = dyn_cast<AllocaInst>(Val))
3559*9880d681SAndroid Build Coastguard Worker if (AI->isStaticAlloca())
3560*9880d681SAndroid Build Coastguard Worker return true;
3561*9880d681SAndroid Build Coastguard Worker
3562*9880d681SAndroid Build Coastguard Worker // Check to see if this value is already used in the memory instruction's
3563*9880d681SAndroid Build Coastguard Worker // block. If so, it's already live into the block at the very least, so we
3564*9880d681SAndroid Build Coastguard Worker // can reasonably fold it.
3565*9880d681SAndroid Build Coastguard Worker return Val->isUsedInBasicBlock(MemoryInst->getParent());
3566*9880d681SAndroid Build Coastguard Worker }
3567*9880d681SAndroid Build Coastguard Worker
3568*9880d681SAndroid Build Coastguard Worker /// It is possible for the addressing mode of the machine to fold the specified
3569*9880d681SAndroid Build Coastguard Worker /// instruction into a load or store that ultimately uses it.
3570*9880d681SAndroid Build Coastguard Worker /// However, the specified instruction has multiple uses.
3571*9880d681SAndroid Build Coastguard Worker /// Given this, it may actually increase register pressure to fold it
3572*9880d681SAndroid Build Coastguard Worker /// into the load. For example, consider this code:
3573*9880d681SAndroid Build Coastguard Worker ///
3574*9880d681SAndroid Build Coastguard Worker /// X = ...
3575*9880d681SAndroid Build Coastguard Worker /// Y = X+1
3576*9880d681SAndroid Build Coastguard Worker /// use(Y) -> nonload/store
3577*9880d681SAndroid Build Coastguard Worker /// Z = Y+1
3578*9880d681SAndroid Build Coastguard Worker /// load Z
3579*9880d681SAndroid Build Coastguard Worker ///
3580*9880d681SAndroid Build Coastguard Worker /// In this case, Y has multiple uses, and can be folded into the load of Z
3581*9880d681SAndroid Build Coastguard Worker /// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to
3582*9880d681SAndroid Build Coastguard Worker /// be live at the use(Y) line. If we don't fold Y into load Z, we use one
3583*9880d681SAndroid Build Coastguard Worker /// fewer register. Since Y can't be folded into "use(Y)" we don't increase the
3584*9880d681SAndroid Build Coastguard Worker /// number of computations either.
3585*9880d681SAndroid Build Coastguard Worker ///
3586*9880d681SAndroid Build Coastguard Worker /// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If
3587*9880d681SAndroid Build Coastguard Worker /// X was live across 'load Z' for other reasons, we actually *would* want to
3588*9880d681SAndroid Build Coastguard Worker /// fold the addressing mode in the Z case. This would make Y die earlier.
3589*9880d681SAndroid Build Coastguard Worker bool AddressingModeMatcher::
isProfitableToFoldIntoAddressingMode(Instruction * I,ExtAddrMode & AMBefore,ExtAddrMode & AMAfter)3590*9880d681SAndroid Build Coastguard Worker isProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore,
3591*9880d681SAndroid Build Coastguard Worker ExtAddrMode &AMAfter) {
3592*9880d681SAndroid Build Coastguard Worker if (IgnoreProfitability) return true;
3593*9880d681SAndroid Build Coastguard Worker
3594*9880d681SAndroid Build Coastguard Worker // AMBefore is the addressing mode before this instruction was folded into it,
3595*9880d681SAndroid Build Coastguard Worker // and AMAfter is the addressing mode after the instruction was folded. Get
3596*9880d681SAndroid Build Coastguard Worker // the set of registers referenced by AMAfter and subtract out those
3597*9880d681SAndroid Build Coastguard Worker // referenced by AMBefore: this is the set of values which folding in this
3598*9880d681SAndroid Build Coastguard Worker // address extends the lifetime of.
3599*9880d681SAndroid Build Coastguard Worker //
3600*9880d681SAndroid Build Coastguard Worker // Note that there are only two potential values being referenced here,
3601*9880d681SAndroid Build Coastguard Worker // BaseReg and ScaleReg (global addresses are always available, as are any
3602*9880d681SAndroid Build Coastguard Worker // folded immediates).
3603*9880d681SAndroid Build Coastguard Worker Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg;
3604*9880d681SAndroid Build Coastguard Worker
3605*9880d681SAndroid Build Coastguard Worker // If the BaseReg or ScaledReg was referenced by the previous addrmode, their
3606*9880d681SAndroid Build Coastguard Worker // lifetime wasn't extended by adding this instruction.
3607*9880d681SAndroid Build Coastguard Worker if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg))
3608*9880d681SAndroid Build Coastguard Worker BaseReg = nullptr;
3609*9880d681SAndroid Build Coastguard Worker if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg))
3610*9880d681SAndroid Build Coastguard Worker ScaledReg = nullptr;
3611*9880d681SAndroid Build Coastguard Worker
3612*9880d681SAndroid Build Coastguard Worker // If folding this instruction (and it's subexprs) didn't extend any live
3613*9880d681SAndroid Build Coastguard Worker // ranges, we're ok with it.
3614*9880d681SAndroid Build Coastguard Worker if (!BaseReg && !ScaledReg)
3615*9880d681SAndroid Build Coastguard Worker return true;
3616*9880d681SAndroid Build Coastguard Worker
3617*9880d681SAndroid Build Coastguard Worker // If all uses of this instruction can have the address mode sunk into them,
3618*9880d681SAndroid Build Coastguard Worker // we can remove the addressing mode and effectively trade one live register
3619*9880d681SAndroid Build Coastguard Worker // for another (at worst.) In this context, folding an addressing mode into
3620*9880d681SAndroid Build Coastguard Worker // the use is just a particularly nice way of sinking it.
3621*9880d681SAndroid Build Coastguard Worker SmallVector<std::pair<Instruction*,unsigned>, 16> MemoryUses;
3622*9880d681SAndroid Build Coastguard Worker SmallPtrSet<Instruction*, 16> ConsideredInsts;
3623*9880d681SAndroid Build Coastguard Worker if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TM))
3624*9880d681SAndroid Build Coastguard Worker return false; // Has a non-memory, non-foldable use!
3625*9880d681SAndroid Build Coastguard Worker
3626*9880d681SAndroid Build Coastguard Worker // Now that we know that all uses of this instruction are part of a chain of
3627*9880d681SAndroid Build Coastguard Worker // computation involving only operations that could theoretically be folded
3628*9880d681SAndroid Build Coastguard Worker // into a memory use, loop over each of these memory operation uses and see
3629*9880d681SAndroid Build Coastguard Worker // if they could *actually* fold the instruction. The assumption is that
3630*9880d681SAndroid Build Coastguard Worker // addressing modes are cheap and that duplicating the computation involved
3631*9880d681SAndroid Build Coastguard Worker // many times is worthwhile, even on a fastpath. For sinking candidates
3632*9880d681SAndroid Build Coastguard Worker // (i.e. cold call sites), this serves as a way to prevent excessive code
3633*9880d681SAndroid Build Coastguard Worker // growth since most architectures have some reasonable small and fast way to
3634*9880d681SAndroid Build Coastguard Worker // compute an effective address. (i.e LEA on x86)
3635*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction*, 32> MatchedAddrModeInsts;
3636*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = MemoryUses.size(); i != e; ++i) {
3637*9880d681SAndroid Build Coastguard Worker Instruction *User = MemoryUses[i].first;
3638*9880d681SAndroid Build Coastguard Worker unsigned OpNo = MemoryUses[i].second;
3639*9880d681SAndroid Build Coastguard Worker
3640*9880d681SAndroid Build Coastguard Worker // Get the access type of this use. If the use isn't a pointer, we don't
3641*9880d681SAndroid Build Coastguard Worker // know what it accesses.
3642*9880d681SAndroid Build Coastguard Worker Value *Address = User->getOperand(OpNo);
3643*9880d681SAndroid Build Coastguard Worker PointerType *AddrTy = dyn_cast<PointerType>(Address->getType());
3644*9880d681SAndroid Build Coastguard Worker if (!AddrTy)
3645*9880d681SAndroid Build Coastguard Worker return false;
3646*9880d681SAndroid Build Coastguard Worker Type *AddressAccessTy = AddrTy->getElementType();
3647*9880d681SAndroid Build Coastguard Worker unsigned AS = AddrTy->getAddressSpace();
3648*9880d681SAndroid Build Coastguard Worker
3649*9880d681SAndroid Build Coastguard Worker // Do a match against the root of this address, ignoring profitability. This
3650*9880d681SAndroid Build Coastguard Worker // will tell us if the addressing mode for the memory operation will
3651*9880d681SAndroid Build Coastguard Worker // *actually* cover the shared instruction.
3652*9880d681SAndroid Build Coastguard Worker ExtAddrMode Result;
3653*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3654*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
3655*9880d681SAndroid Build Coastguard Worker AddressingModeMatcher Matcher(MatchedAddrModeInsts, TM, AddressAccessTy, AS,
3656*9880d681SAndroid Build Coastguard Worker MemoryInst, Result, InsertedInsts,
3657*9880d681SAndroid Build Coastguard Worker PromotedInsts, TPT);
3658*9880d681SAndroid Build Coastguard Worker Matcher.IgnoreProfitability = true;
3659*9880d681SAndroid Build Coastguard Worker bool Success = Matcher.matchAddr(Address, 0);
3660*9880d681SAndroid Build Coastguard Worker (void)Success; assert(Success && "Couldn't select *anything*?");
3661*9880d681SAndroid Build Coastguard Worker
3662*9880d681SAndroid Build Coastguard Worker // The match was to check the profitability, the changes made are not
3663*9880d681SAndroid Build Coastguard Worker // part of the original matcher. Therefore, they should be dropped
3664*9880d681SAndroid Build Coastguard Worker // otherwise the original matcher will not present the right state.
3665*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3666*9880d681SAndroid Build Coastguard Worker
3667*9880d681SAndroid Build Coastguard Worker // If the match didn't cover I, then it won't be shared by it.
3668*9880d681SAndroid Build Coastguard Worker if (std::find(MatchedAddrModeInsts.begin(), MatchedAddrModeInsts.end(),
3669*9880d681SAndroid Build Coastguard Worker I) == MatchedAddrModeInsts.end())
3670*9880d681SAndroid Build Coastguard Worker return false;
3671*9880d681SAndroid Build Coastguard Worker
3672*9880d681SAndroid Build Coastguard Worker MatchedAddrModeInsts.clear();
3673*9880d681SAndroid Build Coastguard Worker }
3674*9880d681SAndroid Build Coastguard Worker
3675*9880d681SAndroid Build Coastguard Worker return true;
3676*9880d681SAndroid Build Coastguard Worker }
3677*9880d681SAndroid Build Coastguard Worker
3678*9880d681SAndroid Build Coastguard Worker } // end anonymous namespace
3679*9880d681SAndroid Build Coastguard Worker
3680*9880d681SAndroid Build Coastguard Worker /// Return true if the specified values are defined in a
3681*9880d681SAndroid Build Coastguard Worker /// different basic block than BB.
IsNonLocalValue(Value * V,BasicBlock * BB)3682*9880d681SAndroid Build Coastguard Worker static bool IsNonLocalValue(Value *V, BasicBlock *BB) {
3683*9880d681SAndroid Build Coastguard Worker if (Instruction *I = dyn_cast<Instruction>(V))
3684*9880d681SAndroid Build Coastguard Worker return I->getParent() != BB;
3685*9880d681SAndroid Build Coastguard Worker return false;
3686*9880d681SAndroid Build Coastguard Worker }
3687*9880d681SAndroid Build Coastguard Worker
3688*9880d681SAndroid Build Coastguard Worker /// Sink addressing mode computation immediate before MemoryInst if doing so
3689*9880d681SAndroid Build Coastguard Worker /// can be done without increasing register pressure. The need for the
3690*9880d681SAndroid Build Coastguard Worker /// register pressure constraint means this can end up being an all or nothing
3691*9880d681SAndroid Build Coastguard Worker /// decision for all uses of the same addressing computation.
3692*9880d681SAndroid Build Coastguard Worker ///
3693*9880d681SAndroid Build Coastguard Worker /// Load and Store Instructions often have addressing modes that can do
3694*9880d681SAndroid Build Coastguard Worker /// significant amounts of computation. As such, instruction selection will try
3695*9880d681SAndroid Build Coastguard Worker /// to get the load or store to do as much computation as possible for the
3696*9880d681SAndroid Build Coastguard Worker /// program. The problem is that isel can only see within a single block. As
3697*9880d681SAndroid Build Coastguard Worker /// such, we sink as much legal addressing mode work into the block as possible.
3698*9880d681SAndroid Build Coastguard Worker ///
3699*9880d681SAndroid Build Coastguard Worker /// This method is used to optimize both load/store and inline asms with memory
3700*9880d681SAndroid Build Coastguard Worker /// operands. It's also used to sink addressing computations feeding into cold
3701*9880d681SAndroid Build Coastguard Worker /// call sites into their (cold) basic block.
3702*9880d681SAndroid Build Coastguard Worker ///
3703*9880d681SAndroid Build Coastguard Worker /// The motivation for handling sinking into cold blocks is that doing so can
3704*9880d681SAndroid Build Coastguard Worker /// both enable other address mode sinking (by satisfying the register pressure
3705*9880d681SAndroid Build Coastguard Worker /// constraint above), and reduce register pressure globally (by removing the
3706*9880d681SAndroid Build Coastguard Worker /// addressing mode computation from the fast path entirely.).
optimizeMemoryInst(Instruction * MemoryInst,Value * Addr,Type * AccessTy,unsigned AddrSpace)3707*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
3708*9880d681SAndroid Build Coastguard Worker Type *AccessTy, unsigned AddrSpace) {
3709*9880d681SAndroid Build Coastguard Worker Value *Repl = Addr;
3710*9880d681SAndroid Build Coastguard Worker
3711*9880d681SAndroid Build Coastguard Worker // Try to collapse single-value PHI nodes. This is necessary to undo
3712*9880d681SAndroid Build Coastguard Worker // unprofitable PRE transformations.
3713*9880d681SAndroid Build Coastguard Worker SmallVector<Value*, 8> worklist;
3714*9880d681SAndroid Build Coastguard Worker SmallPtrSet<Value*, 16> Visited;
3715*9880d681SAndroid Build Coastguard Worker worklist.push_back(Addr);
3716*9880d681SAndroid Build Coastguard Worker
3717*9880d681SAndroid Build Coastguard Worker // Use a worklist to iteratively look through PHI nodes, and ensure that
3718*9880d681SAndroid Build Coastguard Worker // the addressing mode obtained from the non-PHI roots of the graph
3719*9880d681SAndroid Build Coastguard Worker // are equivalent.
3720*9880d681SAndroid Build Coastguard Worker Value *Consensus = nullptr;
3721*9880d681SAndroid Build Coastguard Worker unsigned NumUsesConsensus = 0;
3722*9880d681SAndroid Build Coastguard Worker bool IsNumUsesConsensusValid = false;
3723*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction*, 16> AddrModeInsts;
3724*9880d681SAndroid Build Coastguard Worker ExtAddrMode AddrMode;
3725*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction TPT;
3726*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3727*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
3728*9880d681SAndroid Build Coastguard Worker while (!worklist.empty()) {
3729*9880d681SAndroid Build Coastguard Worker Value *V = worklist.back();
3730*9880d681SAndroid Build Coastguard Worker worklist.pop_back();
3731*9880d681SAndroid Build Coastguard Worker
3732*9880d681SAndroid Build Coastguard Worker // Break use-def graph loops.
3733*9880d681SAndroid Build Coastguard Worker if (!Visited.insert(V).second) {
3734*9880d681SAndroid Build Coastguard Worker Consensus = nullptr;
3735*9880d681SAndroid Build Coastguard Worker break;
3736*9880d681SAndroid Build Coastguard Worker }
3737*9880d681SAndroid Build Coastguard Worker
3738*9880d681SAndroid Build Coastguard Worker // For a PHI node, push all of its incoming values.
3739*9880d681SAndroid Build Coastguard Worker if (PHINode *P = dyn_cast<PHINode>(V)) {
3740*9880d681SAndroid Build Coastguard Worker for (Value *IncValue : P->incoming_values())
3741*9880d681SAndroid Build Coastguard Worker worklist.push_back(IncValue);
3742*9880d681SAndroid Build Coastguard Worker continue;
3743*9880d681SAndroid Build Coastguard Worker }
3744*9880d681SAndroid Build Coastguard Worker
3745*9880d681SAndroid Build Coastguard Worker // For non-PHIs, determine the addressing mode being computed. Note that
3746*9880d681SAndroid Build Coastguard Worker // the result may differ depending on what other uses our candidate
3747*9880d681SAndroid Build Coastguard Worker // addressing instructions might have.
3748*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction*, 16> NewAddrModeInsts;
3749*9880d681SAndroid Build Coastguard Worker ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
3750*9880d681SAndroid Build Coastguard Worker V, AccessTy, AddrSpace, MemoryInst, NewAddrModeInsts, *TM,
3751*9880d681SAndroid Build Coastguard Worker InsertedInsts, PromotedInsts, TPT);
3752*9880d681SAndroid Build Coastguard Worker
3753*9880d681SAndroid Build Coastguard Worker // This check is broken into two cases with very similar code to avoid using
3754*9880d681SAndroid Build Coastguard Worker // getNumUses() as much as possible. Some values have a lot of uses, so
3755*9880d681SAndroid Build Coastguard Worker // calling getNumUses() unconditionally caused a significant compile-time
3756*9880d681SAndroid Build Coastguard Worker // regression.
3757*9880d681SAndroid Build Coastguard Worker if (!Consensus) {
3758*9880d681SAndroid Build Coastguard Worker Consensus = V;
3759*9880d681SAndroid Build Coastguard Worker AddrMode = NewAddrMode;
3760*9880d681SAndroid Build Coastguard Worker AddrModeInsts = NewAddrModeInsts;
3761*9880d681SAndroid Build Coastguard Worker continue;
3762*9880d681SAndroid Build Coastguard Worker } else if (NewAddrMode == AddrMode) {
3763*9880d681SAndroid Build Coastguard Worker if (!IsNumUsesConsensusValid) {
3764*9880d681SAndroid Build Coastguard Worker NumUsesConsensus = Consensus->getNumUses();
3765*9880d681SAndroid Build Coastguard Worker IsNumUsesConsensusValid = true;
3766*9880d681SAndroid Build Coastguard Worker }
3767*9880d681SAndroid Build Coastguard Worker
3768*9880d681SAndroid Build Coastguard Worker // Ensure that the obtained addressing mode is equivalent to that obtained
3769*9880d681SAndroid Build Coastguard Worker // for all other roots of the PHI traversal. Also, when choosing one
3770*9880d681SAndroid Build Coastguard Worker // such root as representative, select the one with the most uses in order
3771*9880d681SAndroid Build Coastguard Worker // to keep the cost modeling heuristics in AddressingModeMatcher
3772*9880d681SAndroid Build Coastguard Worker // applicable.
3773*9880d681SAndroid Build Coastguard Worker unsigned NumUses = V->getNumUses();
3774*9880d681SAndroid Build Coastguard Worker if (NumUses > NumUsesConsensus) {
3775*9880d681SAndroid Build Coastguard Worker Consensus = V;
3776*9880d681SAndroid Build Coastguard Worker NumUsesConsensus = NumUses;
3777*9880d681SAndroid Build Coastguard Worker AddrModeInsts = NewAddrModeInsts;
3778*9880d681SAndroid Build Coastguard Worker }
3779*9880d681SAndroid Build Coastguard Worker continue;
3780*9880d681SAndroid Build Coastguard Worker }
3781*9880d681SAndroid Build Coastguard Worker
3782*9880d681SAndroid Build Coastguard Worker Consensus = nullptr;
3783*9880d681SAndroid Build Coastguard Worker break;
3784*9880d681SAndroid Build Coastguard Worker }
3785*9880d681SAndroid Build Coastguard Worker
3786*9880d681SAndroid Build Coastguard Worker // If the addressing mode couldn't be determined, or if multiple different
3787*9880d681SAndroid Build Coastguard Worker // ones were determined, bail out now.
3788*9880d681SAndroid Build Coastguard Worker if (!Consensus) {
3789*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
3790*9880d681SAndroid Build Coastguard Worker return false;
3791*9880d681SAndroid Build Coastguard Worker }
3792*9880d681SAndroid Build Coastguard Worker TPT.commit();
3793*9880d681SAndroid Build Coastguard Worker
3794*9880d681SAndroid Build Coastguard Worker // Check to see if any of the instructions supersumed by this addr mode are
3795*9880d681SAndroid Build Coastguard Worker // non-local to I's BB.
3796*9880d681SAndroid Build Coastguard Worker bool AnyNonLocal = false;
3797*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = AddrModeInsts.size(); i != e; ++i) {
3798*9880d681SAndroid Build Coastguard Worker if (IsNonLocalValue(AddrModeInsts[i], MemoryInst->getParent())) {
3799*9880d681SAndroid Build Coastguard Worker AnyNonLocal = true;
3800*9880d681SAndroid Build Coastguard Worker break;
3801*9880d681SAndroid Build Coastguard Worker }
3802*9880d681SAndroid Build Coastguard Worker }
3803*9880d681SAndroid Build Coastguard Worker
3804*9880d681SAndroid Build Coastguard Worker // If all the instructions matched are already in this BB, don't do anything.
3805*9880d681SAndroid Build Coastguard Worker if (!AnyNonLocal) {
3806*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode << "\n");
3807*9880d681SAndroid Build Coastguard Worker return false;
3808*9880d681SAndroid Build Coastguard Worker }
3809*9880d681SAndroid Build Coastguard Worker
3810*9880d681SAndroid Build Coastguard Worker // Insert this computation right after this user. Since our caller is
3811*9880d681SAndroid Build Coastguard Worker // scanning from the top of the BB to the bottom, reuse of the expr are
3812*9880d681SAndroid Build Coastguard Worker // guaranteed to happen later.
3813*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(MemoryInst);
3814*9880d681SAndroid Build Coastguard Worker
3815*9880d681SAndroid Build Coastguard Worker // Now that we determined the addressing expression we want to use and know
3816*9880d681SAndroid Build Coastguard Worker // that we have to sink it into this block. Check to see if we have already
3817*9880d681SAndroid Build Coastguard Worker // done this for some other load/store instr in this block. If so, reuse the
3818*9880d681SAndroid Build Coastguard Worker // computation.
3819*9880d681SAndroid Build Coastguard Worker Value *&SunkAddr = SunkAddrs[Addr];
3820*9880d681SAndroid Build Coastguard Worker if (SunkAddr) {
3821*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for "
3822*9880d681SAndroid Build Coastguard Worker << *MemoryInst << "\n");
3823*9880d681SAndroid Build Coastguard Worker if (SunkAddr->getType() != Addr->getType())
3824*9880d681SAndroid Build Coastguard Worker SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType());
3825*9880d681SAndroid Build Coastguard Worker } else if (AddrSinkUsingGEPs ||
3826*9880d681SAndroid Build Coastguard Worker (!AddrSinkUsingGEPs.getNumOccurrences() && TM &&
3827*9880d681SAndroid Build Coastguard Worker TM->getSubtargetImpl(*MemoryInst->getParent()->getParent())
3828*9880d681SAndroid Build Coastguard Worker ->useAA())) {
3829*9880d681SAndroid Build Coastguard Worker // By default, we use the GEP-based method when AA is used later. This
3830*9880d681SAndroid Build Coastguard Worker // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities.
3831*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
3832*9880d681SAndroid Build Coastguard Worker << *MemoryInst << "\n");
3833*9880d681SAndroid Build Coastguard Worker Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
3834*9880d681SAndroid Build Coastguard Worker Value *ResultPtr = nullptr, *ResultIndex = nullptr;
3835*9880d681SAndroid Build Coastguard Worker
3836*9880d681SAndroid Build Coastguard Worker // First, find the pointer.
3837*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) {
3838*9880d681SAndroid Build Coastguard Worker ResultPtr = AddrMode.BaseReg;
3839*9880d681SAndroid Build Coastguard Worker AddrMode.BaseReg = nullptr;
3840*9880d681SAndroid Build Coastguard Worker }
3841*9880d681SAndroid Build Coastguard Worker
3842*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) {
3843*9880d681SAndroid Build Coastguard Worker // We can't add more than one pointer together, nor can we scale a
3844*9880d681SAndroid Build Coastguard Worker // pointer (both of which seem meaningless).
3845*9880d681SAndroid Build Coastguard Worker if (ResultPtr || AddrMode.Scale != 1)
3846*9880d681SAndroid Build Coastguard Worker return false;
3847*9880d681SAndroid Build Coastguard Worker
3848*9880d681SAndroid Build Coastguard Worker ResultPtr = AddrMode.ScaledReg;
3849*9880d681SAndroid Build Coastguard Worker AddrMode.Scale = 0;
3850*9880d681SAndroid Build Coastguard Worker }
3851*9880d681SAndroid Build Coastguard Worker
3852*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseGV) {
3853*9880d681SAndroid Build Coastguard Worker if (ResultPtr)
3854*9880d681SAndroid Build Coastguard Worker return false;
3855*9880d681SAndroid Build Coastguard Worker
3856*9880d681SAndroid Build Coastguard Worker ResultPtr = AddrMode.BaseGV;
3857*9880d681SAndroid Build Coastguard Worker }
3858*9880d681SAndroid Build Coastguard Worker
3859*9880d681SAndroid Build Coastguard Worker // If the real base value actually came from an inttoptr, then the matcher
3860*9880d681SAndroid Build Coastguard Worker // will look through it and provide only the integer value. In that case,
3861*9880d681SAndroid Build Coastguard Worker // use it here.
3862*9880d681SAndroid Build Coastguard Worker if (!ResultPtr && AddrMode.BaseReg) {
3863*9880d681SAndroid Build Coastguard Worker ResultPtr =
3864*9880d681SAndroid Build Coastguard Worker Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), "sunkaddr");
3865*9880d681SAndroid Build Coastguard Worker AddrMode.BaseReg = nullptr;
3866*9880d681SAndroid Build Coastguard Worker } else if (!ResultPtr && AddrMode.Scale == 1) {
3867*9880d681SAndroid Build Coastguard Worker ResultPtr =
3868*9880d681SAndroid Build Coastguard Worker Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), "sunkaddr");
3869*9880d681SAndroid Build Coastguard Worker AddrMode.Scale = 0;
3870*9880d681SAndroid Build Coastguard Worker }
3871*9880d681SAndroid Build Coastguard Worker
3872*9880d681SAndroid Build Coastguard Worker if (!ResultPtr &&
3873*9880d681SAndroid Build Coastguard Worker !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) {
3874*9880d681SAndroid Build Coastguard Worker SunkAddr = Constant::getNullValue(Addr->getType());
3875*9880d681SAndroid Build Coastguard Worker } else if (!ResultPtr) {
3876*9880d681SAndroid Build Coastguard Worker return false;
3877*9880d681SAndroid Build Coastguard Worker } else {
3878*9880d681SAndroid Build Coastguard Worker Type *I8PtrTy =
3879*9880d681SAndroid Build Coastguard Worker Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace());
3880*9880d681SAndroid Build Coastguard Worker Type *I8Ty = Builder.getInt8Ty();
3881*9880d681SAndroid Build Coastguard Worker
3882*9880d681SAndroid Build Coastguard Worker // Start with the base register. Do this first so that subsequent address
3883*9880d681SAndroid Build Coastguard Worker // matching finds it last, which will prevent it from trying to match it
3884*9880d681SAndroid Build Coastguard Worker // as the scaled value in case it happens to be a mul. That would be
3885*9880d681SAndroid Build Coastguard Worker // problematic if we've sunk a different mul for the scale, because then
3886*9880d681SAndroid Build Coastguard Worker // we'd end up sinking both muls.
3887*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseReg) {
3888*9880d681SAndroid Build Coastguard Worker Value *V = AddrMode.BaseReg;
3889*9880d681SAndroid Build Coastguard Worker if (V->getType() != IntPtrTy)
3890*9880d681SAndroid Build Coastguard Worker V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
3891*9880d681SAndroid Build Coastguard Worker
3892*9880d681SAndroid Build Coastguard Worker ResultIndex = V;
3893*9880d681SAndroid Build Coastguard Worker }
3894*9880d681SAndroid Build Coastguard Worker
3895*9880d681SAndroid Build Coastguard Worker // Add the scale value.
3896*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale) {
3897*9880d681SAndroid Build Coastguard Worker Value *V = AddrMode.ScaledReg;
3898*9880d681SAndroid Build Coastguard Worker if (V->getType() == IntPtrTy) {
3899*9880d681SAndroid Build Coastguard Worker // done.
3900*9880d681SAndroid Build Coastguard Worker } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
3901*9880d681SAndroid Build Coastguard Worker cast<IntegerType>(V->getType())->getBitWidth()) {
3902*9880d681SAndroid Build Coastguard Worker V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
3903*9880d681SAndroid Build Coastguard Worker } else {
3904*9880d681SAndroid Build Coastguard Worker // It is only safe to sign extend the BaseReg if we know that the math
3905*9880d681SAndroid Build Coastguard Worker // required to create it did not overflow before we extend it. Since
3906*9880d681SAndroid Build Coastguard Worker // the original IR value was tossed in favor of a constant back when
3907*9880d681SAndroid Build Coastguard Worker // the AddrMode was created we need to bail out gracefully if widths
3908*9880d681SAndroid Build Coastguard Worker // do not match instead of extending it.
3909*9880d681SAndroid Build Coastguard Worker Instruction *I = dyn_cast_or_null<Instruction>(ResultIndex);
3910*9880d681SAndroid Build Coastguard Worker if (I && (ResultIndex != AddrMode.BaseReg))
3911*9880d681SAndroid Build Coastguard Worker I->eraseFromParent();
3912*9880d681SAndroid Build Coastguard Worker return false;
3913*9880d681SAndroid Build Coastguard Worker }
3914*9880d681SAndroid Build Coastguard Worker
3915*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale != 1)
3916*9880d681SAndroid Build Coastguard Worker V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
3917*9880d681SAndroid Build Coastguard Worker "sunkaddr");
3918*9880d681SAndroid Build Coastguard Worker if (ResultIndex)
3919*9880d681SAndroid Build Coastguard Worker ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr");
3920*9880d681SAndroid Build Coastguard Worker else
3921*9880d681SAndroid Build Coastguard Worker ResultIndex = V;
3922*9880d681SAndroid Build Coastguard Worker }
3923*9880d681SAndroid Build Coastguard Worker
3924*9880d681SAndroid Build Coastguard Worker // Add in the Base Offset if present.
3925*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseOffs) {
3926*9880d681SAndroid Build Coastguard Worker Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
3927*9880d681SAndroid Build Coastguard Worker if (ResultIndex) {
3928*9880d681SAndroid Build Coastguard Worker // We need to add this separately from the scale above to help with
3929*9880d681SAndroid Build Coastguard Worker // SDAG consecutive load/store merging.
3930*9880d681SAndroid Build Coastguard Worker if (ResultPtr->getType() != I8PtrTy)
3931*9880d681SAndroid Build Coastguard Worker ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
3932*9880d681SAndroid Build Coastguard Worker ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr");
3933*9880d681SAndroid Build Coastguard Worker }
3934*9880d681SAndroid Build Coastguard Worker
3935*9880d681SAndroid Build Coastguard Worker ResultIndex = V;
3936*9880d681SAndroid Build Coastguard Worker }
3937*9880d681SAndroid Build Coastguard Worker
3938*9880d681SAndroid Build Coastguard Worker if (!ResultIndex) {
3939*9880d681SAndroid Build Coastguard Worker SunkAddr = ResultPtr;
3940*9880d681SAndroid Build Coastguard Worker } else {
3941*9880d681SAndroid Build Coastguard Worker if (ResultPtr->getType() != I8PtrTy)
3942*9880d681SAndroid Build Coastguard Worker ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
3943*9880d681SAndroid Build Coastguard Worker SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr");
3944*9880d681SAndroid Build Coastguard Worker }
3945*9880d681SAndroid Build Coastguard Worker
3946*9880d681SAndroid Build Coastguard Worker if (SunkAddr->getType() != Addr->getType())
3947*9880d681SAndroid Build Coastguard Worker SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType());
3948*9880d681SAndroid Build Coastguard Worker }
3949*9880d681SAndroid Build Coastguard Worker } else {
3950*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
3951*9880d681SAndroid Build Coastguard Worker << *MemoryInst << "\n");
3952*9880d681SAndroid Build Coastguard Worker Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
3953*9880d681SAndroid Build Coastguard Worker Value *Result = nullptr;
3954*9880d681SAndroid Build Coastguard Worker
3955*9880d681SAndroid Build Coastguard Worker // Start with the base register. Do this first so that subsequent address
3956*9880d681SAndroid Build Coastguard Worker // matching finds it last, which will prevent it from trying to match it
3957*9880d681SAndroid Build Coastguard Worker // as the scaled value in case it happens to be a mul. That would be
3958*9880d681SAndroid Build Coastguard Worker // problematic if we've sunk a different mul for the scale, because then
3959*9880d681SAndroid Build Coastguard Worker // we'd end up sinking both muls.
3960*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseReg) {
3961*9880d681SAndroid Build Coastguard Worker Value *V = AddrMode.BaseReg;
3962*9880d681SAndroid Build Coastguard Worker if (V->getType()->isPointerTy())
3963*9880d681SAndroid Build Coastguard Worker V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
3964*9880d681SAndroid Build Coastguard Worker if (V->getType() != IntPtrTy)
3965*9880d681SAndroid Build Coastguard Worker V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
3966*9880d681SAndroid Build Coastguard Worker Result = V;
3967*9880d681SAndroid Build Coastguard Worker }
3968*9880d681SAndroid Build Coastguard Worker
3969*9880d681SAndroid Build Coastguard Worker // Add the scale value.
3970*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale) {
3971*9880d681SAndroid Build Coastguard Worker Value *V = AddrMode.ScaledReg;
3972*9880d681SAndroid Build Coastguard Worker if (V->getType() == IntPtrTy) {
3973*9880d681SAndroid Build Coastguard Worker // done.
3974*9880d681SAndroid Build Coastguard Worker } else if (V->getType()->isPointerTy()) {
3975*9880d681SAndroid Build Coastguard Worker V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
3976*9880d681SAndroid Build Coastguard Worker } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
3977*9880d681SAndroid Build Coastguard Worker cast<IntegerType>(V->getType())->getBitWidth()) {
3978*9880d681SAndroid Build Coastguard Worker V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
3979*9880d681SAndroid Build Coastguard Worker } else {
3980*9880d681SAndroid Build Coastguard Worker // It is only safe to sign extend the BaseReg if we know that the math
3981*9880d681SAndroid Build Coastguard Worker // required to create it did not overflow before we extend it. Since
3982*9880d681SAndroid Build Coastguard Worker // the original IR value was tossed in favor of a constant back when
3983*9880d681SAndroid Build Coastguard Worker // the AddrMode was created we need to bail out gracefully if widths
3984*9880d681SAndroid Build Coastguard Worker // do not match instead of extending it.
3985*9880d681SAndroid Build Coastguard Worker Instruction *I = dyn_cast_or_null<Instruction>(Result);
3986*9880d681SAndroid Build Coastguard Worker if (I && (Result != AddrMode.BaseReg))
3987*9880d681SAndroid Build Coastguard Worker I->eraseFromParent();
3988*9880d681SAndroid Build Coastguard Worker return false;
3989*9880d681SAndroid Build Coastguard Worker }
3990*9880d681SAndroid Build Coastguard Worker if (AddrMode.Scale != 1)
3991*9880d681SAndroid Build Coastguard Worker V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
3992*9880d681SAndroid Build Coastguard Worker "sunkaddr");
3993*9880d681SAndroid Build Coastguard Worker if (Result)
3994*9880d681SAndroid Build Coastguard Worker Result = Builder.CreateAdd(Result, V, "sunkaddr");
3995*9880d681SAndroid Build Coastguard Worker else
3996*9880d681SAndroid Build Coastguard Worker Result = V;
3997*9880d681SAndroid Build Coastguard Worker }
3998*9880d681SAndroid Build Coastguard Worker
3999*9880d681SAndroid Build Coastguard Worker // Add in the BaseGV if present.
4000*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseGV) {
4001*9880d681SAndroid Build Coastguard Worker Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr");
4002*9880d681SAndroid Build Coastguard Worker if (Result)
4003*9880d681SAndroid Build Coastguard Worker Result = Builder.CreateAdd(Result, V, "sunkaddr");
4004*9880d681SAndroid Build Coastguard Worker else
4005*9880d681SAndroid Build Coastguard Worker Result = V;
4006*9880d681SAndroid Build Coastguard Worker }
4007*9880d681SAndroid Build Coastguard Worker
4008*9880d681SAndroid Build Coastguard Worker // Add in the Base Offset if present.
4009*9880d681SAndroid Build Coastguard Worker if (AddrMode.BaseOffs) {
4010*9880d681SAndroid Build Coastguard Worker Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
4011*9880d681SAndroid Build Coastguard Worker if (Result)
4012*9880d681SAndroid Build Coastguard Worker Result = Builder.CreateAdd(Result, V, "sunkaddr");
4013*9880d681SAndroid Build Coastguard Worker else
4014*9880d681SAndroid Build Coastguard Worker Result = V;
4015*9880d681SAndroid Build Coastguard Worker }
4016*9880d681SAndroid Build Coastguard Worker
4017*9880d681SAndroid Build Coastguard Worker if (!Result)
4018*9880d681SAndroid Build Coastguard Worker SunkAddr = Constant::getNullValue(Addr->getType());
4019*9880d681SAndroid Build Coastguard Worker else
4020*9880d681SAndroid Build Coastguard Worker SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr");
4021*9880d681SAndroid Build Coastguard Worker }
4022*9880d681SAndroid Build Coastguard Worker
4023*9880d681SAndroid Build Coastguard Worker MemoryInst->replaceUsesOfWith(Repl, SunkAddr);
4024*9880d681SAndroid Build Coastguard Worker
4025*9880d681SAndroid Build Coastguard Worker // If we have no uses, recursively delete the value and all dead instructions
4026*9880d681SAndroid Build Coastguard Worker // using it.
4027*9880d681SAndroid Build Coastguard Worker if (Repl->use_empty()) {
4028*9880d681SAndroid Build Coastguard Worker // This can cause recursive deletion, which can invalidate our iterator.
4029*9880d681SAndroid Build Coastguard Worker // Use a WeakVH to hold onto it in case this happens.
4030*9880d681SAndroid Build Coastguard Worker Value *CurValue = &*CurInstIterator;
4031*9880d681SAndroid Build Coastguard Worker WeakVH IterHandle(CurValue);
4032*9880d681SAndroid Build Coastguard Worker BasicBlock *BB = CurInstIterator->getParent();
4033*9880d681SAndroid Build Coastguard Worker
4034*9880d681SAndroid Build Coastguard Worker RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo);
4035*9880d681SAndroid Build Coastguard Worker
4036*9880d681SAndroid Build Coastguard Worker if (IterHandle != CurValue) {
4037*9880d681SAndroid Build Coastguard Worker // If the iterator instruction was recursively deleted, start over at the
4038*9880d681SAndroid Build Coastguard Worker // start of the block.
4039*9880d681SAndroid Build Coastguard Worker CurInstIterator = BB->begin();
4040*9880d681SAndroid Build Coastguard Worker SunkAddrs.clear();
4041*9880d681SAndroid Build Coastguard Worker }
4042*9880d681SAndroid Build Coastguard Worker }
4043*9880d681SAndroid Build Coastguard Worker ++NumMemoryInsts;
4044*9880d681SAndroid Build Coastguard Worker return true;
4045*9880d681SAndroid Build Coastguard Worker }
4046*9880d681SAndroid Build Coastguard Worker
4047*9880d681SAndroid Build Coastguard Worker /// If there are any memory operands, use OptimizeMemoryInst to sink their
4048*9880d681SAndroid Build Coastguard Worker /// address computing into the block when possible / profitable.
optimizeInlineAsmInst(CallInst * CS)4049*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
4050*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
4051*9880d681SAndroid Build Coastguard Worker
4052*9880d681SAndroid Build Coastguard Worker const TargetRegisterInfo *TRI =
4053*9880d681SAndroid Build Coastguard Worker TM->getSubtargetImpl(*CS->getParent()->getParent())->getRegisterInfo();
4054*9880d681SAndroid Build Coastguard Worker TargetLowering::AsmOperandInfoVector TargetConstraints =
4055*9880d681SAndroid Build Coastguard Worker TLI->ParseConstraints(*DL, TRI, CS);
4056*9880d681SAndroid Build Coastguard Worker unsigned ArgNo = 0;
4057*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
4058*9880d681SAndroid Build Coastguard Worker TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
4059*9880d681SAndroid Build Coastguard Worker
4060*9880d681SAndroid Build Coastguard Worker // Compute the constraint code and ConstraintType to use.
4061*9880d681SAndroid Build Coastguard Worker TLI->ComputeConstraintToUse(OpInfo, SDValue());
4062*9880d681SAndroid Build Coastguard Worker
4063*9880d681SAndroid Build Coastguard Worker if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
4064*9880d681SAndroid Build Coastguard Worker OpInfo.isIndirect) {
4065*9880d681SAndroid Build Coastguard Worker Value *OpVal = CS->getArgOperand(ArgNo++);
4066*9880d681SAndroid Build Coastguard Worker MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u);
4067*9880d681SAndroid Build Coastguard Worker } else if (OpInfo.Type == InlineAsm::isInput)
4068*9880d681SAndroid Build Coastguard Worker ArgNo++;
4069*9880d681SAndroid Build Coastguard Worker }
4070*9880d681SAndroid Build Coastguard Worker
4071*9880d681SAndroid Build Coastguard Worker return MadeChange;
4072*9880d681SAndroid Build Coastguard Worker }
4073*9880d681SAndroid Build Coastguard Worker
4074*9880d681SAndroid Build Coastguard Worker /// \brief Check if all the uses of \p Inst are equivalent (or free) zero or
4075*9880d681SAndroid Build Coastguard Worker /// sign extensions.
hasSameExtUse(Instruction * Inst,const TargetLowering & TLI)4076*9880d681SAndroid Build Coastguard Worker static bool hasSameExtUse(Instruction *Inst, const TargetLowering &TLI) {
4077*9880d681SAndroid Build Coastguard Worker assert(!Inst->use_empty() && "Input must have at least one use");
4078*9880d681SAndroid Build Coastguard Worker const Instruction *FirstUser = cast<Instruction>(*Inst->user_begin());
4079*9880d681SAndroid Build Coastguard Worker bool IsSExt = isa<SExtInst>(FirstUser);
4080*9880d681SAndroid Build Coastguard Worker Type *ExtTy = FirstUser->getType();
4081*9880d681SAndroid Build Coastguard Worker for (const User *U : Inst->users()) {
4082*9880d681SAndroid Build Coastguard Worker const Instruction *UI = cast<Instruction>(U);
4083*9880d681SAndroid Build Coastguard Worker if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI)))
4084*9880d681SAndroid Build Coastguard Worker return false;
4085*9880d681SAndroid Build Coastguard Worker Type *CurTy = UI->getType();
4086*9880d681SAndroid Build Coastguard Worker // Same input and output types: Same instruction after CSE.
4087*9880d681SAndroid Build Coastguard Worker if (CurTy == ExtTy)
4088*9880d681SAndroid Build Coastguard Worker continue;
4089*9880d681SAndroid Build Coastguard Worker
4090*9880d681SAndroid Build Coastguard Worker // If IsSExt is true, we are in this situation:
4091*9880d681SAndroid Build Coastguard Worker // a = Inst
4092*9880d681SAndroid Build Coastguard Worker // b = sext ty1 a to ty2
4093*9880d681SAndroid Build Coastguard Worker // c = sext ty1 a to ty3
4094*9880d681SAndroid Build Coastguard Worker // Assuming ty2 is shorter than ty3, this could be turned into:
4095*9880d681SAndroid Build Coastguard Worker // a = Inst
4096*9880d681SAndroid Build Coastguard Worker // b = sext ty1 a to ty2
4097*9880d681SAndroid Build Coastguard Worker // c = sext ty2 b to ty3
4098*9880d681SAndroid Build Coastguard Worker // However, the last sext is not free.
4099*9880d681SAndroid Build Coastguard Worker if (IsSExt)
4100*9880d681SAndroid Build Coastguard Worker return false;
4101*9880d681SAndroid Build Coastguard Worker
4102*9880d681SAndroid Build Coastguard Worker // This is a ZExt, maybe this is free to extend from one type to another.
4103*9880d681SAndroid Build Coastguard Worker // In that case, we would not account for a different use.
4104*9880d681SAndroid Build Coastguard Worker Type *NarrowTy;
4105*9880d681SAndroid Build Coastguard Worker Type *LargeTy;
4106*9880d681SAndroid Build Coastguard Worker if (ExtTy->getScalarType()->getIntegerBitWidth() >
4107*9880d681SAndroid Build Coastguard Worker CurTy->getScalarType()->getIntegerBitWidth()) {
4108*9880d681SAndroid Build Coastguard Worker NarrowTy = CurTy;
4109*9880d681SAndroid Build Coastguard Worker LargeTy = ExtTy;
4110*9880d681SAndroid Build Coastguard Worker } else {
4111*9880d681SAndroid Build Coastguard Worker NarrowTy = ExtTy;
4112*9880d681SAndroid Build Coastguard Worker LargeTy = CurTy;
4113*9880d681SAndroid Build Coastguard Worker }
4114*9880d681SAndroid Build Coastguard Worker
4115*9880d681SAndroid Build Coastguard Worker if (!TLI.isZExtFree(NarrowTy, LargeTy))
4116*9880d681SAndroid Build Coastguard Worker return false;
4117*9880d681SAndroid Build Coastguard Worker }
4118*9880d681SAndroid Build Coastguard Worker // All uses are the same or can be derived from one another for free.
4119*9880d681SAndroid Build Coastguard Worker return true;
4120*9880d681SAndroid Build Coastguard Worker }
4121*9880d681SAndroid Build Coastguard Worker
4122*9880d681SAndroid Build Coastguard Worker /// \brief Try to form ExtLd by promoting \p Exts until they reach a
4123*9880d681SAndroid Build Coastguard Worker /// load instruction.
4124*9880d681SAndroid Build Coastguard Worker /// If an ext(load) can be formed, it is returned via \p LI for the load
4125*9880d681SAndroid Build Coastguard Worker /// and \p Inst for the extension.
4126*9880d681SAndroid Build Coastguard Worker /// Otherwise LI == nullptr and Inst == nullptr.
4127*9880d681SAndroid Build Coastguard Worker /// When some promotion happened, \p TPT contains the proper state to
4128*9880d681SAndroid Build Coastguard Worker /// revert them.
4129*9880d681SAndroid Build Coastguard Worker ///
4130*9880d681SAndroid Build Coastguard Worker /// \return true when promoting was necessary to expose the ext(load)
4131*9880d681SAndroid Build Coastguard Worker /// opportunity, false otherwise.
4132*9880d681SAndroid Build Coastguard Worker ///
4133*9880d681SAndroid Build Coastguard Worker /// Example:
4134*9880d681SAndroid Build Coastguard Worker /// \code
4135*9880d681SAndroid Build Coastguard Worker /// %ld = load i32* %addr
4136*9880d681SAndroid Build Coastguard Worker /// %add = add nuw i32 %ld, 4
4137*9880d681SAndroid Build Coastguard Worker /// %zext = zext i32 %add to i64
4138*9880d681SAndroid Build Coastguard Worker /// \endcode
4139*9880d681SAndroid Build Coastguard Worker /// =>
4140*9880d681SAndroid Build Coastguard Worker /// \code
4141*9880d681SAndroid Build Coastguard Worker /// %ld = load i32* %addr
4142*9880d681SAndroid Build Coastguard Worker /// %zext = zext i32 %ld to i64
4143*9880d681SAndroid Build Coastguard Worker /// %add = add nuw i64 %zext, 4
4144*9880d681SAndroid Build Coastguard Worker /// \encode
4145*9880d681SAndroid Build Coastguard Worker /// Thanks to the promotion, we can match zext(load i32*) to i64.
extLdPromotion(TypePromotionTransaction & TPT,LoadInst * & LI,Instruction * & Inst,const SmallVectorImpl<Instruction * > & Exts,unsigned CreatedInstsCost=0)4146*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::extLdPromotion(TypePromotionTransaction &TPT,
4147*9880d681SAndroid Build Coastguard Worker LoadInst *&LI, Instruction *&Inst,
4148*9880d681SAndroid Build Coastguard Worker const SmallVectorImpl<Instruction *> &Exts,
4149*9880d681SAndroid Build Coastguard Worker unsigned CreatedInstsCost = 0) {
4150*9880d681SAndroid Build Coastguard Worker // Iterate over all the extensions to see if one form an ext(load).
4151*9880d681SAndroid Build Coastguard Worker for (auto I : Exts) {
4152*9880d681SAndroid Build Coastguard Worker // Check if we directly have ext(load).
4153*9880d681SAndroid Build Coastguard Worker if ((LI = dyn_cast<LoadInst>(I->getOperand(0)))) {
4154*9880d681SAndroid Build Coastguard Worker Inst = I;
4155*9880d681SAndroid Build Coastguard Worker // No promotion happened here.
4156*9880d681SAndroid Build Coastguard Worker return false;
4157*9880d681SAndroid Build Coastguard Worker }
4158*9880d681SAndroid Build Coastguard Worker // Check whether or not we want to do any promotion.
4159*9880d681SAndroid Build Coastguard Worker if (!TLI || !TLI->enableExtLdPromotion() || DisableExtLdPromotion)
4160*9880d681SAndroid Build Coastguard Worker continue;
4161*9880d681SAndroid Build Coastguard Worker // Get the action to perform the promotion.
4162*9880d681SAndroid Build Coastguard Worker TypePromotionHelper::Action TPH = TypePromotionHelper::getAction(
4163*9880d681SAndroid Build Coastguard Worker I, InsertedInsts, *TLI, PromotedInsts);
4164*9880d681SAndroid Build Coastguard Worker // Check if we can promote.
4165*9880d681SAndroid Build Coastguard Worker if (!TPH)
4166*9880d681SAndroid Build Coastguard Worker continue;
4167*9880d681SAndroid Build Coastguard Worker // Save the current state.
4168*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4169*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
4170*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction *, 4> NewExts;
4171*9880d681SAndroid Build Coastguard Worker unsigned NewCreatedInstsCost = 0;
4172*9880d681SAndroid Build Coastguard Worker unsigned ExtCost = !TLI->isExtFree(I);
4173*9880d681SAndroid Build Coastguard Worker // Promote.
4174*9880d681SAndroid Build Coastguard Worker Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost,
4175*9880d681SAndroid Build Coastguard Worker &NewExts, nullptr, *TLI);
4176*9880d681SAndroid Build Coastguard Worker assert(PromotedVal &&
4177*9880d681SAndroid Build Coastguard Worker "TypePromotionHelper should have filtered out those cases");
4178*9880d681SAndroid Build Coastguard Worker
4179*9880d681SAndroid Build Coastguard Worker // We would be able to merge only one extension in a load.
4180*9880d681SAndroid Build Coastguard Worker // Therefore, if we have more than 1 new extension we heuristically
4181*9880d681SAndroid Build Coastguard Worker // cut this search path, because it means we degrade the code quality.
4182*9880d681SAndroid Build Coastguard Worker // With exactly 2, the transformation is neutral, because we will merge
4183*9880d681SAndroid Build Coastguard Worker // one extension but leave one. However, we optimistically keep going,
4184*9880d681SAndroid Build Coastguard Worker // because the new extension may be removed too.
4185*9880d681SAndroid Build Coastguard Worker long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;
4186*9880d681SAndroid Build Coastguard Worker TotalCreatedInstsCost -= ExtCost;
4187*9880d681SAndroid Build Coastguard Worker if (!StressExtLdPromotion &&
4188*9880d681SAndroid Build Coastguard Worker (TotalCreatedInstsCost > 1 ||
4189*9880d681SAndroid Build Coastguard Worker !isPromotedInstructionLegal(*TLI, *DL, PromotedVal))) {
4190*9880d681SAndroid Build Coastguard Worker // The promotion is not profitable, rollback to the previous state.
4191*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
4192*9880d681SAndroid Build Coastguard Worker continue;
4193*9880d681SAndroid Build Coastguard Worker }
4194*9880d681SAndroid Build Coastguard Worker // The promotion is profitable.
4195*9880d681SAndroid Build Coastguard Worker // Check if it exposes an ext(load).
4196*9880d681SAndroid Build Coastguard Worker (void)extLdPromotion(TPT, LI, Inst, NewExts, TotalCreatedInstsCost);
4197*9880d681SAndroid Build Coastguard Worker if (LI && (StressExtLdPromotion || NewCreatedInstsCost <= ExtCost ||
4198*9880d681SAndroid Build Coastguard Worker // If we have created a new extension, i.e., now we have two
4199*9880d681SAndroid Build Coastguard Worker // extensions. We must make sure one of them is merged with
4200*9880d681SAndroid Build Coastguard Worker // the load, otherwise we may degrade the code quality.
4201*9880d681SAndroid Build Coastguard Worker (LI->hasOneUse() || hasSameExtUse(LI, *TLI))))
4202*9880d681SAndroid Build Coastguard Worker // Promotion happened.
4203*9880d681SAndroid Build Coastguard Worker return true;
4204*9880d681SAndroid Build Coastguard Worker // If this does not help to expose an ext(load) then, rollback.
4205*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
4206*9880d681SAndroid Build Coastguard Worker }
4207*9880d681SAndroid Build Coastguard Worker // None of the extension can form an ext(load).
4208*9880d681SAndroid Build Coastguard Worker LI = nullptr;
4209*9880d681SAndroid Build Coastguard Worker Inst = nullptr;
4210*9880d681SAndroid Build Coastguard Worker return false;
4211*9880d681SAndroid Build Coastguard Worker }
4212*9880d681SAndroid Build Coastguard Worker
4213*9880d681SAndroid Build Coastguard Worker /// Move a zext or sext fed by a load into the same basic block as the load,
4214*9880d681SAndroid Build Coastguard Worker /// unless conditions are unfavorable. This allows SelectionDAG to fold the
4215*9880d681SAndroid Build Coastguard Worker /// extend into the load.
4216*9880d681SAndroid Build Coastguard Worker /// \p I[in/out] the extension may be modified during the process if some
4217*9880d681SAndroid Build Coastguard Worker /// promotions apply.
4218*9880d681SAndroid Build Coastguard Worker ///
moveExtToFormExtLoad(Instruction * & I)4219*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::moveExtToFormExtLoad(Instruction *&I) {
4220*9880d681SAndroid Build Coastguard Worker // Try to promote a chain of computation if it allows to form
4221*9880d681SAndroid Build Coastguard Worker // an extended load.
4222*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction TPT;
4223*9880d681SAndroid Build Coastguard Worker TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4224*9880d681SAndroid Build Coastguard Worker TPT.getRestorationPoint();
4225*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction *, 1> Exts;
4226*9880d681SAndroid Build Coastguard Worker Exts.push_back(I);
4227*9880d681SAndroid Build Coastguard Worker // Look for a load being extended.
4228*9880d681SAndroid Build Coastguard Worker LoadInst *LI = nullptr;
4229*9880d681SAndroid Build Coastguard Worker Instruction *OldExt = I;
4230*9880d681SAndroid Build Coastguard Worker bool HasPromoted = extLdPromotion(TPT, LI, I, Exts);
4231*9880d681SAndroid Build Coastguard Worker if (!LI || !I) {
4232*9880d681SAndroid Build Coastguard Worker assert(!HasPromoted && !LI && "If we did not match any load instruction "
4233*9880d681SAndroid Build Coastguard Worker "the code must remain the same");
4234*9880d681SAndroid Build Coastguard Worker I = OldExt;
4235*9880d681SAndroid Build Coastguard Worker return false;
4236*9880d681SAndroid Build Coastguard Worker }
4237*9880d681SAndroid Build Coastguard Worker
4238*9880d681SAndroid Build Coastguard Worker // If they're already in the same block, there's nothing to do.
4239*9880d681SAndroid Build Coastguard Worker // Make the cheap checks first if we did not promote.
4240*9880d681SAndroid Build Coastguard Worker // If we promoted, we need to check if it is indeed profitable.
4241*9880d681SAndroid Build Coastguard Worker if (!HasPromoted && LI->getParent() == I->getParent())
4242*9880d681SAndroid Build Coastguard Worker return false;
4243*9880d681SAndroid Build Coastguard Worker
4244*9880d681SAndroid Build Coastguard Worker EVT VT = TLI->getValueType(*DL, I->getType());
4245*9880d681SAndroid Build Coastguard Worker EVT LoadVT = TLI->getValueType(*DL, LI->getType());
4246*9880d681SAndroid Build Coastguard Worker
4247*9880d681SAndroid Build Coastguard Worker // If the load has other users and the truncate is not free, this probably
4248*9880d681SAndroid Build Coastguard Worker // isn't worthwhile.
4249*9880d681SAndroid Build Coastguard Worker if (!LI->hasOneUse() && TLI &&
4250*9880d681SAndroid Build Coastguard Worker (TLI->isTypeLegal(LoadVT) || !TLI->isTypeLegal(VT)) &&
4251*9880d681SAndroid Build Coastguard Worker !TLI->isTruncateFree(I->getType(), LI->getType())) {
4252*9880d681SAndroid Build Coastguard Worker I = OldExt;
4253*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
4254*9880d681SAndroid Build Coastguard Worker return false;
4255*9880d681SAndroid Build Coastguard Worker }
4256*9880d681SAndroid Build Coastguard Worker
4257*9880d681SAndroid Build Coastguard Worker // Check whether the target supports casts folded into loads.
4258*9880d681SAndroid Build Coastguard Worker unsigned LType;
4259*9880d681SAndroid Build Coastguard Worker if (isa<ZExtInst>(I))
4260*9880d681SAndroid Build Coastguard Worker LType = ISD::ZEXTLOAD;
4261*9880d681SAndroid Build Coastguard Worker else {
4262*9880d681SAndroid Build Coastguard Worker assert(isa<SExtInst>(I) && "Unexpected ext type!");
4263*9880d681SAndroid Build Coastguard Worker LType = ISD::SEXTLOAD;
4264*9880d681SAndroid Build Coastguard Worker }
4265*9880d681SAndroid Build Coastguard Worker if (TLI && !TLI->isLoadExtLegal(LType, VT, LoadVT)) {
4266*9880d681SAndroid Build Coastguard Worker I = OldExt;
4267*9880d681SAndroid Build Coastguard Worker TPT.rollback(LastKnownGood);
4268*9880d681SAndroid Build Coastguard Worker return false;
4269*9880d681SAndroid Build Coastguard Worker }
4270*9880d681SAndroid Build Coastguard Worker
4271*9880d681SAndroid Build Coastguard Worker // Move the extend into the same block as the load, so that SelectionDAG
4272*9880d681SAndroid Build Coastguard Worker // can fold it.
4273*9880d681SAndroid Build Coastguard Worker TPT.commit();
4274*9880d681SAndroid Build Coastguard Worker I->removeFromParent();
4275*9880d681SAndroid Build Coastguard Worker I->insertAfter(LI);
4276*9880d681SAndroid Build Coastguard Worker ++NumExtsMoved;
4277*9880d681SAndroid Build Coastguard Worker return true;
4278*9880d681SAndroid Build Coastguard Worker }
4279*9880d681SAndroid Build Coastguard Worker
optimizeExtUses(Instruction * I)4280*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeExtUses(Instruction *I) {
4281*9880d681SAndroid Build Coastguard Worker BasicBlock *DefBB = I->getParent();
4282*9880d681SAndroid Build Coastguard Worker
4283*9880d681SAndroid Build Coastguard Worker // If the result of a {s|z}ext and its source are both live out, rewrite all
4284*9880d681SAndroid Build Coastguard Worker // other uses of the source with result of extension.
4285*9880d681SAndroid Build Coastguard Worker Value *Src = I->getOperand(0);
4286*9880d681SAndroid Build Coastguard Worker if (Src->hasOneUse())
4287*9880d681SAndroid Build Coastguard Worker return false;
4288*9880d681SAndroid Build Coastguard Worker
4289*9880d681SAndroid Build Coastguard Worker // Only do this xform if truncating is free.
4290*9880d681SAndroid Build Coastguard Worker if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType()))
4291*9880d681SAndroid Build Coastguard Worker return false;
4292*9880d681SAndroid Build Coastguard Worker
4293*9880d681SAndroid Build Coastguard Worker // Only safe to perform the optimization if the source is also defined in
4294*9880d681SAndroid Build Coastguard Worker // this block.
4295*9880d681SAndroid Build Coastguard Worker if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent())
4296*9880d681SAndroid Build Coastguard Worker return false;
4297*9880d681SAndroid Build Coastguard Worker
4298*9880d681SAndroid Build Coastguard Worker bool DefIsLiveOut = false;
4299*9880d681SAndroid Build Coastguard Worker for (User *U : I->users()) {
4300*9880d681SAndroid Build Coastguard Worker Instruction *UI = cast<Instruction>(U);
4301*9880d681SAndroid Build Coastguard Worker
4302*9880d681SAndroid Build Coastguard Worker // Figure out which BB this ext is used in.
4303*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = UI->getParent();
4304*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) continue;
4305*9880d681SAndroid Build Coastguard Worker DefIsLiveOut = true;
4306*9880d681SAndroid Build Coastguard Worker break;
4307*9880d681SAndroid Build Coastguard Worker }
4308*9880d681SAndroid Build Coastguard Worker if (!DefIsLiveOut)
4309*9880d681SAndroid Build Coastguard Worker return false;
4310*9880d681SAndroid Build Coastguard Worker
4311*9880d681SAndroid Build Coastguard Worker // Make sure none of the uses are PHI nodes.
4312*9880d681SAndroid Build Coastguard Worker for (User *U : Src->users()) {
4313*9880d681SAndroid Build Coastguard Worker Instruction *UI = cast<Instruction>(U);
4314*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = UI->getParent();
4315*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) continue;
4316*9880d681SAndroid Build Coastguard Worker // Be conservative. We don't want this xform to end up introducing
4317*9880d681SAndroid Build Coastguard Worker // reloads just before load / store instructions.
4318*9880d681SAndroid Build Coastguard Worker if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI))
4319*9880d681SAndroid Build Coastguard Worker return false;
4320*9880d681SAndroid Build Coastguard Worker }
4321*9880d681SAndroid Build Coastguard Worker
4322*9880d681SAndroid Build Coastguard Worker // InsertedTruncs - Only insert one trunc in each block once.
4323*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock*, Instruction*> InsertedTruncs;
4324*9880d681SAndroid Build Coastguard Worker
4325*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
4326*9880d681SAndroid Build Coastguard Worker for (Use &U : Src->uses()) {
4327*9880d681SAndroid Build Coastguard Worker Instruction *User = cast<Instruction>(U.getUser());
4328*9880d681SAndroid Build Coastguard Worker
4329*9880d681SAndroid Build Coastguard Worker // Figure out which BB this ext is used in.
4330*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = User->getParent();
4331*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) continue;
4332*9880d681SAndroid Build Coastguard Worker
4333*9880d681SAndroid Build Coastguard Worker // Both src and def are live in this block. Rewrite the use.
4334*9880d681SAndroid Build Coastguard Worker Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
4335*9880d681SAndroid Build Coastguard Worker
4336*9880d681SAndroid Build Coastguard Worker if (!InsertedTrunc) {
4337*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
4338*9880d681SAndroid Build Coastguard Worker assert(InsertPt != UserBB->end());
4339*9880d681SAndroid Build Coastguard Worker InsertedTrunc = new TruncInst(I, Src->getType(), "", &*InsertPt);
4340*9880d681SAndroid Build Coastguard Worker InsertedInsts.insert(InsertedTrunc);
4341*9880d681SAndroid Build Coastguard Worker }
4342*9880d681SAndroid Build Coastguard Worker
4343*9880d681SAndroid Build Coastguard Worker // Replace a use of the {s|z}ext source with a use of the result.
4344*9880d681SAndroid Build Coastguard Worker U = InsertedTrunc;
4345*9880d681SAndroid Build Coastguard Worker ++NumExtUses;
4346*9880d681SAndroid Build Coastguard Worker MadeChange = true;
4347*9880d681SAndroid Build Coastguard Worker }
4348*9880d681SAndroid Build Coastguard Worker
4349*9880d681SAndroid Build Coastguard Worker return MadeChange;
4350*9880d681SAndroid Build Coastguard Worker }
4351*9880d681SAndroid Build Coastguard Worker
4352*9880d681SAndroid Build Coastguard Worker // Find loads whose uses only use some of the loaded value's bits. Add an "and"
4353*9880d681SAndroid Build Coastguard Worker // just after the load if the target can fold this into one extload instruction,
4354*9880d681SAndroid Build Coastguard Worker // with the hope of eliminating some of the other later "and" instructions using
4355*9880d681SAndroid Build Coastguard Worker // the loaded value. "and"s that are made trivially redundant by the insertion
4356*9880d681SAndroid Build Coastguard Worker // of the new "and" are removed by this function, while others (e.g. those whose
4357*9880d681SAndroid Build Coastguard Worker // path from the load goes through a phi) are left for isel to potentially
4358*9880d681SAndroid Build Coastguard Worker // remove.
4359*9880d681SAndroid Build Coastguard Worker //
4360*9880d681SAndroid Build Coastguard Worker // For example:
4361*9880d681SAndroid Build Coastguard Worker //
4362*9880d681SAndroid Build Coastguard Worker // b0:
4363*9880d681SAndroid Build Coastguard Worker // x = load i32
4364*9880d681SAndroid Build Coastguard Worker // ...
4365*9880d681SAndroid Build Coastguard Worker // b1:
4366*9880d681SAndroid Build Coastguard Worker // y = and x, 0xff
4367*9880d681SAndroid Build Coastguard Worker // z = use y
4368*9880d681SAndroid Build Coastguard Worker //
4369*9880d681SAndroid Build Coastguard Worker // becomes:
4370*9880d681SAndroid Build Coastguard Worker //
4371*9880d681SAndroid Build Coastguard Worker // b0:
4372*9880d681SAndroid Build Coastguard Worker // x = load i32
4373*9880d681SAndroid Build Coastguard Worker // x' = and x, 0xff
4374*9880d681SAndroid Build Coastguard Worker // ...
4375*9880d681SAndroid Build Coastguard Worker // b1:
4376*9880d681SAndroid Build Coastguard Worker // z = use x'
4377*9880d681SAndroid Build Coastguard Worker //
4378*9880d681SAndroid Build Coastguard Worker // whereas:
4379*9880d681SAndroid Build Coastguard Worker //
4380*9880d681SAndroid Build Coastguard Worker // b0:
4381*9880d681SAndroid Build Coastguard Worker // x1 = load i32
4382*9880d681SAndroid Build Coastguard Worker // ...
4383*9880d681SAndroid Build Coastguard Worker // b1:
4384*9880d681SAndroid Build Coastguard Worker // x2 = load i32
4385*9880d681SAndroid Build Coastguard Worker // ...
4386*9880d681SAndroid Build Coastguard Worker // b2:
4387*9880d681SAndroid Build Coastguard Worker // x = phi x1, x2
4388*9880d681SAndroid Build Coastguard Worker // y = and x, 0xff
4389*9880d681SAndroid Build Coastguard Worker //
4390*9880d681SAndroid Build Coastguard Worker // becomes (after a call to optimizeLoadExt for each load):
4391*9880d681SAndroid Build Coastguard Worker //
4392*9880d681SAndroid Build Coastguard Worker // b0:
4393*9880d681SAndroid Build Coastguard Worker // x1 = load i32
4394*9880d681SAndroid Build Coastguard Worker // x1' = and x1, 0xff
4395*9880d681SAndroid Build Coastguard Worker // ...
4396*9880d681SAndroid Build Coastguard Worker // b1:
4397*9880d681SAndroid Build Coastguard Worker // x2 = load i32
4398*9880d681SAndroid Build Coastguard Worker // x2' = and x2, 0xff
4399*9880d681SAndroid Build Coastguard Worker // ...
4400*9880d681SAndroid Build Coastguard Worker // b2:
4401*9880d681SAndroid Build Coastguard Worker // x = phi x1', x2'
4402*9880d681SAndroid Build Coastguard Worker // y = and x, 0xff
4403*9880d681SAndroid Build Coastguard Worker //
4404*9880d681SAndroid Build Coastguard Worker
optimizeLoadExt(LoadInst * Load)4405*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
4406*9880d681SAndroid Build Coastguard Worker
4407*9880d681SAndroid Build Coastguard Worker if (!Load->isSimple() ||
4408*9880d681SAndroid Build Coastguard Worker !(Load->getType()->isIntegerTy() || Load->getType()->isPointerTy()))
4409*9880d681SAndroid Build Coastguard Worker return false;
4410*9880d681SAndroid Build Coastguard Worker
4411*9880d681SAndroid Build Coastguard Worker // Skip loads we've already transformed or have no reason to transform.
4412*9880d681SAndroid Build Coastguard Worker if (Load->hasOneUse()) {
4413*9880d681SAndroid Build Coastguard Worker User *LoadUser = *Load->user_begin();
4414*9880d681SAndroid Build Coastguard Worker if (cast<Instruction>(LoadUser)->getParent() == Load->getParent() &&
4415*9880d681SAndroid Build Coastguard Worker !dyn_cast<PHINode>(LoadUser))
4416*9880d681SAndroid Build Coastguard Worker return false;
4417*9880d681SAndroid Build Coastguard Worker }
4418*9880d681SAndroid Build Coastguard Worker
4419*9880d681SAndroid Build Coastguard Worker // Look at all uses of Load, looking through phis, to determine how many bits
4420*9880d681SAndroid Build Coastguard Worker // of the loaded value are needed.
4421*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction *, 8> WorkList;
4422*9880d681SAndroid Build Coastguard Worker SmallPtrSet<Instruction *, 16> Visited;
4423*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction *, 8> AndsToMaybeRemove;
4424*9880d681SAndroid Build Coastguard Worker for (auto *U : Load->users())
4425*9880d681SAndroid Build Coastguard Worker WorkList.push_back(cast<Instruction>(U));
4426*9880d681SAndroid Build Coastguard Worker
4427*9880d681SAndroid Build Coastguard Worker EVT LoadResultVT = TLI->getValueType(*DL, Load->getType());
4428*9880d681SAndroid Build Coastguard Worker unsigned BitWidth = LoadResultVT.getSizeInBits();
4429*9880d681SAndroid Build Coastguard Worker APInt DemandBits(BitWidth, 0);
4430*9880d681SAndroid Build Coastguard Worker APInt WidestAndBits(BitWidth, 0);
4431*9880d681SAndroid Build Coastguard Worker
4432*9880d681SAndroid Build Coastguard Worker while (!WorkList.empty()) {
4433*9880d681SAndroid Build Coastguard Worker Instruction *I = WorkList.back();
4434*9880d681SAndroid Build Coastguard Worker WorkList.pop_back();
4435*9880d681SAndroid Build Coastguard Worker
4436*9880d681SAndroid Build Coastguard Worker // Break use-def graph loops.
4437*9880d681SAndroid Build Coastguard Worker if (!Visited.insert(I).second)
4438*9880d681SAndroid Build Coastguard Worker continue;
4439*9880d681SAndroid Build Coastguard Worker
4440*9880d681SAndroid Build Coastguard Worker // For a PHI node, push all of its users.
4441*9880d681SAndroid Build Coastguard Worker if (auto *Phi = dyn_cast<PHINode>(I)) {
4442*9880d681SAndroid Build Coastguard Worker for (auto *U : Phi->users())
4443*9880d681SAndroid Build Coastguard Worker WorkList.push_back(cast<Instruction>(U));
4444*9880d681SAndroid Build Coastguard Worker continue;
4445*9880d681SAndroid Build Coastguard Worker }
4446*9880d681SAndroid Build Coastguard Worker
4447*9880d681SAndroid Build Coastguard Worker switch (I->getOpcode()) {
4448*9880d681SAndroid Build Coastguard Worker case llvm::Instruction::And: {
4449*9880d681SAndroid Build Coastguard Worker auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1));
4450*9880d681SAndroid Build Coastguard Worker if (!AndC)
4451*9880d681SAndroid Build Coastguard Worker return false;
4452*9880d681SAndroid Build Coastguard Worker APInt AndBits = AndC->getValue();
4453*9880d681SAndroid Build Coastguard Worker DemandBits |= AndBits;
4454*9880d681SAndroid Build Coastguard Worker // Keep track of the widest and mask we see.
4455*9880d681SAndroid Build Coastguard Worker if (AndBits.ugt(WidestAndBits))
4456*9880d681SAndroid Build Coastguard Worker WidestAndBits = AndBits;
4457*9880d681SAndroid Build Coastguard Worker if (AndBits == WidestAndBits && I->getOperand(0) == Load)
4458*9880d681SAndroid Build Coastguard Worker AndsToMaybeRemove.push_back(I);
4459*9880d681SAndroid Build Coastguard Worker break;
4460*9880d681SAndroid Build Coastguard Worker }
4461*9880d681SAndroid Build Coastguard Worker
4462*9880d681SAndroid Build Coastguard Worker case llvm::Instruction::Shl: {
4463*9880d681SAndroid Build Coastguard Worker auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1));
4464*9880d681SAndroid Build Coastguard Worker if (!ShlC)
4465*9880d681SAndroid Build Coastguard Worker return false;
4466*9880d681SAndroid Build Coastguard Worker uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1);
4467*9880d681SAndroid Build Coastguard Worker auto ShlDemandBits = APInt::getAllOnesValue(BitWidth).lshr(ShiftAmt);
4468*9880d681SAndroid Build Coastguard Worker DemandBits |= ShlDemandBits;
4469*9880d681SAndroid Build Coastguard Worker break;
4470*9880d681SAndroid Build Coastguard Worker }
4471*9880d681SAndroid Build Coastguard Worker
4472*9880d681SAndroid Build Coastguard Worker case llvm::Instruction::Trunc: {
4473*9880d681SAndroid Build Coastguard Worker EVT TruncVT = TLI->getValueType(*DL, I->getType());
4474*9880d681SAndroid Build Coastguard Worker unsigned TruncBitWidth = TruncVT.getSizeInBits();
4475*9880d681SAndroid Build Coastguard Worker auto TruncBits = APInt::getAllOnesValue(TruncBitWidth).zext(BitWidth);
4476*9880d681SAndroid Build Coastguard Worker DemandBits |= TruncBits;
4477*9880d681SAndroid Build Coastguard Worker break;
4478*9880d681SAndroid Build Coastguard Worker }
4479*9880d681SAndroid Build Coastguard Worker
4480*9880d681SAndroid Build Coastguard Worker default:
4481*9880d681SAndroid Build Coastguard Worker return false;
4482*9880d681SAndroid Build Coastguard Worker }
4483*9880d681SAndroid Build Coastguard Worker }
4484*9880d681SAndroid Build Coastguard Worker
4485*9880d681SAndroid Build Coastguard Worker uint32_t ActiveBits = DemandBits.getActiveBits();
4486*9880d681SAndroid Build Coastguard Worker // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the
4487*9880d681SAndroid Build Coastguard Worker // target even if isLoadExtLegal says an i1 EXTLOAD is valid. For example,
4488*9880d681SAndroid Build Coastguard Worker // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but
4489*9880d681SAndroid Build Coastguard Worker // (and (load x) 1) is not matched as a single instruction, rather as a LDR
4490*9880d681SAndroid Build Coastguard Worker // followed by an AND.
4491*9880d681SAndroid Build Coastguard Worker // TODO: Look into removing this restriction by fixing backends to either
4492*9880d681SAndroid Build Coastguard Worker // return false for isLoadExtLegal for i1 or have them select this pattern to
4493*9880d681SAndroid Build Coastguard Worker // a single instruction.
4494*9880d681SAndroid Build Coastguard Worker //
4495*9880d681SAndroid Build Coastguard Worker // Also avoid hoisting if we didn't see any ands with the exact DemandBits
4496*9880d681SAndroid Build Coastguard Worker // mask, since these are the only ands that will be removed by isel.
4497*9880d681SAndroid Build Coastguard Worker if (ActiveBits <= 1 || !APIntOps::isMask(ActiveBits, DemandBits) ||
4498*9880d681SAndroid Build Coastguard Worker WidestAndBits != DemandBits)
4499*9880d681SAndroid Build Coastguard Worker return false;
4500*9880d681SAndroid Build Coastguard Worker
4501*9880d681SAndroid Build Coastguard Worker LLVMContext &Ctx = Load->getType()->getContext();
4502*9880d681SAndroid Build Coastguard Worker Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits);
4503*9880d681SAndroid Build Coastguard Worker EVT TruncVT = TLI->getValueType(*DL, TruncTy);
4504*9880d681SAndroid Build Coastguard Worker
4505*9880d681SAndroid Build Coastguard Worker // Reject cases that won't be matched as extloads.
4506*9880d681SAndroid Build Coastguard Worker if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() ||
4507*9880d681SAndroid Build Coastguard Worker !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT))
4508*9880d681SAndroid Build Coastguard Worker return false;
4509*9880d681SAndroid Build Coastguard Worker
4510*9880d681SAndroid Build Coastguard Worker IRBuilder<> Builder(Load->getNextNode());
4511*9880d681SAndroid Build Coastguard Worker auto *NewAnd = dyn_cast<Instruction>(
4512*9880d681SAndroid Build Coastguard Worker Builder.CreateAnd(Load, ConstantInt::get(Ctx, DemandBits)));
4513*9880d681SAndroid Build Coastguard Worker
4514*9880d681SAndroid Build Coastguard Worker // Replace all uses of load with new and (except for the use of load in the
4515*9880d681SAndroid Build Coastguard Worker // new and itself).
4516*9880d681SAndroid Build Coastguard Worker Load->replaceAllUsesWith(NewAnd);
4517*9880d681SAndroid Build Coastguard Worker NewAnd->setOperand(0, Load);
4518*9880d681SAndroid Build Coastguard Worker
4519*9880d681SAndroid Build Coastguard Worker // Remove any and instructions that are now redundant.
4520*9880d681SAndroid Build Coastguard Worker for (auto *And : AndsToMaybeRemove)
4521*9880d681SAndroid Build Coastguard Worker // Check that the and mask is the same as the one we decided to put on the
4522*9880d681SAndroid Build Coastguard Worker // new and.
4523*9880d681SAndroid Build Coastguard Worker if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) {
4524*9880d681SAndroid Build Coastguard Worker And->replaceAllUsesWith(NewAnd);
4525*9880d681SAndroid Build Coastguard Worker if (&*CurInstIterator == And)
4526*9880d681SAndroid Build Coastguard Worker CurInstIterator = std::next(And->getIterator());
4527*9880d681SAndroid Build Coastguard Worker And->eraseFromParent();
4528*9880d681SAndroid Build Coastguard Worker ++NumAndUses;
4529*9880d681SAndroid Build Coastguard Worker }
4530*9880d681SAndroid Build Coastguard Worker
4531*9880d681SAndroid Build Coastguard Worker ++NumAndsAdded;
4532*9880d681SAndroid Build Coastguard Worker return true;
4533*9880d681SAndroid Build Coastguard Worker }
4534*9880d681SAndroid Build Coastguard Worker
4535*9880d681SAndroid Build Coastguard Worker /// Check if V (an operand of a select instruction) is an expensive instruction
4536*9880d681SAndroid Build Coastguard Worker /// that is only used once.
sinkSelectOperand(const TargetTransformInfo * TTI,Value * V)4537*9880d681SAndroid Build Coastguard Worker static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V) {
4538*9880d681SAndroid Build Coastguard Worker auto *I = dyn_cast<Instruction>(V);
4539*9880d681SAndroid Build Coastguard Worker // If it's safe to speculatively execute, then it should not have side
4540*9880d681SAndroid Build Coastguard Worker // effects; therefore, it's safe to sink and possibly *not* execute.
4541*9880d681SAndroid Build Coastguard Worker return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) &&
4542*9880d681SAndroid Build Coastguard Worker TTI->getUserCost(I) >= TargetTransformInfo::TCC_Expensive;
4543*9880d681SAndroid Build Coastguard Worker }
4544*9880d681SAndroid Build Coastguard Worker
4545*9880d681SAndroid Build Coastguard Worker /// Returns true if a SelectInst should be turned into an explicit branch.
isFormingBranchFromSelectProfitable(const TargetTransformInfo * TTI,const TargetLowering * TLI,SelectInst * SI)4546*9880d681SAndroid Build Coastguard Worker static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
4547*9880d681SAndroid Build Coastguard Worker const TargetLowering *TLI,
4548*9880d681SAndroid Build Coastguard Worker SelectInst *SI) {
4549*9880d681SAndroid Build Coastguard Worker // If even a predictable select is cheap, then a branch can't be cheaper.
4550*9880d681SAndroid Build Coastguard Worker if (!TLI->isPredictableSelectExpensive())
4551*9880d681SAndroid Build Coastguard Worker return false;
4552*9880d681SAndroid Build Coastguard Worker
4553*9880d681SAndroid Build Coastguard Worker // FIXME: This should use the same heuristics as IfConversion to determine
4554*9880d681SAndroid Build Coastguard Worker // whether a select is better represented as a branch.
4555*9880d681SAndroid Build Coastguard Worker
4556*9880d681SAndroid Build Coastguard Worker // If metadata tells us that the select condition is obviously predictable,
4557*9880d681SAndroid Build Coastguard Worker // then we want to replace the select with a branch.
4558*9880d681SAndroid Build Coastguard Worker uint64_t TrueWeight, FalseWeight;
4559*9880d681SAndroid Build Coastguard Worker if (SI->extractProfMetadata(TrueWeight, FalseWeight)) {
4560*9880d681SAndroid Build Coastguard Worker uint64_t Max = std::max(TrueWeight, FalseWeight);
4561*9880d681SAndroid Build Coastguard Worker uint64_t Sum = TrueWeight + FalseWeight;
4562*9880d681SAndroid Build Coastguard Worker if (Sum != 0) {
4563*9880d681SAndroid Build Coastguard Worker auto Probability = BranchProbability::getBranchProbability(Max, Sum);
4564*9880d681SAndroid Build Coastguard Worker if (Probability > TLI->getPredictableBranchThreshold())
4565*9880d681SAndroid Build Coastguard Worker return true;
4566*9880d681SAndroid Build Coastguard Worker }
4567*9880d681SAndroid Build Coastguard Worker }
4568*9880d681SAndroid Build Coastguard Worker
4569*9880d681SAndroid Build Coastguard Worker CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition());
4570*9880d681SAndroid Build Coastguard Worker
4571*9880d681SAndroid Build Coastguard Worker // If a branch is predictable, an out-of-order CPU can avoid blocking on its
4572*9880d681SAndroid Build Coastguard Worker // comparison condition. If the compare has more than one use, there's
4573*9880d681SAndroid Build Coastguard Worker // probably another cmov or setcc around, so it's not worth emitting a branch.
4574*9880d681SAndroid Build Coastguard Worker if (!Cmp || !Cmp->hasOneUse())
4575*9880d681SAndroid Build Coastguard Worker return false;
4576*9880d681SAndroid Build Coastguard Worker
4577*9880d681SAndroid Build Coastguard Worker // If either operand of the select is expensive and only needed on one side
4578*9880d681SAndroid Build Coastguard Worker // of the select, we should form a branch.
4579*9880d681SAndroid Build Coastguard Worker if (sinkSelectOperand(TTI, SI->getTrueValue()) ||
4580*9880d681SAndroid Build Coastguard Worker sinkSelectOperand(TTI, SI->getFalseValue()))
4581*9880d681SAndroid Build Coastguard Worker return true;
4582*9880d681SAndroid Build Coastguard Worker
4583*9880d681SAndroid Build Coastguard Worker return false;
4584*9880d681SAndroid Build Coastguard Worker }
4585*9880d681SAndroid Build Coastguard Worker
4586*9880d681SAndroid Build Coastguard Worker
4587*9880d681SAndroid Build Coastguard Worker /// If we have a SelectInst that will likely profit from branch prediction,
4588*9880d681SAndroid Build Coastguard Worker /// turn it into a branch.
optimizeSelectInst(SelectInst * SI)4589*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
4590*9880d681SAndroid Build Coastguard Worker bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1);
4591*9880d681SAndroid Build Coastguard Worker
4592*9880d681SAndroid Build Coastguard Worker // Can we convert the 'select' to CF ?
4593*9880d681SAndroid Build Coastguard Worker if (DisableSelectToBranch || OptSize || !TLI || VectorCond ||
4594*9880d681SAndroid Build Coastguard Worker SI->getMetadata(LLVMContext::MD_unpredictable))
4595*9880d681SAndroid Build Coastguard Worker return false;
4596*9880d681SAndroid Build Coastguard Worker
4597*9880d681SAndroid Build Coastguard Worker TargetLowering::SelectSupportKind SelectKind;
4598*9880d681SAndroid Build Coastguard Worker if (VectorCond)
4599*9880d681SAndroid Build Coastguard Worker SelectKind = TargetLowering::VectorMaskSelect;
4600*9880d681SAndroid Build Coastguard Worker else if (SI->getType()->isVectorTy())
4601*9880d681SAndroid Build Coastguard Worker SelectKind = TargetLowering::ScalarCondVectorVal;
4602*9880d681SAndroid Build Coastguard Worker else
4603*9880d681SAndroid Build Coastguard Worker SelectKind = TargetLowering::ScalarValSelect;
4604*9880d681SAndroid Build Coastguard Worker
4605*9880d681SAndroid Build Coastguard Worker if (TLI->isSelectSupported(SelectKind) &&
4606*9880d681SAndroid Build Coastguard Worker !isFormingBranchFromSelectProfitable(TTI, TLI, SI))
4607*9880d681SAndroid Build Coastguard Worker return false;
4608*9880d681SAndroid Build Coastguard Worker
4609*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
4610*9880d681SAndroid Build Coastguard Worker
4611*9880d681SAndroid Build Coastguard Worker // Transform a sequence like this:
4612*9880d681SAndroid Build Coastguard Worker // start:
4613*9880d681SAndroid Build Coastguard Worker // %cmp = cmp uge i32 %a, %b
4614*9880d681SAndroid Build Coastguard Worker // %sel = select i1 %cmp, i32 %c, i32 %d
4615*9880d681SAndroid Build Coastguard Worker //
4616*9880d681SAndroid Build Coastguard Worker // Into:
4617*9880d681SAndroid Build Coastguard Worker // start:
4618*9880d681SAndroid Build Coastguard Worker // %cmp = cmp uge i32 %a, %b
4619*9880d681SAndroid Build Coastguard Worker // br i1 %cmp, label %select.true, label %select.false
4620*9880d681SAndroid Build Coastguard Worker // select.true:
4621*9880d681SAndroid Build Coastguard Worker // br label %select.end
4622*9880d681SAndroid Build Coastguard Worker // select.false:
4623*9880d681SAndroid Build Coastguard Worker // br label %select.end
4624*9880d681SAndroid Build Coastguard Worker // select.end:
4625*9880d681SAndroid Build Coastguard Worker // %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ]
4626*9880d681SAndroid Build Coastguard Worker //
4627*9880d681SAndroid Build Coastguard Worker // In addition, we may sink instructions that produce %c or %d from
4628*9880d681SAndroid Build Coastguard Worker // the entry block into the destination(s) of the new branch.
4629*9880d681SAndroid Build Coastguard Worker // If the true or false blocks do not contain a sunken instruction, that
4630*9880d681SAndroid Build Coastguard Worker // block and its branch may be optimized away. In that case, one side of the
4631*9880d681SAndroid Build Coastguard Worker // first branch will point directly to select.end, and the corresponding PHI
4632*9880d681SAndroid Build Coastguard Worker // predecessor block will be the start block.
4633*9880d681SAndroid Build Coastguard Worker
4634*9880d681SAndroid Build Coastguard Worker // First, we split the block containing the select into 2 blocks.
4635*9880d681SAndroid Build Coastguard Worker BasicBlock *StartBlock = SI->getParent();
4636*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(SI));
4637*9880d681SAndroid Build Coastguard Worker BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end");
4638*9880d681SAndroid Build Coastguard Worker
4639*9880d681SAndroid Build Coastguard Worker // Delete the unconditional branch that was just created by the split.
4640*9880d681SAndroid Build Coastguard Worker StartBlock->getTerminator()->eraseFromParent();
4641*9880d681SAndroid Build Coastguard Worker
4642*9880d681SAndroid Build Coastguard Worker // These are the new basic blocks for the conditional branch.
4643*9880d681SAndroid Build Coastguard Worker // At least one will become an actual new basic block.
4644*9880d681SAndroid Build Coastguard Worker BasicBlock *TrueBlock = nullptr;
4645*9880d681SAndroid Build Coastguard Worker BasicBlock *FalseBlock = nullptr;
4646*9880d681SAndroid Build Coastguard Worker
4647*9880d681SAndroid Build Coastguard Worker // Sink expensive instructions into the conditional blocks to avoid executing
4648*9880d681SAndroid Build Coastguard Worker // them speculatively.
4649*9880d681SAndroid Build Coastguard Worker if (sinkSelectOperand(TTI, SI->getTrueValue())) {
4650*9880d681SAndroid Build Coastguard Worker TrueBlock = BasicBlock::Create(SI->getContext(), "select.true.sink",
4651*9880d681SAndroid Build Coastguard Worker EndBlock->getParent(), EndBlock);
4652*9880d681SAndroid Build Coastguard Worker auto *TrueBranch = BranchInst::Create(EndBlock, TrueBlock);
4653*9880d681SAndroid Build Coastguard Worker auto *TrueInst = cast<Instruction>(SI->getTrueValue());
4654*9880d681SAndroid Build Coastguard Worker TrueInst->moveBefore(TrueBranch);
4655*9880d681SAndroid Build Coastguard Worker }
4656*9880d681SAndroid Build Coastguard Worker if (sinkSelectOperand(TTI, SI->getFalseValue())) {
4657*9880d681SAndroid Build Coastguard Worker FalseBlock = BasicBlock::Create(SI->getContext(), "select.false.sink",
4658*9880d681SAndroid Build Coastguard Worker EndBlock->getParent(), EndBlock);
4659*9880d681SAndroid Build Coastguard Worker auto *FalseBranch = BranchInst::Create(EndBlock, FalseBlock);
4660*9880d681SAndroid Build Coastguard Worker auto *FalseInst = cast<Instruction>(SI->getFalseValue());
4661*9880d681SAndroid Build Coastguard Worker FalseInst->moveBefore(FalseBranch);
4662*9880d681SAndroid Build Coastguard Worker }
4663*9880d681SAndroid Build Coastguard Worker
4664*9880d681SAndroid Build Coastguard Worker // If there was nothing to sink, then arbitrarily choose the 'false' side
4665*9880d681SAndroid Build Coastguard Worker // for a new input value to the PHI.
4666*9880d681SAndroid Build Coastguard Worker if (TrueBlock == FalseBlock) {
4667*9880d681SAndroid Build Coastguard Worker assert(TrueBlock == nullptr &&
4668*9880d681SAndroid Build Coastguard Worker "Unexpected basic block transform while optimizing select");
4669*9880d681SAndroid Build Coastguard Worker
4670*9880d681SAndroid Build Coastguard Worker FalseBlock = BasicBlock::Create(SI->getContext(), "select.false",
4671*9880d681SAndroid Build Coastguard Worker EndBlock->getParent(), EndBlock);
4672*9880d681SAndroid Build Coastguard Worker BranchInst::Create(EndBlock, FalseBlock);
4673*9880d681SAndroid Build Coastguard Worker }
4674*9880d681SAndroid Build Coastguard Worker
4675*9880d681SAndroid Build Coastguard Worker // Insert the real conditional branch based on the original condition.
4676*9880d681SAndroid Build Coastguard Worker // If we did not create a new block for one of the 'true' or 'false' paths
4677*9880d681SAndroid Build Coastguard Worker // of the condition, it means that side of the branch goes to the end block
4678*9880d681SAndroid Build Coastguard Worker // directly and the path originates from the start block from the point of
4679*9880d681SAndroid Build Coastguard Worker // view of the new PHI.
4680*9880d681SAndroid Build Coastguard Worker if (TrueBlock == nullptr) {
4681*9880d681SAndroid Build Coastguard Worker BranchInst::Create(EndBlock, FalseBlock, SI->getCondition(), SI);
4682*9880d681SAndroid Build Coastguard Worker TrueBlock = StartBlock;
4683*9880d681SAndroid Build Coastguard Worker } else if (FalseBlock == nullptr) {
4684*9880d681SAndroid Build Coastguard Worker BranchInst::Create(TrueBlock, EndBlock, SI->getCondition(), SI);
4685*9880d681SAndroid Build Coastguard Worker FalseBlock = StartBlock;
4686*9880d681SAndroid Build Coastguard Worker } else {
4687*9880d681SAndroid Build Coastguard Worker BranchInst::Create(TrueBlock, FalseBlock, SI->getCondition(), SI);
4688*9880d681SAndroid Build Coastguard Worker }
4689*9880d681SAndroid Build Coastguard Worker
4690*9880d681SAndroid Build Coastguard Worker // The select itself is replaced with a PHI Node.
4691*9880d681SAndroid Build Coastguard Worker PHINode *PN = PHINode::Create(SI->getType(), 2, "", &EndBlock->front());
4692*9880d681SAndroid Build Coastguard Worker PN->takeName(SI);
4693*9880d681SAndroid Build Coastguard Worker PN->addIncoming(SI->getTrueValue(), TrueBlock);
4694*9880d681SAndroid Build Coastguard Worker PN->addIncoming(SI->getFalseValue(), FalseBlock);
4695*9880d681SAndroid Build Coastguard Worker
4696*9880d681SAndroid Build Coastguard Worker SI->replaceAllUsesWith(PN);
4697*9880d681SAndroid Build Coastguard Worker SI->eraseFromParent();
4698*9880d681SAndroid Build Coastguard Worker
4699*9880d681SAndroid Build Coastguard Worker // Instruct OptimizeBlock to skip to the next block.
4700*9880d681SAndroid Build Coastguard Worker CurInstIterator = StartBlock->end();
4701*9880d681SAndroid Build Coastguard Worker ++NumSelectsExpanded;
4702*9880d681SAndroid Build Coastguard Worker return true;
4703*9880d681SAndroid Build Coastguard Worker }
4704*9880d681SAndroid Build Coastguard Worker
isBroadcastShuffle(ShuffleVectorInst * SVI)4705*9880d681SAndroid Build Coastguard Worker static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
4706*9880d681SAndroid Build Coastguard Worker SmallVector<int, 16> Mask(SVI->getShuffleMask());
4707*9880d681SAndroid Build Coastguard Worker int SplatElem = -1;
4708*9880d681SAndroid Build Coastguard Worker for (unsigned i = 0; i < Mask.size(); ++i) {
4709*9880d681SAndroid Build Coastguard Worker if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
4710*9880d681SAndroid Build Coastguard Worker return false;
4711*9880d681SAndroid Build Coastguard Worker SplatElem = Mask[i];
4712*9880d681SAndroid Build Coastguard Worker }
4713*9880d681SAndroid Build Coastguard Worker
4714*9880d681SAndroid Build Coastguard Worker return true;
4715*9880d681SAndroid Build Coastguard Worker }
4716*9880d681SAndroid Build Coastguard Worker
4717*9880d681SAndroid Build Coastguard Worker /// Some targets have expensive vector shifts if the lanes aren't all the same
4718*9880d681SAndroid Build Coastguard Worker /// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases
4719*9880d681SAndroid Build Coastguard Worker /// it's often worth sinking a shufflevector splat down to its use so that
4720*9880d681SAndroid Build Coastguard Worker /// codegen can spot all lanes are identical.
optimizeShuffleVectorInst(ShuffleVectorInst * SVI)4721*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
4722*9880d681SAndroid Build Coastguard Worker BasicBlock *DefBB = SVI->getParent();
4723*9880d681SAndroid Build Coastguard Worker
4724*9880d681SAndroid Build Coastguard Worker // Only do this xform if variable vector shifts are particularly expensive.
4725*9880d681SAndroid Build Coastguard Worker if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType()))
4726*9880d681SAndroid Build Coastguard Worker return false;
4727*9880d681SAndroid Build Coastguard Worker
4728*9880d681SAndroid Build Coastguard Worker // We only expect better codegen by sinking a shuffle if we can recognise a
4729*9880d681SAndroid Build Coastguard Worker // constant splat.
4730*9880d681SAndroid Build Coastguard Worker if (!isBroadcastShuffle(SVI))
4731*9880d681SAndroid Build Coastguard Worker return false;
4732*9880d681SAndroid Build Coastguard Worker
4733*9880d681SAndroid Build Coastguard Worker // InsertedShuffles - Only insert a shuffle in each block once.
4734*9880d681SAndroid Build Coastguard Worker DenseMap<BasicBlock*, Instruction*> InsertedShuffles;
4735*9880d681SAndroid Build Coastguard Worker
4736*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
4737*9880d681SAndroid Build Coastguard Worker for (User *U : SVI->users()) {
4738*9880d681SAndroid Build Coastguard Worker Instruction *UI = cast<Instruction>(U);
4739*9880d681SAndroid Build Coastguard Worker
4740*9880d681SAndroid Build Coastguard Worker // Figure out which BB this ext is used in.
4741*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = UI->getParent();
4742*9880d681SAndroid Build Coastguard Worker if (UserBB == DefBB) continue;
4743*9880d681SAndroid Build Coastguard Worker
4744*9880d681SAndroid Build Coastguard Worker // For now only apply this when the splat is used by a shift instruction.
4745*9880d681SAndroid Build Coastguard Worker if (!UI->isShift()) continue;
4746*9880d681SAndroid Build Coastguard Worker
4747*9880d681SAndroid Build Coastguard Worker // Everything checks out, sink the shuffle if the user's block doesn't
4748*9880d681SAndroid Build Coastguard Worker // already have a copy.
4749*9880d681SAndroid Build Coastguard Worker Instruction *&InsertedShuffle = InsertedShuffles[UserBB];
4750*9880d681SAndroid Build Coastguard Worker
4751*9880d681SAndroid Build Coastguard Worker if (!InsertedShuffle) {
4752*9880d681SAndroid Build Coastguard Worker BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
4753*9880d681SAndroid Build Coastguard Worker assert(InsertPt != UserBB->end());
4754*9880d681SAndroid Build Coastguard Worker InsertedShuffle =
4755*9880d681SAndroid Build Coastguard Worker new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
4756*9880d681SAndroid Build Coastguard Worker SVI->getOperand(2), "", &*InsertPt);
4757*9880d681SAndroid Build Coastguard Worker }
4758*9880d681SAndroid Build Coastguard Worker
4759*9880d681SAndroid Build Coastguard Worker UI->replaceUsesOfWith(SVI, InsertedShuffle);
4760*9880d681SAndroid Build Coastguard Worker MadeChange = true;
4761*9880d681SAndroid Build Coastguard Worker }
4762*9880d681SAndroid Build Coastguard Worker
4763*9880d681SAndroid Build Coastguard Worker // If we removed all uses, nuke the shuffle.
4764*9880d681SAndroid Build Coastguard Worker if (SVI->use_empty()) {
4765*9880d681SAndroid Build Coastguard Worker SVI->eraseFromParent();
4766*9880d681SAndroid Build Coastguard Worker MadeChange = true;
4767*9880d681SAndroid Build Coastguard Worker }
4768*9880d681SAndroid Build Coastguard Worker
4769*9880d681SAndroid Build Coastguard Worker return MadeChange;
4770*9880d681SAndroid Build Coastguard Worker }
4771*9880d681SAndroid Build Coastguard Worker
optimizeSwitchInst(SwitchInst * SI)4772*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {
4773*9880d681SAndroid Build Coastguard Worker if (!TLI || !DL)
4774*9880d681SAndroid Build Coastguard Worker return false;
4775*9880d681SAndroid Build Coastguard Worker
4776*9880d681SAndroid Build Coastguard Worker Value *Cond = SI->getCondition();
4777*9880d681SAndroid Build Coastguard Worker Type *OldType = Cond->getType();
4778*9880d681SAndroid Build Coastguard Worker LLVMContext &Context = Cond->getContext();
4779*9880d681SAndroid Build Coastguard Worker MVT RegType = TLI->getRegisterType(Context, TLI->getValueType(*DL, OldType));
4780*9880d681SAndroid Build Coastguard Worker unsigned RegWidth = RegType.getSizeInBits();
4781*9880d681SAndroid Build Coastguard Worker
4782*9880d681SAndroid Build Coastguard Worker if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth())
4783*9880d681SAndroid Build Coastguard Worker return false;
4784*9880d681SAndroid Build Coastguard Worker
4785*9880d681SAndroid Build Coastguard Worker // If the register width is greater than the type width, expand the condition
4786*9880d681SAndroid Build Coastguard Worker // of the switch instruction and each case constant to the width of the
4787*9880d681SAndroid Build Coastguard Worker // register. By widening the type of the switch condition, subsequent
4788*9880d681SAndroid Build Coastguard Worker // comparisons (for case comparisons) will not need to be extended to the
4789*9880d681SAndroid Build Coastguard Worker // preferred register width, so we will potentially eliminate N-1 extends,
4790*9880d681SAndroid Build Coastguard Worker // where N is the number of cases in the switch.
4791*9880d681SAndroid Build Coastguard Worker auto *NewType = Type::getIntNTy(Context, RegWidth);
4792*9880d681SAndroid Build Coastguard Worker
4793*9880d681SAndroid Build Coastguard Worker // Zero-extend the switch condition and case constants unless the switch
4794*9880d681SAndroid Build Coastguard Worker // condition is a function argument that is already being sign-extended.
4795*9880d681SAndroid Build Coastguard Worker // In that case, we can avoid an unnecessary mask/extension by sign-extending
4796*9880d681SAndroid Build Coastguard Worker // everything instead.
4797*9880d681SAndroid Build Coastguard Worker Instruction::CastOps ExtType = Instruction::ZExt;
4798*9880d681SAndroid Build Coastguard Worker if (auto *Arg = dyn_cast<Argument>(Cond))
4799*9880d681SAndroid Build Coastguard Worker if (Arg->hasSExtAttr())
4800*9880d681SAndroid Build Coastguard Worker ExtType = Instruction::SExt;
4801*9880d681SAndroid Build Coastguard Worker
4802*9880d681SAndroid Build Coastguard Worker auto *ExtInst = CastInst::Create(ExtType, Cond, NewType);
4803*9880d681SAndroid Build Coastguard Worker ExtInst->insertBefore(SI);
4804*9880d681SAndroid Build Coastguard Worker SI->setCondition(ExtInst);
4805*9880d681SAndroid Build Coastguard Worker for (SwitchInst::CaseIt Case : SI->cases()) {
4806*9880d681SAndroid Build Coastguard Worker APInt NarrowConst = Case.getCaseValue()->getValue();
4807*9880d681SAndroid Build Coastguard Worker APInt WideConst = (ExtType == Instruction::ZExt) ?
4808*9880d681SAndroid Build Coastguard Worker NarrowConst.zext(RegWidth) : NarrowConst.sext(RegWidth);
4809*9880d681SAndroid Build Coastguard Worker Case.setValue(ConstantInt::get(Context, WideConst));
4810*9880d681SAndroid Build Coastguard Worker }
4811*9880d681SAndroid Build Coastguard Worker
4812*9880d681SAndroid Build Coastguard Worker return true;
4813*9880d681SAndroid Build Coastguard Worker }
4814*9880d681SAndroid Build Coastguard Worker
4815*9880d681SAndroid Build Coastguard Worker namespace {
4816*9880d681SAndroid Build Coastguard Worker /// \brief Helper class to promote a scalar operation to a vector one.
4817*9880d681SAndroid Build Coastguard Worker /// This class is used to move downward extractelement transition.
4818*9880d681SAndroid Build Coastguard Worker /// E.g.,
4819*9880d681SAndroid Build Coastguard Worker /// a = vector_op <2 x i32>
4820*9880d681SAndroid Build Coastguard Worker /// b = extractelement <2 x i32> a, i32 0
4821*9880d681SAndroid Build Coastguard Worker /// c = scalar_op b
4822*9880d681SAndroid Build Coastguard Worker /// store c
4823*9880d681SAndroid Build Coastguard Worker ///
4824*9880d681SAndroid Build Coastguard Worker /// =>
4825*9880d681SAndroid Build Coastguard Worker /// a = vector_op <2 x i32>
4826*9880d681SAndroid Build Coastguard Worker /// c = vector_op a (equivalent to scalar_op on the related lane)
4827*9880d681SAndroid Build Coastguard Worker /// * d = extractelement <2 x i32> c, i32 0
4828*9880d681SAndroid Build Coastguard Worker /// * store d
4829*9880d681SAndroid Build Coastguard Worker /// Assuming both extractelement and store can be combine, we get rid of the
4830*9880d681SAndroid Build Coastguard Worker /// transition.
4831*9880d681SAndroid Build Coastguard Worker class VectorPromoteHelper {
4832*9880d681SAndroid Build Coastguard Worker /// DataLayout associated with the current module.
4833*9880d681SAndroid Build Coastguard Worker const DataLayout &DL;
4834*9880d681SAndroid Build Coastguard Worker
4835*9880d681SAndroid Build Coastguard Worker /// Used to perform some checks on the legality of vector operations.
4836*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI;
4837*9880d681SAndroid Build Coastguard Worker
4838*9880d681SAndroid Build Coastguard Worker /// Used to estimated the cost of the promoted chain.
4839*9880d681SAndroid Build Coastguard Worker const TargetTransformInfo &TTI;
4840*9880d681SAndroid Build Coastguard Worker
4841*9880d681SAndroid Build Coastguard Worker /// The transition being moved downwards.
4842*9880d681SAndroid Build Coastguard Worker Instruction *Transition;
4843*9880d681SAndroid Build Coastguard Worker /// The sequence of instructions to be promoted.
4844*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction *, 4> InstsToBePromoted;
4845*9880d681SAndroid Build Coastguard Worker /// Cost of combining a store and an extract.
4846*9880d681SAndroid Build Coastguard Worker unsigned StoreExtractCombineCost;
4847*9880d681SAndroid Build Coastguard Worker /// Instruction that will be combined with the transition.
4848*9880d681SAndroid Build Coastguard Worker Instruction *CombineInst;
4849*9880d681SAndroid Build Coastguard Worker
4850*9880d681SAndroid Build Coastguard Worker /// \brief The instruction that represents the current end of the transition.
4851*9880d681SAndroid Build Coastguard Worker /// Since we are faking the promotion until we reach the end of the chain
4852*9880d681SAndroid Build Coastguard Worker /// of computation, we need a way to get the current end of the transition.
getEndOfTransition() const4853*9880d681SAndroid Build Coastguard Worker Instruction *getEndOfTransition() const {
4854*9880d681SAndroid Build Coastguard Worker if (InstsToBePromoted.empty())
4855*9880d681SAndroid Build Coastguard Worker return Transition;
4856*9880d681SAndroid Build Coastguard Worker return InstsToBePromoted.back();
4857*9880d681SAndroid Build Coastguard Worker }
4858*9880d681SAndroid Build Coastguard Worker
4859*9880d681SAndroid Build Coastguard Worker /// \brief Return the index of the original value in the transition.
4860*9880d681SAndroid Build Coastguard Worker /// E.g., for "extractelement <2 x i32> c, i32 1" the original value,
4861*9880d681SAndroid Build Coastguard Worker /// c, is at index 0.
getTransitionOriginalValueIdx() const4862*9880d681SAndroid Build Coastguard Worker unsigned getTransitionOriginalValueIdx() const {
4863*9880d681SAndroid Build Coastguard Worker assert(isa<ExtractElementInst>(Transition) &&
4864*9880d681SAndroid Build Coastguard Worker "Other kind of transitions are not supported yet");
4865*9880d681SAndroid Build Coastguard Worker return 0;
4866*9880d681SAndroid Build Coastguard Worker }
4867*9880d681SAndroid Build Coastguard Worker
4868*9880d681SAndroid Build Coastguard Worker /// \brief Return the index of the index in the transition.
4869*9880d681SAndroid Build Coastguard Worker /// E.g., for "extractelement <2 x i32> c, i32 0" the index
4870*9880d681SAndroid Build Coastguard Worker /// is at index 1.
getTransitionIdx() const4871*9880d681SAndroid Build Coastguard Worker unsigned getTransitionIdx() const {
4872*9880d681SAndroid Build Coastguard Worker assert(isa<ExtractElementInst>(Transition) &&
4873*9880d681SAndroid Build Coastguard Worker "Other kind of transitions are not supported yet");
4874*9880d681SAndroid Build Coastguard Worker return 1;
4875*9880d681SAndroid Build Coastguard Worker }
4876*9880d681SAndroid Build Coastguard Worker
4877*9880d681SAndroid Build Coastguard Worker /// \brief Get the type of the transition.
4878*9880d681SAndroid Build Coastguard Worker /// This is the type of the original value.
4879*9880d681SAndroid Build Coastguard Worker /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the
4880*9880d681SAndroid Build Coastguard Worker /// transition is <2 x i32>.
getTransitionType() const4881*9880d681SAndroid Build Coastguard Worker Type *getTransitionType() const {
4882*9880d681SAndroid Build Coastguard Worker return Transition->getOperand(getTransitionOriginalValueIdx())->getType();
4883*9880d681SAndroid Build Coastguard Worker }
4884*9880d681SAndroid Build Coastguard Worker
4885*9880d681SAndroid Build Coastguard Worker /// \brief Promote \p ToBePromoted by moving \p Def downward through.
4886*9880d681SAndroid Build Coastguard Worker /// I.e., we have the following sequence:
4887*9880d681SAndroid Build Coastguard Worker /// Def = Transition <ty1> a to <ty2>
4888*9880d681SAndroid Build Coastguard Worker /// b = ToBePromoted <ty2> Def, ...
4889*9880d681SAndroid Build Coastguard Worker /// =>
4890*9880d681SAndroid Build Coastguard Worker /// b = ToBePromoted <ty1> a, ...
4891*9880d681SAndroid Build Coastguard Worker /// Def = Transition <ty1> ToBePromoted to <ty2>
4892*9880d681SAndroid Build Coastguard Worker void promoteImpl(Instruction *ToBePromoted);
4893*9880d681SAndroid Build Coastguard Worker
4894*9880d681SAndroid Build Coastguard Worker /// \brief Check whether or not it is profitable to promote all the
4895*9880d681SAndroid Build Coastguard Worker /// instructions enqueued to be promoted.
isProfitableToPromote()4896*9880d681SAndroid Build Coastguard Worker bool isProfitableToPromote() {
4897*9880d681SAndroid Build Coastguard Worker Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx());
4898*9880d681SAndroid Build Coastguard Worker unsigned Index = isa<ConstantInt>(ValIdx)
4899*9880d681SAndroid Build Coastguard Worker ? cast<ConstantInt>(ValIdx)->getZExtValue()
4900*9880d681SAndroid Build Coastguard Worker : -1;
4901*9880d681SAndroid Build Coastguard Worker Type *PromotedType = getTransitionType();
4902*9880d681SAndroid Build Coastguard Worker
4903*9880d681SAndroid Build Coastguard Worker StoreInst *ST = cast<StoreInst>(CombineInst);
4904*9880d681SAndroid Build Coastguard Worker unsigned AS = ST->getPointerAddressSpace();
4905*9880d681SAndroid Build Coastguard Worker unsigned Align = ST->getAlignment();
4906*9880d681SAndroid Build Coastguard Worker // Check if this store is supported.
4907*9880d681SAndroid Build Coastguard Worker if (!TLI.allowsMisalignedMemoryAccesses(
4908*9880d681SAndroid Build Coastguard Worker TLI.getValueType(DL, ST->getValueOperand()->getType()), AS,
4909*9880d681SAndroid Build Coastguard Worker Align)) {
4910*9880d681SAndroid Build Coastguard Worker // If this is not supported, there is no way we can combine
4911*9880d681SAndroid Build Coastguard Worker // the extract with the store.
4912*9880d681SAndroid Build Coastguard Worker return false;
4913*9880d681SAndroid Build Coastguard Worker }
4914*9880d681SAndroid Build Coastguard Worker
4915*9880d681SAndroid Build Coastguard Worker // The scalar chain of computation has to pay for the transition
4916*9880d681SAndroid Build Coastguard Worker // scalar to vector.
4917*9880d681SAndroid Build Coastguard Worker // The vector chain has to account for the combining cost.
4918*9880d681SAndroid Build Coastguard Worker uint64_t ScalarCost =
4919*9880d681SAndroid Build Coastguard Worker TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index);
4920*9880d681SAndroid Build Coastguard Worker uint64_t VectorCost = StoreExtractCombineCost;
4921*9880d681SAndroid Build Coastguard Worker for (const auto &Inst : InstsToBePromoted) {
4922*9880d681SAndroid Build Coastguard Worker // Compute the cost.
4923*9880d681SAndroid Build Coastguard Worker // By construction, all instructions being promoted are arithmetic ones.
4924*9880d681SAndroid Build Coastguard Worker // Moreover, one argument is a constant that can be viewed as a splat
4925*9880d681SAndroid Build Coastguard Worker // constant.
4926*9880d681SAndroid Build Coastguard Worker Value *Arg0 = Inst->getOperand(0);
4927*9880d681SAndroid Build Coastguard Worker bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) ||
4928*9880d681SAndroid Build Coastguard Worker isa<ConstantFP>(Arg0);
4929*9880d681SAndroid Build Coastguard Worker TargetTransformInfo::OperandValueKind Arg0OVK =
4930*9880d681SAndroid Build Coastguard Worker IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
4931*9880d681SAndroid Build Coastguard Worker : TargetTransformInfo::OK_AnyValue;
4932*9880d681SAndroid Build Coastguard Worker TargetTransformInfo::OperandValueKind Arg1OVK =
4933*9880d681SAndroid Build Coastguard Worker !IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
4934*9880d681SAndroid Build Coastguard Worker : TargetTransformInfo::OK_AnyValue;
4935*9880d681SAndroid Build Coastguard Worker ScalarCost += TTI.getArithmeticInstrCost(
4936*9880d681SAndroid Build Coastguard Worker Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK);
4937*9880d681SAndroid Build Coastguard Worker VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType,
4938*9880d681SAndroid Build Coastguard Worker Arg0OVK, Arg1OVK);
4939*9880d681SAndroid Build Coastguard Worker }
4940*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Estimated cost of computation to be promoted:\nScalar: "
4941*9880d681SAndroid Build Coastguard Worker << ScalarCost << "\nVector: " << VectorCost << '\n');
4942*9880d681SAndroid Build Coastguard Worker return ScalarCost > VectorCost;
4943*9880d681SAndroid Build Coastguard Worker }
4944*9880d681SAndroid Build Coastguard Worker
4945*9880d681SAndroid Build Coastguard Worker /// \brief Generate a constant vector with \p Val with the same
4946*9880d681SAndroid Build Coastguard Worker /// number of elements as the transition.
4947*9880d681SAndroid Build Coastguard Worker /// \p UseSplat defines whether or not \p Val should be replicated
4948*9880d681SAndroid Build Coastguard Worker /// across the whole vector.
4949*9880d681SAndroid Build Coastguard Worker /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>,
4950*9880d681SAndroid Build Coastguard Worker /// otherwise we generate a vector with as many undef as possible:
4951*9880d681SAndroid Build Coastguard Worker /// <undef, ..., undef, Val, undef, ..., undef> where \p Val is only
4952*9880d681SAndroid Build Coastguard Worker /// used at the index of the extract.
getConstantVector(Constant * Val,bool UseSplat) const4953*9880d681SAndroid Build Coastguard Worker Value *getConstantVector(Constant *Val, bool UseSplat) const {
4954*9880d681SAndroid Build Coastguard Worker unsigned ExtractIdx = UINT_MAX;
4955*9880d681SAndroid Build Coastguard Worker if (!UseSplat) {
4956*9880d681SAndroid Build Coastguard Worker // If we cannot determine where the constant must be, we have to
4957*9880d681SAndroid Build Coastguard Worker // use a splat constant.
4958*9880d681SAndroid Build Coastguard Worker Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());
4959*9880d681SAndroid Build Coastguard Worker if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))
4960*9880d681SAndroid Build Coastguard Worker ExtractIdx = CstVal->getSExtValue();
4961*9880d681SAndroid Build Coastguard Worker else
4962*9880d681SAndroid Build Coastguard Worker UseSplat = true;
4963*9880d681SAndroid Build Coastguard Worker }
4964*9880d681SAndroid Build Coastguard Worker
4965*9880d681SAndroid Build Coastguard Worker unsigned End = getTransitionType()->getVectorNumElements();
4966*9880d681SAndroid Build Coastguard Worker if (UseSplat)
4967*9880d681SAndroid Build Coastguard Worker return ConstantVector::getSplat(End, Val);
4968*9880d681SAndroid Build Coastguard Worker
4969*9880d681SAndroid Build Coastguard Worker SmallVector<Constant *, 4> ConstVec;
4970*9880d681SAndroid Build Coastguard Worker UndefValue *UndefVal = UndefValue::get(Val->getType());
4971*9880d681SAndroid Build Coastguard Worker for (unsigned Idx = 0; Idx != End; ++Idx) {
4972*9880d681SAndroid Build Coastguard Worker if (Idx == ExtractIdx)
4973*9880d681SAndroid Build Coastguard Worker ConstVec.push_back(Val);
4974*9880d681SAndroid Build Coastguard Worker else
4975*9880d681SAndroid Build Coastguard Worker ConstVec.push_back(UndefVal);
4976*9880d681SAndroid Build Coastguard Worker }
4977*9880d681SAndroid Build Coastguard Worker return ConstantVector::get(ConstVec);
4978*9880d681SAndroid Build Coastguard Worker }
4979*9880d681SAndroid Build Coastguard Worker
4980*9880d681SAndroid Build Coastguard Worker /// \brief Check if promoting to a vector type an operand at \p OperandIdx
4981*9880d681SAndroid Build Coastguard Worker /// in \p Use can trigger undefined behavior.
canCauseUndefinedBehavior(const Instruction * Use,unsigned OperandIdx)4982*9880d681SAndroid Build Coastguard Worker static bool canCauseUndefinedBehavior(const Instruction *Use,
4983*9880d681SAndroid Build Coastguard Worker unsigned OperandIdx) {
4984*9880d681SAndroid Build Coastguard Worker // This is not safe to introduce undef when the operand is on
4985*9880d681SAndroid Build Coastguard Worker // the right hand side of a division-like instruction.
4986*9880d681SAndroid Build Coastguard Worker if (OperandIdx != 1)
4987*9880d681SAndroid Build Coastguard Worker return false;
4988*9880d681SAndroid Build Coastguard Worker switch (Use->getOpcode()) {
4989*9880d681SAndroid Build Coastguard Worker default:
4990*9880d681SAndroid Build Coastguard Worker return false;
4991*9880d681SAndroid Build Coastguard Worker case Instruction::SDiv:
4992*9880d681SAndroid Build Coastguard Worker case Instruction::UDiv:
4993*9880d681SAndroid Build Coastguard Worker case Instruction::SRem:
4994*9880d681SAndroid Build Coastguard Worker case Instruction::URem:
4995*9880d681SAndroid Build Coastguard Worker return true;
4996*9880d681SAndroid Build Coastguard Worker case Instruction::FDiv:
4997*9880d681SAndroid Build Coastguard Worker case Instruction::FRem:
4998*9880d681SAndroid Build Coastguard Worker return !Use->hasNoNaNs();
4999*9880d681SAndroid Build Coastguard Worker }
5000*9880d681SAndroid Build Coastguard Worker llvm_unreachable(nullptr);
5001*9880d681SAndroid Build Coastguard Worker }
5002*9880d681SAndroid Build Coastguard Worker
5003*9880d681SAndroid Build Coastguard Worker public:
VectorPromoteHelper(const DataLayout & DL,const TargetLowering & TLI,const TargetTransformInfo & TTI,Instruction * Transition,unsigned CombineCost)5004*9880d681SAndroid Build Coastguard Worker VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI,
5005*9880d681SAndroid Build Coastguard Worker const TargetTransformInfo &TTI, Instruction *Transition,
5006*9880d681SAndroid Build Coastguard Worker unsigned CombineCost)
5007*9880d681SAndroid Build Coastguard Worker : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),
5008*9880d681SAndroid Build Coastguard Worker StoreExtractCombineCost(CombineCost), CombineInst(nullptr) {
5009*9880d681SAndroid Build Coastguard Worker assert(Transition && "Do not know how to promote null");
5010*9880d681SAndroid Build Coastguard Worker }
5011*9880d681SAndroid Build Coastguard Worker
5012*9880d681SAndroid Build Coastguard Worker /// \brief Check if we can promote \p ToBePromoted to \p Type.
canPromote(const Instruction * ToBePromoted) const5013*9880d681SAndroid Build Coastguard Worker bool canPromote(const Instruction *ToBePromoted) const {
5014*9880d681SAndroid Build Coastguard Worker // We could support CastInst too.
5015*9880d681SAndroid Build Coastguard Worker return isa<BinaryOperator>(ToBePromoted);
5016*9880d681SAndroid Build Coastguard Worker }
5017*9880d681SAndroid Build Coastguard Worker
5018*9880d681SAndroid Build Coastguard Worker /// \brief Check if it is profitable to promote \p ToBePromoted
5019*9880d681SAndroid Build Coastguard Worker /// by moving downward the transition through.
shouldPromote(const Instruction * ToBePromoted) const5020*9880d681SAndroid Build Coastguard Worker bool shouldPromote(const Instruction *ToBePromoted) const {
5021*9880d681SAndroid Build Coastguard Worker // Promote only if all the operands can be statically expanded.
5022*9880d681SAndroid Build Coastguard Worker // Indeed, we do not want to introduce any new kind of transitions.
5023*9880d681SAndroid Build Coastguard Worker for (const Use &U : ToBePromoted->operands()) {
5024*9880d681SAndroid Build Coastguard Worker const Value *Val = U.get();
5025*9880d681SAndroid Build Coastguard Worker if (Val == getEndOfTransition()) {
5026*9880d681SAndroid Build Coastguard Worker // If the use is a division and the transition is on the rhs,
5027*9880d681SAndroid Build Coastguard Worker // we cannot promote the operation, otherwise we may create a
5028*9880d681SAndroid Build Coastguard Worker // division by zero.
5029*9880d681SAndroid Build Coastguard Worker if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()))
5030*9880d681SAndroid Build Coastguard Worker return false;
5031*9880d681SAndroid Build Coastguard Worker continue;
5032*9880d681SAndroid Build Coastguard Worker }
5033*9880d681SAndroid Build Coastguard Worker if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) &&
5034*9880d681SAndroid Build Coastguard Worker !isa<ConstantFP>(Val))
5035*9880d681SAndroid Build Coastguard Worker return false;
5036*9880d681SAndroid Build Coastguard Worker }
5037*9880d681SAndroid Build Coastguard Worker // Check that the resulting operation is legal.
5038*9880d681SAndroid Build Coastguard Worker int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode());
5039*9880d681SAndroid Build Coastguard Worker if (!ISDOpcode)
5040*9880d681SAndroid Build Coastguard Worker return false;
5041*9880d681SAndroid Build Coastguard Worker return StressStoreExtract ||
5042*9880d681SAndroid Build Coastguard Worker TLI.isOperationLegalOrCustom(
5043*9880d681SAndroid Build Coastguard Worker ISDOpcode, TLI.getValueType(DL, getTransitionType(), true));
5044*9880d681SAndroid Build Coastguard Worker }
5045*9880d681SAndroid Build Coastguard Worker
5046*9880d681SAndroid Build Coastguard Worker /// \brief Check whether or not \p Use can be combined
5047*9880d681SAndroid Build Coastguard Worker /// with the transition.
5048*9880d681SAndroid Build Coastguard Worker /// I.e., is it possible to do Use(Transition) => AnotherUse?
canCombine(const Instruction * Use)5049*9880d681SAndroid Build Coastguard Worker bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); }
5050*9880d681SAndroid Build Coastguard Worker
5051*9880d681SAndroid Build Coastguard Worker /// \brief Record \p ToBePromoted as part of the chain to be promoted.
enqueueForPromotion(Instruction * ToBePromoted)5052*9880d681SAndroid Build Coastguard Worker void enqueueForPromotion(Instruction *ToBePromoted) {
5053*9880d681SAndroid Build Coastguard Worker InstsToBePromoted.push_back(ToBePromoted);
5054*9880d681SAndroid Build Coastguard Worker }
5055*9880d681SAndroid Build Coastguard Worker
5056*9880d681SAndroid Build Coastguard Worker /// \brief Set the instruction that will be combined with the transition.
recordCombineInstruction(Instruction * ToBeCombined)5057*9880d681SAndroid Build Coastguard Worker void recordCombineInstruction(Instruction *ToBeCombined) {
5058*9880d681SAndroid Build Coastguard Worker assert(canCombine(ToBeCombined) && "Unsupported instruction to combine");
5059*9880d681SAndroid Build Coastguard Worker CombineInst = ToBeCombined;
5060*9880d681SAndroid Build Coastguard Worker }
5061*9880d681SAndroid Build Coastguard Worker
5062*9880d681SAndroid Build Coastguard Worker /// \brief Promote all the instructions enqueued for promotion if it is
5063*9880d681SAndroid Build Coastguard Worker /// is profitable.
5064*9880d681SAndroid Build Coastguard Worker /// \return True if the promotion happened, false otherwise.
promote()5065*9880d681SAndroid Build Coastguard Worker bool promote() {
5066*9880d681SAndroid Build Coastguard Worker // Check if there is something to promote.
5067*9880d681SAndroid Build Coastguard Worker // Right now, if we do not have anything to combine with,
5068*9880d681SAndroid Build Coastguard Worker // we assume the promotion is not profitable.
5069*9880d681SAndroid Build Coastguard Worker if (InstsToBePromoted.empty() || !CombineInst)
5070*9880d681SAndroid Build Coastguard Worker return false;
5071*9880d681SAndroid Build Coastguard Worker
5072*9880d681SAndroid Build Coastguard Worker // Check cost.
5073*9880d681SAndroid Build Coastguard Worker if (!StressStoreExtract && !isProfitableToPromote())
5074*9880d681SAndroid Build Coastguard Worker return false;
5075*9880d681SAndroid Build Coastguard Worker
5076*9880d681SAndroid Build Coastguard Worker // Promote.
5077*9880d681SAndroid Build Coastguard Worker for (auto &ToBePromoted : InstsToBePromoted)
5078*9880d681SAndroid Build Coastguard Worker promoteImpl(ToBePromoted);
5079*9880d681SAndroid Build Coastguard Worker InstsToBePromoted.clear();
5080*9880d681SAndroid Build Coastguard Worker return true;
5081*9880d681SAndroid Build Coastguard Worker }
5082*9880d681SAndroid Build Coastguard Worker };
5083*9880d681SAndroid Build Coastguard Worker } // End of anonymous namespace.
5084*9880d681SAndroid Build Coastguard Worker
promoteImpl(Instruction * ToBePromoted)5085*9880d681SAndroid Build Coastguard Worker void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) {
5086*9880d681SAndroid Build Coastguard Worker // At this point, we know that all the operands of ToBePromoted but Def
5087*9880d681SAndroid Build Coastguard Worker // can be statically promoted.
5088*9880d681SAndroid Build Coastguard Worker // For Def, we need to use its parameter in ToBePromoted:
5089*9880d681SAndroid Build Coastguard Worker // b = ToBePromoted ty1 a
5090*9880d681SAndroid Build Coastguard Worker // Def = Transition ty1 b to ty2
5091*9880d681SAndroid Build Coastguard Worker // Move the transition down.
5092*9880d681SAndroid Build Coastguard Worker // 1. Replace all uses of the promoted operation by the transition.
5093*9880d681SAndroid Build Coastguard Worker // = ... b => = ... Def.
5094*9880d681SAndroid Build Coastguard Worker assert(ToBePromoted->getType() == Transition->getType() &&
5095*9880d681SAndroid Build Coastguard Worker "The type of the result of the transition does not match "
5096*9880d681SAndroid Build Coastguard Worker "the final type");
5097*9880d681SAndroid Build Coastguard Worker ToBePromoted->replaceAllUsesWith(Transition);
5098*9880d681SAndroid Build Coastguard Worker // 2. Update the type of the uses.
5099*9880d681SAndroid Build Coastguard Worker // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def.
5100*9880d681SAndroid Build Coastguard Worker Type *TransitionTy = getTransitionType();
5101*9880d681SAndroid Build Coastguard Worker ToBePromoted->mutateType(TransitionTy);
5102*9880d681SAndroid Build Coastguard Worker // 3. Update all the operands of the promoted operation with promoted
5103*9880d681SAndroid Build Coastguard Worker // operands.
5104*9880d681SAndroid Build Coastguard Worker // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a.
5105*9880d681SAndroid Build Coastguard Worker for (Use &U : ToBePromoted->operands()) {
5106*9880d681SAndroid Build Coastguard Worker Value *Val = U.get();
5107*9880d681SAndroid Build Coastguard Worker Value *NewVal = nullptr;
5108*9880d681SAndroid Build Coastguard Worker if (Val == Transition)
5109*9880d681SAndroid Build Coastguard Worker NewVal = Transition->getOperand(getTransitionOriginalValueIdx());
5110*9880d681SAndroid Build Coastguard Worker else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) ||
5111*9880d681SAndroid Build Coastguard Worker isa<ConstantFP>(Val)) {
5112*9880d681SAndroid Build Coastguard Worker // Use a splat constant if it is not safe to use undef.
5113*9880d681SAndroid Build Coastguard Worker NewVal = getConstantVector(
5114*9880d681SAndroid Build Coastguard Worker cast<Constant>(Val),
5115*9880d681SAndroid Build Coastguard Worker isa<UndefValue>(Val) ||
5116*9880d681SAndroid Build Coastguard Worker canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()));
5117*9880d681SAndroid Build Coastguard Worker } else
5118*9880d681SAndroid Build Coastguard Worker llvm_unreachable("Did you modified shouldPromote and forgot to update "
5119*9880d681SAndroid Build Coastguard Worker "this?");
5120*9880d681SAndroid Build Coastguard Worker ToBePromoted->setOperand(U.getOperandNo(), NewVal);
5121*9880d681SAndroid Build Coastguard Worker }
5122*9880d681SAndroid Build Coastguard Worker Transition->removeFromParent();
5123*9880d681SAndroid Build Coastguard Worker Transition->insertAfter(ToBePromoted);
5124*9880d681SAndroid Build Coastguard Worker Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted);
5125*9880d681SAndroid Build Coastguard Worker }
5126*9880d681SAndroid Build Coastguard Worker
5127*9880d681SAndroid Build Coastguard Worker /// Some targets can do store(extractelement) with one instruction.
5128*9880d681SAndroid Build Coastguard Worker /// Try to push the extractelement towards the stores when the target
5129*9880d681SAndroid Build Coastguard Worker /// has this feature and this is profitable.
optimizeExtractElementInst(Instruction * Inst)5130*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) {
5131*9880d681SAndroid Build Coastguard Worker unsigned CombineCost = UINT_MAX;
5132*9880d681SAndroid Build Coastguard Worker if (DisableStoreExtract || !TLI ||
5133*9880d681SAndroid Build Coastguard Worker (!StressStoreExtract &&
5134*9880d681SAndroid Build Coastguard Worker !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(),
5135*9880d681SAndroid Build Coastguard Worker Inst->getOperand(1), CombineCost)))
5136*9880d681SAndroid Build Coastguard Worker return false;
5137*9880d681SAndroid Build Coastguard Worker
5138*9880d681SAndroid Build Coastguard Worker // At this point we know that Inst is a vector to scalar transition.
5139*9880d681SAndroid Build Coastguard Worker // Try to move it down the def-use chain, until:
5140*9880d681SAndroid Build Coastguard Worker // - We can combine the transition with its single use
5141*9880d681SAndroid Build Coastguard Worker // => we got rid of the transition.
5142*9880d681SAndroid Build Coastguard Worker // - We escape the current basic block
5143*9880d681SAndroid Build Coastguard Worker // => we would need to check that we are moving it at a cheaper place and
5144*9880d681SAndroid Build Coastguard Worker // we do not do that for now.
5145*9880d681SAndroid Build Coastguard Worker BasicBlock *Parent = Inst->getParent();
5146*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n');
5147*9880d681SAndroid Build Coastguard Worker VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost);
5148*9880d681SAndroid Build Coastguard Worker // If the transition has more than one use, assume this is not going to be
5149*9880d681SAndroid Build Coastguard Worker // beneficial.
5150*9880d681SAndroid Build Coastguard Worker while (Inst->hasOneUse()) {
5151*9880d681SAndroid Build Coastguard Worker Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin());
5152*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n');
5153*9880d681SAndroid Build Coastguard Worker
5154*9880d681SAndroid Build Coastguard Worker if (ToBePromoted->getParent() != Parent) {
5155*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Instruction to promote is in a different block ("
5156*9880d681SAndroid Build Coastguard Worker << ToBePromoted->getParent()->getName()
5157*9880d681SAndroid Build Coastguard Worker << ") than the transition (" << Parent->getName() << ").\n");
5158*9880d681SAndroid Build Coastguard Worker return false;
5159*9880d681SAndroid Build Coastguard Worker }
5160*9880d681SAndroid Build Coastguard Worker
5161*9880d681SAndroid Build Coastguard Worker if (VPH.canCombine(ToBePromoted)) {
5162*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Assume " << *Inst << '\n'
5163*9880d681SAndroid Build Coastguard Worker << "will be combined with: " << *ToBePromoted << '\n');
5164*9880d681SAndroid Build Coastguard Worker VPH.recordCombineInstruction(ToBePromoted);
5165*9880d681SAndroid Build Coastguard Worker bool Changed = VPH.promote();
5166*9880d681SAndroid Build Coastguard Worker NumStoreExtractExposed += Changed;
5167*9880d681SAndroid Build Coastguard Worker return Changed;
5168*9880d681SAndroid Build Coastguard Worker }
5169*9880d681SAndroid Build Coastguard Worker
5170*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Try promoting.\n");
5171*9880d681SAndroid Build Coastguard Worker if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))
5172*9880d681SAndroid Build Coastguard Worker return false;
5173*9880d681SAndroid Build Coastguard Worker
5174*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n");
5175*9880d681SAndroid Build Coastguard Worker
5176*9880d681SAndroid Build Coastguard Worker VPH.enqueueForPromotion(ToBePromoted);
5177*9880d681SAndroid Build Coastguard Worker Inst = ToBePromoted;
5178*9880d681SAndroid Build Coastguard Worker }
5179*9880d681SAndroid Build Coastguard Worker return false;
5180*9880d681SAndroid Build Coastguard Worker }
5181*9880d681SAndroid Build Coastguard Worker
optimizeInst(Instruction * I,bool & ModifiedDT)5182*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeInst(Instruction *I, bool& ModifiedDT) {
5183*9880d681SAndroid Build Coastguard Worker // Bail out if we inserted the instruction to prevent optimizations from
5184*9880d681SAndroid Build Coastguard Worker // stepping on each other's toes.
5185*9880d681SAndroid Build Coastguard Worker if (InsertedInsts.count(I))
5186*9880d681SAndroid Build Coastguard Worker return false;
5187*9880d681SAndroid Build Coastguard Worker
5188*9880d681SAndroid Build Coastguard Worker if (PHINode *P = dyn_cast<PHINode>(I)) {
5189*9880d681SAndroid Build Coastguard Worker // It is possible for very late stage optimizations (such as SimplifyCFG)
5190*9880d681SAndroid Build Coastguard Worker // to introduce PHI nodes too late to be cleaned up. If we detect such a
5191*9880d681SAndroid Build Coastguard Worker // trivial PHI, go ahead and zap it here.
5192*9880d681SAndroid Build Coastguard Worker if (Value *V = SimplifyInstruction(P, *DL, TLInfo, nullptr)) {
5193*9880d681SAndroid Build Coastguard Worker P->replaceAllUsesWith(V);
5194*9880d681SAndroid Build Coastguard Worker P->eraseFromParent();
5195*9880d681SAndroid Build Coastguard Worker ++NumPHIsElim;
5196*9880d681SAndroid Build Coastguard Worker return true;
5197*9880d681SAndroid Build Coastguard Worker }
5198*9880d681SAndroid Build Coastguard Worker return false;
5199*9880d681SAndroid Build Coastguard Worker }
5200*9880d681SAndroid Build Coastguard Worker
5201*9880d681SAndroid Build Coastguard Worker if (CastInst *CI = dyn_cast<CastInst>(I)) {
5202*9880d681SAndroid Build Coastguard Worker // If the source of the cast is a constant, then this should have
5203*9880d681SAndroid Build Coastguard Worker // already been constant folded. The only reason NOT to constant fold
5204*9880d681SAndroid Build Coastguard Worker // it is if something (e.g. LSR) was careful to place the constant
5205*9880d681SAndroid Build Coastguard Worker // evaluation in a block other than then one that uses it (e.g. to hoist
5206*9880d681SAndroid Build Coastguard Worker // the address of globals out of a loop). If this is the case, we don't
5207*9880d681SAndroid Build Coastguard Worker // want to forward-subst the cast.
5208*9880d681SAndroid Build Coastguard Worker if (isa<Constant>(CI->getOperand(0)))
5209*9880d681SAndroid Build Coastguard Worker return false;
5210*9880d681SAndroid Build Coastguard Worker
5211*9880d681SAndroid Build Coastguard Worker if (TLI && OptimizeNoopCopyExpression(CI, *TLI, *DL))
5212*9880d681SAndroid Build Coastguard Worker return true;
5213*9880d681SAndroid Build Coastguard Worker
5214*9880d681SAndroid Build Coastguard Worker if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {
5215*9880d681SAndroid Build Coastguard Worker /// Sink a zext or sext into its user blocks if the target type doesn't
5216*9880d681SAndroid Build Coastguard Worker /// fit in one register
5217*9880d681SAndroid Build Coastguard Worker if (TLI &&
5218*9880d681SAndroid Build Coastguard Worker TLI->getTypeAction(CI->getContext(),
5219*9880d681SAndroid Build Coastguard Worker TLI->getValueType(*DL, CI->getType())) ==
5220*9880d681SAndroid Build Coastguard Worker TargetLowering::TypeExpandInteger) {
5221*9880d681SAndroid Build Coastguard Worker return SinkCast(CI);
5222*9880d681SAndroid Build Coastguard Worker } else {
5223*9880d681SAndroid Build Coastguard Worker bool MadeChange = moveExtToFormExtLoad(I);
5224*9880d681SAndroid Build Coastguard Worker return MadeChange | optimizeExtUses(I);
5225*9880d681SAndroid Build Coastguard Worker }
5226*9880d681SAndroid Build Coastguard Worker }
5227*9880d681SAndroid Build Coastguard Worker return false;
5228*9880d681SAndroid Build Coastguard Worker }
5229*9880d681SAndroid Build Coastguard Worker
5230*9880d681SAndroid Build Coastguard Worker if (CmpInst *CI = dyn_cast<CmpInst>(I))
5231*9880d681SAndroid Build Coastguard Worker if (!TLI || !TLI->hasMultipleConditionRegisters())
5232*9880d681SAndroid Build Coastguard Worker return OptimizeCmpExpression(CI, TLI);
5233*9880d681SAndroid Build Coastguard Worker
5234*9880d681SAndroid Build Coastguard Worker if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
5235*9880d681SAndroid Build Coastguard Worker stripInvariantGroupMetadata(*LI);
5236*9880d681SAndroid Build Coastguard Worker if (TLI) {
5237*9880d681SAndroid Build Coastguard Worker bool Modified = optimizeLoadExt(LI);
5238*9880d681SAndroid Build Coastguard Worker unsigned AS = LI->getPointerAddressSpace();
5239*9880d681SAndroid Build Coastguard Worker Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS);
5240*9880d681SAndroid Build Coastguard Worker return Modified;
5241*9880d681SAndroid Build Coastguard Worker }
5242*9880d681SAndroid Build Coastguard Worker return false;
5243*9880d681SAndroid Build Coastguard Worker }
5244*9880d681SAndroid Build Coastguard Worker
5245*9880d681SAndroid Build Coastguard Worker if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
5246*9880d681SAndroid Build Coastguard Worker stripInvariantGroupMetadata(*SI);
5247*9880d681SAndroid Build Coastguard Worker if (TLI) {
5248*9880d681SAndroid Build Coastguard Worker unsigned AS = SI->getPointerAddressSpace();
5249*9880d681SAndroid Build Coastguard Worker return optimizeMemoryInst(I, SI->getOperand(1),
5250*9880d681SAndroid Build Coastguard Worker SI->getOperand(0)->getType(), AS);
5251*9880d681SAndroid Build Coastguard Worker }
5252*9880d681SAndroid Build Coastguard Worker return false;
5253*9880d681SAndroid Build Coastguard Worker }
5254*9880d681SAndroid Build Coastguard Worker
5255*9880d681SAndroid Build Coastguard Worker BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I);
5256*9880d681SAndroid Build Coastguard Worker
5257*9880d681SAndroid Build Coastguard Worker if (BinOp && (BinOp->getOpcode() == Instruction::AShr ||
5258*9880d681SAndroid Build Coastguard Worker BinOp->getOpcode() == Instruction::LShr)) {
5259*9880d681SAndroid Build Coastguard Worker ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1));
5260*9880d681SAndroid Build Coastguard Worker if (TLI && CI && TLI->hasExtractBitsInsn())
5261*9880d681SAndroid Build Coastguard Worker return OptimizeExtractBits(BinOp, CI, *TLI, *DL);
5262*9880d681SAndroid Build Coastguard Worker
5263*9880d681SAndroid Build Coastguard Worker return false;
5264*9880d681SAndroid Build Coastguard Worker }
5265*9880d681SAndroid Build Coastguard Worker
5266*9880d681SAndroid Build Coastguard Worker if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
5267*9880d681SAndroid Build Coastguard Worker if (GEPI->hasAllZeroIndices()) {
5268*9880d681SAndroid Build Coastguard Worker /// The GEP operand must be a pointer, so must its result -> BitCast
5269*9880d681SAndroid Build Coastguard Worker Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(),
5270*9880d681SAndroid Build Coastguard Worker GEPI->getName(), GEPI);
5271*9880d681SAndroid Build Coastguard Worker GEPI->replaceAllUsesWith(NC);
5272*9880d681SAndroid Build Coastguard Worker GEPI->eraseFromParent();
5273*9880d681SAndroid Build Coastguard Worker ++NumGEPsElim;
5274*9880d681SAndroid Build Coastguard Worker optimizeInst(NC, ModifiedDT);
5275*9880d681SAndroid Build Coastguard Worker return true;
5276*9880d681SAndroid Build Coastguard Worker }
5277*9880d681SAndroid Build Coastguard Worker return false;
5278*9880d681SAndroid Build Coastguard Worker }
5279*9880d681SAndroid Build Coastguard Worker
5280*9880d681SAndroid Build Coastguard Worker if (CallInst *CI = dyn_cast<CallInst>(I))
5281*9880d681SAndroid Build Coastguard Worker return optimizeCallInst(CI, ModifiedDT);
5282*9880d681SAndroid Build Coastguard Worker
5283*9880d681SAndroid Build Coastguard Worker if (SelectInst *SI = dyn_cast<SelectInst>(I))
5284*9880d681SAndroid Build Coastguard Worker return optimizeSelectInst(SI);
5285*9880d681SAndroid Build Coastguard Worker
5286*9880d681SAndroid Build Coastguard Worker if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
5287*9880d681SAndroid Build Coastguard Worker return optimizeShuffleVectorInst(SVI);
5288*9880d681SAndroid Build Coastguard Worker
5289*9880d681SAndroid Build Coastguard Worker if (auto *Switch = dyn_cast<SwitchInst>(I))
5290*9880d681SAndroid Build Coastguard Worker return optimizeSwitchInst(Switch);
5291*9880d681SAndroid Build Coastguard Worker
5292*9880d681SAndroid Build Coastguard Worker if (isa<ExtractElementInst>(I))
5293*9880d681SAndroid Build Coastguard Worker return optimizeExtractElementInst(I);
5294*9880d681SAndroid Build Coastguard Worker
5295*9880d681SAndroid Build Coastguard Worker return false;
5296*9880d681SAndroid Build Coastguard Worker }
5297*9880d681SAndroid Build Coastguard Worker
5298*9880d681SAndroid Build Coastguard Worker /// Given an OR instruction, check to see if this is a bitreverse
5299*9880d681SAndroid Build Coastguard Worker /// idiom. If so, insert the new intrinsic and return true.
makeBitReverse(Instruction & I,const DataLayout & DL,const TargetLowering & TLI)5300*9880d681SAndroid Build Coastguard Worker static bool makeBitReverse(Instruction &I, const DataLayout &DL,
5301*9880d681SAndroid Build Coastguard Worker const TargetLowering &TLI) {
5302*9880d681SAndroid Build Coastguard Worker if (!I.getType()->isIntegerTy() ||
5303*9880d681SAndroid Build Coastguard Worker !TLI.isOperationLegalOrCustom(ISD::BITREVERSE,
5304*9880d681SAndroid Build Coastguard Worker TLI.getValueType(DL, I.getType(), true)))
5305*9880d681SAndroid Build Coastguard Worker return false;
5306*9880d681SAndroid Build Coastguard Worker
5307*9880d681SAndroid Build Coastguard Worker SmallVector<Instruction*, 4> Insts;
5308*9880d681SAndroid Build Coastguard Worker if (!recognizeBSwapOrBitReverseIdiom(&I, false, true, Insts))
5309*9880d681SAndroid Build Coastguard Worker return false;
5310*9880d681SAndroid Build Coastguard Worker Instruction *LastInst = Insts.back();
5311*9880d681SAndroid Build Coastguard Worker I.replaceAllUsesWith(LastInst);
5312*9880d681SAndroid Build Coastguard Worker RecursivelyDeleteTriviallyDeadInstructions(&I);
5313*9880d681SAndroid Build Coastguard Worker return true;
5314*9880d681SAndroid Build Coastguard Worker }
5315*9880d681SAndroid Build Coastguard Worker
5316*9880d681SAndroid Build Coastguard Worker // In this pass we look for GEP and cast instructions that are used
5317*9880d681SAndroid Build Coastguard Worker // across basic blocks and rewrite them to improve basic-block-at-a-time
5318*9880d681SAndroid Build Coastguard Worker // selection.
optimizeBlock(BasicBlock & BB,bool & ModifiedDT)5319*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, bool& ModifiedDT) {
5320*9880d681SAndroid Build Coastguard Worker SunkAddrs.clear();
5321*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
5322*9880d681SAndroid Build Coastguard Worker
5323*9880d681SAndroid Build Coastguard Worker CurInstIterator = BB.begin();
5324*9880d681SAndroid Build Coastguard Worker while (CurInstIterator != BB.end()) {
5325*9880d681SAndroid Build Coastguard Worker MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);
5326*9880d681SAndroid Build Coastguard Worker if (ModifiedDT)
5327*9880d681SAndroid Build Coastguard Worker return true;
5328*9880d681SAndroid Build Coastguard Worker }
5329*9880d681SAndroid Build Coastguard Worker
5330*9880d681SAndroid Build Coastguard Worker bool MadeBitReverse = true;
5331*9880d681SAndroid Build Coastguard Worker while (TLI && MadeBitReverse) {
5332*9880d681SAndroid Build Coastguard Worker MadeBitReverse = false;
5333*9880d681SAndroid Build Coastguard Worker for (auto &I : reverse(BB)) {
5334*9880d681SAndroid Build Coastguard Worker if (makeBitReverse(I, *DL, *TLI)) {
5335*9880d681SAndroid Build Coastguard Worker MadeBitReverse = MadeChange = true;
5336*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
5337*9880d681SAndroid Build Coastguard Worker break;
5338*9880d681SAndroid Build Coastguard Worker }
5339*9880d681SAndroid Build Coastguard Worker }
5340*9880d681SAndroid Build Coastguard Worker }
5341*9880d681SAndroid Build Coastguard Worker MadeChange |= dupRetToEnableTailCallOpts(&BB);
5342*9880d681SAndroid Build Coastguard Worker
5343*9880d681SAndroid Build Coastguard Worker return MadeChange;
5344*9880d681SAndroid Build Coastguard Worker }
5345*9880d681SAndroid Build Coastguard Worker
5346*9880d681SAndroid Build Coastguard Worker // llvm.dbg.value is far away from the value then iSel may not be able
5347*9880d681SAndroid Build Coastguard Worker // handle it properly. iSel will drop llvm.dbg.value if it can not
5348*9880d681SAndroid Build Coastguard Worker // find a node corresponding to the value.
placeDbgValues(Function & F)5349*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::placeDbgValues(Function &F) {
5350*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
5351*9880d681SAndroid Build Coastguard Worker for (BasicBlock &BB : F) {
5352*9880d681SAndroid Build Coastguard Worker Instruction *PrevNonDbgInst = nullptr;
5353*9880d681SAndroid Build Coastguard Worker for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) {
5354*9880d681SAndroid Build Coastguard Worker Instruction *Insn = &*BI++;
5355*9880d681SAndroid Build Coastguard Worker DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn);
5356*9880d681SAndroid Build Coastguard Worker // Leave dbg.values that refer to an alloca alone. These
5357*9880d681SAndroid Build Coastguard Worker // instrinsics describe the address of a variable (= the alloca)
5358*9880d681SAndroid Build Coastguard Worker // being taken. They should not be moved next to the alloca
5359*9880d681SAndroid Build Coastguard Worker // (and to the beginning of the scope), but rather stay close to
5360*9880d681SAndroid Build Coastguard Worker // where said address is used.
5361*9880d681SAndroid Build Coastguard Worker if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) {
5362*9880d681SAndroid Build Coastguard Worker PrevNonDbgInst = Insn;
5363*9880d681SAndroid Build Coastguard Worker continue;
5364*9880d681SAndroid Build Coastguard Worker }
5365*9880d681SAndroid Build Coastguard Worker
5366*9880d681SAndroid Build Coastguard Worker Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue());
5367*9880d681SAndroid Build Coastguard Worker if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) {
5368*9880d681SAndroid Build Coastguard Worker // If VI is a phi in a block with an EHPad terminator, we can't insert
5369*9880d681SAndroid Build Coastguard Worker // after it.
5370*9880d681SAndroid Build Coastguard Worker if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad())
5371*9880d681SAndroid Build Coastguard Worker continue;
5372*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI);
5373*9880d681SAndroid Build Coastguard Worker DVI->removeFromParent();
5374*9880d681SAndroid Build Coastguard Worker if (isa<PHINode>(VI))
5375*9880d681SAndroid Build Coastguard Worker DVI->insertBefore(&*VI->getParent()->getFirstInsertionPt());
5376*9880d681SAndroid Build Coastguard Worker else
5377*9880d681SAndroid Build Coastguard Worker DVI->insertAfter(VI);
5378*9880d681SAndroid Build Coastguard Worker MadeChange = true;
5379*9880d681SAndroid Build Coastguard Worker ++NumDbgValueMoved;
5380*9880d681SAndroid Build Coastguard Worker }
5381*9880d681SAndroid Build Coastguard Worker }
5382*9880d681SAndroid Build Coastguard Worker }
5383*9880d681SAndroid Build Coastguard Worker return MadeChange;
5384*9880d681SAndroid Build Coastguard Worker }
5385*9880d681SAndroid Build Coastguard Worker
5386*9880d681SAndroid Build Coastguard Worker // If there is a sequence that branches based on comparing a single bit
5387*9880d681SAndroid Build Coastguard Worker // against zero that can be combined into a single instruction, and the
5388*9880d681SAndroid Build Coastguard Worker // target supports folding these into a single instruction, sink the
5389*9880d681SAndroid Build Coastguard Worker // mask and compare into the branch uses. Do this before OptimizeBlock ->
5390*9880d681SAndroid Build Coastguard Worker // OptimizeInst -> OptimizeCmpExpression, which perturbs the pattern being
5391*9880d681SAndroid Build Coastguard Worker // searched for.
sinkAndCmp(Function & F)5392*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::sinkAndCmp(Function &F) {
5393*9880d681SAndroid Build Coastguard Worker if (!EnableAndCmpSinking)
5394*9880d681SAndroid Build Coastguard Worker return false;
5395*9880d681SAndroid Build Coastguard Worker if (!TLI || !TLI->isMaskAndBranchFoldingLegal())
5396*9880d681SAndroid Build Coastguard Worker return false;
5397*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
5398*9880d681SAndroid Build Coastguard Worker for (BasicBlock &BB : F) {
5399*9880d681SAndroid Build Coastguard Worker // Does this BB end with the following?
5400*9880d681SAndroid Build Coastguard Worker // %andVal = and %val, #single-bit-set
5401*9880d681SAndroid Build Coastguard Worker // %icmpVal = icmp %andResult, 0
5402*9880d681SAndroid Build Coastguard Worker // br i1 %cmpVal label %dest1, label %dest2"
5403*9880d681SAndroid Build Coastguard Worker BranchInst *Brcc = dyn_cast<BranchInst>(BB.getTerminator());
5404*9880d681SAndroid Build Coastguard Worker if (!Brcc || !Brcc->isConditional())
5405*9880d681SAndroid Build Coastguard Worker continue;
5406*9880d681SAndroid Build Coastguard Worker ICmpInst *Cmp = dyn_cast<ICmpInst>(Brcc->getOperand(0));
5407*9880d681SAndroid Build Coastguard Worker if (!Cmp || Cmp->getParent() != &BB)
5408*9880d681SAndroid Build Coastguard Worker continue;
5409*9880d681SAndroid Build Coastguard Worker ConstantInt *Zero = dyn_cast<ConstantInt>(Cmp->getOperand(1));
5410*9880d681SAndroid Build Coastguard Worker if (!Zero || !Zero->isZero())
5411*9880d681SAndroid Build Coastguard Worker continue;
5412*9880d681SAndroid Build Coastguard Worker Instruction *And = dyn_cast<Instruction>(Cmp->getOperand(0));
5413*9880d681SAndroid Build Coastguard Worker if (!And || And->getOpcode() != Instruction::And || And->getParent() != &BB)
5414*9880d681SAndroid Build Coastguard Worker continue;
5415*9880d681SAndroid Build Coastguard Worker ConstantInt* Mask = dyn_cast<ConstantInt>(And->getOperand(1));
5416*9880d681SAndroid Build Coastguard Worker if (!Mask || !Mask->getUniqueInteger().isPowerOf2())
5417*9880d681SAndroid Build Coastguard Worker continue;
5418*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "found and; icmp ?,0; brcc\n"); DEBUG(BB.dump());
5419*9880d681SAndroid Build Coastguard Worker
5420*9880d681SAndroid Build Coastguard Worker // Push the "and; icmp" for any users that are conditional branches.
5421*9880d681SAndroid Build Coastguard Worker // Since there can only be one branch use per BB, we don't need to keep
5422*9880d681SAndroid Build Coastguard Worker // track of which BBs we insert into.
5423*9880d681SAndroid Build Coastguard Worker for (Use &TheUse : Cmp->uses()) {
5424*9880d681SAndroid Build Coastguard Worker // Find brcc use.
5425*9880d681SAndroid Build Coastguard Worker BranchInst *BrccUser = dyn_cast<BranchInst>(TheUse);
5426*9880d681SAndroid Build Coastguard Worker if (!BrccUser || !BrccUser->isConditional())
5427*9880d681SAndroid Build Coastguard Worker continue;
5428*9880d681SAndroid Build Coastguard Worker BasicBlock *UserBB = BrccUser->getParent();
5429*9880d681SAndroid Build Coastguard Worker if (UserBB == &BB) continue;
5430*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "found Brcc use\n");
5431*9880d681SAndroid Build Coastguard Worker
5432*9880d681SAndroid Build Coastguard Worker // Sink the "and; icmp" to use.
5433*9880d681SAndroid Build Coastguard Worker MadeChange = true;
5434*9880d681SAndroid Build Coastguard Worker BinaryOperator *NewAnd =
5435*9880d681SAndroid Build Coastguard Worker BinaryOperator::CreateAnd(And->getOperand(0), And->getOperand(1), "",
5436*9880d681SAndroid Build Coastguard Worker BrccUser);
5437*9880d681SAndroid Build Coastguard Worker CmpInst *NewCmp =
5438*9880d681SAndroid Build Coastguard Worker CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), NewAnd, Zero,
5439*9880d681SAndroid Build Coastguard Worker "", BrccUser);
5440*9880d681SAndroid Build Coastguard Worker TheUse = NewCmp;
5441*9880d681SAndroid Build Coastguard Worker ++NumAndCmpsMoved;
5442*9880d681SAndroid Build Coastguard Worker DEBUG(BrccUser->getParent()->dump());
5443*9880d681SAndroid Build Coastguard Worker }
5444*9880d681SAndroid Build Coastguard Worker }
5445*9880d681SAndroid Build Coastguard Worker return MadeChange;
5446*9880d681SAndroid Build Coastguard Worker }
5447*9880d681SAndroid Build Coastguard Worker
5448*9880d681SAndroid Build Coastguard Worker /// \brief Scale down both weights to fit into uint32_t.
scaleWeights(uint64_t & NewTrue,uint64_t & NewFalse)5449*9880d681SAndroid Build Coastguard Worker static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
5450*9880d681SAndroid Build Coastguard Worker uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
5451*9880d681SAndroid Build Coastguard Worker uint32_t Scale = (NewMax / UINT32_MAX) + 1;
5452*9880d681SAndroid Build Coastguard Worker NewTrue = NewTrue / Scale;
5453*9880d681SAndroid Build Coastguard Worker NewFalse = NewFalse / Scale;
5454*9880d681SAndroid Build Coastguard Worker }
5455*9880d681SAndroid Build Coastguard Worker
5456*9880d681SAndroid Build Coastguard Worker /// \brief Some targets prefer to split a conditional branch like:
5457*9880d681SAndroid Build Coastguard Worker /// \code
5458*9880d681SAndroid Build Coastguard Worker /// %0 = icmp ne i32 %a, 0
5459*9880d681SAndroid Build Coastguard Worker /// %1 = icmp ne i32 %b, 0
5460*9880d681SAndroid Build Coastguard Worker /// %or.cond = or i1 %0, %1
5461*9880d681SAndroid Build Coastguard Worker /// br i1 %or.cond, label %TrueBB, label %FalseBB
5462*9880d681SAndroid Build Coastguard Worker /// \endcode
5463*9880d681SAndroid Build Coastguard Worker /// into multiple branch instructions like:
5464*9880d681SAndroid Build Coastguard Worker /// \code
5465*9880d681SAndroid Build Coastguard Worker /// bb1:
5466*9880d681SAndroid Build Coastguard Worker /// %0 = icmp ne i32 %a, 0
5467*9880d681SAndroid Build Coastguard Worker /// br i1 %0, label %TrueBB, label %bb2
5468*9880d681SAndroid Build Coastguard Worker /// bb2:
5469*9880d681SAndroid Build Coastguard Worker /// %1 = icmp ne i32 %b, 0
5470*9880d681SAndroid Build Coastguard Worker /// br i1 %1, label %TrueBB, label %FalseBB
5471*9880d681SAndroid Build Coastguard Worker /// \endcode
5472*9880d681SAndroid Build Coastguard Worker /// This usually allows instruction selection to do even further optimizations
5473*9880d681SAndroid Build Coastguard Worker /// and combine the compare with the branch instruction. Currently this is
5474*9880d681SAndroid Build Coastguard Worker /// applied for targets which have "cheap" jump instructions.
5475*9880d681SAndroid Build Coastguard Worker ///
5476*9880d681SAndroid Build Coastguard Worker /// FIXME: Remove the (equivalent?) implementation in SelectionDAG.
5477*9880d681SAndroid Build Coastguard Worker ///
splitBranchCondition(Function & F)5478*9880d681SAndroid Build Coastguard Worker bool CodeGenPrepare::splitBranchCondition(Function &F) {
5479*9880d681SAndroid Build Coastguard Worker if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive())
5480*9880d681SAndroid Build Coastguard Worker return false;
5481*9880d681SAndroid Build Coastguard Worker
5482*9880d681SAndroid Build Coastguard Worker bool MadeChange = false;
5483*9880d681SAndroid Build Coastguard Worker for (auto &BB : F) {
5484*9880d681SAndroid Build Coastguard Worker // Does this BB end with the following?
5485*9880d681SAndroid Build Coastguard Worker // %cond1 = icmp|fcmp|binary instruction ...
5486*9880d681SAndroid Build Coastguard Worker // %cond2 = icmp|fcmp|binary instruction ...
5487*9880d681SAndroid Build Coastguard Worker // %cond.or = or|and i1 %cond1, cond2
5488*9880d681SAndroid Build Coastguard Worker // br i1 %cond.or label %dest1, label %dest2"
5489*9880d681SAndroid Build Coastguard Worker BinaryOperator *LogicOp;
5490*9880d681SAndroid Build Coastguard Worker BasicBlock *TBB, *FBB;
5491*9880d681SAndroid Build Coastguard Worker if (!match(BB.getTerminator(), m_Br(m_OneUse(m_BinOp(LogicOp)), TBB, FBB)))
5492*9880d681SAndroid Build Coastguard Worker continue;
5493*9880d681SAndroid Build Coastguard Worker
5494*9880d681SAndroid Build Coastguard Worker auto *Br1 = cast<BranchInst>(BB.getTerminator());
5495*9880d681SAndroid Build Coastguard Worker if (Br1->getMetadata(LLVMContext::MD_unpredictable))
5496*9880d681SAndroid Build Coastguard Worker continue;
5497*9880d681SAndroid Build Coastguard Worker
5498*9880d681SAndroid Build Coastguard Worker unsigned Opc;
5499*9880d681SAndroid Build Coastguard Worker Value *Cond1, *Cond2;
5500*9880d681SAndroid Build Coastguard Worker if (match(LogicOp, m_And(m_OneUse(m_Value(Cond1)),
5501*9880d681SAndroid Build Coastguard Worker m_OneUse(m_Value(Cond2)))))
5502*9880d681SAndroid Build Coastguard Worker Opc = Instruction::And;
5503*9880d681SAndroid Build Coastguard Worker else if (match(LogicOp, m_Or(m_OneUse(m_Value(Cond1)),
5504*9880d681SAndroid Build Coastguard Worker m_OneUse(m_Value(Cond2)))))
5505*9880d681SAndroid Build Coastguard Worker Opc = Instruction::Or;
5506*9880d681SAndroid Build Coastguard Worker else
5507*9880d681SAndroid Build Coastguard Worker continue;
5508*9880d681SAndroid Build Coastguard Worker
5509*9880d681SAndroid Build Coastguard Worker if (!match(Cond1, m_CombineOr(m_Cmp(), m_BinOp())) ||
5510*9880d681SAndroid Build Coastguard Worker !match(Cond2, m_CombineOr(m_Cmp(), m_BinOp())) )
5511*9880d681SAndroid Build Coastguard Worker continue;
5512*9880d681SAndroid Build Coastguard Worker
5513*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump());
5514*9880d681SAndroid Build Coastguard Worker
5515*9880d681SAndroid Build Coastguard Worker // Create a new BB.
5516*9880d681SAndroid Build Coastguard Worker auto TmpBB =
5517*9880d681SAndroid Build Coastguard Worker BasicBlock::Create(BB.getContext(), BB.getName() + ".cond.split",
5518*9880d681SAndroid Build Coastguard Worker BB.getParent(), BB.getNextNode());
5519*9880d681SAndroid Build Coastguard Worker
5520*9880d681SAndroid Build Coastguard Worker // Update original basic block by using the first condition directly by the
5521*9880d681SAndroid Build Coastguard Worker // branch instruction and removing the no longer needed and/or instruction.
5522*9880d681SAndroid Build Coastguard Worker Br1->setCondition(Cond1);
5523*9880d681SAndroid Build Coastguard Worker LogicOp->eraseFromParent();
5524*9880d681SAndroid Build Coastguard Worker
5525*9880d681SAndroid Build Coastguard Worker // Depending on the conditon we have to either replace the true or the false
5526*9880d681SAndroid Build Coastguard Worker // successor of the original branch instruction.
5527*9880d681SAndroid Build Coastguard Worker if (Opc == Instruction::And)
5528*9880d681SAndroid Build Coastguard Worker Br1->setSuccessor(0, TmpBB);
5529*9880d681SAndroid Build Coastguard Worker else
5530*9880d681SAndroid Build Coastguard Worker Br1->setSuccessor(1, TmpBB);
5531*9880d681SAndroid Build Coastguard Worker
5532*9880d681SAndroid Build Coastguard Worker // Fill in the new basic block.
5533*9880d681SAndroid Build Coastguard Worker auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB);
5534*9880d681SAndroid Build Coastguard Worker if (auto *I = dyn_cast<Instruction>(Cond2)) {
5535*9880d681SAndroid Build Coastguard Worker I->removeFromParent();
5536*9880d681SAndroid Build Coastguard Worker I->insertBefore(Br2);
5537*9880d681SAndroid Build Coastguard Worker }
5538*9880d681SAndroid Build Coastguard Worker
5539*9880d681SAndroid Build Coastguard Worker // Update PHI nodes in both successors. The original BB needs to be
5540*9880d681SAndroid Build Coastguard Worker // replaced in one succesor's PHI nodes, because the branch comes now from
5541*9880d681SAndroid Build Coastguard Worker // the newly generated BB (NewBB). In the other successor we need to add one
5542*9880d681SAndroid Build Coastguard Worker // incoming edge to the PHI nodes, because both branch instructions target
5543*9880d681SAndroid Build Coastguard Worker // now the same successor. Depending on the original branch condition
5544*9880d681SAndroid Build Coastguard Worker // (and/or) we have to swap the successors (TrueDest, FalseDest), so that
5545*9880d681SAndroid Build Coastguard Worker // we perfrom the correct update for the PHI nodes.
5546*9880d681SAndroid Build Coastguard Worker // This doesn't change the successor order of the just created branch
5547*9880d681SAndroid Build Coastguard Worker // instruction (or any other instruction).
5548*9880d681SAndroid Build Coastguard Worker if (Opc == Instruction::Or)
5549*9880d681SAndroid Build Coastguard Worker std::swap(TBB, FBB);
5550*9880d681SAndroid Build Coastguard Worker
5551*9880d681SAndroid Build Coastguard Worker // Replace the old BB with the new BB.
5552*9880d681SAndroid Build Coastguard Worker for (auto &I : *TBB) {
5553*9880d681SAndroid Build Coastguard Worker PHINode *PN = dyn_cast<PHINode>(&I);
5554*9880d681SAndroid Build Coastguard Worker if (!PN)
5555*9880d681SAndroid Build Coastguard Worker break;
5556*9880d681SAndroid Build Coastguard Worker int i;
5557*9880d681SAndroid Build Coastguard Worker while ((i = PN->getBasicBlockIndex(&BB)) >= 0)
5558*9880d681SAndroid Build Coastguard Worker PN->setIncomingBlock(i, TmpBB);
5559*9880d681SAndroid Build Coastguard Worker }
5560*9880d681SAndroid Build Coastguard Worker
5561*9880d681SAndroid Build Coastguard Worker // Add another incoming edge form the new BB.
5562*9880d681SAndroid Build Coastguard Worker for (auto &I : *FBB) {
5563*9880d681SAndroid Build Coastguard Worker PHINode *PN = dyn_cast<PHINode>(&I);
5564*9880d681SAndroid Build Coastguard Worker if (!PN)
5565*9880d681SAndroid Build Coastguard Worker break;
5566*9880d681SAndroid Build Coastguard Worker auto *Val = PN->getIncomingValueForBlock(&BB);
5567*9880d681SAndroid Build Coastguard Worker PN->addIncoming(Val, TmpBB);
5568*9880d681SAndroid Build Coastguard Worker }
5569*9880d681SAndroid Build Coastguard Worker
5570*9880d681SAndroid Build Coastguard Worker // Update the branch weights (from SelectionDAGBuilder::
5571*9880d681SAndroid Build Coastguard Worker // FindMergedConditions).
5572*9880d681SAndroid Build Coastguard Worker if (Opc == Instruction::Or) {
5573*9880d681SAndroid Build Coastguard Worker // Codegen X | Y as:
5574*9880d681SAndroid Build Coastguard Worker // BB1:
5575*9880d681SAndroid Build Coastguard Worker // jmp_if_X TBB
5576*9880d681SAndroid Build Coastguard Worker // jmp TmpBB
5577*9880d681SAndroid Build Coastguard Worker // TmpBB:
5578*9880d681SAndroid Build Coastguard Worker // jmp_if_Y TBB
5579*9880d681SAndroid Build Coastguard Worker // jmp FBB
5580*9880d681SAndroid Build Coastguard Worker //
5581*9880d681SAndroid Build Coastguard Worker
5582*9880d681SAndroid Build Coastguard Worker // We have flexibility in setting Prob for BB1 and Prob for NewBB.
5583*9880d681SAndroid Build Coastguard Worker // The requirement is that
5584*9880d681SAndroid Build Coastguard Worker // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
5585*9880d681SAndroid Build Coastguard Worker // = TrueProb for orignal BB.
5586*9880d681SAndroid Build Coastguard Worker // Assuming the orignal weights are A and B, one choice is to set BB1's
5587*9880d681SAndroid Build Coastguard Worker // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
5588*9880d681SAndroid Build Coastguard Worker // assumes that
5589*9880d681SAndroid Build Coastguard Worker // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
5590*9880d681SAndroid Build Coastguard Worker // Another choice is to assume TrueProb for BB1 equals to TrueProb for
5591*9880d681SAndroid Build Coastguard Worker // TmpBB, but the math is more complicated.
5592*9880d681SAndroid Build Coastguard Worker uint64_t TrueWeight, FalseWeight;
5593*9880d681SAndroid Build Coastguard Worker if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
5594*9880d681SAndroid Build Coastguard Worker uint64_t NewTrueWeight = TrueWeight;
5595*9880d681SAndroid Build Coastguard Worker uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;
5596*9880d681SAndroid Build Coastguard Worker scaleWeights(NewTrueWeight, NewFalseWeight);
5597*9880d681SAndroid Build Coastguard Worker Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext())
5598*9880d681SAndroid Build Coastguard Worker .createBranchWeights(TrueWeight, FalseWeight));
5599*9880d681SAndroid Build Coastguard Worker
5600*9880d681SAndroid Build Coastguard Worker NewTrueWeight = TrueWeight;
5601*9880d681SAndroid Build Coastguard Worker NewFalseWeight = 2 * FalseWeight;
5602*9880d681SAndroid Build Coastguard Worker scaleWeights(NewTrueWeight, NewFalseWeight);
5603*9880d681SAndroid Build Coastguard Worker Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext())
5604*9880d681SAndroid Build Coastguard Worker .createBranchWeights(TrueWeight, FalseWeight));
5605*9880d681SAndroid Build Coastguard Worker }
5606*9880d681SAndroid Build Coastguard Worker } else {
5607*9880d681SAndroid Build Coastguard Worker // Codegen X & Y as:
5608*9880d681SAndroid Build Coastguard Worker // BB1:
5609*9880d681SAndroid Build Coastguard Worker // jmp_if_X TmpBB
5610*9880d681SAndroid Build Coastguard Worker // jmp FBB
5611*9880d681SAndroid Build Coastguard Worker // TmpBB:
5612*9880d681SAndroid Build Coastguard Worker // jmp_if_Y TBB
5613*9880d681SAndroid Build Coastguard Worker // jmp FBB
5614*9880d681SAndroid Build Coastguard Worker //
5615*9880d681SAndroid Build Coastguard Worker // This requires creation of TmpBB after CurBB.
5616*9880d681SAndroid Build Coastguard Worker
5617*9880d681SAndroid Build Coastguard Worker // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
5618*9880d681SAndroid Build Coastguard Worker // The requirement is that
5619*9880d681SAndroid Build Coastguard Worker // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
5620*9880d681SAndroid Build Coastguard Worker // = FalseProb for orignal BB.
5621*9880d681SAndroid Build Coastguard Worker // Assuming the orignal weights are A and B, one choice is to set BB1's
5622*9880d681SAndroid Build Coastguard Worker // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
5623*9880d681SAndroid Build Coastguard Worker // assumes that
5624*9880d681SAndroid Build Coastguard Worker // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
5625*9880d681SAndroid Build Coastguard Worker uint64_t TrueWeight, FalseWeight;
5626*9880d681SAndroid Build Coastguard Worker if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
5627*9880d681SAndroid Build Coastguard Worker uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;
5628*9880d681SAndroid Build Coastguard Worker uint64_t NewFalseWeight = FalseWeight;
5629*9880d681SAndroid Build Coastguard Worker scaleWeights(NewTrueWeight, NewFalseWeight);
5630*9880d681SAndroid Build Coastguard Worker Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext())
5631*9880d681SAndroid Build Coastguard Worker .createBranchWeights(TrueWeight, FalseWeight));
5632*9880d681SAndroid Build Coastguard Worker
5633*9880d681SAndroid Build Coastguard Worker NewTrueWeight = 2 * TrueWeight;
5634*9880d681SAndroid Build Coastguard Worker NewFalseWeight = FalseWeight;
5635*9880d681SAndroid Build Coastguard Worker scaleWeights(NewTrueWeight, NewFalseWeight);
5636*9880d681SAndroid Build Coastguard Worker Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext())
5637*9880d681SAndroid Build Coastguard Worker .createBranchWeights(TrueWeight, FalseWeight));
5638*9880d681SAndroid Build Coastguard Worker }
5639*9880d681SAndroid Build Coastguard Worker }
5640*9880d681SAndroid Build Coastguard Worker
5641*9880d681SAndroid Build Coastguard Worker // Note: No point in getting fancy here, since the DT info is never
5642*9880d681SAndroid Build Coastguard Worker // available to CodeGenPrepare.
5643*9880d681SAndroid Build Coastguard Worker ModifiedDT = true;
5644*9880d681SAndroid Build Coastguard Worker
5645*9880d681SAndroid Build Coastguard Worker MadeChange = true;
5646*9880d681SAndroid Build Coastguard Worker
5647*9880d681SAndroid Build Coastguard Worker DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump();
5648*9880d681SAndroid Build Coastguard Worker TmpBB->dump());
5649*9880d681SAndroid Build Coastguard Worker }
5650*9880d681SAndroid Build Coastguard Worker return MadeChange;
5651*9880d681SAndroid Build Coastguard Worker }
5652*9880d681SAndroid Build Coastguard Worker
stripInvariantGroupMetadata(Instruction & I)5653*9880d681SAndroid Build Coastguard Worker void CodeGenPrepare::stripInvariantGroupMetadata(Instruction &I) {
5654*9880d681SAndroid Build Coastguard Worker if (auto *InvariantMD = I.getMetadata(LLVMContext::MD_invariant_group))
5655*9880d681SAndroid Build Coastguard Worker I.dropUnknownNonDebugMetadata(InvariantMD->getMetadataID());
5656*9880d681SAndroid Build Coastguard Worker }
5657