1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers atomic intrinsics to non-atomic form for use in a known
11 // non-preemptible environment.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/Transforms/Scalar/LowerAtomic.h"
16 #include "llvm/IR/Function.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/IntrinsicInst.h"
19 #include "llvm/Pass.h"
20 #include "llvm/Transforms/Scalar.h"
21 using namespace llvm;
22
23 #define DEBUG_TYPE "loweratomic"
24
LowerAtomicCmpXchgInst(AtomicCmpXchgInst * CXI)25 static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
26 IRBuilder<> Builder(CXI);
27 Value *Ptr = CXI->getPointerOperand();
28 Value *Cmp = CXI->getCompareOperand();
29 Value *Val = CXI->getNewValOperand();
30
31 LoadInst *Orig = Builder.CreateLoad(Ptr);
32 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
33 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
34 Builder.CreateStore(Res, Ptr);
35
36 Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
37 Res = Builder.CreateInsertValue(Res, Equal, 1);
38
39 CXI->replaceAllUsesWith(Res);
40 CXI->eraseFromParent();
41 return true;
42 }
43
LowerAtomicRMWInst(AtomicRMWInst * RMWI)44 static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
45 IRBuilder<> Builder(RMWI);
46 Value *Ptr = RMWI->getPointerOperand();
47 Value *Val = RMWI->getValOperand();
48
49 LoadInst *Orig = Builder.CreateLoad(Ptr);
50 Value *Res = nullptr;
51
52 switch (RMWI->getOperation()) {
53 default: llvm_unreachable("Unexpected RMW operation");
54 case AtomicRMWInst::Xchg:
55 Res = Val;
56 break;
57 case AtomicRMWInst::Add:
58 Res = Builder.CreateAdd(Orig, Val);
59 break;
60 case AtomicRMWInst::Sub:
61 Res = Builder.CreateSub(Orig, Val);
62 break;
63 case AtomicRMWInst::And:
64 Res = Builder.CreateAnd(Orig, Val);
65 break;
66 case AtomicRMWInst::Nand:
67 Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
68 break;
69 case AtomicRMWInst::Or:
70 Res = Builder.CreateOr(Orig, Val);
71 break;
72 case AtomicRMWInst::Xor:
73 Res = Builder.CreateXor(Orig, Val);
74 break;
75 case AtomicRMWInst::Max:
76 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
77 Val, Orig);
78 break;
79 case AtomicRMWInst::Min:
80 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
81 Orig, Val);
82 break;
83 case AtomicRMWInst::UMax:
84 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
85 Val, Orig);
86 break;
87 case AtomicRMWInst::UMin:
88 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
89 Orig, Val);
90 break;
91 }
92 Builder.CreateStore(Res, Ptr);
93 RMWI->replaceAllUsesWith(Orig);
94 RMWI->eraseFromParent();
95 return true;
96 }
97
LowerFenceInst(FenceInst * FI)98 static bool LowerFenceInst(FenceInst *FI) {
99 FI->eraseFromParent();
100 return true;
101 }
102
LowerLoadInst(LoadInst * LI)103 static bool LowerLoadInst(LoadInst *LI) {
104 LI->setAtomic(AtomicOrdering::NotAtomic);
105 return true;
106 }
107
LowerStoreInst(StoreInst * SI)108 static bool LowerStoreInst(StoreInst *SI) {
109 SI->setAtomic(AtomicOrdering::NotAtomic);
110 return true;
111 }
112
runOnBasicBlock(BasicBlock & BB)113 static bool runOnBasicBlock(BasicBlock &BB) {
114 bool Changed = false;
115 for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE;) {
116 Instruction *Inst = &*DI++;
117 if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
118 Changed |= LowerFenceInst(FI);
119 else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
120 Changed |= LowerAtomicCmpXchgInst(CXI);
121 else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
122 Changed |= LowerAtomicRMWInst(RMWI);
123 else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
124 if (LI->isAtomic())
125 LowerLoadInst(LI);
126 } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
127 if (SI->isAtomic())
128 LowerStoreInst(SI);
129 }
130 }
131 return Changed;
132 }
133
lowerAtomics(Function & F)134 static bool lowerAtomics(Function &F) {
135 bool Changed = false;
136 for (BasicBlock &BB : F) {
137 Changed |= runOnBasicBlock(BB);
138 }
139 return Changed;
140 }
141
run(Function & F,FunctionAnalysisManager &)142 PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) {
143 if (lowerAtomics(F))
144 return PreservedAnalyses::none();
145 return PreservedAnalyses::all();
146 }
147
148 namespace {
149 class LowerAtomicLegacyPass : public FunctionPass {
150 public:
151 static char ID;
152
LowerAtomicLegacyPass()153 LowerAtomicLegacyPass() : FunctionPass(ID) {
154 initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry());
155 }
156
runOnFunction(Function & F)157 bool runOnFunction(Function &F) override {
158 if (skipFunction(F))
159 return false;
160 FunctionAnalysisManager DummyFAM;
161 auto PA = Impl.run(F, DummyFAM);
162 return !PA.areAllPreserved();
163 }
164
165 private:
166 LowerAtomicPass Impl;
167 };
168 }
169
170 char LowerAtomicLegacyPass::ID = 0;
171 INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic",
172 "Lower atomic intrinsics to non-atomic form", false, false)
173
createLowerAtomicPass()174 Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); }
175