1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CGCall.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Operator.h"
24 
25 using namespace clang;
26 using namespace CodeGen;
27 
28 namespace {
29   class AtomicInfo {
30     CodeGenFunction &CGF;
31     QualType AtomicTy;
32     QualType ValueTy;
33     uint64_t AtomicSizeInBits;
34     uint64_t ValueSizeInBits;
35     CharUnits AtomicAlign;
36     CharUnits ValueAlign;
37     CharUnits LValueAlign;
38     TypeEvaluationKind EvaluationKind;
39     bool UseLibcall;
40     LValue LVal;
41     CGBitFieldInfo BFI;
42   public:
AtomicInfo(CodeGenFunction & CGF,LValue & lvalue)43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
46       assert(!lvalue.isGlobalReg());
47       ASTContext &C = CGF.getContext();
48       if (lvalue.isSimple()) {
49         AtomicTy = lvalue.getType();
50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
51           ValueTy = ATy->getValueType();
52         else
53           ValueTy = AtomicTy;
54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56         uint64_t ValueAlignInBits;
57         uint64_t AtomicAlignInBits;
58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59         ValueSizeInBits = ValueTI.Width;
60         ValueAlignInBits = ValueTI.Align;
61 
62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63         AtomicSizeInBits = AtomicTI.Width;
64         AtomicAlignInBits = AtomicTI.Align;
65 
66         assert(ValueSizeInBits <= AtomicSizeInBits);
67         assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71         if (lvalue.getAlignment().isZero())
72           lvalue.setAlignment(AtomicAlign);
73 
74         LVal = lvalue;
75       } else if (lvalue.isBitField()) {
76         ValueTy = lvalue.getType();
77         ValueSizeInBits = C.getTypeSize(ValueTy);
78         auto &OrigBFI = lvalue.getBitFieldInfo();
79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80         AtomicSizeInBits = C.toBits(
81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82                 .RoundUpToAlignment(lvalue.getAlignment()));
83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
84         auto OffsetInChars =
85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86             lvalue.getAlignment();
87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88             VoidPtrAddr, OffsetInChars.getQuantity());
89         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90             VoidPtrAddr,
91             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92             "atomic_bitfield_base");
93         BFI = OrigBFI;
94         BFI.Offset = Offset;
95         BFI.StorageSize = AtomicSizeInBits;
96         LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
97                                     lvalue.getAlignment());
98         LVal.setTBAAInfo(lvalue.getTBAAInfo());
99         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100         if (AtomicTy.isNull()) {
101           llvm::APInt Size(
102               /*numBits=*/32,
103               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104           AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
105                                             /*IndexTypeQuals=*/0);
106         }
107         AtomicAlign = ValueAlign = lvalue.getAlignment();
108       } else if (lvalue.isVectorElt()) {
109         ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
110         ValueSizeInBits = C.getTypeSize(ValueTy);
111         AtomicTy = lvalue.getType();
112         AtomicSizeInBits = C.getTypeSize(AtomicTy);
113         AtomicAlign = ValueAlign = lvalue.getAlignment();
114         LVal = lvalue;
115       } else {
116         assert(lvalue.isExtVectorElt());
117         ValueTy = lvalue.getType();
118         ValueSizeInBits = C.getTypeSize(ValueTy);
119         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120             lvalue.getType(), lvalue.getExtVectorAddr()
121                                   ->getType()
122                                   ->getPointerElementType()
123                                   ->getVectorNumElements());
124         AtomicSizeInBits = C.getTypeSize(AtomicTy);
125         AtomicAlign = ValueAlign = lvalue.getAlignment();
126         LVal = lvalue;
127       }
128       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130     }
131 
getAtomicType() const132     QualType getAtomicType() const { return AtomicTy; }
getValueType() const133     QualType getValueType() const { return ValueTy; }
getAtomicAlignment() const134     CharUnits getAtomicAlignment() const { return AtomicAlign; }
getValueAlignment() const135     CharUnits getValueAlignment() const { return ValueAlign; }
getAtomicSizeInBits() const136     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
getValueSizeInBits() const137     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
getEvaluationKind() const138     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
shouldUseLibcall() const139     bool shouldUseLibcall() const { return UseLibcall; }
getAtomicLValue() const140     const LValue &getAtomicLValue() const { return LVal; }
getAtomicAddress() const141     llvm::Value *getAtomicAddress() const {
142       if (LVal.isSimple())
143         return LVal.getAddress();
144       else if (LVal.isBitField())
145         return LVal.getBitFieldAddr();
146       else if (LVal.isVectorElt())
147         return LVal.getVectorAddr();
148       assert(LVal.isExtVectorElt());
149       return LVal.getExtVectorAddr();
150     }
151 
152     /// Is the atomic size larger than the underlying value type?
153     ///
154     /// Note that the absence of padding does not mean that atomic
155     /// objects are completely interchangeable with non-atomic
156     /// objects: we might have promoted the alignment of a type
157     /// without making it bigger.
hasPadding() const158     bool hasPadding() const {
159       return (ValueSizeInBits != AtomicSizeInBits);
160     }
161 
162     bool emitMemSetZeroIfNecessary() const;
163 
getAtomicSizeValue() const164     llvm::Value *getAtomicSizeValue() const {
165       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
166       return CGF.CGM.getSize(size);
167     }
168 
169     /// Cast the given pointer to an integer pointer suitable for
170     /// atomic operations.
171     llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
172 
173     /// Turn an atomic-layout object into an r-value.
174     RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
175                                SourceLocation loc, bool AsValue) const;
176 
177     /// \brief Converts a rvalue to integer value.
178     llvm::Value *convertRValueToInt(RValue RVal) const;
179 
180     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
181                                      AggValueSlot ResultSlot,
182                                      SourceLocation Loc, bool AsValue) const;
183 
184     /// Copy an atomic r-value into atomic-layout memory.
185     void emitCopyIntoMemory(RValue rvalue) const;
186 
187     /// Project an l-value down to the value field.
projectValue() const188     LValue projectValue() const {
189       assert(LVal.isSimple());
190       llvm::Value *addr = getAtomicAddress();
191       if (hasPadding())
192         addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
193 
194       return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
195                               CGF.getContext(), LVal.getTBAAInfo());
196     }
197 
198     /// \brief Emits atomic load.
199     /// \returns Loaded value.
200     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
201                           bool AsValue, llvm::AtomicOrdering AO,
202                           bool IsVolatile);
203 
204     /// \brief Emits atomic compare-and-exchange sequence.
205     /// \param Expected Expected value.
206     /// \param Desired Desired value.
207     /// \param Success Atomic ordering for success operation.
208     /// \param Failure Atomic ordering for failed operation.
209     /// \param IsWeak true if atomic operation is weak, false otherwise.
210     /// \returns Pair of values: previous value from storage (value type) and
211     /// boolean flag (i1 type) with true if success and false otherwise.
212     std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
213         RValue Expected, RValue Desired,
214         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
215         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
216         bool IsWeak = false);
217 
218     /// Materialize an atomic r-value in atomic-layout memory.
219     llvm::Value *materializeRValue(RValue rvalue) const;
220 
221     /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
222     /// libcalls.
223     static AtomicExpr::AtomicOrderingKind
224     translateAtomicOrdering(const llvm::AtomicOrdering AO);
225 
226   private:
227     bool requiresMemSetZero(llvm::Type *type) const;
228 
229     /// \brief Creates temp alloca for intermediate operations on atomic value.
230     llvm::Value *CreateTempAlloca() const;
231 
232     /// \brief Emits atomic load as a libcall.
233     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
234                                llvm::AtomicOrdering AO, bool IsVolatile);
235     /// \brief Emits atomic load as LLVM instruction.
236     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
237     /// \brief Emits atomic compare-and-exchange op as a libcall.
238     std::pair<RValue, llvm::Value *> EmitAtomicCompareExchangeLibcall(
239         RValue Expected, RValue DesiredAddr,
240         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
241         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
242     /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
243     std::pair<RValue, llvm::Value *> EmitAtomicCompareExchangeOp(
244         RValue Expected, RValue Desired,
245         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
246         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
247         bool IsWeak = false);
248   };
249 }
250 
251 AtomicExpr::AtomicOrderingKind
translateAtomicOrdering(const llvm::AtomicOrdering AO)252 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
253   switch (AO) {
254   case llvm::Unordered:
255   case llvm::NotAtomic:
256   case llvm::Monotonic:
257     return AtomicExpr::AO_ABI_memory_order_relaxed;
258   case llvm::Acquire:
259     return AtomicExpr::AO_ABI_memory_order_acquire;
260   case llvm::Release:
261     return AtomicExpr::AO_ABI_memory_order_release;
262   case llvm::AcquireRelease:
263     return AtomicExpr::AO_ABI_memory_order_acq_rel;
264   case llvm::SequentiallyConsistent:
265     return AtomicExpr::AO_ABI_memory_order_seq_cst;
266   }
267   llvm_unreachable("Unhandled AtomicOrdering");
268 }
269 
CreateTempAlloca() const270 llvm::Value *AtomicInfo::CreateTempAlloca() const {
271   auto *TempAlloca = CGF.CreateMemTemp(
272       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
273                                                                 : AtomicTy,
274       "atomic-temp");
275   TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
276   // Cast to pointer to value type for bitfields.
277   if (LVal.isBitField())
278     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
279         TempAlloca, getAtomicAddress()->getType());
280   return TempAlloca;
281 }
282 
emitAtomicLibcall(CodeGenFunction & CGF,StringRef fnName,QualType resultType,CallArgList & args)283 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
284                                 StringRef fnName,
285                                 QualType resultType,
286                                 CallArgList &args) {
287   const CGFunctionInfo &fnInfo =
288     CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
289             FunctionType::ExtInfo(), RequiredArgs::All);
290   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
291   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
292   return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
293 }
294 
295 /// Does a store of the given IR type modify the full expected width?
isFullSizeType(CodeGenModule & CGM,llvm::Type * type,uint64_t expectedSize)296 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
297                            uint64_t expectedSize) {
298   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
299 }
300 
301 /// Does the atomic type require memsetting to zero before initialization?
302 ///
303 /// The IR type is provided as a way of making certain queries faster.
requiresMemSetZero(llvm::Type * type) const304 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
305   // If the atomic type has size padding, we definitely need a memset.
306   if (hasPadding()) return true;
307 
308   // Otherwise, do some simple heuristics to try to avoid it:
309   switch (getEvaluationKind()) {
310   // For scalars and complexes, check whether the store size of the
311   // type uses the full size.
312   case TEK_Scalar:
313     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
314   case TEK_Complex:
315     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
316                            AtomicSizeInBits / 2);
317 
318   // Padding in structs has an undefined bit pattern.  User beware.
319   case TEK_Aggregate:
320     return false;
321   }
322   llvm_unreachable("bad evaluation kind");
323 }
324 
emitMemSetZeroIfNecessary() const325 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
326   assert(LVal.isSimple());
327   llvm::Value *addr = LVal.getAddress();
328   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
329     return false;
330 
331   CGF.Builder.CreateMemSet(
332       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
333       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
334       LVal.getAlignment().getQuantity());
335   return true;
336 }
337 
emitAtomicCmpXchg(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,llvm::Value * Dest,llvm::Value * Ptr,llvm::Value * Val1,llvm::Value * Val2,uint64_t Size,unsigned Align,llvm::AtomicOrdering SuccessOrder,llvm::AtomicOrdering FailureOrder)338 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
339                               llvm::Value *Dest, llvm::Value *Ptr,
340                               llvm::Value *Val1, llvm::Value *Val2,
341                               uint64_t Size, unsigned Align,
342                               llvm::AtomicOrdering SuccessOrder,
343                               llvm::AtomicOrdering FailureOrder) {
344   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
345   llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
346   Expected->setAlignment(Align);
347   llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
348   Desired->setAlignment(Align);
349 
350   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
351       Ptr, Expected, Desired, SuccessOrder, FailureOrder);
352   Pair->setVolatile(E->isVolatile());
353   Pair->setWeak(IsWeak);
354 
355   // Cmp holds the result of the compare-exchange operation: true on success,
356   // false on failure.
357   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
358   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
359 
360   // This basic block is used to hold the store instruction if the operation
361   // failed.
362   llvm::BasicBlock *StoreExpectedBB =
363       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
364 
365   // This basic block is the exit point of the operation, we should end up
366   // here regardless of whether or not the operation succeeded.
367   llvm::BasicBlock *ContinueBB =
368       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
369 
370   // Update Expected if Expected isn't equal to Old, otherwise branch to the
371   // exit point.
372   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
373 
374   CGF.Builder.SetInsertPoint(StoreExpectedBB);
375   // Update the memory at Expected with Old's value.
376   llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
377   StoreExpected->setAlignment(Align);
378   // Finally, branch to the exit point.
379   CGF.Builder.CreateBr(ContinueBB);
380 
381   CGF.Builder.SetInsertPoint(ContinueBB);
382   // Update the memory at Dest with Cmp's value.
383   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
384   return;
385 }
386 
387 /// Given an ordering required on success, emit all possible cmpxchg
388 /// instructions to cope with the provided (but possibly only dynamically known)
389 /// FailureOrder.
emitAtomicCmpXchgFailureSet(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,llvm::Value * Dest,llvm::Value * Ptr,llvm::Value * Val1,llvm::Value * Val2,llvm::Value * FailureOrderVal,uint64_t Size,unsigned Align,llvm::AtomicOrdering SuccessOrder)390 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
391                                         bool IsWeak, llvm::Value *Dest,
392                                         llvm::Value *Ptr, llvm::Value *Val1,
393                                         llvm::Value *Val2,
394                                         llvm::Value *FailureOrderVal,
395                                         uint64_t Size, unsigned Align,
396                                         llvm::AtomicOrdering SuccessOrder) {
397   llvm::AtomicOrdering FailureOrder;
398   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
399     switch (FO->getSExtValue()) {
400     default:
401       FailureOrder = llvm::Monotonic;
402       break;
403     case AtomicExpr::AO_ABI_memory_order_consume:
404     case AtomicExpr::AO_ABI_memory_order_acquire:
405       FailureOrder = llvm::Acquire;
406       break;
407     case AtomicExpr::AO_ABI_memory_order_seq_cst:
408       FailureOrder = llvm::SequentiallyConsistent;
409       break;
410     }
411     if (FailureOrder >= SuccessOrder) {
412       // Don't assert on undefined behaviour.
413       FailureOrder =
414         llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
415     }
416     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
417                       SuccessOrder, FailureOrder);
418     return;
419   }
420 
421   // Create all the relevant BB's
422   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
423                    *SeqCstBB = nullptr;
424   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
425   if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
426     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
427   if (SuccessOrder == llvm::SequentiallyConsistent)
428     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
429 
430   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
431 
432   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
433 
434   // Emit all the different atomics
435 
436   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
437   // doesn't matter unless someone is crazy enough to use something that
438   // doesn't fold to a constant for the ordering.
439   CGF.Builder.SetInsertPoint(MonotonicBB);
440   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
441                     Size, Align, SuccessOrder, llvm::Monotonic);
442   CGF.Builder.CreateBr(ContBB);
443 
444   if (AcquireBB) {
445     CGF.Builder.SetInsertPoint(AcquireBB);
446     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
447                       Size, Align, SuccessOrder, llvm::Acquire);
448     CGF.Builder.CreateBr(ContBB);
449     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
450                 AcquireBB);
451     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
452                 AcquireBB);
453   }
454   if (SeqCstBB) {
455     CGF.Builder.SetInsertPoint(SeqCstBB);
456     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
457                       Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
458     CGF.Builder.CreateBr(ContBB);
459     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
460                 SeqCstBB);
461   }
462 
463   CGF.Builder.SetInsertPoint(ContBB);
464 }
465 
EmitAtomicOp(CodeGenFunction & CGF,AtomicExpr * E,llvm::Value * Dest,llvm::Value * Ptr,llvm::Value * Val1,llvm::Value * Val2,llvm::Value * IsWeak,llvm::Value * FailureOrder,uint64_t Size,unsigned Align,llvm::AtomicOrdering Order)466 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
467                          llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
468                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
469                          uint64_t Size, unsigned Align,
470                          llvm::AtomicOrdering Order) {
471   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
472   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
473 
474   switch (E->getOp()) {
475   case AtomicExpr::AO__c11_atomic_init:
476     llvm_unreachable("Already handled!");
477 
478   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
479     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
480                                 FailureOrder, Size, Align, Order);
481     return;
482   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
483     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
484                                 FailureOrder, Size, Align, Order);
485     return;
486   case AtomicExpr::AO__atomic_compare_exchange:
487   case AtomicExpr::AO__atomic_compare_exchange_n: {
488     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
489       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
490                                   Val1, Val2, FailureOrder, Size, Align, Order);
491     } else {
492       // Create all the relevant BB's
493       llvm::BasicBlock *StrongBB =
494           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
495       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
496       llvm::BasicBlock *ContBB =
497           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
498 
499       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
500       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
501 
502       CGF.Builder.SetInsertPoint(StrongBB);
503       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
504                                   FailureOrder, Size, Align, Order);
505       CGF.Builder.CreateBr(ContBB);
506 
507       CGF.Builder.SetInsertPoint(WeakBB);
508       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
509                                   FailureOrder, Size, Align, Order);
510       CGF.Builder.CreateBr(ContBB);
511 
512       CGF.Builder.SetInsertPoint(ContBB);
513     }
514     return;
515   }
516   case AtomicExpr::AO__c11_atomic_load:
517   case AtomicExpr::AO__atomic_load_n:
518   case AtomicExpr::AO__atomic_load: {
519     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
520     Load->setAtomic(Order);
521     Load->setAlignment(Size);
522     Load->setVolatile(E->isVolatile());
523     llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
524     StoreDest->setAlignment(Align);
525     return;
526   }
527 
528   case AtomicExpr::AO__c11_atomic_store:
529   case AtomicExpr::AO__atomic_store:
530   case AtomicExpr::AO__atomic_store_n: {
531     assert(!Dest && "Store does not return a value");
532     llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
533     LoadVal1->setAlignment(Align);
534     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
535     Store->setAtomic(Order);
536     Store->setAlignment(Size);
537     Store->setVolatile(E->isVolatile());
538     return;
539   }
540 
541   case AtomicExpr::AO__c11_atomic_exchange:
542   case AtomicExpr::AO__atomic_exchange_n:
543   case AtomicExpr::AO__atomic_exchange:
544     Op = llvm::AtomicRMWInst::Xchg;
545     break;
546 
547   case AtomicExpr::AO__atomic_add_fetch:
548     PostOp = llvm::Instruction::Add;
549     // Fall through.
550   case AtomicExpr::AO__c11_atomic_fetch_add:
551   case AtomicExpr::AO__atomic_fetch_add:
552     Op = llvm::AtomicRMWInst::Add;
553     break;
554 
555   case AtomicExpr::AO__atomic_sub_fetch:
556     PostOp = llvm::Instruction::Sub;
557     // Fall through.
558   case AtomicExpr::AO__c11_atomic_fetch_sub:
559   case AtomicExpr::AO__atomic_fetch_sub:
560     Op = llvm::AtomicRMWInst::Sub;
561     break;
562 
563   case AtomicExpr::AO__atomic_and_fetch:
564     PostOp = llvm::Instruction::And;
565     // Fall through.
566   case AtomicExpr::AO__c11_atomic_fetch_and:
567   case AtomicExpr::AO__atomic_fetch_and:
568     Op = llvm::AtomicRMWInst::And;
569     break;
570 
571   case AtomicExpr::AO__atomic_or_fetch:
572     PostOp = llvm::Instruction::Or;
573     // Fall through.
574   case AtomicExpr::AO__c11_atomic_fetch_or:
575   case AtomicExpr::AO__atomic_fetch_or:
576     Op = llvm::AtomicRMWInst::Or;
577     break;
578 
579   case AtomicExpr::AO__atomic_xor_fetch:
580     PostOp = llvm::Instruction::Xor;
581     // Fall through.
582   case AtomicExpr::AO__c11_atomic_fetch_xor:
583   case AtomicExpr::AO__atomic_fetch_xor:
584     Op = llvm::AtomicRMWInst::Xor;
585     break;
586 
587   case AtomicExpr::AO__atomic_nand_fetch:
588     PostOp = llvm::Instruction::And;
589     // Fall through.
590   case AtomicExpr::AO__atomic_fetch_nand:
591     Op = llvm::AtomicRMWInst::Nand;
592     break;
593   }
594 
595   llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
596   LoadVal1->setAlignment(Align);
597   llvm::AtomicRMWInst *RMWI =
598       CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
599   RMWI->setVolatile(E->isVolatile());
600 
601   // For __atomic_*_fetch operations, perform the operation again to
602   // determine the value which was written.
603   llvm::Value *Result = RMWI;
604   if (PostOp)
605     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
606   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
607     Result = CGF.Builder.CreateNot(Result);
608   llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
609   StoreDest->setAlignment(Align);
610 }
611 
612 // This function emits any expression (scalar, complex, or aggregate)
613 // into a temporary alloca.
614 static llvm::Value *
EmitValToTemp(CodeGenFunction & CGF,Expr * E)615 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
616   llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
617   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
618                        /*Init*/ true);
619   return DeclPtr;
620 }
621 
622 static void
AddDirectArgument(CodeGenFunction & CGF,CallArgList & Args,bool UseOptimizedLibcall,llvm::Value * Val,QualType ValTy,SourceLocation Loc,CharUnits SizeInChars)623 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
624                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
625                   SourceLocation Loc, CharUnits SizeInChars) {
626   if (UseOptimizedLibcall) {
627     // Load value and pass it to the function directly.
628     unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
629     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
630     ValTy =
631         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
632     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
633                                                 SizeInBits)->getPointerTo();
634     Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
635                                Align, CGF.getContext().getPointerType(ValTy),
636                                Loc);
637     // Coerce the value into an appropriately sized integer type.
638     Args.add(RValue::get(Val), ValTy);
639   } else {
640     // Non-optimized functions always take a reference.
641     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
642                          CGF.getContext().VoidPtrTy);
643   }
644 }
645 
EmitAtomicExpr(AtomicExpr * E,llvm::Value * Dest)646 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
647   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
648   QualType MemTy = AtomicTy;
649   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
650     MemTy = AT->getValueType();
651   CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
652   uint64_t Size = sizeChars.getQuantity();
653   CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
654   unsigned Align = alignChars.getQuantity();
655   unsigned MaxInlineWidthInBits =
656     getTarget().getMaxAtomicInlineWidth();
657   bool UseLibcall = (Size != Align ||
658                      getContext().toBits(sizeChars) > MaxInlineWidthInBits);
659 
660   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
661               *Val2 = nullptr;
662   llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
663 
664   if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
665     assert(!Dest && "Init does not return a value");
666     LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
667     EmitAtomicInit(E->getVal1(), lvalue);
668     return RValue::get(nullptr);
669   }
670 
671   llvm::Value *Order = EmitScalarExpr(E->getOrder());
672 
673   switch (E->getOp()) {
674   case AtomicExpr::AO__c11_atomic_init:
675     llvm_unreachable("Already handled!");
676 
677   case AtomicExpr::AO__c11_atomic_load:
678   case AtomicExpr::AO__atomic_load_n:
679     break;
680 
681   case AtomicExpr::AO__atomic_load:
682     Dest = EmitScalarExpr(E->getVal1());
683     break;
684 
685   case AtomicExpr::AO__atomic_store:
686     Val1 = EmitScalarExpr(E->getVal1());
687     break;
688 
689   case AtomicExpr::AO__atomic_exchange:
690     Val1 = EmitScalarExpr(E->getVal1());
691     Dest = EmitScalarExpr(E->getVal2());
692     break;
693 
694   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
695   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
696   case AtomicExpr::AO__atomic_compare_exchange_n:
697   case AtomicExpr::AO__atomic_compare_exchange:
698     Val1 = EmitScalarExpr(E->getVal1());
699     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
700       Val2 = EmitScalarExpr(E->getVal2());
701     else
702       Val2 = EmitValToTemp(*this, E->getVal2());
703     OrderFail = EmitScalarExpr(E->getOrderFail());
704     if (E->getNumSubExprs() == 6)
705       IsWeak = EmitScalarExpr(E->getWeak());
706     break;
707 
708   case AtomicExpr::AO__c11_atomic_fetch_add:
709   case AtomicExpr::AO__c11_atomic_fetch_sub:
710     if (MemTy->isPointerType()) {
711       // For pointer arithmetic, we're required to do a bit of math:
712       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
713       // ... but only for the C11 builtins. The GNU builtins expect the
714       // user to multiply by sizeof(T).
715       QualType Val1Ty = E->getVal1()->getType();
716       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
717       CharUnits PointeeIncAmt =
718           getContext().getTypeSizeInChars(MemTy->getPointeeType());
719       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
720       Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
721       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
722       break;
723     }
724     // Fall through.
725   case AtomicExpr::AO__atomic_fetch_add:
726   case AtomicExpr::AO__atomic_fetch_sub:
727   case AtomicExpr::AO__atomic_add_fetch:
728   case AtomicExpr::AO__atomic_sub_fetch:
729   case AtomicExpr::AO__c11_atomic_store:
730   case AtomicExpr::AO__c11_atomic_exchange:
731   case AtomicExpr::AO__atomic_store_n:
732   case AtomicExpr::AO__atomic_exchange_n:
733   case AtomicExpr::AO__c11_atomic_fetch_and:
734   case AtomicExpr::AO__c11_atomic_fetch_or:
735   case AtomicExpr::AO__c11_atomic_fetch_xor:
736   case AtomicExpr::AO__atomic_fetch_and:
737   case AtomicExpr::AO__atomic_fetch_or:
738   case AtomicExpr::AO__atomic_fetch_xor:
739   case AtomicExpr::AO__atomic_fetch_nand:
740   case AtomicExpr::AO__atomic_and_fetch:
741   case AtomicExpr::AO__atomic_or_fetch:
742   case AtomicExpr::AO__atomic_xor_fetch:
743   case AtomicExpr::AO__atomic_nand_fetch:
744     Val1 = EmitValToTemp(*this, E->getVal1());
745     break;
746   }
747 
748   QualType RValTy = E->getType().getUnqualifiedType();
749 
750   auto GetDest = [&] {
751     if (!RValTy->isVoidType() && !Dest) {
752       Dest = CreateMemTemp(RValTy, ".atomicdst");
753     }
754     return Dest;
755   };
756 
757   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
758   if (UseLibcall) {
759     bool UseOptimizedLibcall = false;
760     switch (E->getOp()) {
761     case AtomicExpr::AO__c11_atomic_fetch_add:
762     case AtomicExpr::AO__atomic_fetch_add:
763     case AtomicExpr::AO__c11_atomic_fetch_and:
764     case AtomicExpr::AO__atomic_fetch_and:
765     case AtomicExpr::AO__c11_atomic_fetch_or:
766     case AtomicExpr::AO__atomic_fetch_or:
767     case AtomicExpr::AO__c11_atomic_fetch_sub:
768     case AtomicExpr::AO__atomic_fetch_sub:
769     case AtomicExpr::AO__c11_atomic_fetch_xor:
770     case AtomicExpr::AO__atomic_fetch_xor:
771       // For these, only library calls for certain sizes exist.
772       UseOptimizedLibcall = true;
773       break;
774     default:
775       // Only use optimized library calls for sizes for which they exist.
776       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
777         UseOptimizedLibcall = true;
778       break;
779     }
780 
781     CallArgList Args;
782     if (!UseOptimizedLibcall) {
783       // For non-optimized library calls, the size is the first parameter
784       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
785                getContext().getSizeType());
786     }
787     // Atomic address is the first or second parameter
788     Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
789 
790     std::string LibCallName;
791     QualType LoweredMemTy =
792       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
793     QualType RetTy;
794     bool HaveRetTy = false;
795     switch (E->getOp()) {
796     // There is only one libcall for compare an exchange, because there is no
797     // optimisation benefit possible from a libcall version of a weak compare
798     // and exchange.
799     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
800     //                                void *desired, int success, int failure)
801     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
802     //                                  int success, int failure)
803     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
804     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
805     case AtomicExpr::AO__atomic_compare_exchange:
806     case AtomicExpr::AO__atomic_compare_exchange_n:
807       LibCallName = "__atomic_compare_exchange";
808       RetTy = getContext().BoolTy;
809       HaveRetTy = true;
810       Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
811       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
812                         E->getExprLoc(), sizeChars);
813       Args.add(RValue::get(Order), getContext().IntTy);
814       Order = OrderFail;
815       break;
816     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
817     //                        int order)
818     // T __atomic_exchange_N(T *mem, T val, int order)
819     case AtomicExpr::AO__c11_atomic_exchange:
820     case AtomicExpr::AO__atomic_exchange_n:
821     case AtomicExpr::AO__atomic_exchange:
822       LibCallName = "__atomic_exchange";
823       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
824                         E->getExprLoc(), sizeChars);
825       break;
826     // void __atomic_store(size_t size, void *mem, void *val, int order)
827     // void __atomic_store_N(T *mem, T val, int order)
828     case AtomicExpr::AO__c11_atomic_store:
829     case AtomicExpr::AO__atomic_store:
830     case AtomicExpr::AO__atomic_store_n:
831       LibCallName = "__atomic_store";
832       RetTy = getContext().VoidTy;
833       HaveRetTy = true;
834       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
835                         E->getExprLoc(), sizeChars);
836       break;
837     // void __atomic_load(size_t size, void *mem, void *return, int order)
838     // T __atomic_load_N(T *mem, int order)
839     case AtomicExpr::AO__c11_atomic_load:
840     case AtomicExpr::AO__atomic_load:
841     case AtomicExpr::AO__atomic_load_n:
842       LibCallName = "__atomic_load";
843       break;
844     // T __atomic_fetch_add_N(T *mem, T val, int order)
845     case AtomicExpr::AO__c11_atomic_fetch_add:
846     case AtomicExpr::AO__atomic_fetch_add:
847       LibCallName = "__atomic_fetch_add";
848       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
849                         E->getExprLoc(), sizeChars);
850       break;
851     // T __atomic_fetch_and_N(T *mem, T val, int order)
852     case AtomicExpr::AO__c11_atomic_fetch_and:
853     case AtomicExpr::AO__atomic_fetch_and:
854       LibCallName = "__atomic_fetch_and";
855       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
856                         E->getExprLoc(), sizeChars);
857       break;
858     // T __atomic_fetch_or_N(T *mem, T val, int order)
859     case AtomicExpr::AO__c11_atomic_fetch_or:
860     case AtomicExpr::AO__atomic_fetch_or:
861       LibCallName = "__atomic_fetch_or";
862       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
863                         E->getExprLoc(), sizeChars);
864       break;
865     // T __atomic_fetch_sub_N(T *mem, T val, int order)
866     case AtomicExpr::AO__c11_atomic_fetch_sub:
867     case AtomicExpr::AO__atomic_fetch_sub:
868       LibCallName = "__atomic_fetch_sub";
869       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
870                         E->getExprLoc(), sizeChars);
871       break;
872     // T __atomic_fetch_xor_N(T *mem, T val, int order)
873     case AtomicExpr::AO__c11_atomic_fetch_xor:
874     case AtomicExpr::AO__atomic_fetch_xor:
875       LibCallName = "__atomic_fetch_xor";
876       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
877                         E->getExprLoc(), sizeChars);
878       break;
879     default: return EmitUnsupportedRValue(E, "atomic library call");
880     }
881 
882     // Optimized functions have the size in their name.
883     if (UseOptimizedLibcall)
884       LibCallName += "_" + llvm::utostr(Size);
885     // By default, assume we return a value of the atomic type.
886     if (!HaveRetTy) {
887       if (UseOptimizedLibcall) {
888         // Value is returned directly.
889         // The function returns an appropriately sized integer type.
890         RetTy = getContext().getIntTypeForBitwidth(
891             getContext().toBits(sizeChars), /*Signed=*/false);
892       } else {
893         // Value is returned through parameter before the order.
894         RetTy = getContext().VoidTy;
895         Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
896       }
897     }
898     // order is always the last parameter
899     Args.add(RValue::get(Order),
900              getContext().IntTy);
901 
902     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
903     // The value is returned directly from the libcall.
904     if (HaveRetTy && !RetTy->isVoidType())
905       return Res;
906     // The value is returned via an explicit out param.
907     if (RetTy->isVoidType())
908       return RValue::get(nullptr);
909     // The value is returned directly for optimized libcalls but the caller is
910     // expected an out-param.
911     if (UseOptimizedLibcall) {
912       llvm::Value *ResVal = Res.getScalarVal();
913       llvm::StoreInst *StoreDest = Builder.CreateStore(
914           ResVal,
915           Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
916       StoreDest->setAlignment(Align);
917     }
918     return convertTempToRValue(Dest, RValTy, E->getExprLoc());
919   }
920 
921   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
922                  E->getOp() == AtomicExpr::AO__atomic_store ||
923                  E->getOp() == AtomicExpr::AO__atomic_store_n;
924   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
925                 E->getOp() == AtomicExpr::AO__atomic_load ||
926                 E->getOp() == AtomicExpr::AO__atomic_load_n;
927 
928   llvm::Type *ITy =
929       llvm::IntegerType::get(getLLVMContext(), Size * 8);
930   llvm::Value *OrigDest = GetDest();
931   Ptr = Builder.CreateBitCast(
932       Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
933   if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
934   if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
935   if (Dest && !E->isCmpXChg())
936     Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
937 
938   if (isa<llvm::ConstantInt>(Order)) {
939     int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
940     switch (ord) {
941     case AtomicExpr::AO_ABI_memory_order_relaxed:
942       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
943                    Size, Align, llvm::Monotonic);
944       break;
945     case AtomicExpr::AO_ABI_memory_order_consume:
946     case AtomicExpr::AO_ABI_memory_order_acquire:
947       if (IsStore)
948         break; // Avoid crashing on code with undefined behavior
949       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
950                    Size, Align, llvm::Acquire);
951       break;
952     case AtomicExpr::AO_ABI_memory_order_release:
953       if (IsLoad)
954         break; // Avoid crashing on code with undefined behavior
955       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
956                    Size, Align, llvm::Release);
957       break;
958     case AtomicExpr::AO_ABI_memory_order_acq_rel:
959       if (IsLoad || IsStore)
960         break; // Avoid crashing on code with undefined behavior
961       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
962                    Size, Align, llvm::AcquireRelease);
963       break;
964     case AtomicExpr::AO_ABI_memory_order_seq_cst:
965       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
966                    Size, Align, llvm::SequentiallyConsistent);
967       break;
968     default: // invalid order
969       // We should not ever get here normally, but it's hard to
970       // enforce that in general.
971       break;
972     }
973     if (RValTy->isVoidType())
974       return RValue::get(nullptr);
975     return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
976   }
977 
978   // Long case, when Order isn't obviously constant.
979 
980   // Create all the relevant BB's
981   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
982                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
983                    *SeqCstBB = nullptr;
984   MonotonicBB = createBasicBlock("monotonic", CurFn);
985   if (!IsStore)
986     AcquireBB = createBasicBlock("acquire", CurFn);
987   if (!IsLoad)
988     ReleaseBB = createBasicBlock("release", CurFn);
989   if (!IsLoad && !IsStore)
990     AcqRelBB = createBasicBlock("acqrel", CurFn);
991   SeqCstBB = createBasicBlock("seqcst", CurFn);
992   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
993 
994   // Create the switch for the split
995   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
996   // doesn't matter unless someone is crazy enough to use something that
997   // doesn't fold to a constant for the ordering.
998   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
999   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1000 
1001   // Emit all the different atomics
1002   Builder.SetInsertPoint(MonotonicBB);
1003   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1004                Size, Align, llvm::Monotonic);
1005   Builder.CreateBr(ContBB);
1006   if (!IsStore) {
1007     Builder.SetInsertPoint(AcquireBB);
1008     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1009                  Size, Align, llvm::Acquire);
1010     Builder.CreateBr(ContBB);
1011     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1012                 AcquireBB);
1013     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1014                 AcquireBB);
1015   }
1016   if (!IsLoad) {
1017     Builder.SetInsertPoint(ReleaseBB);
1018     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1019                  Size, Align, llvm::Release);
1020     Builder.CreateBr(ContBB);
1021     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1022                 ReleaseBB);
1023   }
1024   if (!IsLoad && !IsStore) {
1025     Builder.SetInsertPoint(AcqRelBB);
1026     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1027                  Size, Align, llvm::AcquireRelease);
1028     Builder.CreateBr(ContBB);
1029     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1030                 AcqRelBB);
1031   }
1032   Builder.SetInsertPoint(SeqCstBB);
1033   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1034                Size, Align, llvm::SequentiallyConsistent);
1035   Builder.CreateBr(ContBB);
1036   SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1037               SeqCstBB);
1038 
1039   // Cleanup and return
1040   Builder.SetInsertPoint(ContBB);
1041   if (RValTy->isVoidType())
1042     return RValue::get(nullptr);
1043   return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1044 }
1045 
emitCastToAtomicIntPointer(llvm::Value * addr) const1046 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
1047   unsigned addrspace =
1048     cast<llvm::PointerType>(addr->getType())->getAddressSpace();
1049   llvm::IntegerType *ty =
1050     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1051   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1052 }
1053 
convertTempToRValue(llvm::Value * addr,AggValueSlot resultSlot,SourceLocation loc,bool AsValue) const1054 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
1055                                        AggValueSlot resultSlot,
1056                                        SourceLocation loc, bool AsValue) const {
1057   if (LVal.isSimple()) {
1058     if (EvaluationKind == TEK_Aggregate)
1059       return resultSlot.asRValue();
1060 
1061     // Drill into the padding structure if we have one.
1062     if (hasPadding())
1063       addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
1064 
1065     // Otherwise, just convert the temporary to an r-value using the
1066     // normal conversion routine.
1067     return CGF.convertTempToRValue(addr, getValueType(), loc);
1068   }
1069   if (!AsValue)
1070     // Get RValue from temp memory as atomic for non-simple lvalues
1071     return RValue::get(
1072         CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
1073   if (LVal.isBitField())
1074     return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
1075         addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
1076   if (LVal.isVectorElt())
1077     return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
1078                                                       LVal.getType(),
1079                                                       LVal.getAlignment()),
1080                                 loc);
1081   assert(LVal.isExtVectorElt());
1082   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1083       addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
1084 }
1085 
ConvertIntToValueOrAtomic(llvm::Value * IntVal,AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue) const1086 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1087                                              AggValueSlot ResultSlot,
1088                                              SourceLocation Loc,
1089                                              bool AsValue) const {
1090   // Try not to in some easy cases.
1091   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1092   if (getEvaluationKind() == TEK_Scalar &&
1093       (((!LVal.isBitField() ||
1094          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1095         !hasPadding()) ||
1096        !AsValue)) {
1097     auto *ValTy = AsValue
1098                       ? CGF.ConvertTypeForMem(ValueTy)
1099                       : getAtomicAddress()->getType()->getPointerElementType();
1100     if (ValTy->isIntegerTy()) {
1101       assert(IntVal->getType() == ValTy && "Different integer types.");
1102       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1103     } else if (ValTy->isPointerTy())
1104       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1105     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1106       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1107   }
1108 
1109   // Create a temporary.  This needs to be big enough to hold the
1110   // atomic integer.
1111   llvm::Value *Temp;
1112   bool TempIsVolatile = false;
1113   CharUnits TempAlignment;
1114   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1115     assert(!ResultSlot.isIgnored());
1116     Temp = ResultSlot.getAddr();
1117     TempAlignment = getValueAlignment();
1118     TempIsVolatile = ResultSlot.isVolatile();
1119   } else {
1120     Temp = CreateTempAlloca();
1121     TempAlignment = getAtomicAlignment();
1122   }
1123 
1124   // Slam the integer into the temporary.
1125   llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
1126   CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
1127       ->setVolatile(TempIsVolatile);
1128 
1129   return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
1130 }
1131 
EmitAtomicLoadLibcall(llvm::Value * AddForLoaded,llvm::AtomicOrdering AO,bool)1132 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1133                                        llvm::AtomicOrdering AO, bool) {
1134   // void __atomic_load(size_t size, void *mem, void *return, int order);
1135   CallArgList Args;
1136   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1137   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1138            CGF.getContext().VoidPtrTy);
1139   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1140            CGF.getContext().VoidPtrTy);
1141   Args.add(RValue::get(
1142                llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1143            CGF.getContext().IntTy);
1144   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1145 }
1146 
EmitAtomicLoadOp(llvm::AtomicOrdering AO,bool IsVolatile)1147 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1148                                           bool IsVolatile) {
1149   // Okay, we're doing this natively.
1150   llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1151   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1152   Load->setAtomic(AO);
1153 
1154   // Other decoration.
1155   Load->setAlignment(getAtomicAlignment().getQuantity());
1156   if (IsVolatile)
1157     Load->setVolatile(true);
1158   if (LVal.getTBAAInfo())
1159     CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
1160   return Load;
1161 }
1162 
1163 /// An LValue is a candidate for having its loads and stores be made atomic if
1164 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1165 /// performing such an operation can be performed without a libcall.
LValueIsSuitableForInlineAtomic(LValue LV)1166 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1167   AtomicInfo AI(*this, LV);
1168   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1169   // An atomic is inline if we don't need to use a libcall.
1170   bool AtomicIsInline = !AI.shouldUseLibcall();
1171   return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1172 }
1173 
1174 /// An type is a candidate for having its loads and stores be made atomic if
1175 /// we are operating under /volatile:ms *and* we know the access is volatile and
1176 /// performing such an operation can be performed without a libcall.
typeIsSuitableForInlineAtomic(QualType Ty,bool IsVolatile) const1177 bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
1178                                                     bool IsVolatile) const {
1179   // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1180   bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1181       getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1182   return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1183 }
1184 
EmitAtomicLoad(LValue LV,SourceLocation SL,AggValueSlot Slot)1185 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1186                                        AggValueSlot Slot) {
1187   llvm::AtomicOrdering AO;
1188   bool IsVolatile = LV.isVolatileQualified();
1189   if (LV.getType()->isAtomicType()) {
1190     AO = llvm::SequentiallyConsistent;
1191   } else {
1192     AO = llvm::Acquire;
1193     IsVolatile = true;
1194   }
1195   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1196 }
1197 
EmitAtomicLoad(AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue,llvm::AtomicOrdering AO,bool IsVolatile)1198 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1199                                   bool AsValue, llvm::AtomicOrdering AO,
1200                                   bool IsVolatile) {
1201   // Check whether we should use a library call.
1202   if (shouldUseLibcall()) {
1203     llvm::Value *TempAddr;
1204     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1205       assert(getEvaluationKind() == TEK_Aggregate);
1206       TempAddr = ResultSlot.getAddr();
1207     } else
1208       TempAddr = CreateTempAlloca();
1209 
1210     EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
1211 
1212     // Okay, turn that back into the original value or whole atomic (for
1213     // non-simple lvalues) type.
1214     return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1215   }
1216 
1217   // Okay, we're doing this natively.
1218   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1219 
1220   // If we're ignoring an aggregate return, don't do anything.
1221   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1222     return RValue::getAggregate(nullptr, false);
1223 
1224   // Okay, turn that back into the original value or atomic (for non-simple
1225   // lvalues) type.
1226   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1227 }
1228 
1229 /// Emit a load from an l-value of atomic type.  Note that the r-value
1230 /// we produce is an r-value of the atomic *value* type.
EmitAtomicLoad(LValue src,SourceLocation loc,llvm::AtomicOrdering AO,bool IsVolatile,AggValueSlot resultSlot)1231 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1232                                        llvm::AtomicOrdering AO, bool IsVolatile,
1233                                        AggValueSlot resultSlot) {
1234   AtomicInfo Atomics(*this, src);
1235   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1236                                 IsVolatile);
1237 }
1238 
1239 /// Copy an r-value into memory as part of storing to an atomic type.
1240 /// This needs to create a bit-pattern suitable for atomic operations.
emitCopyIntoMemory(RValue rvalue) const1241 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1242   assert(LVal.isSimple());
1243   // If we have an r-value, the rvalue should be of the atomic type,
1244   // which means that the caller is responsible for having zeroed
1245   // any padding.  Just do an aggregate copy of that type.
1246   if (rvalue.isAggregate()) {
1247     CGF.EmitAggregateCopy(getAtomicAddress(),
1248                           rvalue.getAggregateAddr(),
1249                           getAtomicType(),
1250                           (rvalue.isVolatileQualified()
1251                            || LVal.isVolatileQualified()),
1252                           LVal.getAlignment());
1253     return;
1254   }
1255 
1256   // Okay, otherwise we're copying stuff.
1257 
1258   // Zero out the buffer if necessary.
1259   emitMemSetZeroIfNecessary();
1260 
1261   // Drill past the padding if present.
1262   LValue TempLVal = projectValue();
1263 
1264   // Okay, store the rvalue in.
1265   if (rvalue.isScalar()) {
1266     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1267   } else {
1268     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1269   }
1270 }
1271 
1272 
1273 /// Materialize an r-value into memory for the purposes of storing it
1274 /// to an atomic type.
materializeRValue(RValue rvalue) const1275 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1276   // Aggregate r-values are already in memory, and EmitAtomicStore
1277   // requires them to be values of the atomic type.
1278   if (rvalue.isAggregate())
1279     return rvalue.getAggregateAddr();
1280 
1281   // Otherwise, make a temporary and materialize into it.
1282   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
1283                                      getAtomicAlignment());
1284   AtomicInfo Atomics(CGF, TempLV);
1285   Atomics.emitCopyIntoMemory(rvalue);
1286   return TempLV.getAddress();
1287 }
1288 
convertRValueToInt(RValue RVal) const1289 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1290   // If we've got a scalar value of the right size, try to avoid going
1291   // through memory.
1292   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1293     llvm::Value *Value = RVal.getScalarVal();
1294     if (isa<llvm::IntegerType>(Value->getType()))
1295       return CGF.EmitToMemory(Value, ValueTy);
1296     else {
1297       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1298           CGF.getLLVMContext(),
1299           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1300       if (isa<llvm::PointerType>(Value->getType()))
1301         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1302       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1303         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1304     }
1305   }
1306   // Otherwise, we need to go through memory.
1307   // Put the r-value in memory.
1308   llvm::Value *Addr = materializeRValue(RVal);
1309 
1310   // Cast the temporary to the atomic int type and pull a value out.
1311   Addr = emitCastToAtomicIntPointer(Addr);
1312   return CGF.Builder.CreateAlignedLoad(Addr,
1313                                        getAtomicAlignment().getQuantity());
1314 }
1315 
EmitAtomicCompareExchangeOp(RValue Expected,RValue Desired,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1316 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1317     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1318     llvm::AtomicOrdering Failure, bool IsWeak) {
1319   // Do the atomic store.
1320   auto *ExpectedVal = convertRValueToInt(Expected);
1321   auto *DesiredVal = convertRValueToInt(Desired);
1322   auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1323   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
1324                                                Success, Failure);
1325   // Other decoration.
1326   Inst->setVolatile(LVal.isVolatileQualified());
1327   Inst->setWeak(IsWeak);
1328 
1329   // Okay, turn that back into the original value type.
1330   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1331   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1332   return std::make_pair(
1333       ConvertIntToValueOrAtomic(PreviousVal, AggValueSlot::ignored(),
1334                                 SourceLocation(), /*AsValue=*/false),
1335       SuccessFailureVal);
1336 }
1337 
1338 std::pair<RValue, llvm::Value *>
EmitAtomicCompareExchangeLibcall(RValue Expected,RValue Desired,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure)1339 AtomicInfo::EmitAtomicCompareExchangeLibcall(RValue Expected, RValue Desired,
1340                                              llvm::AtomicOrdering Success,
1341                                              llvm::AtomicOrdering Failure) {
1342   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1343   // void *desired, int success, int failure);
1344   auto *ExpectedAddr = materializeRValue(Expected);
1345   auto *DesiredAddr = materializeRValue(Desired);
1346   CallArgList Args;
1347   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1348   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1349            CGF.getContext().VoidPtrTy);
1350   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1351            CGF.getContext().VoidPtrTy);
1352   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1353            CGF.getContext().VoidPtrTy);
1354   Args.add(RValue::get(llvm::ConstantInt::get(
1355                CGF.IntTy, translateAtomicOrdering(Success))),
1356            CGF.getContext().IntTy);
1357   Args.add(RValue::get(llvm::ConstantInt::get(
1358                CGF.IntTy, translateAtomicOrdering(Failure))),
1359            CGF.getContext().IntTy);
1360   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1361                                               CGF.getContext().BoolTy, Args);
1362 
1363   return std::make_pair(
1364       convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1365                           SourceLocation(), /*AsValue=*/false),
1366       SuccessFailureRVal.getScalarVal());
1367 }
1368 
EmitAtomicCompareExchange(RValue Expected,RValue Desired,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1369 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1370     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1371     llvm::AtomicOrdering Failure, bool IsWeak) {
1372   if (Failure >= Success)
1373     // Don't assert on undefined behavior.
1374     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1375 
1376   // Check whether we should use a library call.
1377   if (shouldUseLibcall()) {
1378     // Produce a source address.
1379     return EmitAtomicCompareExchangeLibcall(Expected, Desired, Success,
1380                                             Failure);
1381   }
1382 
1383   // If we've got a scalar value of the right size, try to avoid going
1384   // through memory.
1385   return EmitAtomicCompareExchangeOp(Expected, Desired, Success, Failure,
1386                                      IsWeak);
1387 }
1388 
EmitAtomicStore(RValue rvalue,LValue lvalue,bool isInit)1389 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1390                                       bool isInit) {
1391   bool IsVolatile = lvalue.isVolatileQualified();
1392   llvm::AtomicOrdering AO;
1393   if (lvalue.getType()->isAtomicType()) {
1394     AO = llvm::SequentiallyConsistent;
1395   } else {
1396     AO = llvm::Release;
1397     IsVolatile = true;
1398   }
1399   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1400 }
1401 
1402 /// Emit a store to an l-value of atomic type.
1403 ///
1404 /// Note that the r-value is expected to be an r-value *of the atomic
1405 /// type*; this means that for aggregate r-values, it should include
1406 /// storage for any padding that was necessary.
EmitAtomicStore(RValue rvalue,LValue dest,llvm::AtomicOrdering AO,bool IsVolatile,bool isInit)1407 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1408                                       llvm::AtomicOrdering AO, bool IsVolatile,
1409                                       bool isInit) {
1410   // If this is an aggregate r-value, it should agree in type except
1411   // maybe for address-space qualification.
1412   assert(!rvalue.isAggregate() ||
1413          rvalue.getAggregateAddr()->getType()->getPointerElementType()
1414            == dest.getAddress()->getType()->getPointerElementType());
1415 
1416   AtomicInfo atomics(*this, dest);
1417   LValue LVal = atomics.getAtomicLValue();
1418 
1419   // If this is an initialization, just put the value there normally.
1420   if (LVal.isSimple()) {
1421     if (isInit) {
1422       atomics.emitCopyIntoMemory(rvalue);
1423       return;
1424     }
1425 
1426     // Check whether we should use a library call.
1427     if (atomics.shouldUseLibcall()) {
1428       // Produce a source address.
1429       llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1430 
1431       // void __atomic_store(size_t size, void *mem, void *val, int order)
1432       CallArgList args;
1433       args.add(RValue::get(atomics.getAtomicSizeValue()),
1434                getContext().getSizeType());
1435       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
1436                getContext().VoidPtrTy);
1437       args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
1438       args.add(RValue::get(llvm::ConstantInt::get(
1439                    IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1440                getContext().IntTy);
1441       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1442       return;
1443     }
1444 
1445     // Okay, we're doing this natively.
1446     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1447 
1448     // Do the atomic store.
1449     llvm::Value *addr =
1450         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1451     intValue = Builder.CreateIntCast(
1452         intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
1453     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1454 
1455     // Initializations don't need to be atomic.
1456     if (!isInit)
1457       store->setAtomic(AO);
1458 
1459     // Other decoration.
1460     store->setAlignment(dest.getAlignment().getQuantity());
1461     if (IsVolatile)
1462       store->setVolatile(true);
1463     if (dest.getTBAAInfo())
1464       CGM.DecorateInstruction(store, dest.getTBAAInfo());
1465     return;
1466   }
1467 
1468   // Atomic load of prev value.
1469   RValue OldRVal =
1470       atomics.EmitAtomicLoad(AggValueSlot::ignored(), SourceLocation(),
1471                              /*AsValue=*/false, AO, IsVolatile);
1472   // For non-simple lvalues perform compare-and-swap procedure.
1473   auto *ContBB = createBasicBlock("atomic_cont");
1474   auto *ExitBB = createBasicBlock("atomic_exit");
1475   auto *CurBB = Builder.GetInsertBlock();
1476   EmitBlock(ContBB);
1477   llvm::PHINode *PHI = Builder.CreatePHI(OldRVal.getScalarVal()->getType(),
1478                                          /*NumReservedValues=*/2);
1479   PHI->addIncoming(OldRVal.getScalarVal(), CurBB);
1480   RValue OriginalRValue = RValue::get(PHI);
1481   // Build new lvalue for temp address
1482   auto *Ptr = atomics.materializeRValue(OriginalRValue);
1483   // Build new lvalue for temp address
1484   LValue UpdateLVal;
1485   if (LVal.isBitField())
1486     UpdateLVal = LValue::MakeBitfield(Ptr, LVal.getBitFieldInfo(),
1487                                       LVal.getType(), LVal.getAlignment());
1488   else if (LVal.isVectorElt())
1489     UpdateLVal = LValue::MakeVectorElt(Ptr, LVal.getVectorIdx(), LVal.getType(),
1490                                        LVal.getAlignment());
1491   else {
1492     assert(LVal.isExtVectorElt());
1493     UpdateLVal = LValue::MakeExtVectorElt(Ptr, LVal.getExtVectorElts(),
1494                                           LVal.getType(), LVal.getAlignment());
1495   }
1496   UpdateLVal.setTBAAInfo(LVal.getTBAAInfo());
1497   // Store new value in the corresponding memory area
1498   EmitStoreThroughLValue(rvalue, UpdateLVal);
1499   // Load new value
1500   RValue NewRValue = RValue::get(EmitLoadOfScalar(
1501       Ptr, LVal.isVolatile(), atomics.getAtomicAlignment().getQuantity(),
1502       atomics.getAtomicType(), SourceLocation()));
1503   // Try to write new value using cmpxchg operation
1504   auto Pair = atomics.EmitAtomicCompareExchange(OriginalRValue, NewRValue, AO);
1505   PHI->addIncoming(Pair.first.getScalarVal(), ContBB);
1506   Builder.CreateCondBr(Pair.second, ExitBB, ContBB);
1507   EmitBlock(ExitBB, /*IsFinished=*/true);
1508 }
1509 
1510 /// Emit a compare-and-exchange op for atomic type.
1511 ///
EmitAtomicCompareExchange(LValue Obj,RValue Expected,RValue Desired,SourceLocation Loc,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak,AggValueSlot Slot)1512 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1513     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1514     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1515     AggValueSlot Slot) {
1516   // If this is an aggregate r-value, it should agree in type except
1517   // maybe for address-space qualification.
1518   assert(!Expected.isAggregate() ||
1519          Expected.getAggregateAddr()->getType()->getPointerElementType() ==
1520              Obj.getAddress()->getType()->getPointerElementType());
1521   assert(!Desired.isAggregate() ||
1522          Desired.getAggregateAddr()->getType()->getPointerElementType() ==
1523              Obj.getAddress()->getType()->getPointerElementType());
1524   AtomicInfo Atomics(*this, Obj);
1525 
1526   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1527                                            IsWeak);
1528 }
1529 
EmitAtomicUpdate(LValue LVal,llvm::AtomicOrdering AO,const std::function<RValue (RValue)> & UpdateOp,bool IsVolatile)1530 void CodeGenFunction::EmitAtomicUpdate(
1531     LValue LVal, llvm::AtomicOrdering AO,
1532     const std::function<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1533   AtomicInfo Atomics(*this, LVal);
1534   LValue AtomicLVal = Atomics.getAtomicLValue();
1535 
1536   // Atomic load of prev value.
1537   RValue OldRVal =
1538       Atomics.EmitAtomicLoad(AggValueSlot::ignored(), SourceLocation(),
1539                              /*AsValue=*/false, AO, IsVolatile);
1540   bool IsScalar = OldRVal.isScalar();
1541   auto *OldVal =
1542       IsScalar ? OldRVal.getScalarVal() : Atomics.convertRValueToInt(OldRVal);
1543   // For non-simple lvalues perform compare-and-swap procedure.
1544   auto *ContBB = createBasicBlock("atomic_cont");
1545   auto *ExitBB = createBasicBlock("atomic_exit");
1546   auto *CurBB = Builder.GetInsertBlock();
1547   EmitBlock(ContBB);
1548   llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(),
1549                                          /*NumReservedValues=*/2);
1550   PHI->addIncoming(OldVal, CurBB);
1551   RValue OriginalRValue =
1552       IsScalar ? RValue::get(PHI) : Atomics.ConvertIntToValueOrAtomic(
1553                                         PHI, AggValueSlot::ignored(),
1554                                         SourceLocation(), /*AsValue=*/false);
1555   // Build new lvalue for temp address
1556   LValue UpdateLVal;
1557   llvm::Value *Ptr = nullptr;
1558   RValue UpRVal;
1559   if (AtomicLVal.isSimple()) {
1560     UpRVal = OriginalRValue;
1561   } else {
1562     // Build new lvalue for temp address
1563     Ptr = Atomics.materializeRValue(OriginalRValue);
1564     if (AtomicLVal.isBitField())
1565       UpdateLVal =
1566           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1567                                AtomicLVal.getType(), AtomicLVal.getAlignment());
1568     else if (AtomicLVal.isVectorElt())
1569       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1570                                          AtomicLVal.getType(),
1571                                          AtomicLVal.getAlignment());
1572     else {
1573       assert(AtomicLVal.isExtVectorElt());
1574       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1575                                             AtomicLVal.getType(),
1576                                             AtomicLVal.getAlignment());
1577     }
1578     UpdateLVal.setTBAAInfo(LVal.getTBAAInfo());
1579     UpRVal = EmitLoadOfLValue(UpdateLVal, SourceLocation());
1580   }
1581   // Store new value in the corresponding memory area
1582   RValue NewRVal = UpdateOp(UpRVal);
1583   if (!AtomicLVal.isSimple()) {
1584     EmitStoreThroughLValue(NewRVal, UpdateLVal);
1585     // Load new value
1586     NewRVal = RValue::get(
1587         EmitLoadOfScalar(Ptr, AtomicLVal.isVolatile(),
1588                          Atomics.getAtomicAlignment().getQuantity(),
1589                          Atomics.getAtomicType(), SourceLocation()));
1590   }
1591   // Try to write new value using cmpxchg operation
1592   auto Pair = Atomics.EmitAtomicCompareExchange(OriginalRValue, NewRVal, AO);
1593   OldVal = IsScalar ? Pair.first.getScalarVal()
1594                     : Atomics.convertRValueToInt(Pair.first);
1595   PHI->addIncoming(OldVal, ContBB);
1596   Builder.CreateCondBr(Pair.second, ExitBB, ContBB);
1597   EmitBlock(ExitBB, /*IsFinished=*/true);
1598 }
1599 
EmitAtomicInit(Expr * init,LValue dest)1600 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1601   AtomicInfo atomics(*this, dest);
1602 
1603   switch (atomics.getEvaluationKind()) {
1604   case TEK_Scalar: {
1605     llvm::Value *value = EmitScalarExpr(init);
1606     atomics.emitCopyIntoMemory(RValue::get(value));
1607     return;
1608   }
1609 
1610   case TEK_Complex: {
1611     ComplexPairTy value = EmitComplexExpr(init);
1612     atomics.emitCopyIntoMemory(RValue::getComplex(value));
1613     return;
1614   }
1615 
1616   case TEK_Aggregate: {
1617     // Fix up the destination if the initializer isn't an expression
1618     // of atomic type.
1619     bool Zeroed = false;
1620     if (!init->getType()->isAtomicType()) {
1621       Zeroed = atomics.emitMemSetZeroIfNecessary();
1622       dest = atomics.projectValue();
1623     }
1624 
1625     // Evaluate the expression directly into the destination.
1626     AggValueSlot slot = AggValueSlot::forLValue(dest,
1627                                         AggValueSlot::IsNotDestructed,
1628                                         AggValueSlot::DoesNotNeedGCBarriers,
1629                                         AggValueSlot::IsNotAliased,
1630                                         Zeroed ? AggValueSlot::IsZeroed :
1631                                                  AggValueSlot::IsNotZeroed);
1632 
1633     EmitAggExpr(init, slot);
1634     return;
1635   }
1636   }
1637   llvm_unreachable("bad evaluation kind");
1638 }
1639