21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
26 using namespace clang;
27 using namespace CodeGen;
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
51 ValueTy = ATy->getValueType();
58 TypeInfo ValueTI =
C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.
Width;
60 ValueAlignInBits = ValueTI.
Align;
62 TypeInfo AtomicTI =
C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.
Width;
64 AtomicAlignInBits = AtomicTI.
Align;
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
69 AtomicAlign =
C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign =
C.toCharUnitsFromBits(ValueAlignInBits);
77 ValueSizeInBits =
C.getTypeSize(ValueTy);
80 AtomicSizeInBits =
C.toBits(
81 C.toCharUnitsFromBits(
Offset + OrigBFI.Size +
C.getCharWidth() - 1)
85 (
C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.
getAlignment()) *
87 llvm::Value *StoragePtr = CGF.
Builder.CreateConstGEP1_64(
88 CGF.
Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
90 StoragePtr, CGF.
UnqualPtrTy,
"atomic_bitfield_base");
95 llvm::Type *StorageTy = CGF.
Builder.getIntNTy(AtomicSizeInBits);
100 AtomicTy =
C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105 AtomicTy =
C.getConstantArrayType(
C.CharTy, Size,
nullptr,
112 ValueSizeInBits =
C.getTypeSize(ValueTy);
114 AtomicSizeInBits =
C.getTypeSize(AtomicTy);
120 ValueSizeInBits =
C.getTypeSize(ValueTy);
122 lvalue.
getType(), cast<llvm::FixedVectorType>(
125 AtomicSizeInBits =
C.getTypeSize(AtomicTy);
129 UseLibcall = !
C.getTargetInfo().hasBuiltinAtomic(
133 QualType getAtomicType()
const {
return AtomicTy; }
134 QualType getValueType()
const {
return ValueTy; }
135 CharUnits getAtomicAlignment()
const {
return AtomicAlign; }
136 uint64_t getAtomicSizeInBits()
const {
return AtomicSizeInBits; }
137 uint64_t getValueSizeInBits()
const {
return ValueSizeInBits; }
139 bool shouldUseLibcall()
const {
return UseLibcall; }
140 const LValue &getAtomicLValue()
const {
return LVal; }
141 llvm::Value *getAtomicPointer()
const {
151 Address getAtomicAddress()
const {
161 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
164 Address getAtomicAddressAsAtomicIntPointer()
const {
165 return castToAtomicIntPointer(getAtomicAddress());
174 bool hasPadding()
const {
175 return (ValueSizeInBits != AtomicSizeInBits);
178 bool emitMemSetZeroIfNecessary()
const;
180 llvm::Value *getAtomicSizeValue()
const {
198 llvm::Value *getScalarRValValueOrNull(
RValue RVal)
const;
201 llvm::Value *convertRValueToInt(
RValue RVal,
bool CmpXchg =
false)
const;
205 bool CmpXchg =
false)
const;
208 void emitCopyIntoMemory(
RValue rvalue)
const;
211 LValue projectValue()
const {
213 Address addr = getAtomicAddress();
224 bool AsValue, llvm::AtomicOrdering AO,
235 std::pair<RValue, llvm::Value *>
237 llvm::AtomicOrdering Success =
238 llvm::AtomicOrdering::SequentiallyConsistent,
239 llvm::AtomicOrdering Failure =
240 llvm::AtomicOrdering::SequentiallyConsistent,
241 bool IsWeak =
false);
246 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
251 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
RValue UpdateRVal,
258 Address CreateTempAlloca()
const;
260 bool requiresMemSetZero(llvm::Type *
type)
const;
264 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
265 llvm::AtomicOrdering AO,
bool IsVolatile);
267 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO,
bool IsVolatile,
268 bool CmpXchg =
false);
270 llvm::Value *EmitAtomicCompareExchangeLibcall(
271 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
272 llvm::AtomicOrdering Success =
273 llvm::AtomicOrdering::SequentiallyConsistent,
274 llvm::AtomicOrdering Failure =
275 llvm::AtomicOrdering::SequentiallyConsistent);
277 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
278 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
279 llvm::AtomicOrdering Success =
280 llvm::AtomicOrdering::SequentiallyConsistent,
281 llvm::AtomicOrdering Failure =
282 llvm::AtomicOrdering::SequentiallyConsistent,
283 bool IsWeak =
false);
286 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
290 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
294 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
RValue UpdateRVal,
297 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
RValue UpdateRal,
302 Address AtomicInfo::CreateTempAlloca()
const {
304 (LVal.
isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
306 getAtomicAlignment(),
311 TempAlloca, getAtomicAddress().getType(),
312 getAtomicAddress().getElementType());
324 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
325 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
326 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
327 CGF.
getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
329 llvm::FunctionCallee fn =
344 bool AtomicInfo::requiresMemSetZero(llvm::Type *
type)
const {
346 if (hasPadding())
return true;
349 switch (getEvaluationKind()) {
356 AtomicSizeInBits / 2);
362 llvm_unreachable(
"bad evaluation kind");
365 bool AtomicInfo::emitMemSetZeroIfNecessary()
const {
382 llvm::AtomicOrdering SuccessOrder,
383 llvm::AtomicOrdering FailureOrder,
392 Pair->setWeak(IsWeak);
396 llvm::Value *Old = CGF.
Builder.CreateExtractValue(Pair, 0);
397 llvm::Value *Cmp = CGF.
Builder.CreateExtractValue(Pair, 1);
401 llvm::BasicBlock *StoreExpectedBB =
406 llvm::BasicBlock *ContinueBB =
411 CGF.
Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
413 CGF.
Builder.SetInsertPoint(StoreExpectedBB);
417 CGF.
Builder.CreateBr(ContinueBB);
419 CGF.
Builder.SetInsertPoint(ContinueBB);
430 llvm::Value *FailureOrderVal,
432 llvm::AtomicOrdering SuccessOrder,
434 llvm::AtomicOrdering FailureOrder;
435 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
436 auto FOS = FO->getSExtValue();
437 if (!llvm::isValidAtomicOrderingCABI(FOS))
438 FailureOrder = llvm::AtomicOrdering::Monotonic;
440 switch ((llvm::AtomicOrderingCABI)FOS) {
441 case llvm::AtomicOrderingCABI::relaxed:
444 case llvm::AtomicOrderingCABI::release:
445 case llvm::AtomicOrderingCABI::acq_rel:
446 FailureOrder = llvm::AtomicOrdering::Monotonic;
448 case llvm::AtomicOrderingCABI::consume:
449 case llvm::AtomicOrderingCABI::acquire:
450 FailureOrder = llvm::AtomicOrdering::Acquire;
452 case llvm::AtomicOrderingCABI::seq_cst:
453 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
461 FailureOrder,
Scope);
474 llvm::SwitchInst *SI = CGF.
Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
476 SI->addCase(CGF.
Builder.getInt32((
int)llvm::AtomicOrderingCABI::consume),
478 SI->addCase(CGF.
Builder.getInt32((
int)llvm::AtomicOrderingCABI::acquire),
480 SI->addCase(CGF.
Builder.getInt32((
int)llvm::AtomicOrderingCABI::seq_cst),
484 CGF.
Builder.SetInsertPoint(MonotonicBB);
486 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic,
Scope);
489 CGF.
Builder.SetInsertPoint(AcquireBB);
491 llvm::AtomicOrdering::Acquire,
Scope);
494 CGF.
Builder.SetInsertPoint(SeqCstBB);
496 llvm::AtomicOrdering::SequentiallyConsistent,
Scope);
499 CGF.
Builder.SetInsertPoint(ContBB);
509 llvm::CmpInst::Predicate Pred;
512 llvm_unreachable(
"Unexpected min/max operation");
513 case AtomicExpr::AO__atomic_max_fetch:
514 case AtomicExpr::AO__scoped_atomic_max_fetch:
515 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
517 case AtomicExpr::AO__atomic_min_fetch:
518 case AtomicExpr::AO__scoped_atomic_min_fetch:
519 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
522 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS,
"tst");
523 return Builder.CreateSelect(Cmp, OldVal, RHS,
"newval");
528 llvm::Value *IsWeak, llvm::Value *FailureOrder,
529 uint64_t Size, llvm::AtomicOrdering Order,
532 bool PostOpMinMax =
false;
535 switch (E->
getOp()) {
536 case AtomicExpr::AO__c11_atomic_init:
537 case AtomicExpr::AO__opencl_atomic_init:
538 llvm_unreachable(
"Already handled!");
540 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
541 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
542 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
544 FailureOrder, Size, Order,
Scope);
546 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
547 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
548 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
550 FailureOrder, Size, Order,
Scope);
552 case AtomicExpr::AO__atomic_compare_exchange:
553 case AtomicExpr::AO__atomic_compare_exchange_n:
554 case AtomicExpr::AO__scoped_atomic_compare_exchange:
555 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
556 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
558 Val1, Val2, FailureOrder, Size, Order,
Scope);
561 llvm::BasicBlock *StrongBB =
564 llvm::BasicBlock *ContBB =
567 llvm::SwitchInst *SI = CGF.
Builder.CreateSwitch(IsWeak, WeakBB);
568 SI->addCase(CGF.
Builder.getInt1(
false), StrongBB);
570 CGF.
Builder.SetInsertPoint(StrongBB);
572 FailureOrder, Size, Order,
Scope);
575 CGF.
Builder.SetInsertPoint(WeakBB);
577 FailureOrder, Size, Order,
Scope);
580 CGF.
Builder.SetInsertPoint(ContBB);
584 case AtomicExpr::AO__c11_atomic_load:
585 case AtomicExpr::AO__opencl_atomic_load:
586 case AtomicExpr::AO__hip_atomic_load:
587 case AtomicExpr::AO__atomic_load_n:
588 case AtomicExpr::AO__atomic_load:
589 case AtomicExpr::AO__scoped_atomic_load_n:
590 case AtomicExpr::AO__scoped_atomic_load: {
598 case AtomicExpr::AO__c11_atomic_store:
599 case AtomicExpr::AO__opencl_atomic_store:
600 case AtomicExpr::AO__hip_atomic_store:
601 case AtomicExpr::AO__atomic_store:
602 case AtomicExpr::AO__atomic_store_n:
603 case AtomicExpr::AO__scoped_atomic_store:
604 case AtomicExpr::AO__scoped_atomic_store_n: {
612 case AtomicExpr::AO__c11_atomic_exchange:
613 case AtomicExpr::AO__hip_atomic_exchange:
614 case AtomicExpr::AO__opencl_atomic_exchange:
615 case AtomicExpr::AO__atomic_exchange_n:
616 case AtomicExpr::AO__atomic_exchange:
617 case AtomicExpr::AO__scoped_atomic_exchange_n:
618 case AtomicExpr::AO__scoped_atomic_exchange:
619 Op = llvm::AtomicRMWInst::Xchg;
622 case AtomicExpr::AO__atomic_add_fetch:
623 case AtomicExpr::AO__scoped_atomic_add_fetch:
627 case AtomicExpr::AO__c11_atomic_fetch_add:
628 case AtomicExpr::AO__hip_atomic_fetch_add:
629 case AtomicExpr::AO__opencl_atomic_fetch_add:
630 case AtomicExpr::AO__atomic_fetch_add:
631 case AtomicExpr::AO__scoped_atomic_fetch_add:
636 case AtomicExpr::AO__atomic_sub_fetch:
637 case AtomicExpr::AO__scoped_atomic_sub_fetch:
641 case AtomicExpr::AO__c11_atomic_fetch_sub:
642 case AtomicExpr::AO__hip_atomic_fetch_sub:
643 case AtomicExpr::AO__opencl_atomic_fetch_sub:
644 case AtomicExpr::AO__atomic_fetch_sub:
645 case AtomicExpr::AO__scoped_atomic_fetch_sub:
650 case AtomicExpr::AO__atomic_min_fetch:
651 case AtomicExpr::AO__scoped_atomic_min_fetch:
654 case AtomicExpr::AO__c11_atomic_fetch_min:
655 case AtomicExpr::AO__hip_atomic_fetch_min:
656 case AtomicExpr::AO__opencl_atomic_fetch_min:
657 case AtomicExpr::AO__atomic_fetch_min:
658 case AtomicExpr::AO__scoped_atomic_fetch_min:
660 ? llvm::AtomicRMWInst::FMin
662 ? llvm::AtomicRMWInst::Min
663 : llvm::AtomicRMWInst::UMin);
666 case AtomicExpr::AO__atomic_max_fetch:
667 case AtomicExpr::AO__scoped_atomic_max_fetch:
670 case AtomicExpr::AO__c11_atomic_fetch_max:
671 case AtomicExpr::AO__hip_atomic_fetch_max:
672 case AtomicExpr::AO__opencl_atomic_fetch_max:
673 case AtomicExpr::AO__atomic_fetch_max:
674 case AtomicExpr::AO__scoped_atomic_fetch_max:
676 ? llvm::AtomicRMWInst::FMax
678 ? llvm::AtomicRMWInst::Max
679 : llvm::AtomicRMWInst::UMax);
682 case AtomicExpr::AO__atomic_and_fetch:
683 case AtomicExpr::AO__scoped_atomic_and_fetch:
686 case AtomicExpr::AO__c11_atomic_fetch_and:
687 case AtomicExpr::AO__hip_atomic_fetch_and:
688 case AtomicExpr::AO__opencl_atomic_fetch_and:
689 case AtomicExpr::AO__atomic_fetch_and:
690 case AtomicExpr::AO__scoped_atomic_fetch_and:
694 case AtomicExpr::AO__atomic_or_fetch:
695 case AtomicExpr::AO__scoped_atomic_or_fetch:
696 PostOp = llvm::Instruction::Or;
698 case AtomicExpr::AO__c11_atomic_fetch_or:
699 case AtomicExpr::AO__hip_atomic_fetch_or:
700 case AtomicExpr::AO__opencl_atomic_fetch_or:
701 case AtomicExpr::AO__atomic_fetch_or:
702 case AtomicExpr::AO__scoped_atomic_fetch_or:
703 Op = llvm::AtomicRMWInst::Or;
706 case AtomicExpr::AO__atomic_xor_fetch:
707 case AtomicExpr::AO__scoped_atomic_xor_fetch:
708 PostOp = llvm::Instruction::Xor;
710 case AtomicExpr::AO__c11_atomic_fetch_xor:
711 case AtomicExpr::AO__hip_atomic_fetch_xor:
712 case AtomicExpr::AO__opencl_atomic_fetch_xor:
713 case AtomicExpr::AO__atomic_fetch_xor:
714 case AtomicExpr::AO__scoped_atomic_fetch_xor:
715 Op = llvm::AtomicRMWInst::Xor;
718 case AtomicExpr::AO__atomic_nand_fetch:
719 case AtomicExpr::AO__scoped_atomic_nand_fetch:
722 case AtomicExpr::AO__c11_atomic_fetch_nand:
723 case AtomicExpr::AO__atomic_fetch_nand:
724 case AtomicExpr::AO__scoped_atomic_fetch_nand:
725 Op = llvm::AtomicRMWInst::Nand;
730 llvm::AtomicRMWInst *RMWI =
736 llvm::Value *Result = RMWI;
742 Result = CGF.
Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
744 if (E->
getOp() == AtomicExpr::AO__atomic_nand_fetch ||
745 E->
getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
746 Result = CGF.
Builder.CreateNot(Result);
762 llvm::Value *IsWeak, llvm::Value *FailureOrder,
763 uint64_t Size, llvm::AtomicOrdering Order,
764 llvm::Value *
Scope) {
765 auto ScopeModel =
Expr->getScopeModel();
776 if (
auto SC = dyn_cast<llvm::ConstantInt>(
Scope)) {
787 auto Scopes = ScopeModel->getRuntimeValues();
788 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
789 for (
auto S : Scopes)
792 llvm::BasicBlock *ContBB =
795 auto *SC = Builder.CreateIntCast(
Scope, Builder.getInt32Ty(),
false);
798 auto FallBack = ScopeModel->getFallBackValue();
799 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
800 for (
auto S : Scopes) {
803 SI->addCase(Builder.getInt32(S), B);
805 Builder.SetInsertPoint(B);
812 Builder.CreateBr(ContBB);
815 Builder.SetInsertPoint(ContBB);
822 MemTy = AT->getValueType();
823 llvm::Value *IsWeak =
nullptr, *OrderFail =
nullptr;
830 if (E->
getOp() == AtomicExpr::AO__c11_atomic_init ||
831 E->
getOp() == AtomicExpr::AO__opencl_atomic_init) {
838 uint64_t Size = TInfo.Width.getQuantity();
844 bool Misaligned = (Ptr.
getAlignment() % TInfo.Width) != 0;
848 << (
int)TInfo.Width.getQuantity()
853 << (
int)TInfo.Width.getQuantity() << (
int)MaxInlineWidth.
getQuantity();
859 bool ShouldCastToIntPtrTy =
true;
861 switch (E->
getOp()) {
862 case AtomicExpr::AO__c11_atomic_init:
863 case AtomicExpr::AO__opencl_atomic_init:
864 llvm_unreachable(
"Already handled above with EmitAtomicInit!");
866 case AtomicExpr::AO__atomic_load_n:
867 case AtomicExpr::AO__scoped_atomic_load_n:
868 case AtomicExpr::AO__c11_atomic_load:
869 case AtomicExpr::AO__opencl_atomic_load:
870 case AtomicExpr::AO__hip_atomic_load:
873 case AtomicExpr::AO__atomic_load:
874 case AtomicExpr::AO__scoped_atomic_load:
878 case AtomicExpr::AO__atomic_store:
879 case AtomicExpr::AO__scoped_atomic_store:
883 case AtomicExpr::AO__atomic_exchange:
884 case AtomicExpr::AO__scoped_atomic_exchange:
889 case AtomicExpr::AO__atomic_compare_exchange:
890 case AtomicExpr::AO__atomic_compare_exchange_n:
891 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
892 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
893 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
894 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
895 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
896 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
897 case AtomicExpr::AO__scoped_atomic_compare_exchange:
898 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
900 if (E->
getOp() == AtomicExpr::AO__atomic_compare_exchange ||
901 E->
getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
906 if (E->
getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
907 E->
getOp() == AtomicExpr::AO__atomic_compare_exchange ||
908 E->
getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
909 E->
getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
913 case AtomicExpr::AO__c11_atomic_fetch_add:
914 case AtomicExpr::AO__c11_atomic_fetch_sub:
915 case AtomicExpr::AO__hip_atomic_fetch_add:
916 case AtomicExpr::AO__hip_atomic_fetch_sub:
917 case AtomicExpr::AO__opencl_atomic_fetch_add:
918 case AtomicExpr::AO__opencl_atomic_fetch_sub:
935 case AtomicExpr::AO__atomic_fetch_add:
936 case AtomicExpr::AO__atomic_fetch_max:
937 case AtomicExpr::AO__atomic_fetch_min:
938 case AtomicExpr::AO__atomic_fetch_sub:
939 case AtomicExpr::AO__atomic_add_fetch:
940 case AtomicExpr::AO__atomic_max_fetch:
941 case AtomicExpr::AO__atomic_min_fetch:
942 case AtomicExpr::AO__atomic_sub_fetch:
943 case AtomicExpr::AO__c11_atomic_fetch_max:
944 case AtomicExpr::AO__c11_atomic_fetch_min:
945 case AtomicExpr::AO__opencl_atomic_fetch_max:
946 case AtomicExpr::AO__opencl_atomic_fetch_min:
947 case AtomicExpr::AO__hip_atomic_fetch_max:
948 case AtomicExpr::AO__hip_atomic_fetch_min:
949 case AtomicExpr::AO__scoped_atomic_fetch_add:
950 case AtomicExpr::AO__scoped_atomic_fetch_max:
951 case AtomicExpr::AO__scoped_atomic_fetch_min:
952 case AtomicExpr::AO__scoped_atomic_fetch_sub:
953 case AtomicExpr::AO__scoped_atomic_add_fetch:
954 case AtomicExpr::AO__scoped_atomic_max_fetch:
955 case AtomicExpr::AO__scoped_atomic_min_fetch:
956 case AtomicExpr::AO__scoped_atomic_sub_fetch:
960 case AtomicExpr::AO__atomic_fetch_and:
961 case AtomicExpr::AO__atomic_fetch_nand:
962 case AtomicExpr::AO__atomic_fetch_or:
963 case AtomicExpr::AO__atomic_fetch_xor:
964 case AtomicExpr::AO__atomic_and_fetch:
965 case AtomicExpr::AO__atomic_nand_fetch:
966 case AtomicExpr::AO__atomic_or_fetch:
967 case AtomicExpr::AO__atomic_xor_fetch:
968 case AtomicExpr::AO__atomic_store_n:
969 case AtomicExpr::AO__atomic_exchange_n:
970 case AtomicExpr::AO__c11_atomic_fetch_and:
971 case AtomicExpr::AO__c11_atomic_fetch_nand:
972 case AtomicExpr::AO__c11_atomic_fetch_or:
973 case AtomicExpr::AO__c11_atomic_fetch_xor:
974 case AtomicExpr::AO__c11_atomic_store:
975 case AtomicExpr::AO__c11_atomic_exchange:
976 case AtomicExpr::AO__hip_atomic_fetch_and:
977 case AtomicExpr::AO__hip_atomic_fetch_or:
978 case AtomicExpr::AO__hip_atomic_fetch_xor:
979 case AtomicExpr::AO__hip_atomic_store:
980 case AtomicExpr::AO__hip_atomic_exchange:
981 case AtomicExpr::AO__opencl_atomic_fetch_and:
982 case AtomicExpr::AO__opencl_atomic_fetch_or:
983 case AtomicExpr::AO__opencl_atomic_fetch_xor:
984 case AtomicExpr::AO__opencl_atomic_store:
985 case AtomicExpr::AO__opencl_atomic_exchange:
986 case AtomicExpr::AO__scoped_atomic_fetch_and:
987 case AtomicExpr::AO__scoped_atomic_fetch_nand:
988 case AtomicExpr::AO__scoped_atomic_fetch_or:
989 case AtomicExpr::AO__scoped_atomic_fetch_xor:
990 case AtomicExpr::AO__scoped_atomic_and_fetch:
991 case AtomicExpr::AO__scoped_atomic_nand_fetch:
992 case AtomicExpr::AO__scoped_atomic_or_fetch:
993 case AtomicExpr::AO__scoped_atomic_xor_fetch:
994 case AtomicExpr::AO__scoped_atomic_store_n:
995 case AtomicExpr::AO__scoped_atomic_exchange_n:
1006 AtomicInfo Atomics(*
this, AtomicVal);
1008 if (ShouldCastToIntPtrTy) {
1009 Ptr = Atomics.castToAtomicIntPointer(Ptr);
1011 Val1 = Atomics.convertToAtomicIntPointer(Val1);
1013 Val2 = Atomics.convertToAtomicIntPointer(Val2);
1016 if (ShouldCastToIntPtrTy)
1017 Dest = Atomics.castToAtomicIntPointer(Dest);
1021 Dest = Atomics.CreateTempAlloca();
1022 if (ShouldCastToIntPtrTy)
1023 Dest = Atomics.castToAtomicIntPointer(Dest);
1026 bool PowerOf2Size = (Size & (Size - 1)) == 0;
1027 bool UseLibcall = !PowerOf2Size || (Size > 16);
1047 auto CastToGenericAddrSpace = [&](llvm::Value *
V,
QualType PT) {
1050 auto AS = PT->castAs<
PointerType>()->getPointeeType().getAddressSpace();
1054 auto *DestType = llvm::PointerType::get(
getLLVMContext(), DestAS);
1065 std::string LibCallName;
1067 bool HaveRetTy =
false;
1068 switch (E->
getOp()) {
1069 case AtomicExpr::AO__c11_atomic_init:
1070 case AtomicExpr::AO__opencl_atomic_init:
1071 llvm_unreachable(
"Already handled!");
1078 case AtomicExpr::AO__atomic_compare_exchange:
1079 case AtomicExpr::AO__atomic_compare_exchange_n:
1080 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1081 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1082 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1083 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1084 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1085 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1086 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1087 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1088 LibCallName =
"__atomic_compare_exchange";
1102 case AtomicExpr::AO__atomic_exchange:
1103 case AtomicExpr::AO__atomic_exchange_n:
1104 case AtomicExpr::AO__c11_atomic_exchange:
1105 case AtomicExpr::AO__hip_atomic_exchange:
1106 case AtomicExpr::AO__opencl_atomic_exchange:
1107 case AtomicExpr::AO__scoped_atomic_exchange:
1108 case AtomicExpr::AO__scoped_atomic_exchange_n:
1109 LibCallName =
"__atomic_exchange";
1115 case AtomicExpr::AO__atomic_store:
1116 case AtomicExpr::AO__atomic_store_n:
1117 case AtomicExpr::AO__c11_atomic_store:
1118 case AtomicExpr::AO__hip_atomic_store:
1119 case AtomicExpr::AO__opencl_atomic_store:
1120 case AtomicExpr::AO__scoped_atomic_store:
1121 case AtomicExpr::AO__scoped_atomic_store_n:
1122 LibCallName =
"__atomic_store";
1130 case AtomicExpr::AO__atomic_load:
1131 case AtomicExpr::AO__atomic_load_n:
1132 case AtomicExpr::AO__c11_atomic_load:
1133 case AtomicExpr::AO__hip_atomic_load:
1134 case AtomicExpr::AO__opencl_atomic_load:
1135 case AtomicExpr::AO__scoped_atomic_load:
1136 case AtomicExpr::AO__scoped_atomic_load_n:
1137 LibCallName =
"__atomic_load";
1139 case AtomicExpr::AO__atomic_add_fetch:
1140 case AtomicExpr::AO__scoped_atomic_add_fetch:
1141 case AtomicExpr::AO__atomic_fetch_add:
1142 case AtomicExpr::AO__c11_atomic_fetch_add:
1143 case AtomicExpr::AO__hip_atomic_fetch_add:
1144 case AtomicExpr::AO__opencl_atomic_fetch_add:
1145 case AtomicExpr::AO__scoped_atomic_fetch_add:
1146 case AtomicExpr::AO__atomic_and_fetch:
1147 case AtomicExpr::AO__scoped_atomic_and_fetch:
1148 case AtomicExpr::AO__atomic_fetch_and:
1149 case AtomicExpr::AO__c11_atomic_fetch_and:
1150 case AtomicExpr::AO__hip_atomic_fetch_and:
1151 case AtomicExpr::AO__opencl_atomic_fetch_and:
1152 case AtomicExpr::AO__scoped_atomic_fetch_and:
1153 case AtomicExpr::AO__atomic_or_fetch:
1154 case AtomicExpr::AO__scoped_atomic_or_fetch:
1155 case AtomicExpr::AO__atomic_fetch_or:
1156 case AtomicExpr::AO__c11_atomic_fetch_or:
1157 case AtomicExpr::AO__hip_atomic_fetch_or:
1158 case AtomicExpr::AO__opencl_atomic_fetch_or:
1159 case AtomicExpr::AO__scoped_atomic_fetch_or:
1160 case AtomicExpr::AO__atomic_sub_fetch:
1161 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1162 case AtomicExpr::AO__atomic_fetch_sub:
1163 case AtomicExpr::AO__c11_atomic_fetch_sub:
1164 case AtomicExpr::AO__hip_atomic_fetch_sub:
1165 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1166 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1167 case AtomicExpr::AO__atomic_xor_fetch:
1168 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1169 case AtomicExpr::AO__atomic_fetch_xor:
1170 case AtomicExpr::AO__c11_atomic_fetch_xor:
1171 case AtomicExpr::AO__hip_atomic_fetch_xor:
1172 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1173 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1174 case AtomicExpr::AO__atomic_nand_fetch:
1175 case AtomicExpr::AO__atomic_fetch_nand:
1176 case AtomicExpr::AO__c11_atomic_fetch_nand:
1177 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1178 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1179 case AtomicExpr::AO__atomic_min_fetch:
1180 case AtomicExpr::AO__atomic_fetch_min:
1181 case AtomicExpr::AO__c11_atomic_fetch_min:
1182 case AtomicExpr::AO__hip_atomic_fetch_min:
1183 case AtomicExpr::AO__opencl_atomic_fetch_min:
1184 case AtomicExpr::AO__scoped_atomic_fetch_min:
1185 case AtomicExpr::AO__scoped_atomic_min_fetch:
1186 case AtomicExpr::AO__atomic_max_fetch:
1187 case AtomicExpr::AO__atomic_fetch_max:
1188 case AtomicExpr::AO__c11_atomic_fetch_max:
1189 case AtomicExpr::AO__hip_atomic_fetch_max:
1190 case AtomicExpr::AO__opencl_atomic_fetch_max:
1191 case AtomicExpr::AO__scoped_atomic_fetch_max:
1192 case AtomicExpr::AO__scoped_atomic_max_fetch:
1193 llvm_unreachable(
"Integral atomic operations always become atomicrmw!");
1198 std::string(
"__opencl") + StringRef(LibCallName).drop_front(1).str();
1226 bool IsStore = E->
getOp() == AtomicExpr::AO__c11_atomic_store ||
1227 E->
getOp() == AtomicExpr::AO__opencl_atomic_store ||
1228 E->
getOp() == AtomicExpr::AO__hip_atomic_store ||
1229 E->
getOp() == AtomicExpr::AO__atomic_store ||
1230 E->
getOp() == AtomicExpr::AO__atomic_store_n ||
1231 E->
getOp() == AtomicExpr::AO__scoped_atomic_store ||
1232 E->
getOp() == AtomicExpr::AO__scoped_atomic_store_n;
1233 bool IsLoad = E->
getOp() == AtomicExpr::AO__c11_atomic_load ||
1234 E->
getOp() == AtomicExpr::AO__opencl_atomic_load ||
1235 E->
getOp() == AtomicExpr::AO__hip_atomic_load ||
1236 E->
getOp() == AtomicExpr::AO__atomic_load ||
1237 E->
getOp() == AtomicExpr::AO__atomic_load_n ||
1238 E->
getOp() == AtomicExpr::AO__scoped_atomic_load ||
1239 E->
getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1241 if (isa<llvm::ConstantInt>(Order)) {
1242 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1245 if (llvm::isValidAtomicOrderingCABI(ord))
1246 switch ((llvm::AtomicOrderingCABI)ord) {
1247 case llvm::AtomicOrderingCABI::relaxed:
1248 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1249 llvm::AtomicOrdering::Monotonic,
Scope);
1251 case llvm::AtomicOrderingCABI::consume:
1252 case llvm::AtomicOrderingCABI::acquire:
1255 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1256 llvm::AtomicOrdering::Acquire,
Scope);
1258 case llvm::AtomicOrderingCABI::release:
1261 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1262 llvm::AtomicOrdering::Release,
Scope);
1264 case llvm::AtomicOrderingCABI::acq_rel:
1265 if (IsLoad || IsStore)
1267 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1268 llvm::AtomicOrdering::AcquireRelease,
Scope);
1270 case llvm::AtomicOrderingCABI::seq_cst:
1271 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1272 llvm::AtomicOrdering::SequentiallyConsistent,
Scope);
1285 llvm::BasicBlock *MonotonicBB =
nullptr, *AcquireBB =
nullptr,
1286 *ReleaseBB =
nullptr, *AcqRelBB =
nullptr,
1287 *SeqCstBB =
nullptr;
1293 if (!IsLoad && !IsStore)
1302 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1303 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, MonotonicBB);
1306 Builder.SetInsertPoint(MonotonicBB);
1307 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1308 llvm::AtomicOrdering::Monotonic,
Scope);
1311 Builder.SetInsertPoint(AcquireBB);
1312 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1313 llvm::AtomicOrdering::Acquire,
Scope);
1315 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::consume),
1317 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::acquire),
1321 Builder.SetInsertPoint(ReleaseBB);
1322 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1323 llvm::AtomicOrdering::Release,
Scope);
1325 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::release),
1328 if (!IsLoad && !IsStore) {
1329 Builder.SetInsertPoint(AcqRelBB);
1330 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1331 llvm::AtomicOrdering::AcquireRelease,
Scope);
1333 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::acq_rel),
1336 Builder.SetInsertPoint(SeqCstBB);
1337 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1338 llvm::AtomicOrdering::SequentiallyConsistent,
Scope);
1340 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::seq_cst),
1344 Builder.SetInsertPoint(ContBB);
1348 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1354 llvm::IntegerType *ty =
1359 Address AtomicInfo::convertToAtomicIntPointer(
Address Addr)
const {
1362 if (SourceSizeInBits != AtomicSizeInBits) {
1363 Address Tmp = CreateTempAlloca();
1365 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1369 return castToAtomicIntPointer(Addr);
1375 bool asValue)
const {
1411 if (ValTy->isFloatingPointTy())
1412 return ValTy->isX86_FP80Ty() || CmpXchg;
1413 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
1416 RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1419 bool CmpXchg)
const {
1421 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1422 Val->getType()->isIEEELikeFPTy()) &&
1423 "Expected integer, pointer or floating point value when converting "
1430 auto *ValTy = AsValue
1432 : getAtomicAddress().getElementType();
1434 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1435 "Different integer types.");
1438 if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
1445 bool TempIsVolatile =
false;
1451 Temp = CreateTempAlloca();
1455 Address CastTemp = castToAtomicIntPointer(Temp);
1458 return convertAtomicTempToRValue(Temp, ResultSlot,
Loc, AsValue);
1461 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1462 llvm::AtomicOrdering AO,
bool) {
1474 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1475 bool IsVolatile,
bool CmpXchg) {
1477 Address Addr = getAtomicAddress();
1479 Addr = castToAtomicIntPointer(Addr);
1481 Load->setAtomic(AO);
1485 Load->setVolatile(
true);
1495 AtomicInfo AI(*
this, LV);
1498 bool AtomicIsInline = !AI.shouldUseLibcall();
1503 return IsVolatile && AtomicIsInline;
1508 llvm::AtomicOrdering AO;
1511 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1513 AO = llvm::AtomicOrdering::Acquire;
1520 bool AsValue, llvm::AtomicOrdering AO,
1523 if (shouldUseLibcall()) {
1529 TempAddr = CreateTempAlloca();
1531 EmitAtomicLoadLibcall(TempAddr.
emitRawPointer(CGF), AO, IsVolatile);
1535 return convertAtomicTempToRValue(TempAddr, ResultSlot,
Loc, AsValue);
1539 auto *
Load = EmitAtomicLoadOp(AO, IsVolatile);
1547 return ConvertToValueOrAtomic(
Load, ResultSlot,
Loc, AsValue);
1553 llvm::AtomicOrdering AO,
bool IsVolatile,
1555 AtomicInfo Atomics(*
this, src);
1556 return Atomics.EmitAtomicLoad(resultSlot, loc,
true, AO,
1562 void AtomicInfo::emitCopyIntoMemory(
RValue rvalue)
const {
1581 emitMemSetZeroIfNecessary();
1584 LValue TempLVal = projectValue();
1597 Address AtomicInfo::materializeRValue(
RValue rvalue)
const {
1605 AtomicInfo Atomics(CGF, TempLV);
1606 Atomics.emitCopyIntoMemory(rvalue);
1610 llvm::Value *AtomicInfo::getScalarRValValueOrNull(
RValue RVal)
const {
1616 llvm::Value *AtomicInfo::convertRValueToInt(
RValue RVal,
bool CmpXchg)
const {
1619 if (llvm::Value *
Value = getScalarRValValueOrNull(RVal)) {
1623 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1625 LVal.
isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1626 if (llvm::BitCastInst::isBitCastable(
Value->
getType(), InputIntTy))
1632 Address Addr = materializeRValue(RVal);
1635 Addr = castToAtomicIntPointer(Addr);
1639 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1640 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1641 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure,
bool IsWeak) {
1643 Address Addr = getAtomicAddressAsAtomicIntPointer();
1648 Inst->setWeak(IsWeak);
1651 auto *PreviousVal = CGF.
Builder.CreateExtractValue(Inst, 0);
1652 auto *SuccessFailureVal = CGF.
Builder.CreateExtractValue(Inst, 1);
1653 return std::make_pair(PreviousVal, SuccessFailureVal);
1657 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1658 llvm::Value *DesiredAddr,
1659 llvm::AtomicOrdering Success,
1660 llvm::AtomicOrdering Failure) {
1669 llvm::ConstantInt::get(CGF.
IntTy, (
int)llvm::toCABI(Success))),
1672 llvm::ConstantInt::get(CGF.
IntTy, (
int)llvm::toCABI(Failure))),
1677 return SuccessFailureRVal.getScalarVal();
1680 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1682 llvm::AtomicOrdering Failure,
bool IsWeak) {
1684 if (shouldUseLibcall()) {
1688 llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);
1689 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,
1691 return std::make_pair(
1699 auto *ExpectedVal = convertRValueToInt(
Expected,
true);
1700 auto *DesiredVal = convertRValueToInt(Desired,
true);
1701 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1703 return std::make_pair(
1715 LValue AtomicLVal = Atomics.getAtomicLValue();
1722 Address Ptr = Atomics.materializeRValue(OldRVal);
1755 RValue NewRVal = UpdateOp(UpRVal);
1765 void AtomicInfo::EmitAtomicUpdateLibcall(
1766 llvm::AtomicOrdering AO,
const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
1768 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1770 Address ExpectedAddr = CreateTempAlloca();
1772 EmitAtomicLoadLibcall(ExpectedAddr.
emitRawPointer(CGF), AO, IsVolatile);
1776 Address DesiredAddr = CreateTempAlloca();
1778 requiresMemSetZero(getAtomicAddress().getElementType())) {
1782 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1789 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
1790 CGF.
Builder.CreateCondBr(Res, ExitBB, ContBB);
1794 void AtomicInfo::EmitAtomicUpdateOp(
1795 llvm::AtomicOrdering AO,
const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
1797 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1800 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile,
true);
1804 auto *CurBB = CGF.
Builder.GetInsertBlock();
1806 llvm::PHINode *PHI = CGF.
Builder.CreatePHI(OldVal->getType(),
1808 PHI->addIncoming(OldVal, CurBB);
1809 Address NewAtomicAddr = CreateTempAlloca();
1812 ? castToAtomicIntPointer(NewAtomicAddr)
1816 requiresMemSetZero(getAtomicAddress().getElementType())) {
1825 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1826 PHI->addIncoming(Res.first, CGF.
Builder.GetInsertBlock());
1827 CGF.
Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1833 LValue AtomicLVal = Atomics.getAtomicLValue();
1857 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1858 RValue UpdateRVal,
bool IsVolatile) {
1859 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1861 Address ExpectedAddr = CreateTempAlloca();
1863 EmitAtomicLoadLibcall(ExpectedAddr.
emitRawPointer(CGF), AO, IsVolatile);
1867 Address DesiredAddr = CreateTempAlloca();
1869 requiresMemSetZero(getAtomicAddress().getElementType())) {
1877 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
1878 CGF.
Builder.CreateCondBr(Res, ExitBB, ContBB);
1882 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
RValue UpdateRVal,
1884 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1887 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile,
true);
1891 auto *CurBB = CGF.
Builder.GetInsertBlock();
1893 llvm::PHINode *PHI = CGF.
Builder.CreatePHI(OldVal->getType(),
1895 PHI->addIncoming(OldVal, CurBB);
1896 Address NewAtomicAddr = CreateTempAlloca();
1897 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1899 requiresMemSetZero(getAtomicAddress().getElementType())) {
1905 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1906 PHI->addIncoming(Res.first, CGF.
Builder.GetInsertBlock());
1907 CGF.
Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1911 void AtomicInfo::EmitAtomicUpdate(
1912 llvm::AtomicOrdering AO,
const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
1914 if (shouldUseLibcall()) {
1915 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1917 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1921 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO,
RValue UpdateRVal,
1923 if (shouldUseLibcall()) {
1924 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1926 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1933 llvm::AtomicOrdering AO;
1935 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1937 AO = llvm::AtomicOrdering::Release;
1949 llvm::AtomicOrdering AO,
bool IsVolatile,
1957 AtomicInfo atomics(*
this, dest);
1958 LValue LVal = atomics.getAtomicLValue();
1963 atomics.emitCopyIntoMemory(rvalue);
1968 if (atomics.shouldUseLibcall()) {
1970 Address srcAddr = atomics.materializeRValue(rvalue);
1987 llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue);
1990 Address Addr = atomics.getAtomicAddress();
1991 if (llvm::Value *
Value = atomics.getScalarRValValueOrNull(rvalue))
1993 Addr = atomics.castToAtomicIntPointer(Addr);
1999 if (AO == llvm::AtomicOrdering::Acquire)
2000 AO = llvm::AtomicOrdering::Monotonic;
2001 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2002 AO = llvm::AtomicOrdering::Release;
2005 store->setAtomic(AO);
2009 store->setVolatile(
true);
2015 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2022 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure,
bool IsWeak,
2027 Expected.getAggregateAddress().getElementType() ==
2032 AtomicInfo Atomics(*
this, Obj);
2034 return Atomics.EmitAtomicCompareExchange(
Expected, Desired, Success, Failure,
2039 LValue LVal, llvm::AtomicOrdering AO,
2040 const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
bool IsVolatile) {
2041 AtomicInfo Atomics(*
this, LVal);
2042 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2046 AtomicInfo atomics(*
this, dest);
2048 switch (atomics.getEvaluationKind()) {
2064 bool Zeroed =
false;
2066 Zeroed = atomics.emitMemSetZeroIfNecessary();
2067 dest = atomics.projectValue();
2081 llvm_unreachable(
"bad evaluation kind");
Defines the clang::ASTContext interface.
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
static llvm::Value * EmitPostAtomicMinMax(CGBuilderTy &Builder, AtomicExpr::AtomicOp Op, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS)
Duplicate the atomic min/max operation in conventional IR for the builtin variants that return the ne...
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg)
Return true if.
__DEVICE__ int min(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
TypeInfoChars getTypeInfoInChars(const Type *T) const
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
Expr * getOrderFail() const
QualType getValueType() const
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
SourceLocation getBeginLoc() const LLVM_READONLY
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Address getAddress() const
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
CGFunctionInfo - Class to encapsulate the information about a function definition.
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitAtomicInit(Expr *E, LValue lvalue)
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::LLVMContext & getLLVMContext()
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
ASTContext & getContext() const
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
const TargetInfo & getTarget() const
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
llvm::Type * ConvertTypeForMem(QualType T)
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
RValue EmitAtomicExpr(AtomicExpr *E)
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
const TargetCodeGenInfo & getTargetHooks() const
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
RValue EmitLoadOfExtVectorElementLValue(LValue V)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
const LangOptions & getLangOpts() const
llvm::LLVMContext & getLLVMContext()
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
DiagnosticsEngine & getDiags() const
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
LValue - This represents an lvalue references.
llvm::Value * getVectorIdx() const
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
llvm::Value * getRawVectorPointer(CodeGenFunction &CGF) const
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
llvm::Value * getRawExtVectorPointer(CodeGenFunction &CGF) const
void setAlignment(CharUnits A)
const CGBitFieldInfo & getBitFieldInfo() const
bool isVolatileQualified() const
CharUnits getAlignment() const
static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Address getAddress() const
llvm::Constant * getExtVectorElts() const
bool isExtVectorElt() const
LValueBaseInfo getBaseInfo() const
TBAAAccessInfo getTBAAInfo() const
Address getVectorAddress() const
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Address getExtVectorAddress() const
Address getBitFieldAddress() const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
bool isVolatileQualified() const
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr, LangAS SrcAddr, LangAS DestAddr, llvm::Type *DestTy, bool IsNonNull=false) const
virtual llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts, SyncScope Scope, llvm::AtomicOrdering Ordering, llvm::LLVMContext &Ctx) const
Get the syncscope used in LLVM IR.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
PointerType - C99 6.7.5.1 - Pointer Declarators.
A (possibly-)qualified type.
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isAtomicType() const
bool isFloatingType() const
const T * getAs() const
Member-template getAs<specific type>'.
Represents a GCC generic vector type.
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Sub(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Add(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
llvm::StringRef getAsString(SyncScope S)
Structure with information about how a bitfield should be accessed.
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
unsigned Size
The total size of the bit-field, in bits.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * SizeTy
llvm::IntegerType * IntTy
int
llvm::PointerType * UnqualPtrTy