14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
22 using namespace clang;
28 LLVMContext &Ctx = Header->getContext();
30 NewLoopProperties.push_back(
nullptr);
31 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
33 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
34 LoopID->replaceOperandWith(0, LoopID);
38 MDNode *LoopInfo::createPipeliningMetadata(
const LoopAttributes &Attrs,
40 bool &HasUserTransforms) {
41 LLVMContext &Ctx = Header->getContext();
43 std::optional<bool> Enabled;
49 if (Enabled !=
true) {
51 if (Enabled ==
false) {
52 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
53 NewLoopProperties.push_back(
54 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.pipeline.disable"),
55 ConstantAsMetadata::get(ConstantInt::get(
56 llvm::Type::getInt1Ty(Ctx), 1))}));
57 LoopProperties = NewLoopProperties;
59 return createLoopPropertiesMetadata(LoopProperties);
63 Args.push_back(
nullptr);
64 Args.append(LoopProperties.begin(), LoopProperties.end());
68 MDString::get(Ctx,
"llvm.loop.pipeline.initiationinterval"),
69 ConstantAsMetadata::get(ConstantInt::get(
71 Args.push_back(MDNode::get(Ctx, Vals));
76 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
77 LoopID->replaceOperandWith(0, LoopID);
78 HasUserTransforms =
true;
85 bool &HasUserTransforms) {
86 LLVMContext &Ctx = Header->getContext();
88 std::optional<bool> Enabled;
92 Enabled = std::nullopt;
97 if (Enabled !=
true) {
100 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
106 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
109 FollowupLoopProperties.push_back(
110 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
112 bool FollowupHasTransforms =
false;
113 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
114 FollowupHasTransforms);
117 Args.push_back(
nullptr);
118 Args.append(LoopProperties.begin(), LoopProperties.end());
122 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.count"),
123 ConstantAsMetadata::get(ConstantInt::get(
125 Args.push_back(MDNode::get(Ctx, Vals));
130 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.enable")};
131 Args.push_back(MDNode::get(Ctx, Vals));
134 if (FollowupHasTransforms)
135 Args.push_back(MDNode::get(
136 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll.followup_all"), Followup}));
138 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
139 LoopID->replaceOperandWith(0, LoopID);
140 HasUserTransforms =
true;
147 bool &HasUserTransforms) {
148 LLVMContext &Ctx = Header->getContext();
150 std::optional<bool> Enabled;
157 if (Enabled !=
true) {
159 if (Enabled ==
false) {
160 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
161 NewLoopProperties.push_back(MDNode::get(
162 Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
163 LoopProperties = NewLoopProperties;
165 return createPartialUnrollMetadata(Attrs, LoopProperties,
170 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
171 FollowupLoopProperties.push_back(
172 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
174 bool FollowupHasTransforms =
false;
175 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
176 FollowupHasTransforms);
179 Args.push_back(
nullptr);
180 Args.append(LoopProperties.begin(), LoopProperties.end());
185 MDString::get(Ctx,
"llvm.loop.unroll_and_jam.count"),
186 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
188 Args.push_back(MDNode::get(Ctx, Vals));
192 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.enable")};
193 Args.push_back(MDNode::get(Ctx, Vals));
196 if (FollowupHasTransforms)
197 Args.push_back(MDNode::get(
198 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_outer"),
201 if (UnrollAndJamInnerFollowup)
202 Args.push_back(MDNode::get(
203 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_inner"),
204 UnrollAndJamInnerFollowup}));
206 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
207 LoopID->replaceOperandWith(0, LoopID);
208 HasUserTransforms =
true;
213 LoopInfo::createLoopVectorizeMetadata(
const LoopAttributes &Attrs,
215 bool &HasUserTransforms) {
216 LLVMContext &Ctx = Header->getContext();
218 std::optional<bool> Enabled;
227 if (Enabled !=
true) {
229 if (Enabled ==
false) {
230 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
231 NewLoopProperties.push_back(
232 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
233 ConstantAsMetadata::get(ConstantInt::get(
234 llvm::Type::getInt1Ty(Ctx), 0))}));
235 LoopProperties = NewLoopProperties;
237 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
242 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
245 FollowupLoopProperties.push_back(
246 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
248 bool FollowupHasTransforms =
false;
249 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
250 FollowupHasTransforms);
253 Args.push_back(
nullptr);
254 Args.append(LoopProperties.begin(), LoopProperties.end());
258 bool IsVectorPredicateEnabled =
false;
260 IsVectorPredicateEnabled =
264 MDString::get(Ctx,
"llvm.loop.vectorize.predicate.enable"),
265 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
266 IsVectorPredicateEnabled))};
267 Args.push_back(MDNode::get(Ctx, Vals));
273 MDString::get(Ctx,
"llvm.loop.vectorize.width"),
274 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
277 Args.push_back(MDNode::get(Ctx, Vals));
283 MDString::get(Ctx,
"llvm.loop.vectorize.scalable.enable"),
284 ConstantAsMetadata::get(
285 ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
286 Args.push_back(MDNode::get(Ctx, Vals));
292 MDString::get(Ctx,
"llvm.loop.interleave.count"),
293 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
295 Args.push_back(MDNode::get(Ctx, Vals));
314 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
315 ConstantAsMetadata::get(ConstantInt::get(
316 llvm::Type::getInt1Ty(Ctx), AttrVal))}));
319 if (FollowupHasTransforms)
320 Args.push_back(MDNode::get(
322 {MDString::get(Ctx,
"llvm.loop.vectorize.followup_all"), Followup}));
324 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
325 LoopID->replaceOperandWith(0, LoopID);
326 HasUserTransforms =
true;
331 LoopInfo::createLoopDistributeMetadata(
const LoopAttributes &Attrs,
333 bool &HasUserTransforms) {
334 LLVMContext &Ctx = Header->getContext();
336 std::optional<bool> Enabled;
342 if (Enabled !=
true) {
344 if (Enabled ==
false) {
345 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
346 NewLoopProperties.push_back(
347 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
348 ConstantAsMetadata::get(ConstantInt::get(
349 llvm::Type::getInt1Ty(Ctx), 0))}));
350 LoopProperties = NewLoopProperties;
352 return createLoopVectorizeMetadata(Attrs, LoopProperties,
356 bool FollowupHasTransforms =
false;
358 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
361 Args.push_back(
nullptr);
362 Args.append(LoopProperties.begin(), LoopProperties.end());
364 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
365 ConstantAsMetadata::get(ConstantInt::get(
366 llvm::Type::getInt1Ty(Ctx),
368 Args.push_back(MDNode::get(Ctx, Vals));
370 if (FollowupHasTransforms)
371 Args.push_back(MDNode::get(
373 {MDString::get(Ctx,
"llvm.loop.distribute.followup_all"), Followup}));
375 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
376 LoopID->replaceOperandWith(0, LoopID);
377 HasUserTransforms =
true;
381 MDNode *LoopInfo::createFullUnrollMetadata(
const LoopAttributes &Attrs,
383 bool &HasUserTransforms) {
384 LLVMContext &Ctx = Header->getContext();
386 std::optional<bool> Enabled;
392 if (Enabled !=
true) {
394 if (Enabled ==
false) {
395 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
396 NewLoopProperties.push_back(
397 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
398 LoopProperties = NewLoopProperties;
400 return createLoopDistributeMetadata(Attrs, LoopProperties,
405 Args.push_back(
nullptr);
406 Args.append(LoopProperties.begin(), LoopProperties.end());
407 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.full")));
412 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
413 LoopID->replaceOperandWith(0, LoopID);
414 HasUserTransforms =
true;
421 if (StagedAttrs.GlobalSYCLIVDepInfo &&
422 (StagedAttrs.GlobalSYCLIVDepInfo->SafeLen == 0 ||
423 (
SafeLen != 0 && StagedAttrs.GlobalSYCLIVDepInfo->SafeLen >=
SafeLen)))
428 if (!StagedAttrs.GlobalSYCLIVDepInfo)
431 StagedAttrs.GlobalSYCLIVDepInfo->SafeLen =
SafeLen;
435 StagedAttrs.ArraySYCLIVDepInfo.erase(
436 llvm::remove_if(StagedAttrs.ArraySYCLIVDepInfo,
438 return !A.isSafeLenGreaterOrEqual(SafeLen);
440 StagedAttrs.ArraySYCLIVDepInfo.end());
444 auto SafeLenItr = llvm::find_if(
445 StagedAttrs.ArraySYCLIVDepInfo,
446 [
SafeLen](
const auto &Info) { return Info.SafeLen == SafeLen; });
448 llvm::find_if(StagedAttrs.ArraySYCLIVDepInfo,
449 [
Array](
const auto &Info) { return Info.hasArray(Array); });
451 if (ArrayItr != StagedAttrs.ArraySYCLIVDepInfo.end()) {
455 if (ArrayItr->isSafeLenGreaterOrEqual(
SafeLen))
460 ArrayItr->eraseArray(
Array);
464 if (SafeLenItr != StagedAttrs.ArraySYCLIVDepInfo.end()) {
465 SafeLenItr->Arrays.emplace_back(
Array, MDNode::getDistinct(Ctx, {}));
469 StagedAttrs.ArraySYCLIVDepInfo.emplace_back(
SafeLen,
Array,
470 MDNode::getDistinct(Ctx, {}));
480 MD.push_back(MDString::get(Ctx,
"llvm.loop.parallel_access_indices"));
481 std::transform(I.
Arrays.begin(), I.
Arrays.end(), std::back_inserter(MD),
482 [](
const auto &Pair) { return Pair.second; });
485 MD.push_back(ConstantAsMetadata::get(
486 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), I.
SafeLen)));
487 LoopProperties.push_back(MDNode::get(Ctx, MD));
497 Metadata *EnableMDs[] = {MDString::get(Ctx,
"llvm.loop.ivdep.enable")};
498 LoopProperties.push_back(MDNode::get(Ctx, EnableMDs));
502 Metadata *SafelenMDs[] = {MDString::get(Ctx,
"llvm.loop.ivdep.safelen"),
503 ConstantAsMetadata::get(ConstantInt::get(
504 llvm::Type::getInt32Ty(Ctx), I.
SafeLen))};
505 LoopProperties.push_back(MDNode::get(Ctx, SafelenMDs));
508 MDNode *LoopInfo::createMetadata(
511 bool &HasUserTransforms) {
516 LoopProperties.push_back(StartLoc.getAsMDNode());
520 LoopProperties.push_back(EndLoc.getAsMDNode());
523 LLVMContext &Ctx = Header->getContext();
525 LoopProperties.push_back(
526 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.mustprogress")));
529 "There must be an access group iff the loop is parallel");
531 LoopProperties.push_back(MDNode::get(
532 Ctx, {MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccGroup}));
549 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.ii.count"),
550 ConstantAsMetadata::get(ConstantInt::get(
552 LoopProperties.push_back(MDNode::get(Ctx, Vals));
558 MDString::get(Ctx,
"llvm.loop.max_concurrency.count"),
559 ConstantAsMetadata::get(ConstantInt::get(
561 LoopProperties.push_back(MDNode::get(Ctx, Vals));
565 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.coalesce.enable")};
566 LoopProperties.push_back(MDNode::get(Ctx, Vals));
571 MDString::get(Ctx,
"llvm.loop.coalesce.count"),
572 ConstantAsMetadata::get(ConstantInt::get(
574 LoopProperties.push_back(MDNode::get(Ctx, Vals));
580 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.intel.pipelining.enable"),
581 ConstantAsMetadata::get(
582 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 0))};
583 LoopProperties.push_back(MDNode::get(Ctx, Vals));
587 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.max_interleaving.count"),
588 ConstantAsMetadata::get(ConstantInt::get(
589 llvm::Type::getInt32Ty(Ctx),
591 LoopProperties.push_back(MDNode::get(Ctx, Vals));
596 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.fusion.disable")};
597 LoopProperties.push_back(MDNode::get(Ctx, Vals));
602 MDString::get(Ctx,
"llvm.loop.intel.speculated.iterations.count"),
603 ConstantAsMetadata::get(
604 ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
606 LoopProperties.push_back(MDNode::get(Ctx, Vals));
610 Metadata *Vals[] = {MDString::get(Ctx, VC.first),
611 ConstantAsMetadata::get(ConstantInt::get(
612 llvm::Type::getInt32Ty(Ctx), VC.second))};
613 LoopProperties.push_back(MDNode::get(Ctx, Vals));
618 MDString::get(Ctx,
"llvm.loop.intel.max_reinvocation_delay.count"),
619 ConstantAsMetadata::get(
620 ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
622 LoopProperties.push_back(MDNode::get(Ctx, Vals));
628 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.intel.pipelining.enable"),
629 ConstantAsMetadata::get(
630 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
631 LoopProperties.push_back(MDNode::get(Ctx, Vals));
636 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.align"),
637 ConstantAsMetadata::get(ConstantInt::get(
638 llvm::Type::getInt32Ty(Ctx), Attrs.
CodeAlign))};
639 LoopProperties.push_back(MDNode::get(Ctx, Vals));
642 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
643 AdditionalLoopProperties.end());
644 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
653 SYCLIInterval(0), SYCLLoopCoalesceEnable(
false),
654 SYCLLoopCoalesceNLevels(0), SYCLLoopPipeliningDisable(
false),
655 SYCLLoopPipeliningEnable(
false), UnrollCount(0), UnrollAndJamCount(0),
657 PipelineInitiationInterval(0), SYCLNofusionEnable(
false), CodeAlign(0),
658 MustProgress(
false) {}
692 const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc,
694 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
699 LLVMContext &Ctx = Header->getContext();
700 AccGroup = MDNode::getDistinct(Ctx, {});
726 TempLoopID = MDNode::getTemporary(Header->getContext(), std::nullopt);
737 LLVMContext &Ctx = Header->getContext();
785 if (!Parent->UnrollAndJamInnerFollowup) {
795 BeforeLoopProperties.push_back(
796 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
798 bool InnerFollowupHasTransform =
false;
799 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
800 InnerFollowupHasTransform);
801 if (InnerFollowupHasTransform)
802 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
805 CurLoopAttr = BeforeJam;
808 bool HasUserTransforms =
false;
809 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
810 TempLoopID->replaceAllUsesWith(LoopID);
814 const llvm::DebugLoc &EndLoc) {
816 new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
817 Active.empty() ?
nullptr : Active.back().get()));
825 const llvm::DebugLoc &StartLoc,
826 const llvm::DebugLoc &EndLoc,
bool MustProgress) {
828 for (
const auto *
Attr : Attrs) {
829 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(
Attr);
830 const OpenCLUnrollHintAttr *OpenCLHint =
831 dyn_cast<OpenCLUnrollHintAttr>(
Attr);
832 const LoopUnrollHintAttr *UnrollHint = dyn_cast<LoopUnrollHintAttr>(
Attr);
835 if (!LH && !OpenCLHint && !UnrollHint) {
839 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
840 LoopHintAttr::LoopHintState
State = LoopHintAttr::Disable;
841 unsigned ValueInt = 1;
848 if (OpenCLHint || UnrollHint) {
851 ValueInt = OpenCLHint->getUnrollHint();
852 else if (
Expr *E = UnrollHint->getUnrollHintExpr())
853 ValueInt = E->EvaluateKnownConstInt(Ctx).getSExtValue();
856 State = LoopHintAttr::Enable;
857 }
else if (ValueInt != 1) {
858 Option = LoopHintAttr::UnrollCount;
859 State = LoopHintAttr::Numeric;
862 auto *ValueExpr = LH->getValue();
864 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
865 ValueInt = ValueAPS.getSExtValue();
868 Option = LH->getOption();
869 State = LH->getState();
872 case LoopHintAttr::Disable:
874 case LoopHintAttr::Vectorize:
879 case LoopHintAttr::Interleave:
883 case LoopHintAttr::Unroll:
886 case LoopHintAttr::UnrollAndJam:
889 case LoopHintAttr::VectorizePredicate:
892 case LoopHintAttr::Distribute:
895 case LoopHintAttr::PipelineDisabled:
898 case LoopHintAttr::UnrollCount:
899 case LoopHintAttr::UnrollAndJamCount:
900 case LoopHintAttr::VectorizeWidth:
901 case LoopHintAttr::InterleaveCount:
902 case LoopHintAttr::PipelineInitiationInterval:
903 llvm_unreachable(
"Options cannot be disabled.");
907 case LoopHintAttr::Enable:
909 case LoopHintAttr::Vectorize:
910 case LoopHintAttr::Interleave:
913 case LoopHintAttr::Unroll:
916 case LoopHintAttr::UnrollAndJam:
919 case LoopHintAttr::VectorizePredicate:
922 case LoopHintAttr::Distribute:
925 case LoopHintAttr::UnrollCount:
926 case LoopHintAttr::UnrollAndJamCount:
927 case LoopHintAttr::VectorizeWidth:
928 case LoopHintAttr::InterleaveCount:
929 case LoopHintAttr::PipelineDisabled:
930 case LoopHintAttr::PipelineInitiationInterval:
931 llvm_unreachable(
"Options cannot enabled.");
935 case LoopHintAttr::AssumeSafety:
937 case LoopHintAttr::Vectorize:
938 case LoopHintAttr::Interleave:
943 case LoopHintAttr::Unroll:
944 case LoopHintAttr::UnrollAndJam:
945 case LoopHintAttr::VectorizePredicate:
946 case LoopHintAttr::UnrollCount:
947 case LoopHintAttr::UnrollAndJamCount:
948 case LoopHintAttr::VectorizeWidth:
949 case LoopHintAttr::InterleaveCount:
950 case LoopHintAttr::Distribute:
951 case LoopHintAttr::PipelineDisabled:
952 case LoopHintAttr::PipelineInitiationInterval:
953 llvm_unreachable(
"Options cannot be used to assume mem safety.");
957 case LoopHintAttr::Full:
959 case LoopHintAttr::Unroll:
962 case LoopHintAttr::UnrollAndJam:
965 case LoopHintAttr::Vectorize:
966 case LoopHintAttr::Interleave:
967 case LoopHintAttr::UnrollCount:
968 case LoopHintAttr::UnrollAndJamCount:
969 case LoopHintAttr::VectorizeWidth:
970 case LoopHintAttr::InterleaveCount:
971 case LoopHintAttr::Distribute:
972 case LoopHintAttr::PipelineDisabled:
973 case LoopHintAttr::PipelineInitiationInterval:
974 case LoopHintAttr::VectorizePredicate:
975 llvm_unreachable(
"Options cannot be used with 'full' hint.");
979 case LoopHintAttr::FixedWidth:
980 case LoopHintAttr::ScalableWidth:
982 case LoopHintAttr::VectorizeWidth:
990 llvm_unreachable(
"Options cannot be used with 'scalable' hint.");
994 case LoopHintAttr::Numeric:
996 case LoopHintAttr::InterleaveCount:
999 case LoopHintAttr::UnrollCount:
1002 case LoopHintAttr::UnrollAndJamCount:
1005 case LoopHintAttr::PipelineInitiationInterval:
1008 case LoopHintAttr::Unroll:
1009 case LoopHintAttr::UnrollAndJam:
1010 case LoopHintAttr::VectorizePredicate:
1011 case LoopHintAttr::Vectorize:
1012 case LoopHintAttr::VectorizeWidth:
1013 case LoopHintAttr::Interleave:
1014 case LoopHintAttr::Distribute:
1015 case LoopHintAttr::PipelineDisabled:
1016 llvm_unreachable(
"Options cannot be assigned a value.");
1052 for (
const auto *A : Attrs) {
1053 if (
const auto *SYCLIntelIVDep = dyn_cast<SYCLIntelIVDepAttr>(A))
1055 SYCLIntelIVDep->getArrayDecl());
1057 if (
const auto *SYCLIntelII =
1058 dyn_cast<SYCLIntelInitiationIntervalAttr>(A)) {
1059 const auto *CE = cast<ConstantExpr>(SYCLIntelII->getNExpr());
1064 if (
const auto *SYCLIntelMaxConcurrency =
1065 dyn_cast<SYCLIntelMaxConcurrencyAttr>(A)) {
1066 const auto *CE = cast<ConstantExpr>(SYCLIntelMaxConcurrency->getNExpr());
1071 if (
const auto *SYCLIntelLoopCountAvg =
1072 dyn_cast<SYCLIntelLoopCountAttr>(A)) {
1074 cast<ConstantExpr>(SYCLIntelLoopCountAvg->getNTripCount());
1077 SYCLIntelLoopCountAvg->isMax() ?
"llvm.loop.intel.loopcount_max"
1078 : SYCLIntelLoopCountAvg->isMin() ?
"llvm.loop.intel.loopcount_min"
1079 : SYCLIntelLoopCountAvg->isAvg() ?
"llvm.loop.intel.loopcount_avg"
1080 :
"llvm.loop.intel.loopcount";
1084 if (
const auto *SYCLIntelLoopCoalesce =
1085 dyn_cast<SYCLIntelLoopCoalesceAttr>(A)) {
1086 if (
const auto *LCE = SYCLIntelLoopCoalesce->getNExpr()) {
1087 const auto *CE = cast<ConstantExpr>(LCE);
1095 if (isa<SYCLIntelDisableLoopPipeliningAttr>(A))
1098 if (
const auto *SYCLIntelMaxInterleaving =
1099 dyn_cast<SYCLIntelMaxInterleavingAttr>(A)) {
1100 const auto *CE = cast<ConstantExpr>(SYCLIntelMaxInterleaving->getNExpr());
1105 if (
const auto *SYCLIntelSpeculatedIterations =
1106 dyn_cast<SYCLIntelSpeculatedIterationsAttr>(A)) {
1108 cast<ConstantExpr>(SYCLIntelSpeculatedIterations->getNExpr());
1113 if (isa<SYCLIntelNofusionAttr>(A))
1116 if (
const auto *SYCLIntelMaxReinvocationDelay =
1117 dyn_cast<SYCLIntelMaxReinvocationDelayAttr>(A)) {
1118 const auto *CE = cast<ConstantExpr>(
1119 SYCLIntelMaxReinvocationDelay->getNExpr());
1124 if (isa<SYCLIntelEnableLoopPipeliningAttr>(A))
1130 if (
const auto *CodeAlign = getSpecificAttr<const CodeAlignAttr>(Attrs)) {
1131 const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment());
1138 if (CGOpts.OptimizationLevel > 0)
1141 if (!CGOpts.UnrollLoops &&
1147 push(Header, StartLoc, EndLoc);
1151 assert(!Active.empty() &&
"No active loops to pop");
1152 Active.back()->finish();
1157 if (I->mayReadOrWriteMemory()) {
1159 for (
const auto &AL : Active) {
1161 if (MDNode *
Group = AL->getAccessGroup())
1162 AccessGroups.push_back(
Group);
1164 MDNode *UnionMD =
nullptr;
1165 if (AccessGroups.size() == 1)
1166 UnionMD = cast<MDNode>(AccessGroups[0]);
1167 else if (AccessGroups.size() >= 2)
1168 UnionMD = MDNode::get(I->getContext(), AccessGroups);
1169 I->setMetadata(
"llvm.access.group", UnionMD);
1179 if (I->isTerminator()) {
1180 for (BasicBlock *Succ : successors(I))
1182 I->setMetadata(llvm::LLVMContext::MD_loop, L.
getLoopID());
1189 void LoopInfo::collectIVDepMetadata(
1192 Parent->collectIVDepMetadata(Array, MD);
1195 llvm::find_if(Attrs.ArraySYCLIVDepInfo,
1196 [Array](
const auto &Info) { return Info.hasArray(Array); });
1199 if (ArrayIVDep != Attrs.ArraySYCLIVDepInfo.end()) {
1200 MD.push_back(ArrayIVDep->getArrayPairItr(Array)->second);
1204 if (!Attrs.GlobalSYCLIVDepInfo)
1207 auto GlobalArrayPairItr = Attrs.GlobalSYCLIVDepInfo->getArrayPairItr(Array);
1208 if (GlobalArrayPairItr == Attrs.GlobalSYCLIVDepInfo->Arrays.end()) {
1209 Attrs.GlobalSYCLIVDepInfo->Arrays.emplace_back(
1210 Array, MDNode::getDistinct(Header->getContext(), {}));
1211 GlobalArrayPairItr = std::prev(Attrs.GlobalSYCLIVDepInfo->Arrays.end());
1213 MD.push_back(GlobalArrayPairItr->second);
1217 llvm::Instruction *GEP)
const {
1219 collectIVDepMetadata(
Array, MD);
1222 GEP->setMetadata(
"llvm.index.group", cast<llvm::MDNode>(MD.front()));
1223 else if (!MD.empty())
1224 GEP->setMetadata(
"llvm.index.group", MDNode::get(Header->getContext(), MD));
1228 llvm::Instruction *GEP) {
1229 assert(isa<llvm::GetElementPtrInst>(GEP) &&
"Only GEP instructions can be "
1230 "annotated with IVDep attribute "
Defines the clang::ASTContext interface.
static void EmitIVDepLoopMetadata(LLVMContext &Ctx, llvm::SmallVectorImpl< llvm::Metadata * > &LoopProperties, const LoopAttributes::SYCLIVDepInfo &I)
static void EmitLegacyIVDepLoopMetadata(LLVMContext &Ctx, llvm::SmallVectorImpl< llvm::Metadata * > &LoopProperties, const LoopAttributes::SYCLIVDepInfo &I)
Setting the legacy LLVM IR representation of the ivdep attribute.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Attr - This represents one attribute.
CodeGenOptions - Track various options which control how the code is optimized and passed to the back...
void setPipelineDisabled(bool S)
Set the pipeline disabled state.
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
bool hasInfo() const
Returns true if there is LoopInfo on the stack.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setSYCLLoopPipeliningDisable()
Set flag of disable_loop_pipelining for the next loop pushed.
void setSYCLSpeculatedIterationsNIterations(unsigned C)
Set value of speculated iterations for the next loop pushed.
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
void setDistributeState(bool Enable=true)
Set the next pushed loop as a distribution candidate.
void setSYCLLoopPipeliningEnable()
Set flag of enable_loop_pipelining for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void setSYCLMaxReinvocationDelayNCycles(unsigned C)
Set value of max reinvocation delay for the next loop pushed.
void setInterleaveCount(unsigned C)
Set the interleave count for the next loop pushed.
void setSYCLMaxInterleavingNInvocations(unsigned C)
Set value of max interleaved invocations for the next loop pushed.
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
void setSYCLIntelFPGAVariantCount(const char *Var, unsigned int Count)
Set value of variant and loop count for the next loop pushed.
void setSYCLIInterval(unsigned C)
Set value of an initiation interval for the next loop pushed.
void setVectorizeScalable(const LoopAttributes::LVEnableState &State)
void setVectorizePredicateState(const LoopAttributes::LVEnableState &State)
Set the next pushed vectorize predicate state.
void pop()
End the current loop.
void setCodeAlign(unsigned C)
Set value of code align for the next loop pushed.
void addSYCLIVDepInfo(llvm::LLVMContext &Ctx, unsigned SafeLen, const ValueDecl *Array)
Add a safelen value for the next loop pushed.
void setSYCLMaxConcurrencyNThreads(unsigned C)
Set value of max_concurrency for the next loop pushed.
void setSYCLLoopCoalesceEnable()
Set flag of loop_coalesce for the next loop pushed.
void addIVDepMetadata(const ValueDecl *Array, llvm::Instruction *GEP)
void setSYCLNofusionEnable()
Set flag of nofusion for the next loop pushed.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setSYCLLoopCoalesceNLevels(unsigned C)
Set value of coalesced levels for the next loop pushed.
void setMustProgress(bool P)
Set no progress for the next loop pushed.
void setUnrollAndJamState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll_and_jam state.
void setUnrollAndJamCount(unsigned C)
Set the unroll count for the next loop pushed.
const LoopInfo & getInfo() const
Return the LoopInfo for the current loop.
void setPipelineInitiationInterval(unsigned C)
Set the pipeline initiation interval.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Information used when generating a structured loop.
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
void finish()
Create the loop's metadata.
void addIVDepMetadata(const ValueDecl *Array, llvm::Instruction *GEP) const
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
This represents one expression.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
The JSON file list parser is used to communicate input to InstallAPI.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::SmallVector< std::pair< const ValueDecl *, llvm::MDNode * >, 4 > Arrays
Attributes that may be specified on loops.
unsigned UnrollCount
llvm.unroll.
bool MustProgress
Value for whether the loop is required to make progress.
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
LoopAttributes(bool IsParallel=false)
llvm::SmallVector< SYCLIVDepInfo, 4 > ArraySYCLIVDepInfo
unsigned SYCLLoopCoalesceNLevels
Value for llvm.loop.coalesce.count metadata.
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
bool SYCLLoopCoalesceEnable
Flag for llvm.loop.coalesce metadata.
LVEnableState VectorizeScalable
bool SYCLLoopPipeliningEnable
Flag for llvm.loop.intel.pipelining.enable, i32 1 metadata.
llvm::SmallVector< std::pair< const char *, unsigned int >, 2 > SYCLIntelFPGAVariantCount
Value for count variant (min/max/avg) and count metadata.
std::optional< unsigned > SYCLMaxReinvocationDelayNCycles
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
bool SYCLLoopPipeliningDisable
Flag for llvm.loop.intel.pipelining.enable, i32 0 metadata.
std::optional< unsigned > SYCLSpeculatedIterationsNIterations
Value for llvm.loop.intel.speculated.iterations.count metadata.
unsigned UnrollAndJamCount
llvm.unroll.
std::optional< unsigned > SYCLMaxConcurrencyNThreads
Value for llvm.loop.max_concurrency.count metadata.
std::optional< unsigned > SYCLMaxInterleavingNInvocations
Value for llvm.loop.max_interleaving.count metadata.
unsigned SYCLIInterval
Value for llvm.loop.ii.count metadata.
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
unsigned CodeAlign
Value for 'llvm.loop.align' metadata.
bool SYCLNofusionEnable
Flag for llvm.loop.fusion.disable metatdata.
std::optional< SYCLIVDepInfo > GlobalSYCLIVDepInfo
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.