From 4bab7a2c9591e6cf51bf33dfdcea38175a978be5 Mon Sep 17 00:00:00 2001 From: "y00495115]" Date: Fri, 13 Jun 2025 17:07:35 +0800 Subject: [PATCH] [ICP]] Enable promotion of more targets --- .../llvm/Analysis/ProfileSummaryInfo.h | 3 +- llvm/include/llvm/ProfileData/ProfileCommon.h | 1 + llvm/lib/Analysis/ProfileSummaryInfo.cpp | 11 ++- .../lib/ProfileData/ProfileSummaryBuilder.cpp | 16 ++++ .../Instrumentation/IndirectCallPromotion.cpp | 4 +- .../indirect_call_promotion_enhanced.ll | 81 +++++++++++++++++++ 6 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 llvm/test/Transforms/PGOProfile/indirect_call_promotion_enhanced.ll diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h index 38eb71ba271d..4ee54f592e2c 100644 --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -46,6 +46,7 @@ private: void computeThresholds(); // Count thresholds to answer isHotCount and isColdCount queries. std::optional HotCountThreshold, ColdCountThreshold; + std::optional HotCountThresholdICP; // True if the working set size of the code is considered huge, // because the number of profile counts required to reach the hot // percentile is above a huge threshold. @@ -179,7 +180,7 @@ public: PercentileCutoff, F, BFI); } /// Returns true if count \p C is considered hot. - bool isHotCount(uint64_t C) const; + bool isHotCount(uint64_t C, bool isForICP = false) const; /// Returns true if count \p C is considered cold. bool isColdCount(uint64_t C) const; /// Returns true if count \p C is considered hot with regard to a given diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h index 4fe92cef4d72..8a0a5a82cfcd 100644 --- a/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -65,6 +65,7 @@ public: static const ProfileSummaryEntry & getEntryForPercentile(const SummaryEntryVector &DS, uint64_t Percentile); static uint64_t getHotCountThreshold(const SummaryEntryVector &DS); + static uint64_t getHotCountThresholdForICP(const SummaryEntryVector &DS); static uint64_t getColdCountThreshold(const SummaryEntryVector &DS); }; diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp index 203f1e42733f..6bf6dd540d79 100644 --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -53,6 +53,10 @@ static cl::opt PartialSampleProfileWorkingSetSizeScaleFactor( "and the factor to scale the working set size to use the same " "shared thresholds as PGO.")); +static cl::opt ProfileSummaryHotCountForICP( + "profile-summary-hot-count-for-ipc", cl::Hidden, cl::init(false), + cl::desc("A hot count is for IPC.")); + // The profile summary metadata may be attached either by the frontend or by // any backend passes (IR level instrumentation, for example). This method // checks if the Summary is null and if so checks if the summary metadata is now @@ -124,6 +128,8 @@ void ProfileSummaryInfo::computeThresholds() { DetailedSummary, ProfileSummaryCutoffHot); HotCountThreshold = ProfileSummaryBuilder::getHotCountThreshold(DetailedSummary); + HotCountThresholdICP = + ProfileSummaryBuilder::getHotCountThresholdForICP(DetailedSummary); ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold(DetailedSummary); assert(ColdCountThreshold <= HotCountThreshold && @@ -171,7 +177,10 @@ bool ProfileSummaryInfo::hasLargeWorkingSetSize() const { return HasLargeWorkingSetSize && *HasLargeWorkingSetSize; } -bool ProfileSummaryInfo::isHotCount(uint64_t C) const { +bool ProfileSummaryInfo::isHotCount(uint64_t C, bool isForICP) const { + if (ProfileSummaryHotCountForICP) { + return HotCountThresholdICP && C >= *HotCountThresholdICP; + } return HotCountThreshold && C >= *HotCountThreshold; } diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp index 8e07478fb083..29ef5d0d7300 100644 --- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -35,6 +35,12 @@ cl::opt ProfileSummaryCutoffHot( cl::desc("A count is hot if it exceeds the minimum count to" " reach this percentile of total counts.")); +cl::opt ProfileSummaryCutoffHotICP( + "profile-summary-cutoff-hot-icp", cl::Hidden, cl::init(990000), + cl::desc("A count is hot in the context of ICP, if it exceeds the minimum " + "count to" + " reach this percentile of total counts.")); + cl::opt ProfileSummaryCutoffCold( "profile-summary-cutoff-cold", cl::Hidden, cl::init(999999), cl::desc("A count is cold if it is below the minimum count" @@ -172,6 +178,16 @@ ProfileSummaryBuilder::getHotCountThreshold(const SummaryEntryVector &DS) { return HotCountThreshold; } +uint64_t ProfileSummaryBuilder::getHotCountThresholdForICP( + const SummaryEntryVector &DS) { + auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile( + DS, ProfileSummaryCutoffHotICP); + uint64_t HotCountThresholdICP = HotEntry.MinCount; + if (ProfileSummaryHotCount.getNumOccurrences() > 0) + HotCountThresholdICP = ProfileSummaryHotCount; + return HotCountThresholdICP; +} + uint64_t ProfileSummaryBuilder::getColdCountThreshold(const SummaryEntryVector &DS) { auto &ColdEntry = ProfileSummaryBuilder::getEntryForPercentile( diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 5c9799235017..146fa3e06996 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -302,8 +302,8 @@ bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) { uint64_t TotalCount; auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction( CB, NumVals, TotalCount, NumCandidates); - if (!NumCandidates || - (PSI && PSI->hasProfileSummary() && !PSI->isHotCount(TotalCount))) + if (!NumCandidates || (PSI && PSI->hasProfileSummary() && + !PSI->isHotCount(TotalCount, /*isForICP=*/true))) continue; auto PromotionCandidates = getPromotionCandidatesForCallSite( *CB, ICallProfDataRef, TotalCount, NumCandidates); diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion_enhanced.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_enhanced.ll new file mode 100644 index 000000000000..89fc708c9d6a --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_enhanced.ll @@ -0,0 +1,81 @@ +; RUN: opt < %s -passes=pgo-icall-prom -profile-summary-cutoff-hot-icp=200000 profile-summary-hot-count-for-ipc -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt < %s -passes=pgo-icall-prom -profile-summary-cutoff-hot-icp=100000 profile-summary-hot-count-for-ipc -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=FAIL-REMARK + +; PASS-REMARK: remark: :0:0: Promote indirect call to func4 with count 5 out of 14 +; PASS-REMARK: remark: :0:0: Promote indirect call to func2 with count 4 out of 9 +; PASS-REMARK: remark: :0:0: Promote indirect call to func3 with count 3 out of 5 + +; FAIL-REMARK-NOT: remark: :0:0: Promote indirect call to func4 +; FAIL-REMARK-NOT: remark: :0:0: Promote indirect call to func2 +; FAIL-REMARK-NOT: remark: :0:0: Promote indirect call to func3 + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@foo = common global ptr null, align 8 + +define i32 @func1() { +entry: + ret i32 0 +} + +define i32 @func2() { +entry: + ret i32 1 +} + +define i32 @func3() { +entry: + ret i32 2 +} + +define i32 @func4() { +entry: + ret i32 3 +} + +define i32 @bar() { +entry: + %tmp = load ptr, ptr @foo, align 8 + %call = call i32 %tmp(), !prof !34 + ret i32 %call +} + + +!llvm.module.flags = !{!0, !1, !2, !3, !4, !5} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"PIC Level", i32 2} +!2 = !{i32 7, !"PIE Level", i32 2} +!3 = !{i32 7, !"uwtable", i32 2} +!4 = !{i32 7, !"frame-pointer", i32 1} +!5 = !{i32 1, !"ProfileSummary", !6} +!6 = !{!7, !8, !9, !10, !11, !12, !13, !14, !15, !16} +!7 = !{!"ProfileFormat", !"InstrProf"} +!8 = !{!"TotalCount", i64 3} +!9 = !{!"MaxCount", i64 1} +!10 = !{!"MaxInternalCount", i64 1} +!11 = !{!"MaxFunctionCount", i64 1} +!12 = !{!"NumCounts", i64 7} +!13 = !{!"NumFunctions", i64 4} +!14 = !{!"IsPartialProfile", i64 0} +!15 = !{!"PartialProfileRatio", double 0.000000e+00} +!16 = !{!"DetailedSummary", !17} +!17 = !{!18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30, !31, !32, !33} +!18 = !{i32 10000, i64 16, i32 1} +!19 = !{i32 100000, i64 15, i32 2} +!20 = !{i32 200000, i64 14, i32 3} +!21 = !{i32 300000, i64 13, i32 4} +!22 = !{i32 400000, i64 12, i32 5} +!23 = !{i32 500000, i64 11, i32 6} +!24 = !{i32 600000, i64 10, i32 7} +!25 = !{i32 700000, i64 9, i32 8} +!26 = !{i32 800000, i64 8, i32 9} +!27 = !{i32 900000, i64 7, i32 10} +!28 = !{i32 950000, i64 6, i32 11} +!29 = !{i32 990000, i64 5, i32 12} +!30 = !{i32 999000, i64 4, i32 13} +!31 = !{i32 999900, i64 3, i32 14} +!32 = !{i32 999990, i64 2, i32 15} +!33 = !{i32 999999, i64 1, i32 16} +!34 = !{!"VP", i32 0, i64 14, i64 7651369219802541373, i64 5, i64 -4377547752858689819, i64 4, i64 -6929281286627296573, i64 3, i64 -2545542355363006406, i64 2} -- Gitee