From 7eedab2d96c1fdd4053f016b7ecc9f01ef3212e8 Mon Sep 17 00:00:00 2001 From: Ellis Hoag Date: Fri, 4 Aug 2023 11:26:14 -0700 Subject: [PATCH 01/28] [InstrProf] Encode linkage names in IRPGO counter names Prior to this diff, names in the `__llvm_prf_names` section had the format `[:]`, e.g., `main.cpp:foo`, `bar`. `` is used to discriminate between possibly identical function names when linkage is local and `` simply comes from `F.getName()`. This has two problems: * `:` is commonly found in Objective-C functions so that names like `main.mm:-[C foo::]` and `-[C bar::]` are difficult to parse * `` might be different from the linkage name, so it cannot be used to pass a function order to the linker via `-symbol-ordering-file` or `-order_file` (see https://discourse.llvm.org/t/rfc-temporal-profiling-extension-for-irpgo/68068) Instead, this diff changes the format to `[;]`, e.g., `main.cpp;_foo`, `_bar`. The hope is that `;` won't realistically be found in either `` or ``. To prevent invalidating all prior IRPGO profiles, we also lookup the prior name format when a record is not found (see `InstrProfSymtab::create()`, `readMemprof()`, and `getInstrProfRecord()`). It seems that Swift and Clang FE-PGO rely on the original `getPGOFuncName()`, so we cannot simply replace it. Reviewed By: MaskRay Differential Revision: https://reviews.llvm.org/D156569 --- llvm/include/llvm/ProfileData/InstrProf.h | 26 ++- .../llvm/ProfileData/InstrProfReader.h | 5 +- llvm/lib/ProfileData/InstrProf.cpp | 150 +++++++++++++----- llvm/lib/ProfileData/InstrProfReader.cpp | 21 ++- .../Instrumentation/MemProfiler.cpp | 29 +++- .../Instrumentation/PGOInstrumentation.cpp | 10 +- .../Transforms/PGOProfile/comdat_internal.ll | 2 +- .../Transforms/PGOProfile/criticaledge.ll | 2 +- .../PGOProfile/statics_counter_naming.ll | 4 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 14 +- llvm/unittests/ProfileData/InstrProfTest.cpp | 132 ++++++++++++++- 11 files changed, 305 insertions(+), 90 deletions(-) diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index f64d2e6cb739..f9096b461572 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -184,6 +184,15 @@ std::string getPGOFuncName(StringRef RawFuncName, StringRef FileName, uint64_t Version = INSTR_PROF_INDEX_VERSION); +/// \return the modified name for function \c F suitable to be +/// used as the key for IRPGO profile lookup. \c InLTO indicates if this is +/// called from LTO optimization passes. +std::string getIRPGOFuncName(const Function &F, bool InLTO = false); + +/// \return the filename and the function name parsed from the output of +/// \c getIRPGOFuncName() +std::pair getParsedIRPGOFuncName(StringRef IRPGOFuncName); + /// Return the name of the global variable used to store a function /// name in PGO instrumentation. \c FuncName is the name of the function /// returned by the \c getPGOFuncName call. @@ -434,6 +443,8 @@ private: return "** External Symbol **"; } + Error addFuncWithName(Function &F, StringRef PGOFuncName); + // If the symtab is created by a series of calls to \c addFuncName, \c // finalizeSymtab needs to be called before looking up function names. // This is required because the underlying map is a vector (for space @@ -516,11 +527,6 @@ public: /// Return function from the name's md5 hash. Return nullptr if not found. inline Function *getFunction(uint64_t FuncMD5Hash); - /// Return the function's original assembly name by stripping off - /// the prefix attached (to symbols with priviate linkage). For - /// global functions, it returns the same string as getFuncName. - inline StringRef getOrigFuncName(uint64_t FuncMD5Hash); - /// Return the name section data. inline StringRef getNameData() const { return Data; } @@ -586,16 +592,6 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { return nullptr; } -// See also getPGOFuncName implementation. These two need to be -// matched. -StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { - StringRef PGOName = getFuncName(FuncMD5Hash); - size_t S = PGOName.find_first_of(':'); - if (S == StringRef::npos) - return PGOName; - return PGOName.drop_front(S + 1); -} - // To store the sums of profile count values, or the percentage of // the sums of the total count values. struct CountSumOrPercent { diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 80c5284d8a7d..74e921e10c47 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -716,9 +716,12 @@ public: /// When return a hash_mismatch error and MismatchedFuncSum is not nullptr, /// the sum of all counters in the mismatched function will be set to /// MismatchedFuncSum. If there are multiple instances of mismatched - /// functions, MismatchedFuncSum returns the maximum. + /// functions, MismatchedFuncSum returns the maximum. If \c FuncName is not + /// found, try to lookup \c DeprecatedFuncName to handle profiles built by + /// older compilers. Expected getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, + StringRef DeprecatedFuncName = "", uint64_t *MismatchedFuncSum = nullptr); /// Return the memprof record for the function identified by diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 0f9c33de3f52..835dd697bc7b 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -27,6 +27,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Mangler.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" @@ -264,6 +265,67 @@ static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { return PathNameStr.substr(LastPos); } +static StringRef getStrippedSourceFileName(const Function &F) { + StringRef FileName(F.getParent()->getSourceFileName()); + uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; + if (StripLevel < StaticFuncStripDirNamePrefix) + StripLevel = StaticFuncStripDirNamePrefix; + if (StripLevel) + FileName = stripDirPrefix(FileName, StripLevel); + return FileName; +} + +// The PGO name has the format [;] where ; is +// provided if linkage is local and is the mangled function +// name. The filepath is used to discriminate possibly identical function names. +// ; is used because it is unlikely to be found in either or +// . +// +// Older compilers used getPGOFuncName() which has the format +// [:]. is used to discriminate between +// possibly identical function names when linkage is local and +// simply comes from F.getName(). This caused trouble for Objective-C functions +// which commonly have :'s in their names. Also, since is not +// mangled, they cannot be passed to Mach-O linkers via -order_file. We still +// need to compute this name to lookup functions from profiles built by older +// compilers. +static std::string getIRPGOFuncName(const Function &F, + GlobalValue::LinkageTypes Linkage, + StringRef FileName) { + SmallString<64> Name; + if (llvm::GlobalValue::isLocalLinkage(Linkage)) { + Name.append(FileName.empty() ? "" : FileName); + Name.append(";"); + } + Mangler().getNameWithPrefix(Name, &F, /*CannotUsePrivateLabel=*/true); + return Name.str().str(); +} + +static std::optional lookupPGOFuncName(const Function &F) { + if (MDNode *MD = getPGOFuncNameMetadata(F)) { + StringRef S = cast(MD->getOperand(0))->getString(); + return S.str(); + } + return {}; +} + +// See getPGOFuncName() +std::string getIRPGOFuncName(const Function &F, bool InLTO) { + if (!InLTO) { + auto FileName = getStrippedSourceFileName(F); + return getIRPGOFuncName(F, F.getLinkage(), FileName); + } + + // In LTO mode (when InLTO is true), first check if there is a meta data. + if (auto IRPGOFuncName = lookupPGOFuncName(F)) + return *IRPGOFuncName; + + // If there is no meta data, the function must be a global before the value + // profile annotation pass. Its current linkage may be internal if it is + // internalized in LTO mode. + return getIRPGOFuncName(F, GlobalValue::ExternalLinkage, ""); +} + // Return the PGOFuncName. This function has some special handling when called // in LTO optimization. The following only applies when calling in LTO passes // (when \c InLTO is true): LTO's internalization privatizes many global linkage @@ -279,20 +341,13 @@ static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { // data, its original linkage must be non-internal. std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { if (!InLTO) { - StringRef FileName(F.getParent()->getSourceFileName()); - uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; - if (StripLevel < StaticFuncStripDirNamePrefix) - StripLevel = StaticFuncStripDirNamePrefix; - if (StripLevel) - FileName = stripDirPrefix(FileName, StripLevel); + auto FileName = getStrippedSourceFileName(F); return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); } // In LTO mode (when InLTO is true), first check if there is a meta data. - if (MDNode *MD = getPGOFuncNameMetadata(F)) { - StringRef S = cast(MD->getOperand(0))->getString(); - return S.str(); - } + if (auto PGOFuncName = lookupPGOFuncName(F)) + return *PGOFuncName; // If there is no meta data, the function must be a global before the value // profile annotation pass. Its current linkage may be internal if it is @@ -300,6 +355,15 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } +// See getIRPGOFuncName() for a discription of the format. +std::pair +getParsedIRPGOFuncName(StringRef IRPGOFuncName) { + auto [FileName, FuncName] = IRPGOFuncName.split(';'); + if (FuncName.empty()) + return std::make_pair(StringRef(), IRPGOFuncName); + return std::make_pair(FileName, FuncName); +} + StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { if (FileName.empty()) return PGOFuncName; @@ -320,7 +384,7 @@ std::string getPGOFuncNameVarName(StringRef FuncName, return VarName; // Now fix up illegal chars in local VarName that may upset the assembler. - const char *InvalidChars = "-:<>/\"'"; + const char InvalidChars[] = "-:;<>/\"'"; size_t found = VarName.find_first_of(InvalidChars); while (found != std::string::npos) { VarName[found] = '_'; @@ -366,41 +430,49 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) { // Ignore in this case. if (!F.hasName()) continue; - const std::string &PGOFuncName = getPGOFuncName(F, InLTO); - if (Error E = addFuncName(PGOFuncName)) + if (Error E = addFuncWithName(F, getIRPGOFuncName(F, InLTO))) + return E; + // Also use getPGOFuncName() so that we can find records from older profiles + if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO))) return E; - MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); - // In ThinLTO, local function may have been promoted to global and have - // suffix ".llvm." added to the function name. We need to add the - // stripped function name to the symbol table so that we can find a match - // from profile. - // - // We may have other suffixes similar as ".llvm." which are needed to - // be stripped before the matching, but ".__uniq." suffix which is used - // to differentiate internal linkage functions in different modules - // should be kept. Now this is the only suffix with the pattern ".xxx" - // which is kept before matching. - const std::string UniqSuffix = ".__uniq."; - auto pos = PGOFuncName.find(UniqSuffix); - // Search '.' after ".__uniq." if ".__uniq." exists, otherwise - // search '.' from the beginning. - if (pos != std::string::npos) - pos += UniqSuffix.length(); - else - pos = 0; - pos = PGOFuncName.find('.', pos); - if (pos != std::string::npos && pos != 0) { - const std::string &OtherFuncName = PGOFuncName.substr(0, pos); - if (Error E = addFuncName(OtherFuncName)) - return E; - MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); - } } Sorted = false; finalizeSymtab(); return Error::success(); } +Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { + if (Error E = addFuncName(PGOFuncName)) + return E; + MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); + // In ThinLTO, local function may have been promoted to global and have + // suffix ".llvm." added to the function name. We need to add the + // stripped function name to the symbol table so that we can find a match + // from profile. + // + // We may have other suffixes similar as ".llvm." which are needed to + // be stripped before the matching, but ".__uniq." suffix which is used + // to differentiate internal linkage functions in different modules + // should be kept. Now this is the only suffix with the pattern ".xxx" + // which is kept before matching. + const std::string UniqSuffix = ".__uniq."; + auto pos = PGOFuncName.find(UniqSuffix); + // Search '.' after ".__uniq." if ".__uniq." exists, otherwise + // search '.' from the beginning. + if (pos != std::string::npos) + pos += UniqSuffix.length(); + else + pos = 0; + pos = PGOFuncName.find('.', pos); + if (pos != std::string::npos && pos != 0) { + StringRef OtherFuncName = PGOFuncName.substr(0, pos); + if (Error E = addFuncName(OtherFuncName)) + return E; + MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); + } + return Error::success(); +} + uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { finalizeSymtab(); auto It = partition_point(AddrToMD5Map, [=](std::pair A) { diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 4160f7e6dfd5..7f856c4802da 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1214,12 +1214,25 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() { } Expected IndexedInstrProfReader::getInstrProfRecord( - StringRef FuncName, uint64_t FuncHash, uint64_t *MismatchedFuncSum) { + StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName, + uint64_t *MismatchedFuncSum) { ArrayRef Data; uint64_t FuncSum = 0; - Error Err = Remapper->getRecords(FuncName, Data); - if (Err) - return std::move(Err); + auto Err = Remapper->getRecords(FuncName, Data); + if (Err) { + // If we don't find FuncName, try DeprecatedFuncName to handle profiles + // built by older compilers. + auto Err2 = + handleErrors(std::move(Err), [&](const InstrProfError &IE) -> Error { + if (IE.get() != instrprof_error::unknown_function) + return make_error(IE); + if (auto Err = Remapper->getRecords(DeprecatedFuncName, Data)) + return Err; + return Error::success(); + }); + if (Err2) + return std::move(Err2); + } // Found it. Look for counters with the right hash. // A flag to indicate if the records are from the same type diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp index 789ed005d03d..453dd00488b8 100644 --- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp @@ -679,12 +679,26 @@ static void readMemprof(Module &M, Function &F, const TargetLibraryInfo &TLI) { auto &Ctx = M.getContext(); - auto FuncName = getPGOFuncName(F); + auto FuncName = getIRPGOFuncName(F); auto FuncGUID = Function::getGUID(FuncName); - Expected MemProfResult = - MemProfReader->getMemProfRecord(FuncGUID); - if (Error E = MemProfResult.takeError()) { - handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { + std::optional MemProfRec; + auto Err = MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec); + if (Err) { + // If we don't find getIRPGOFuncName(), try getPGOFuncName() to handle + // profiles built by older compilers + Err = handleErrors(std::move(Err), [&](const InstrProfError &IE) -> Error { + if (IE.get() != instrprof_error::unknown_function) + return make_error(IE); + auto FuncName = getPGOFuncName(F); + auto FuncGUID = Function::getGUID(FuncName); + if (auto Err = + MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec)) + return Err; + return Error::success(); + }); + } + if (Err) { + handleAllErrors(std::move(Err), [&](const InstrProfError &IPE) { auto Err = IPE.get(); bool SkipWarning = false; LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncName @@ -722,15 +736,14 @@ static void readMemprof(Module &M, Function &F, // the frame array (see comments below where the map entries are added). std::map *, unsigned>>> LocHashToCallSites; - const auto MemProfRec = std::move(MemProfResult.get()); - for (auto &AI : MemProfRec.AllocSites) { + for (auto &AI : MemProfRec->AllocSites) { // Associate the allocation info with the leaf frame. The later matching // code will match any inlined call sequences in the IR with a longer prefix // of call stack frames. uint64_t StackId = computeStackId(AI.CallStack[0]); LocHashToAllocInfo[StackId].insert(&AI); } - for (auto &CS : MemProfRec.CallSites) { + for (auto &CS : MemProfRec->CallSites) { // Need to record all frames from leaf up to and including this function, // as any of these may or may not have been inlined at this point. unsigned Idx = 0; diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index b9459b59e704..43969427175c 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -525,6 +525,7 @@ public: std::vector> ValueSites; SelectInstVisitor SIVisitor; std::string FuncName; + std::string DeprecatedFuncName; GlobalVariable *FuncNameVar; // CFG hash value for this function. @@ -590,7 +591,8 @@ public: NumOfCSPGOBB += MST.BBInfos.size(); } - FuncName = getPGOFuncName(F); + FuncName = getIRPGOFuncName(F); + DeprecatedFuncName = getPGOFuncName(F); computeCFGHash(); if (!ComdatMembers.empty()) renameComdatFunction(); @@ -1336,7 +1338,8 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros, auto &Ctx = M->getContext(); uint64_t MismatchedFuncSum = 0; Expected Result = PGOReader->getInstrProfRecord( - FuncInfo.FuncName, FuncInfo.FunctionHash, &MismatchedFuncSum); + FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName, + &MismatchedFuncSum); if (Error E = Result.takeError()) { handleInstrProfError(std::move(E), MismatchedFuncSum); return false; @@ -1381,7 +1384,8 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros, void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) { uint64_t MismatchedFuncSum = 0; Expected Result = PGOReader->getInstrProfRecord( - FuncInfo.FuncName, FuncInfo.FunctionHash, &MismatchedFuncSum); + FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName, + &MismatchedFuncSum); if (auto Err = Result.takeError()) { handleInstrProfError(std::move(Err), MismatchedFuncSum); return; diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll index a1cf115f910a..1c44a274f3c0 100644 --- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll +++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll @@ -13,7 +13,7 @@ $foo = comdat any ; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; CHECK-NOT: __profn__stdin__foo ; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 -; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null +; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null ; CHECK-NOT: @foo ; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm diff --git a/llvm/test/Transforms/PGOProfile/criticaledge.ll b/llvm/test/Transforms/PGOProfile/criticaledge.ll index 616d0fa3a4cf..c24925c68fa3 100644 --- a/llvm/test/Transforms/PGOProfile/criticaledge.ll +++ b/llvm/test/Transforms/PGOProfile/criticaledge.ll @@ -11,7 +11,7 @@ target triple = "x86_64-unknown-linux-gnu" ; GEN: $__llvm_profile_raw_version = comdat any ; GEN: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge" -; GEN: @__profn__stdin__bar = private constant [11 x i8] c":bar" +; GEN: @__profn__stdin__bar = private constant [11 x i8] c";bar" define i32 @test_criticalEdge(i32 %i, i32 %j) { entry: diff --git a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll index fe8df33324ab..4bffc87a9390 100644 --- a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll +++ b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll @@ -4,8 +4,8 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -; NOPATH: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func" -; HASPATH-NOT: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func" +; NOPATH: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll;func" +; HASPATH-NOT: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll;func" define internal i32 @func() { entry: diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index da10ddcc58c6..9da5368f5a42 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -3069,17 +3069,11 @@ static int order_main(int argc, const char *argv[]) { WithColor::note() << "# Ordered " << Nodes.size() << " functions\n"; for (auto &N : Nodes) { - auto FuncName = Reader->getSymtab().getFuncName(N.Id); - if (FuncName.contains(':')) { - // GlobalValue::getGlobalIdentifier() prefixes the filename if the symbol - // is local. This logic will break if there is a colon in the filename, - // but we cannot use rsplit() because ObjC symbols can have colons. - auto [Filename, ParsedFuncName] = FuncName.split(':'); - // Emit a comment describing where this symbol came from + auto [Filename, ParsedFuncName] = + getParsedIRPGOFuncName(Reader->getSymtab().getFuncName(N.Id)); + if (!Filename.empty()) OS << "# " << Filename << "\n"; - FuncName = ParsedFuncName; - } - OS << FuncName << "\n"; + OS << ParsedFuncName << "\n"; } return 0; } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 30c4dac0bd2f..76f88c058424 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -17,11 +17,11 @@ #include "llvm/Support/Compression.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Testing/Support/Error.h" -#include "llvm/Testing/Support/SupportHelpers.h" #include "gtest/gtest.h" #include using namespace llvm; +using ::testing::EndsWith; using ::testing::IsSubsetOf; using ::testing::SizeIs; using ::testing::UnorderedElementsAre; @@ -520,6 +520,119 @@ TEST_F(InstrProfTest, test_memprof_merge) { EXPECT_THAT(WantRecord, EqualsRecord(Record)); } +TEST_F(InstrProfTest, test_irpgo_function_name) { + LLVMContext Ctx; + auto M = std::make_unique("MyModule.cpp", Ctx); + // Use Mach-O mangling so that non-private symbols get a `_` prefix. + M->setDataLayout(DataLayout("m:o")); + auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); + + std::vector> Data; + Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "_ExternalFoo"); + Data.emplace_back("InternalFoo", Function::InternalLinkage, + "MyModule.cpp;_InternalFoo"); + Data.emplace_back("PrivateFoo", Function::PrivateLinkage, + "MyModule.cpp;l_PrivateFoo"); + Data.emplace_back("WeakODRFoo", Function::WeakODRLinkage, "_WeakODRFoo"); + // Test Objective-C symbols + Data.emplace_back("\01-[C dynamicFoo:]", Function::ExternalLinkage, + "-[C dynamicFoo:]"); + Data.emplace_back("-", Function::ExternalLinkage, + "_-"); + Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage, + "MyModule.cpp;-[C internalFoo:]"); + + for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) + Function::Create(FTy, Linkage, Name, M.get()); + + for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) { + auto *F = M->getFunction(Name); + auto IRPGOFuncName = getIRPGOFuncName(*F); + EXPECT_EQ(IRPGOFuncName, ExpectedIRPGOFuncName); + + auto [Filename, ParsedIRPGOFuncName] = + getParsedIRPGOFuncName(IRPGOFuncName); + StringRef ExpectedParsedIRPGOFuncName = IRPGOFuncName; + if (ExpectedParsedIRPGOFuncName.consume_front("MyModule.cpp;")) { + EXPECT_EQ(Filename, "MyModule.cpp"); + } else { + EXPECT_EQ(Filename, ""); + } + EXPECT_EQ(ParsedIRPGOFuncName, ExpectedParsedIRPGOFuncName); + } +} + +TEST_F(InstrProfTest, test_pgo_function_name) { + LLVMContext Ctx; + auto M = std::make_unique("MyModule.cpp", Ctx); + auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); + + std::vector> Data; + Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo"); + Data.emplace_back("InternalFoo", Function::InternalLinkage, + "MyModule.cpp:InternalFoo"); + Data.emplace_back("PrivateFoo", Function::PrivateLinkage, + "MyModule.cpp:PrivateFoo"); + Data.emplace_back("WeakODRFoo", Function::WeakODRLinkage, "WeakODRFoo"); + // Test Objective-C symbols + Data.emplace_back("\01-[C externalFoo:]", Function::ExternalLinkage, + "-[C externalFoo:]"); + Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage, + "MyModule.cpp:-[C internalFoo:]"); + + for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) + Function::Create(FTy, Linkage, Name, M.get()); + + for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) { + auto *F = M->getFunction(Name); + EXPECT_EQ(getPGOFuncName(*F), ExpectedPGOFuncName); + } +} + +TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) { + LLVMContext Ctx; + auto M = std::make_unique("MyModule.cpp", Ctx); + // Use Mach-O mangling so that non-private symbols get a `_` prefix. + M->setDataLayout(DataLayout("m:o")); + auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); + auto *InternalFooF = + Function::Create(FTy, Function::InternalLinkage, "InternalFoo", M.get()); + auto *ExternalFooF = + Function::Create(FTy, Function::ExternalLinkage, "ExternalFoo", M.get()); + + auto *InternalBarF = + Function::Create(FTy, Function::InternalLinkage, "InternalBar", M.get()); + auto *ExternalBarF = + Function::Create(FTy, Function::ExternalLinkage, "ExternalBar", M.get()); + + Writer.addRecord({getIRPGOFuncName(*InternalFooF), 0x1234, {1}}, Err); + Writer.addRecord({getIRPGOFuncName(*ExternalFooF), 0x5678, {1}}, Err); + // Write a record with a deprecated name + Writer.addRecord({getPGOFuncName(*InternalBarF), 0x1111, {2}}, Err); + Writer.addRecord({getPGOFuncName(*ExternalBarF), 0x2222, {2}}, Err); + + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + EXPECT_THAT_EXPECTED( + Reader->getInstrProfRecord(getIRPGOFuncName(*InternalFooF), 0x1234, + getPGOFuncName(*InternalFooF)), + Succeeded()); + EXPECT_THAT_EXPECTED( + Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalFooF), 0x5678, + getPGOFuncName(*ExternalFooF)), + Succeeded()); + // Ensure we can still read this old record name + EXPECT_THAT_EXPECTED( + Reader->getInstrProfRecord(getIRPGOFuncName(*InternalBarF), 0x1111, + getPGOFuncName(*InternalBarF)), + Succeeded()); + EXPECT_THAT_EXPECTED( + Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalBarF), 0x2222, + getPGOFuncName(*ExternalBarF)), + Succeeded()); +} + static const char callee1[] = "callee1"; static const char callee2[] = "callee2"; static const char callee3[] = "callee3"; @@ -1215,12 +1328,19 @@ TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) { for (unsigned I = 0; I < std::size(Funcs); I++) { Function *F = M->getFunction(Funcs[I]); - ASSERT_TRUE(F != nullptr); + + std::string IRPGOName = getIRPGOFuncName(*F); + auto IRPGOFuncName = + ProfSymtab.getFuncName(IndexedInstrProf::ComputeHash(IRPGOName)); + EXPECT_EQ(StringRef(IRPGOName), IRPGOFuncName); + EXPECT_EQ(StringRef(Funcs[I]), + getParsedIRPGOFuncName(IRPGOFuncName).second); + // Ensure we can still read this old record name. std::string PGOName = getPGOFuncName(*F); - uint64_t Key = IndexedInstrProf::ComputeHash(PGOName); - ASSERT_EQ(StringRef(PGOName), - ProfSymtab.getFuncName(Key)); - ASSERT_EQ(StringRef(Funcs[I]), ProfSymtab.getOrigFuncName(Key)); + auto PGOFuncName = + ProfSymtab.getFuncName(IndexedInstrProf::ComputeHash(PGOName)); + EXPECT_EQ(StringRef(PGOName), PGOFuncName); + EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str())); } } -- Gitee From 65f2cd1a98ce709209efed48b89ce01d9027ffc4 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Mon, 21 Aug 2023 16:10:20 -0700 Subject: [PATCH 02/28] [NFC][Profile] Rename Counters/DataSize to NumCounters/Data Fixes some FIXMEs. Reviewed By: MaskRay Differential Revision: https://reviews.llvm.org/D158466 --- compiler-rt/include/profile/InstrProfData.inc | 6 +-- compiler-rt/lib/profile/InstrProfilingMerge.c | 38 +++++++++++++++---- .../lib/profile/InstrProfilingWriter.c | 5 --- .../llvm/ProfileData/InstrProfData.inc | 6 +-- llvm/lib/ProfileData/InstrProfReader.cpp | 4 +- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 94261f4705b9..4456bf1ab176 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -128,11 +128,9 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) -/* FIXME: A more accurate name is NumData */ -INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) -/* FIXME: A more accurate name is NumCounters */ -INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 432e824955f8..0064fb4ab65d 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -53,7 +53,7 @@ int __llvm_profile_check_compatibility(const char *ProfileData, SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + Header->BinaryIdsSize); - SrcDataEnd = SrcDataStart + Header->DataSize; + SrcDataEnd = SrcDataStart + Header->NumData; if (ProfileSize < sizeof(__llvm_profile_header)) return 1; @@ -61,10 +61,10 @@ int __llvm_profile_check_compatibility(const char *ProfileData, /* Check the header first. */ if (Header->Magic != __llvm_profile_get_magic() || Header->Version != __llvm_profile_get_version() || - Header->DataSize != + Header->NumData != __llvm_profile_get_num_data(__llvm_profile_begin_data(), __llvm_profile_end_data()) || - Header->CountersSize != + Header->NumCounters != __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), __llvm_profile_end_counters()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - @@ -74,8 +74,8 @@ int __llvm_profile_check_compatibility(const char *ProfileData, if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize + - Header->DataSize * sizeof(__llvm_profile_data) + Header->NamesSize + - Header->CountersSize * __llvm_profile_counter_entry_size()) + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + + Header->NumCounters * __llvm_profile_counter_entry_size()) return 1; for (SrcData = SrcDataStart, @@ -120,16 +120,38 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + Header->BinaryIdsSize); - SrcDataEnd = SrcDataStart + Header->DataSize; + SrcDataEnd = SrcDataStart + Header->NumData; SrcCountersStart = (char *)SrcDataEnd; - SrcNameStart = SrcCountersStart + - Header->CountersSize * __llvm_profile_counter_entry_size(); + SrcCountersEnd = SrcCountersStart + + Header->NumCounters * __llvm_profile_counter_entry_size(); + SrcNameStart = SrcCountersEnd; SrcValueProfDataStart = SrcNameStart + Header->NamesSize + __llvm_profile_get_num_padding_bytes(Header->NamesSize); if (SrcNameStart < SrcCountersStart) return 1; + // Merge counters when there is no data section and debug info correlation is + // enabled. + if (Header->NumData == 0) { + if (!(__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE)) { + PROF_ERR("%s\n", "Missing profile data section."); + return 1; + } + for (SrcCounter = SrcCountersStart, + DstCounter = __llvm_profile_begin_counters(); + SrcCounter < SrcCountersEnd;) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { + *DstCounter &= *SrcCounter; + } else { + *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter; + } + SrcCounter += __llvm_profile_counter_entry_size(); + DstCounter += __llvm_profile_counter_entry_size(); + } + return 0; + } + for (SrcData = SrcDataStart, DstData = (__llvm_profile_data *)__llvm_profile_begin_data(), SrcValueProfData = SrcValueProfDataStart; diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 4a392984fe6b..1e22398a4c0f 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -286,11 +286,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, &PaddingBytesAfterNames); { - // TODO: Unfortunately the header's fields are named DataSize and - // CountersSize when they should be named NumData and NumCounters, - // respectively. - const uint64_t CountersSize = NumCounters; - const uint64_t DataSize = NumData; /* Initialize header structure. */ #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; #include "profile/InstrProfData.inc" diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 94261f4705b9..4456bf1ab176 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -128,11 +128,9 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) -/* FIXME: A more accurate name is NumData */ -INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) -/* FIXME: A more accurate name is NumCounters */ -INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 7f856c4802da..db20441b712c 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -550,9 +550,9 @@ Error RawInstrProfReader::readHeader( CountersDelta = swap(Header.CountersDelta); NamesDelta = swap(Header.NamesDelta); - auto NumData = swap(Header.DataSize); + auto NumData = swap(Header.NumData); auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); - auto CountersSize = swap(Header.CountersSize) * getCounterTypeSize(); + auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize(); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); -- Gitee From 29cc4e44b140df626443f01ee4f6041fedf5200d Mon Sep 17 00:00:00 2001 From: Alan Phipps Date: Mon, 18 Sep 2023 15:18:40 -0500 Subject: [PATCH 03/28] [InstrProf][compiler-rt] Enable MC/DC Support in LLVM Source-based Code Coverage (1/3) Part 1 of 3. This includes the LLVM back-end processing and profile reading/writing components. compiler-rt changes are included. Differential Revision: https://reviews.llvm.org/D138846 --- .../CodeGen/coverage-profile-raw-version.c | 9 + compiler-rt/include/profile/InstrProfData.inc | 22 +- compiler-rt/lib/profile/InstrProfiling.c | 4 + compiler-rt/lib/profile/InstrProfiling.h | 25 +- .../lib/profile/InstrProfilingBuffer.c | 42 +- compiler-rt/lib/profile/InstrProfilingFile.c | 50 ++- .../lib/profile/InstrProfilingInternal.h | 8 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 37 +- .../profile/InstrProfilingPlatformDarwin.c | 9 + .../lib/profile/InstrProfilingPlatformLinux.c | 10 + .../lib/profile/InstrProfilingPlatformOther.c | 4 + .../profile/InstrProfilingPlatformWindows.c | 7 + .../lib/profile/InstrProfilingWriter.c | 18 +- .../profile/instrprof-write-buffer-internal.c | 21 +- llvm/docs/LangRef.rst | 138 +++++++ llvm/include/llvm/IR/IntrinsicInst.h | 94 ++++- llvm/include/llvm/IR/Intrinsics.td | 15 + .../ProfileData/Coverage/CoverageMapping.h | 4 +- llvm/include/llvm/ProfileData/InstrProf.h | 21 +- .../llvm/ProfileData/InstrProfData.inc | 22 +- .../llvm/ProfileData/InstrProfReader.h | 9 + .../Instrumentation/InstrProfiling.h | 46 ++- .../SelectionDAG/SelectionDAGBuilder.cpp | 6 + llvm/lib/IR/IntrinsicInst.cpp | 4 +- .../Coverage/CoverageMappingReader.cpp | 4 + llvm/lib/ProfileData/InstrProf.cpp | 23 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 4 + llvm/lib/ProfileData/InstrProfReader.cpp | 109 ++++- llvm/lib/ProfileData/InstrProfWriter.cpp | 19 + .../Instrumentation/InstrProfiling.cpp | 379 ++++++++++++++---- .../Instrumentation/InstrProfiling/mcdc.ll | 53 +++ .../Transforms/PGOProfile/comdat_internal.ll | 4 +- .../tools/llvm-profdata/Inputs/basic.profraw | Bin 152 -> 192 bytes .../llvm-profdata/Inputs/c-general.profraw | Bin 1800 -> 2016 bytes .../llvm-profdata/Inputs/compressed.profraw | Bin 1768 -> 1968 bytes .../llvm-profdata/binary-ids-padding.test | 13 +- .../llvm-profdata/large-binary-id-size.test | 5 +- ...alformed-not-space-for-another-header.test | 9 +- .../malformed-num-counters-zero.test | 10 +- .../malformed-ptr-to-counter-array.test | 9 +- .../test/tools/llvm-profdata/mcdc-bitmap.test | 201 ++++++++++ .../misaligned-binary-ids-size.test | 2 +- .../mismatched-raw-profile-header.test | 3 + .../tools/llvm-profdata/raw-32-bits-be.test | 28 +- .../tools/llvm-profdata/raw-32-bits-le.test | 28 +- .../tools/llvm-profdata/raw-64-bits-be.test | 24 +- .../tools/llvm-profdata/raw-64-bits-le.test | 24 +- .../tools/llvm-profdata/raw-two-profiles.test | 14 +- 48 files changed, 1420 insertions(+), 170 deletions(-) create mode 100644 clang/test/CodeGen/coverage-profile-raw-version.c create mode 100644 llvm/test/Instrumentation/InstrProfiling/mcdc.ll create mode 100644 llvm/test/tools/llvm-profdata/mcdc-bitmap.test diff --git a/clang/test/CodeGen/coverage-profile-raw-version.c b/clang/test/CodeGen/coverage-profile-raw-version.c new file mode 100644 index 000000000000..bb30fd8c1c70 --- /dev/null +++ b/clang/test/CodeGen/coverage-profile-raw-version.c @@ -0,0 +1,9 @@ +// RUN: %clang_cc1 -debug-info-kind=standalone -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -debug-info-kind=standalone -mllvm -debug-info-correlate -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s --check-prefix=DEBUG_INFO + +// CHECK: @__llvm_profile_raw_version = {{.*}}constant i64 9 +// DEBUG_INFO: @__llvm_profile_raw_version = {{.*}}constant i64 576460752303423497 + +int main() { + return 0; +} diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 4456bf1ab176..fad14576c442 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -76,6 +76,7 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -87,7 +88,9 @@ INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -132,9 +135,13 @@ INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -267,6 +274,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -646,11 +656,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 8 +#define INSTR_PROF_RAW_VERSION 9 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 10 +#define INSTR_PROF_INDEX_VERSION 11 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 5 +#define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -687,6 +697,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -698,6 +709,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -709,6 +721,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -723,6 +736,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c index 0dd5ff5ae633..da04d8ebdec9 100644 --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -60,6 +60,10 @@ COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0; memset(I, ResetValue, E - I); + I = __llvm_profile_begin_bitmap(); + E = __llvm_profile_end_bitmap(); + memset(I, 0x0, E - I); + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const __llvm_profile_data *DI; diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 4433d7bd4887..e143149fca82 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -88,6 +88,8 @@ const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); uint32_t *__llvm_profile_begin_orderfile(); @@ -101,11 +103,11 @@ void __llvm_profile_reset_counters(void); /*! * \brief Merge profile data from buffer. * - * Read profile data form buffer \p Profile and merge with in-process profile - * counters. The client is expected to have checked or already knows the profile - * data in the buffer matches the in-process counter structure before calling - * it. Returns 0 (success) if the profile data is valid. Upon reading - * invalid/corrupted profile data, returns 1 (failure). + * Read profile data from buffer \p Profile and merge with in-process profile + * counters and bitmaps. The client is expected to have checked or already + * know the profile data in the buffer matches the in-process counter + * structure before calling it. Returns 0 (success) if the profile data is + * valid. Upon reading invalid/corrupted profile data, returns 1 (failure). */ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); @@ -113,8 +115,8 @@ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); * * Returns 0 (success) if the profile data in buffer \p Profile with size * \p Size was generated by the same binary and therefore matches - * structurally the in-process counters. If the profile data in buffer is - * not compatible, the interface returns 1 (failure). + * structurally the in-process counters and bitmaps. If the profile data in + * buffer is not compatible, the interface returns 1 (failure). */ int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size); @@ -276,6 +278,10 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End); /*! \brief Get the size of the profile counters section in bytes. */ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); +/*! \brief Get the number of bytes in the profile bitmap section. */ +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End); + /* ! \brief Given the sizes of the data and counter information, return the * number of padding bytes before and after the counters, and after the names, * in the raw profile. @@ -286,8 +292,9 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); * needed to achieve that. */ void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, - uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, + uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames); /*! diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 61ac5d9c0285..c7217b2dfef8 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -43,11 +43,14 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return __llvm_profile_get_size_for_buffer_internal( - DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd); + DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, + NamesBegin, NamesEnd); } COMPILER_RT_VISIBILITY @@ -83,6 +86,12 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { __llvm_profile_counter_entry_size(); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End) { + return (End - Begin); +} + /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { @@ -102,13 +111,16 @@ static int needsCounterPadding(void) { COMPILER_RT_VISIBILITY void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, - uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, + uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames) { if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = __llvm_profile_get_num_padding_bytes(CountersSize); + *PaddingBytesAfterBitmapBytes = + __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); return; } @@ -118,31 +130,37 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); + *PaddingBytesAfterBitmapBytes = + calculateBytesNeededToPageAlign(NumBitmapBytes); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, - const char *NamesEnd) { + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, + &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + - PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames; + PaddingBytesAfterCounters + NumBitmapBytes + + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames; } COMPILER_RT_VISIBILITY @@ -160,9 +178,11 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) { + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, 0, NamesBegin, NamesEnd, 0); + CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin, + NamesEnd, 0); } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 2bd6a49ce065..49fef96578c8 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -108,14 +108,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); - /* Check that the counter and data sections in this image are + /* Check that the counter, bitmap, and data sections in this image are * page-aligned. */ unsigned PageSize = getpagesize(); if ((intptr_t)CountersBegin % PageSize != 0) { @@ -123,6 +127,11 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { CountersBegin, PageSize); return 1; } + if ((intptr_t)BitmapBegin % PageSize != 0) { + PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n", + BitmapBegin, PageSize); + return 1; + } if ((intptr_t)DataBegin % PageSize != 0) { PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", DataBegin, PageSize); @@ -132,10 +141,11 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Determine how much padding is needed before/after the counters and * after the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, + &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; uint64_t FileOffsetToCounters = CurrentFileOffset + @@ -155,6 +165,31 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { FileOffsetToCounters); return 1; } + + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + uint64_t PageAlignedBitmapLength = + NumBitmapBytes + PaddingBytesAfterBitmapBytes; + uint64_t FileOffsetToBitmap = + CurrentFileOffset + sizeof(__llvm_profile_header) + DataSize + + PaddingBytesBeforeCounters + CountersSize + PaddingBytesAfterCounters; + void *BitmapMmap = + mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap); + if (BitmapMmap != BitmapBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - BitmapBegin: %p\n" + " - PageAlignedBitmapLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToBitmap: %" PRIu64 "\n", + strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno, + FileOffsetToBitmap); + return 1; + } return 0; } #elif defined(__ELF__) || defined(_WIN32) @@ -197,6 +232,8 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); /* Get the file size. */ uint64_t FileSize = 0; @@ -218,6 +255,11 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + + /* BIAS MODE not supported yet for Bitmap (MCDC). */ + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); return 0; } #else diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h index 360165e32ab3..03ed67fcfa76 100644 --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -21,8 +21,8 @@ */ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, - const char *NamesEnd); + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -36,7 +36,8 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); /*! * The data structure describing the data to be written by the @@ -153,6 +154,7 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite); diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 0064fb4ab65d..520fb561e4d3 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -67,6 +67,9 @@ int __llvm_profile_check_compatibility(const char *ProfileData, Header->NumCounters != __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), __llvm_profile_end_counters()) || + Header->NumBitmapBytes != + __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), + __llvm_profile_end_bitmap()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()) || Header->ValueKindLast != IPVK_Last) @@ -75,7 +78,8 @@ int __llvm_profile_check_compatibility(const char *ProfileData, if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + - Header->NumCounters * __llvm_profile_counter_entry_size()) + Header->NumCounters * __llvm_profile_counter_entry_size() + + Header->NumBitmapBytes) return 1; for (SrcData = SrcDataStart, @@ -83,7 +87,8 @@ int __llvm_profile_check_compatibility(const char *ProfileData, SrcData < SrcDataEnd; ++SrcData, ++DstData) { if (SrcData->NameRef != DstData->NameRef || SrcData->FuncHash != DstData->FuncHash || - SrcData->NumCounters != DstData->NumCounters) + SrcData->NumCounters != DstData->NumCounters || + SrcData->NumBitmapBytes != DstData->NumBitmapBytes) return 1; } @@ -112,10 +117,14 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; - char *SrcCountersStart; + + char *SrcCountersStart, *DstCounter; + const char *SrcCountersEnd, *SrcCounter; + const char *SrcBitmapStart; const char *SrcNameStart; const char *SrcValueProfDataStart, *SrcValueProfData; uintptr_t CountersDelta = Header->CountersDelta; + uintptr_t BitmapDelta = Header->BitmapDelta; SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + @@ -124,11 +133,12 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, SrcCountersStart = (char *)SrcDataEnd; SrcCountersEnd = SrcCountersStart + Header->NumCounters * __llvm_profile_counter_entry_size(); - SrcNameStart = SrcCountersEnd; + SrcBitmapStart = SrcCountersEnd; + SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; SrcValueProfDataStart = SrcNameStart + Header->NamesSize + __llvm_profile_get_num_padding_bytes(Header->NamesSize); - if (SrcNameStart < SrcCountersStart) + if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; // Merge counters when there is no data section and debug info correlation is @@ -162,6 +172,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // extend CounterPtr to get the original value. char *DstCounters = (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); + char *DstBitmap = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); unsigned NVK = 0; // SrcData is a serialized representation of the memory image. We need to @@ -191,6 +203,21 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, } } + const char *SrcBitmap = + SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); + // BitmapDelta also needs to be decreased as we advance to the next data + // record. + BitmapDelta -= sizeof(*SrcData); + unsigned NB = SrcData->NumBitmapBytes; + // NumBitmapBytes may legitimately be 0. Just keep going. + if (NB != 0) { + if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) + return 1; + // Merge Src and Dst Bitmap bytes by simply ORing them together. + for (unsigned I = 0; I < NB; I++) + DstBitmap[I] |= SrcBitmap[I]; + } + /* Now merge value profile data. */ if (!VPMergeHook) continue; diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c index d9f2a113f5b0..2154d242a817 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -31,6 +31,11 @@ extern char COMPILER_RT_VISIBILITY extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY +extern char + BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -56,6 +61,10 @@ char *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &CountersEnd; } COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index 2cce0a4b2c48..d0c42462e5e3 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -35,6 +35,8 @@ #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) +#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) #define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) #define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) @@ -48,6 +50,8 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -74,6 +78,12 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) { + return &PROF_BITS_START; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { + return &PROF_BITS_STOP; +} COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &PROF_ORDERFILE_START; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index c7b6e842c9fa..5319ca813b43 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -88,6 +88,10 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return CountersLast; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return BitmapLast; } /* TODO: correctly set up OrderFileFirst. */ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index dd576b2f8357..9dbd702865fd 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -14,6 +14,7 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ #pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") @@ -30,6 +31,8 @@ #pragma section(".lprfd$Z", read, write) #pragma section(".lprfc$A", read, write) #pragma section(".lprfc$Z", read, write) +#pragma section(".lprfb$A", read, write) +#pragma section(".lprfb$Z", read, write) #pragma section(".lorderfile$A", read, write) #pragma section(".lprfnd$A", read, write) #pragma section(".lprfnd$Z", read, write) @@ -43,6 +46,8 @@ const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; char COMPILER_RT_SECTION(".lprfc$A") CountersStart; char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfb$A") BitmapStart; +char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd; uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; @@ -58,6 +63,8 @@ const char *__llvm_profile_end_names(void) { return &NamesEnd; } char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } char *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 1e22398a4c0f..3b61f3def9f6 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -246,17 +246,20 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, - CountersEnd, VPDataReader, NamesBegin, NamesEnd, - SkipNameDataWrite); + CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, + NamesBegin, NamesEnd, SkipNameDataWrite); } COMPILER_RT_VISIBILITY int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { int DebugInfoCorrelate = @@ -271,6 +274,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumCounters = __llvm_profile_get_num_counters(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; /* Create the header. */ @@ -279,11 +284,11 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSectionSize, CountersSectionSize, NamesSize, + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterNames); + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); { /* Initialize header structure. */ @@ -295,6 +300,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, * CountersDelta to match. */ #ifdef _WIN64 Header.CountersDelta = (uint32_t)Header.CountersDelta; + Header.BitmapDelta = (uint32_t)Header.BitmapDelta; #endif /* The data and names sections are omitted in lightweight mode. */ @@ -319,6 +325,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c index 7b96c6d91c33..d9670f739ca9 100644 --- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -25,17 +25,18 @@ const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); uint64_t __llvm_profile_get_size_for_buffer_internal( const void *DataBegin, const void *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); -int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin, - const void *DataEnd, - const char *CountersBegin, - const char *CountersEnd, - const char *NamesBegin, - const char *NamesEnd); +int __llvm_profile_write_buffer_internal( + char *Buffer, const void *DataBegin, const void *DataEnd, + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); void __llvm_profile_set_dumped(void); @@ -43,12 +44,14 @@ int main(int argc, const char *argv[]) { uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal( __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), + __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); char *buf = malloc(bufsize); - int ret = __llvm_profile_write_buffer_internal(buf, - __llvm_profile_begin_data(), __llvm_profile_end_data(), + int ret = __llvm_profile_write_buffer_internal( + buf, __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), + __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); if (ret != 0) { diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 2f4d01c4cddd..e9caee30b1a3 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -13820,6 +13820,144 @@ pass will generate the appropriate data structures and replace the ``llvm.instrprof.value.profile`` intrinsic with the call to the profile runtime library with proper arguments. +'``llvm.instrprof.mcdc.parameters``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.parameters(ptr , i64 , + i32 ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.parameters``' intrinsic is used to initiate MC/DC +code coverage instrumentation for a function. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is the number of bitmap bytes required by the function to +record the number of test vectors executed for each boolean expression. + +Semantics: +"""""""""" + +This intrinsic represents basic MC/DC parameters initiating one or more MC/DC +instrumentation sequences in a function. It will cause the ``-instrprof`` pass +to generate the appropriate data structures and the code to instrument MC/DC +test vectors in a format that can be written out by a compiler runtime and +consumed via the ``llvm-profdata`` tool. + +'``llvm.instrprof.mcdc.condbitmap.update``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.condbitmap.update(ptr , i64 , + i32 , + ptr , + i1 ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic is used to track +MC/DC condition evaluation for each condition in a boolean expression. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is an ID of a condition to track. This value is used as a +bit index into the condition bitmap. + +The fourth argument is the address of the condition bitmap. + +The fifth argument is the boolean value representing the evaluation of the +condition (true or false) + +Semantics: +"""""""""" + +This intrinsic represents the update of a condition bitmap that is local to a +function and will cause the ``-instrprof`` pass to generate the code to +instrument the control flow around each condition in a boolean expression. The +ID of each condition corresponds to a bit index in the condition bitmap which +is set based on the evaluation of the condition. + +'``llvm.instrprof.mcdc.tvbitmap.update``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr , i64 , + i32 ) + i32 , + ptr ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.tvbitmap.update``' intrinsic is used to track MC/DC +test vector execution after each boolean expression has been fully executed. +The overall value of the condition bitmap, after it has been successively +updated using the '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic with +the true or false evaluation of each condition, uniquely identifies an executed +MC/DC test vector and is used as a bit index into the global test vector +bitmap. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is the number of bitmap bytes required by the function to +record the number of test vectors executed for each boolean expression. + +The fourth argument is the byte index into the global test vector bitmap +corresponding to the function. + +The fifth argument is the address of the condition bitmap, which contains a +value representing an executed MC/DC test vector. It is loaded and used as the +bit index of the test vector bitmap. + +Semantics: +"""""""""" + +This intrinsic represents the final operation of an MC/DC instrumentation +sequence and will cause the ``-instrprof`` pass to generate the code to +instrument an update of a function's global test vector bitmap to indicate that +a test vector has been executed. The global test vector bitmap can be consumed +by the ``llvm-profdata`` and ``llvm-cov`` tools. + '``llvm.thread.pointer``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 62bd833198f0..f04b5ae152f2 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1399,6 +1399,11 @@ public: ConstantInt *getHash() const { return cast(const_cast(getArgOperand(1))); } +}; + +/// A base class for all instrprof counter intrinsics. +class InstrProfCntrInstBase : public InstrProfInstBase { +public: // The number of counters for the instrumented function. ConstantInt *getNumCounters() const; // The index of the counter that this instruction acts on. @@ -1406,7 +1411,7 @@ public: }; /// This represents the llvm.instrprof.cover intrinsic. -class InstrProfCoverInst : public InstrProfInstBase { +class InstrProfCoverInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_cover; @@ -1417,7 +1422,7 @@ public: }; /// This represents the llvm.instrprof.increment intrinsic. -class InstrProfIncrementInst : public InstrProfInstBase { +class InstrProfIncrementInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_increment || @@ -1441,7 +1446,7 @@ public: }; /// This represents the llvm.instrprof.timestamp intrinsic. -class InstrProfTimestampInst : public InstrProfInstBase { +class InstrProfTimestampInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_timestamp; @@ -1452,7 +1457,7 @@ public: }; /// This represents the llvm.instrprof.value.profile intrinsic. -class InstrProfValueProfileInst : public InstrProfInstBase { +class InstrProfValueProfileInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_value_profile; @@ -1475,6 +1480,87 @@ public: } }; +/// A base class for instrprof mcdc intrinsics that require global bitmap bytes. +class InstrProfMCDCBitmapInstBase : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters || + I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + /// \return The number of bytes used for the MCDC bitmaps for the instrumented + /// function. + ConstantInt *getNumBitmapBytes() const { + return cast(const_cast(getArgOperand(2))); + } +}; + +/// This represents the llvm.instrprof.mcdc.parameters intrinsic. +class InstrProfMCDCBitmapParameters : public InstrProfMCDCBitmapInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + +/// This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic. +class InstrProfMCDCTVBitmapUpdate : public InstrProfMCDCBitmapInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + /// \return The index of the TestVector Bitmap upon which this intrinsic + /// acts. + ConstantInt *getBitmapIndex() const { + return cast(const_cast(getArgOperand(3))); + } + + /// \return The address of the corresponding condition bitmap containing + /// the index of the TestVector to update within the TestVector Bitmap. + Value *getMCDCCondBitmapAddr() const { + return cast(const_cast(getArgOperand(4))); + } +}; + +/// This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic. +/// It does not pertain to global bitmap updates or parameters and so doesn't +/// inherit from InstrProfMCDCBitmapInstBase. +class InstrProfMCDCCondBitmapUpdate : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_condbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + /// \return The ID of the condition to update. + ConstantInt *getCondID() const { + return cast(const_cast(getArgOperand(2))); + } + + /// \return The address of the corresponding condition bitmap. + Value *getMCDCCondBitmapAddr() const { + return cast(const_cast(getArgOperand(3))); + } + + /// \return The boolean value to set in the condition bitmap for the + /// corresponding condition ID. This represents how the condition evaluated. + Value *getCondBool() const { + return cast(const_cast(getArgOperand(4))); + } +}; + class PseudoProbeInst : public IntrinsicInst { public: static bool classof(const IntrinsicInst *I) { diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 20a8fa419465..a47d94452db4 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -909,6 +909,21 @@ def int_instrprof_value_profile : Intrinsic<[], llvm_i64_ty, llvm_i32_ty, llvm_i32_ty]>; +// A parameter configuration for instrumentation based MCDC profiling. +def int_instrprof_mcdc_parameters : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty]>; + +// A test vector bitmap update for instrumentation based MCDC profiling. +def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty]>; + +// A condition bitmap value update for instrumentation based MCDC profiling. +def int_instrprof_mcdc_condbitmap_update : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_ptr_ty, llvm_i1_ty]>; + def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty]>; def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>; diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 3c8f940ba97b..12498630190d 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -1023,7 +1023,9 @@ enum CovMapVersion { // Compilation directory is stored separately and combined with relative // filenames to produce an absolute file path. Version6 = 5, - // The current version is Version6. + // Branch regions extended and Decision Regions added for MC/DC. + Version7 = 6, + // The current version is Version7. CurrentVersion = INSTR_PROF_COVMAP_VERSION }; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index f9096b461572..2a780104b177 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -96,6 +96,9 @@ inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } /// Return the name prefix of profile counter variables. inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } +/// Return the name prefix of profile bitmap variables. +inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; } + /// Return the name prefix of value profile variables. inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } @@ -335,6 +338,7 @@ enum class instrprof_error { invalid_prof, hash_mismatch, count_mismatch, + bitmap_mismatch, counter_overflow, value_site_count_mismatch, compress_failed, @@ -690,18 +694,23 @@ struct InstrProfValueSiteRecord { /// Profiling information for a single function. struct InstrProfRecord { std::vector Counts; + std::vector BitmapBytes; InstrProfRecord() = default; InstrProfRecord(std::vector Counts) : Counts(std::move(Counts)) {} + InstrProfRecord(std::vector Counts, + std::vector BitmapBytes) + : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {} InstrProfRecord(InstrProfRecord &&) = default; InstrProfRecord(const InstrProfRecord &RHS) - : Counts(RHS.Counts), + : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes), ValueData(RHS.ValueData ? std::make_unique(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { Counts = RHS.Counts; + BitmapBytes = RHS.BitmapBytes; if (!RHS.ValueData) { ValueData = nullptr; return *this; @@ -880,6 +889,11 @@ struct NamedInstrProfRecord : InstrProfRecord { NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} + NamedInstrProfRecord(StringRef Name, uint64_t Hash, + std::vector Counts, + std::vector BitmapBytes) + : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name), + Hash(Hash) {} static bool hasCSFlagInHash(uint64_t FuncHash) { return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); @@ -1015,7 +1029,9 @@ enum ProfVersion { Version9 = 9, // An additional (optional) temporal profile traces section is added. Version10 = 10, - // The current version is 10. + // An additional field is used for bitmap bytes. + Version11 = 11, + // The current version is 11. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1153,6 +1169,7 @@ namespace RawInstrProf { // Version 6: Added binary id. // Version 7: Reorder binary id and include version in signature. // Version 8: Use relative counter pointer. +// Version 9: Added relative bitmap bytes pointer and count used by MC/DC. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 4456bf1ab176..fad14576c442 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -76,6 +76,7 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -87,7 +88,9 @@ INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -132,9 +135,13 @@ INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -267,6 +274,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -646,11 +656,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 8 +#define INSTR_PROF_RAW_VERSION 9 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 10 +#define INSTR_PROF_INDEX_VERSION 11 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 5 +#define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -687,6 +697,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -698,6 +709,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -709,6 +721,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -723,6 +736,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 74e921e10c47..51d86ee0bce3 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -323,11 +323,14 @@ private: // the variant types of the profile. uint64_t Version; uint64_t CountersDelta; + uint64_t BitmapDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; const char *CountersStart; const char *CountersEnd; + const char *BitmapStart; + const char *BitmapEnd; const char *NamesStart; const char *NamesEnd; // After value profile is all read, this pointer points to @@ -429,6 +432,7 @@ private: Error readName(NamedInstrProfRecord &Record); Error readFuncHash(NamedInstrProfRecord &Record); Error readRawCounts(InstrProfRecord &Record); + Error readRawBitmapBytes(InstrProfRecord &Record); Error readValueProfilingData(InstrProfRecord &Record); bool atEnd() const { return Data == DataEnd; } @@ -441,6 +445,7 @@ private: // As we advance to the next record, we maintain the correct CountersDelta // with respect to the next record. CountersDelta -= sizeof(*Data); + BitmapDelta -= sizeof(*Data); } Data++; ValueDataStart += CurValueDataSize; @@ -732,6 +737,10 @@ public: Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts); + /// Fill Bitmap Bytes with the profile data for the given function name. + Error getFunctionBitmapBytes(StringRef FuncName, uint64_t FuncHash, + std::vector &BitmapBytes); + /// Return the maximum of all known function counts. /// \c UseCS indicates whether to use the context-sensitive count. uint64_t getMaximumFunctionCount(bool UseCS) { diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index cb0c055dcb74..d8f3e75087ac 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -50,6 +50,7 @@ private: uint32_t NumValueSites[IPVK_Last + 1]; GlobalVariable *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; + GlobalVariable *RegionBitmaps = nullptr; PerFunctionProfileData() { memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); @@ -105,20 +106,59 @@ private: /// Force emitting of name vars for unused functions. void lowerCoverageData(GlobalVariable *CoverageNamesVar); + /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction + /// using the index represented by the a temp value into a bitmap. + void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); + + /// Replace instrprof.mcdc.temp.update with a shift and or instruction using + /// the corresponding condition ID. + void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); + /// Compute the address of the counter value that this profiling instruction /// acts on. - Value *getCounterAddress(InstrProfInstBase *I); + Value *getCounterAddress(InstrProfCntrInstBase *I); /// Get the region counters for an increment, creating them if necessary. /// /// If the counter array doesn't yet exist, the profile data variables /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc); + GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); /// Create the region counters. - GlobalVariable *createRegionCounters(InstrProfInstBase *Inc, StringRef Name, + GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, + StringRef Name, GlobalValue::LinkageTypes Linkage); + /// Compute the address of the test vector bitmap that this profiling + /// instruction acts on. + Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); + + /// Get the region bitmaps for an increment, creating them if necessary. + /// + /// If the bitmap array doesn't yet exist, the profile data variables + /// referring to them will also be created. + GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); + + /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with + /// an MC/DC Decision region. The number of bytes required is indicated by + /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called + /// as part of setupProfileSection() and is conceptually very similar to + /// what is done for profile data counters in createRegionCounters(). + GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage); + + /// Set Comdat property of GV, if required. + void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName); + + /// Setup the sections into which counters and bitmaps are allocated. + GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK); + + /// Create INSTR_PROF_DATA variable for counters and bitmaps. + void createDataVariable(InstrProfCntrInstBase *Inc, + InstrProfMCDCBitmapParameters *Update); + /// Emit the section with compressed function names. void emitNameData(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 20c37eb4cb11..6e11bd28a683 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7153,6 +7153,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, llvm_unreachable("instrprof failed to lower a timestamp"); case Intrinsic::instrprof_value_profile: llvm_unreachable("instrprof failed to lower a value profiling call"); + case Intrinsic::instrprof_mcdc_parameters: + llvm_unreachable("instrprof failed to lower mcdc parameters"); + case Intrinsic::instrprof_mcdc_tvbitmap_update: + llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update"); + case Intrinsic::instrprof_mcdc_condbitmap_update: + llvm_unreachable("instrprof failed to lower an mcdc condbitmap update"); case Intrinsic::localescape: { MachineFunction &MF = DAG.getMachineFunction(); const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index 36d56699c64e..4270247ef0f5 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -270,13 +270,13 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef NameTable, return -1; } -ConstantInt *InstrProfInstBase::getNumCounters() const { +ConstantInt *InstrProfCntrInstBase::getNumCounters() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("InstrProfValueProfileInst does not have counters!"); return cast(const_cast(getArgOperand(2))); } -ConstantInt *InstrProfInstBase::getIndex() const { +ConstantInt *InstrProfCntrInstBase::getIndex() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()"); return cast(const_cast(getArgOperand(3))); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 05737323314a..cd3970c5151d 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -757,6 +757,7 @@ Expected> CovMapFuncRecordReader::get( case CovMapVersion::Version4: case CovMapVersion::Version5: case CovMapVersion::Version6: + case CovMapVersion::Version7: // Decompress the name data. if (Error E = P.create(P.getNameData())) return std::move(E); @@ -775,6 +776,9 @@ Expected> CovMapFuncRecordReader::get( else if (Version == CovMapVersion::Version6) return std::make_unique>(P, R, D, F); + else if (Version == CovMapVersion::Version7) + return std::make_unique>(P, R, D, F); } llvm_unreachable("Unsupported version"); } diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 835dd697bc7b..7e2ee660978a 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -136,6 +136,9 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::count_mismatch: OS << "function basic block count change detected (counter mismatch)"; break; + case instrprof_error::bitmap_mismatch: + OS << "function bitmap size change detected (bitmap size mismatch)"; + break; case instrprof_error::counter_overflow: OS << "counter overflow"; break; @@ -804,6 +807,18 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, Warn(instrprof_error::counter_overflow); } + // If the number of bitmap bytes doesn't match we either have bad data + // or a hash collision. + if (BitmapBytes.size() != Other.BitmapBytes.size()) { + Warn(instrprof_error::bitmap_mismatch); + return; + } + + // Bitmap bytes are merged by simply ORing them together. + for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) { + BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I]; + } + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) mergeValueProfData(Kind, Other, Weight, Warn); } @@ -1476,9 +1491,11 @@ Expected
Header::readFromBuffer(const unsigned char *Buffer) { // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version10, + IndexedInstrProf::ProfVersion::CurrentVersion == Version11, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); + case 11ull: + [[fallthrough]]; case 10ull: H.TemporalProfTracesOffset = read(Buffer, offsetOf(&Header::TemporalProfTracesOffset)); @@ -1502,10 +1519,12 @@ size_t Header::size() const { // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); + case 11ull: + [[fallthrough]]; case 10ull: return offsetOf(&Header::TemporalProfTracesOffset) + sizeof(Header::TemporalProfTracesOffset); diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index c822d81f8bef..c917394520df 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -207,11 +207,15 @@ void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, // In this mode, CounterPtr actually stores the section relative address // of the counter. maybeSwap(CounterOffset), + // TODO: MC/DC is not yet supported. + /*BitmapOffset=*/maybeSwap(0), maybeSwap(FunctionPtr), // TODO: Value profiling is not yet supported. /*ValuesPtr=*/maybeSwap(0), maybeSwap(NumCounters), /*NumValueSites=*/{maybeSwap(0), maybeSwap(0)}, + // TODO: MC/DC is not yet supported. + /*NumBitmapBytes=*/maybeSwap(0), }); NamesVec.push_back(FunctionName.str()); } diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index db20441b712c..e4ef1b52d415 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -433,6 +433,29 @@ Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { Record.Counts.push_back(Count); } + // Bitmap byte information is indicated with special character. + if (Line->startswith("$")) { + Record.BitmapBytes.clear(); + // Read the number of bitmap bytes. + uint64_t NumBitmapBytes; + if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes)) + return error(instrprof_error::malformed, + "number of bitmap bytes is not a valid integer"); + if (NumBitmapBytes != 0) { + // Read each bitmap and fill our internal storage with the values. + Record.BitmapBytes.reserve(NumBitmapBytes); + for (uint8_t I = 0; I < NumBitmapBytes; ++I) { + if (Line.is_at_end()) + return error(instrprof_error::truncated); + uint8_t BitmapByte; + if ((Line++)->getAsInteger(0, BitmapByte)) + return error(instrprof_error::malformed, + "bitmap byte is not a valid integer"); + Record.BitmapBytes.push_back(BitmapByte); + } + } + } + // Check if value profile data exists and read it if so. if (Error E = readValueProfileData(Record)) return error(std::move(E)); @@ -549,11 +572,14 @@ Error RawInstrProfReader::readHeader( return error(instrprof_error::bad_header); CountersDelta = swap(Header.CountersDelta); + BitmapDelta = swap(Header.BitmapDelta); NamesDelta = swap(Header.NamesDelta); auto NumData = swap(Header.NumData); auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize(); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); + auto NumBitmapBytes = swap(Header.NumBitmapBytes); + auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes); auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); @@ -563,8 +589,10 @@ Error RawInstrProfReader::readHeader( // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize; ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters; - ptrdiff_t NamesOffset = + ptrdiff_t BitmapOffset = CountersOffset + CountersSize + PaddingBytesAfterCounters; + ptrdiff_t NamesOffset = + BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; auto *Start = reinterpret_cast(&Header); @@ -593,6 +621,8 @@ Error RawInstrProfReader::readHeader( reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); CountersStart = Start + CountersOffset; CountersEnd = CountersStart + CountersSize; + BitmapStart = Start + BitmapOffset; + BitmapEnd = BitmapStart + NumBitmapBytes; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); @@ -683,6 +713,49 @@ Error RawInstrProfReader::readRawCounts( return success(); } +template +Error RawInstrProfReader::readRawBitmapBytes(InstrProfRecord &Record) { + uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes); + + Record.BitmapBytes.clear(); + Record.BitmapBytes.reserve(NumBitmapBytes); + + // It's possible MCDC is either not enabled or only used for some functions + // and not others. So if we record 0 bytes, just move on. + if (NumBitmapBytes == 0) + return success(); + + // BitmapDelta decreases as we advance to the next data record. + ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta; + if (BitmapOffset < 0) + return error( + instrprof_error::malformed, + ("bitmap offset " + Twine(BitmapOffset) + " is negative").str()); + + if (BitmapOffset >= BitmapEnd - BitmapStart) + return error(instrprof_error::malformed, + ("bitmap offset " + Twine(BitmapOffset) + + " is greater than the maximum bitmap offset " + + Twine(BitmapEnd - BitmapStart - 1)) + .str()); + + uint64_t MaxNumBitmapBytes = + (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t); + if (NumBitmapBytes > MaxNumBitmapBytes) + return error(instrprof_error::malformed, + ("number of bitmap bytes " + Twine(NumBitmapBytes) + + " is greater than the maximum number of bitmap bytes " + + Twine(MaxNumBitmapBytes)) + .str()); + + for (uint32_t I = 0; I < NumBitmapBytes; I++) { + const char *Ptr = BitmapStart + BitmapOffset + I; + Record.BitmapBytes.push_back(swap(*Ptr)); + } + + return success(); +} + template Error RawInstrProfReader::readValueProfilingData( InstrProfRecord &Record) { @@ -733,6 +806,10 @@ Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) if (Error E = readRawCounts(Record)) return error(std::move(E)); + // Read raw bitmap bytes and set Record. + if (Error E = readRawBitmapBytes(Record)) + return error(std::move(E)); + // Read value data and set Record. if (Error E = readValueProfilingData(Record)) return error(std::move(E)); @@ -794,6 +871,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, DataBuffer.clear(); std::vector CounterBuffer; + std::vector BitmapByteBuffer; const unsigned char *End = D + N; while (D < End) { @@ -819,7 +897,24 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); - DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); + // Read bitmap bytes for GET_VERSION(FormatVersion) > 8. + if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version8) { + uint64_t BitmapBytes = 0; + if (D + sizeof(uint64_t) > End) + return data_type(); + BitmapBytes = endian::readNext(D); + // Read bitmap byte values. + if (D + BitmapBytes * sizeof(uint8_t) > End) + return data_type(); + BitmapByteBuffer.clear(); + BitmapByteBuffer.reserve(BitmapBytes); + for (uint64_t J = 0; J < BitmapBytes; ++J) + BitmapByteBuffer.push_back(static_cast( + endian::readNext(D))); + } + + DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer), + std::move(BitmapByteBuffer)); // Read value profiling data. if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && @@ -1319,6 +1414,16 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, return success(); } +Error IndexedInstrProfReader::getFunctionBitmapBytes( + StringRef FuncName, uint64_t FuncHash, std::vector &BitmapBytes) { + Expected Record = getInstrProfRecord(FuncName, FuncHash); + if (Error E = Record.takeError()) + return error(std::move(E)); + + BitmapBytes = Record.get().BitmapBytes; + return success(); +} + Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { ArrayRef Data; diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index b74d5c3862d8..d8b8571bd5c9 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -131,6 +131,8 @@ public: M += sizeof(uint64_t); // The function hash M += sizeof(uint64_t); // The size of the Counts vector M += ProfRecord.Counts.size() * sizeof(uint64_t); + M += sizeof(uint64_t); // The size of the Bitmap vector + M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t); // Value data M += ValueProfData::getSize(ProfileData.second); @@ -160,6 +162,10 @@ public: for (uint64_t I : ProfRecord.Counts) LE.write(I); + LE.write(ProfRecord.BitmapBytes.size()); + for (uint64_t I : ProfRecord.BitmapBytes) + LE.write(I); + // Write value data std::unique_ptr VDataPtr = ValueProfData::serializeFrom(ProfileData.second); @@ -380,6 +386,8 @@ bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { const InstrProfRecord &IPR = Func.second; if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) return true; + if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; })) + return true; } return false; } @@ -703,6 +711,17 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, for (uint64_t Count : Func.Counts) OS << Count << "\n"; + if (Func.BitmapBytes.size() > 0) { + OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n"; + OS << "# Bitmap Byte Values:\n"; + for (uint8_t Byte : Func.BitmapBytes) { + OS << "0x"; + OS.write_hex(Byte); + OS << "\n"; + } + OS << "\n"; + } + uint32_t NumValueKinds = Func.getNumValueKinds(); if (!NumValueKinds) { OS << "\n"; diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 343554241da3..ce287551eed6 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -430,6 +430,15 @@ bool InstrProfiling::lowerIntrinsics(Function *F) { } else if (auto *IPVP = dyn_cast(&Instr)) { lowerValueProfileInst(IPVP); MadeChange = true; + } else if (auto *IPMP = dyn_cast(&Instr)) { + IPMP->eraseFromParent(); + MadeChange = true; + } else if (auto *IPBU = dyn_cast(&Instr)) { + lowerMCDCTestVectorBitmapUpdate(IPBU); + MadeChange = true; + } else if (auto *IPTU = dyn_cast(&Instr)) { + lowerMCDCCondBitmapUpdate(IPTU); + MadeChange = true; } } } @@ -544,19 +553,33 @@ bool InstrProfiling::run( // the instrumented function. This is counting the number of instrumented // target value sites to enter it as field in the profile data variable. for (Function &F : M) { - InstrProfInstBase *FirstProfInst = nullptr; - for (BasicBlock &BB : F) - for (auto I = BB.begin(), E = BB.end(); I != E; I++) + InstrProfCntrInstBase *FirstProfInst = nullptr; + InstrProfMCDCBitmapParameters *FirstProfMCDCParams = nullptr; + for (BasicBlock &BB : F) { + for (auto I = BB.begin(), E = BB.end(); I != E; I++) { if (auto *Ind = dyn_cast(I)) computeNumValueSiteCounts(Ind); - else if (FirstProfInst == nullptr && - (isa(I) || isa(I))) - FirstProfInst = dyn_cast(I); + else { + if (FirstProfInst == nullptr && + (isa(I) || isa(I))) + FirstProfInst = dyn_cast(I); + if (FirstProfMCDCParams == nullptr) + FirstProfMCDCParams = dyn_cast(I); + } + } + } + + // If the MCDCBitmapParameters intrinsic was seen, create the bitmaps. + if (FirstProfMCDCParams != nullptr) { + static_cast(getOrCreateRegionBitmaps(FirstProfMCDCParams)); + } - // Value profiling intrinsic lowering requires per-function profile data - // variable to be created first. - if (FirstProfInst != nullptr) + // Use a profile intrinsic to create the region counters and data variable. + // Also create the data variable based on the MCDCParams. + if (FirstProfInst != nullptr) { static_cast(getOrCreateRegionCounters(FirstProfInst)); + createDataVariable(FirstProfInst, FirstProfMCDCParams); + } } for (Function &F : M) @@ -670,7 +693,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { Ind->eraseFromParent(); } -Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { +Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { auto *Counters = getOrCreateRegionCounters(I); IRBuilder<> Builder(I); @@ -710,6 +733,25 @@ Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { return Builder.CreateIntToPtr(Add, Addr->getType()); } +Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { + auto *Bitmaps = getOrCreateRegionBitmaps(I); + IRBuilder<> Builder(I); + + auto *Addr = Builder.CreateConstInBoundsGEP2_32( + Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue()); + + if (isRuntimeCounterRelocationEnabled()) { + LLVMContext &Ctx = M->getContext(); + Ctx.diagnose(DiagnosticInfoPGOProfile( + M->getName().data(), + Twine("Runtime counter relocation is presently not supported for MC/DC " + "bitmaps."), + DS_Warning)); + } + + return Addr; +} + void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) { auto *Addr = getCounterAddress(CoverInstruction); IRBuilder<> Builder(CoverInstruction); @@ -769,6 +811,86 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { CoverageNamesVar->eraseFromParent(); } +void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( + InstrProfMCDCTVBitmapUpdate *Update) { + IRBuilder<> Builder(Update); + auto *Int8Ty = Type::getInt8Ty(M->getContext()); + auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); + auto *BitmapAddr = getBitmapAddress(Update); + + // Load Temp Val. + // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); + + // Calculate byte offset using div8. + // %1 = lshr i32 %mcdc.temp, 3 + auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3); + + // Add byte offset to section base byte address. + // %2 = zext i32 %1 to i64 + // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2 + auto *BitmapByteAddr = + Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty), + Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty)); + + // Convert to a pointer. + // %4 = inttoptr i32 %3 to ptr + BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy); + + // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8) + // %5 = and i32 %mcdc.temp, 7 + // %6 = trunc i32 %5 to i8 + auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty); + + // Shift bit offset left to form a bitmap. + // %7 = shl i8 1, %6 + auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet); + + // Load profile bitmap byte. + // %mcdc.bits = load i8, ptr %4, align 1 + auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits"); + + // Perform logical OR of profile bitmap byte and shifted bit offset. + // %8 = or i8 %mcdc.bits, %7 + auto *Result = Builder.CreateOr(Bitmap, ShiftedVal); + + // Store the updated profile bitmap byte. + // store i8 %8, ptr %3, align 1 + Builder.CreateStore(Result, BitmapByteAddr); + Update->eraseFromParent(); +} + +void InstrProfiling::lowerMCDCCondBitmapUpdate( + InstrProfMCDCCondBitmapUpdate *Update) { + IRBuilder<> Builder(Update); + auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); + + // Load the MCDC temporary value from the stack. + // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); + + // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits. + // %1 = zext i1 %tobool to i32 + auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty); + + // Shift the boolean value left (by the condition's ID) to form a bitmap. + // %2 = shl i32 %1, getCondID()> + auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID()); + + // Perform logical OR of the bitmap against the loaded MCDC temporary value. + // %3 = or i32 %mcdc.temp, %2 + auto *Result = Builder.CreateOr(Temp, ShiftedVal); + + // Store the updated temporary value back to the stack. + // store i32 %3, ptr %mcdc.addr, align 4 + Builder.CreateStore(Result, MCDCCondBitmapAddr); + Update->eraseFromParent(); +} + /// Get the name of a profiling variable for a particular function. static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed) { @@ -924,37 +1046,31 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { return true; } -GlobalVariable * -InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name, - GlobalValue::LinkageTypes Linkage) { - uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto &Ctx = M->getContext(); - GlobalVariable *GV; - if (isa(Inc)) { - auto *CounterTy = Type::getInt8Ty(Ctx); - auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); - // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. - std::vector InitialValues(NumCounters, - Constant::getAllOnesValue(CounterTy)); - GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, - ConstantArray::get(CounterArrTy, InitialValues), - Name); - GV->setAlignment(Align(1)); - } else { - auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); - GV = new GlobalVariable(*M, CounterTy, false, Linkage, - Constant::getNullValue(CounterTy), Name); - GV->setAlignment(Align(8)); - } - return GV; +void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, + StringRef VarName) { + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); + + if (!UseComdat) + return; + + StringRef GroupName = + TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName; + Comdat *C = M->getOrInsertComdat(GroupName); + if (!NeedComdat) + C->setSelectionKind(Comdat::NoDeduplicate); + GV->setComdat(C); + // COFF doesn't allow the comdat group leader to have private linkage, so + // upgrade private linkage to internal linkage to produce a symbol table + // entry. + if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) + GV->setLinkage(GlobalValue::InternalLinkage); } -GlobalVariable * -InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { +GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK) { GlobalVariable *NamePtr = Inc->getName(); - auto &PD = ProfileDataMap[NamePtr]; - if (PD.RegionCounters) - return PD.RegionCounters; // Match the linkage and visibility of the name global. Function *Fn = Inc->getParent()->getParent(); @@ -993,42 +1109,100 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { // nodeduplicate COMDAT which is lowered to a zero-flag section group. This // allows -z start-stop-gc to discard the entire group when the function is // discarded. - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); bool Renamed; - std::string CntsVarName = - getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); - std::string DataVarName = - getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); - auto MaybeSetComdat = [&](GlobalVariable *GV) { - bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); - if (UseComdat) { - StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode - ? GV->getName() - : CntsVarName; - Comdat *C = M->getOrInsertComdat(GroupName); - if (!NeedComdat) - C->setSelectionKind(Comdat::NoDeduplicate); - GV->setComdat(C); - // COFF doesn't allow the comdat group leader to have private linkage, so - // upgrade private linkage to internal linkage to produce a symbol table - // entry. - if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) - GV->setLinkage(GlobalValue::InternalLinkage); - } - }; + GlobalVariable *Ptr; + StringRef VarPrefix; + std::string VarName; + if (IPSK == IPSK_cnts) { + VarPrefix = getInstrProfCountersVarPrefix(); + VarName = getVarName(Inc, VarPrefix, Renamed); + InstrProfCntrInstBase *CntrIncrement = dyn_cast(Inc); + Ptr = createRegionCounters(CntrIncrement, VarName, Linkage); + } else if (IPSK == IPSK_bitmap) { + VarPrefix = getInstrProfBitmapVarPrefix(); + VarName = getVarName(Inc, VarPrefix, Renamed); + InstrProfMCDCBitmapInstBase *BitmapUpdate = + dyn_cast(Inc); + Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage); + } else { + llvm_unreachable("Profile Section must be for Counters or Bitmaps"); + } + + Ptr->setVisibility(Visibility); + // Put the counters and bitmaps in their own sections so linkers can + // remove unneeded sections. + Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat())); + Ptr->setLinkage(Linkage); + maybeSetComdat(Ptr, Fn, VarName); + return Ptr; +} + +GlobalVariable * +InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage) { + uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); + auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes); + auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage, + Constant::getNullValue(BitmapTy), Name); + GV->setAlignment(Align(1)); + return GV; +} + +GlobalVariable * +InstrProfiling::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) { + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionBitmaps) + return PD.RegionBitmaps; + + // If RegionBitmaps doesn't already exist, create it by first setting up + // the corresponding profile section. + auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap); + PD.RegionBitmaps = BitmapPtr; + return PD.RegionBitmaps; +} +GlobalVariable * +InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, + GlobalValue::LinkageTypes Linkage) { uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - LLVMContext &Ctx = M->getContext(); + auto &Ctx = M->getContext(); + GlobalVariable *GV; + if (isa(Inc)) { + auto *CounterTy = Type::getInt8Ty(Ctx); + auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); + // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. + std::vector InitialValues(NumCounters, + Constant::getAllOnesValue(CounterTy)); + GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, + ConstantArray::get(CounterArrTy, InitialValues), + Name); + GV->setAlignment(Align(1)); + } else { + auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + GV = new GlobalVariable(*M, CounterTy, false, Linkage, + Constant::getNullValue(CounterTy), Name); + GV->setAlignment(Align(8)); + } + return GV; +} + +GlobalVariable * +InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionCounters) + return PD.RegionCounters; - auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage); - CounterPtr->setVisibility(Visibility); - CounterPtr->setSection( - getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); - CounterPtr->setLinkage(Linkage); - MaybeSetComdat(CounterPtr); + // If RegionCounters doesn't already exist, create it by first setting up + // the corresponding profile section. + auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts); PD.RegionCounters = CounterPtr; + if (DebugInfoCorrelate) { + LLVMContext &Ctx = M->getContext(); + Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { @@ -1066,8 +1240,50 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); } + + // Mark the counter variable as used so that it isn't optimized out. + CompilerUsedVars.push_back(PD.RegionCounters); } + return PD.RegionCounters; +} + +void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, + InstrProfMCDCBitmapParameters *Params) { + // When debug information is correlated to profile data, a data variable + // is not needed. + if (DebugInfoCorrelate) + return; + + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + + LLVMContext &Ctx = M->getContext(); + + Function *Fn = Inc->getParent()->getParent(); + GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); + GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); + + // Due to the limitation of binder as of 2021/09/28, the duplicate weak + // symbols in the same csect won't be discarded. When there are duplicate weak + // symbols, we can NOT guarantee that the relocations get resolved to the + // intended weak symbol, so we can not ensure the correctness of the relative + // CounterPtr, so we have to use private linkage for counter and data symbols. + if (TT.isOSBinFormatXCOFF()) { + Linkage = GlobalValue::PrivateLinkage; + Visibility = GlobalValue::DefaultVisibility; + } + + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool Renamed; + + // The Data Variable section is anchored to profile counters. + std::string CntsVarName = + getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); + std::string DataVarName = + getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); + auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for // the current function. @@ -1085,16 +1301,17 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); - MaybeSetComdat(ValuesVar); + maybeSetComdat(ValuesVar, Fn, CntsVarName); ValuesPtrExpr = ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } - if (DebugInfoCorrelate) { - // Mark the counter variable as used so that it isn't optimized out. - CompilerUsedVars.push_back(PD.RegionCounters); - return PD.RegionCounters; - } + uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); + auto *CounterPtr = PD.RegionCounters; + + uint64_t NumBitmapBytes = 0; + if (Params != nullptr) + NumBitmapBytes = Params->getNumBitmapBytes()->getZExtValue(); // Create data variable. auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); @@ -1137,6 +1354,16 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); + // Bitmaps are relative to the same data variable as profile counters. + GlobalVariable *BitmapPtr = PD.RegionBitmaps; + Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); + + if (BitmapPtr != nullptr) { + RelativeBitmapPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + ConstantExpr::getPtrToInt(Data, IntPtrTy)); + } + Constant *DataVals[] = { #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, #include "llvm/ProfileData/InstrProfData.inc" @@ -1146,7 +1373,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { Data->setVisibility(Visibility); Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); - MaybeSetComdat(Data); + maybeSetComdat(Data, Fn, CntsVarName); PD.DataVar = Data; @@ -1158,8 +1385,6 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { NamePtr->setLinkage(GlobalValue::PrivateLinkage); // Collect the referenced names to be used by emitNameData. ReferencedNames.push_back(NamePtr); - - return PD.RegionCounters; } void InstrProfiling::emitVNodes() { diff --git a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll new file mode 100644 index 000000000000..fccb026c25bf --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll @@ -0,0 +1,53 @@ +; Check that MC/DC intrinsics are properly lowered +; RUN: opt < %s -passes=instrprof -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -runtime-counter-relocation -S 2>&1 | FileCheck %s --check-prefix RELOC + +; RELOC: Runtime counter relocation is presently not supported for MC/DC bitmaps + +target triple = "x86_64-unknown-linux-gnu" + +@__profn_test = private constant [4 x i8] c"test" + +; CHECK: @__profbm_test = private global [1 x i8] zeroinitializer, section "__llvm_prf_bits", comdat, align 1 + +define dso_local void @test(i32 noundef %A) { +entry: + %A.addr = alloca i32, align 4 + %mcdc.addr = alloca i32, align 4 + call void @llvm.instrprof.cover(ptr @__profn_test, i64 99278, i32 5, i32 0) + ; CHECK: store i8 0, ptr @__profc_test, align 1 + + call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 99278, i32 1) + store i32 0, ptr %mcdc.addr, align 4 + %0 = load i32, ptr %A.addr, align 4 + %tobool = icmp ne i32 %0, 0 + + call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 99278, i32 0, ptr %mcdc.addr, i1 %tobool) + ; CHECK: %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + ; CHECK-NEXT: %1 = zext i1 %tobool to i32 + ; CHECK-NEXT: %2 = shl i32 %1, 0 + ; CHECK-NEXT: %3 = or i32 %mcdc.temp, %2 + ; CHECK-NEXT: store i32 %3, ptr %mcdc.addr, align 4 + + call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 99278, i32 1, i32 0, ptr %mcdc.addr) + ; CHECK: %mcdc.temp1 = load i32, ptr %mcdc.addr, align 4 + ; CHECK-NEXT: %4 = lshr i32 %mcdc.temp1, 3 + ; CHECK-NEXT: %5 = zext i32 %4 to i64 + ; CHECK-NEXT: %6 = add i64 ptrtoint (ptr @__profbm_test to i64), %5 + ; CHECK-NEXT: %7 = inttoptr i64 %6 to ptr + ; CHECK-NEXT: %8 = and i32 %mcdc.temp1, 7 + ; CHECK-NEXT: %9 = trunc i32 %8 to i8 + ; CHECK-NEXT: %10 = shl i8 1, %9 + ; CHECK-NEXT: %mcdc.bits = load i8, ptr %7, align 1 + ; CHECK-NEXT: %11 = or i8 %mcdc.bits, %10 + ; CHECK-NEXT: store i8 %11, ptr %7, align 1 + ret void +} + +declare void @llvm.instrprof.cover(ptr, i64, i32, i32) + +declare void @llvm.instrprof.mcdc.parameters(ptr, i64, i32) + +declare void @llvm.instrprof.mcdc.condbitmap.update(ptr, i64, i32, ptr, i1) + +declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr, i64, i32, i32, ptr) diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll index 1c44a274f3c0..8c6942c0f527 100644 --- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll +++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll @@ -13,9 +13,9 @@ $foo = comdat any ; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; CHECK-NOT: __profn__stdin__foo ; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 -; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null +; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null ; CHECK-NOT: @foo -; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 +; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm ; CHECK: @llvm.compiler.used diff --git a/llvm/test/tools/llvm-profdata/Inputs/basic.profraw b/llvm/test/tools/llvm-profdata/Inputs/basic.profraw index ad88759398c6020f4ab8a5606258e69d98e36687..1b284b84fad6dd7f9407b1c3b99cb178af0e09c6 100644 GIT binary patch delta 63 zcmbQicz}_!u_!ISs37M*=R{6@Muv%+@_Y__aks4+{{I)d(5P~>exjd}00R^}C!MPiq_)7#Jp|DQPSa2C)!;5z2(oEDX7cnRyHh E0CJEFxc~qF diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw index bc8fc5db1cb154d98ca962e84313463e3298cb92..9cd225587c92511e99f3497ce1d5f47c6fc5f0af 100644 GIT binary patch delta 308 zcmeC+d%(}xSd^AuRFLzZb0Vky_8T_ z2Map_g$4d1%$=MM7A^z|M<5H&2MaF*3U?q0gIzKCB2aV#lIY~~%z~3Sm;_Lj)^mdV hG1&qro`I~m9;i42C}@Bz*w4f{If6-G^8%&}764*Dk$V6D delta 330 zcmaFB-@(V(Sd^AuRFLzZV$QVgXRX-d83sq200i+W)W}ZBd!q5+)yH6rcP6 zC=1puPjh&2EJ diff --git a/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw b/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw index 134b78f7af5b760dc3af7422c1bf7661f4bae14a..9966729d92ddc33bf89eeb3fee87215bbabbbef1 100644 GIT binary patch delta 326 zcmaFCyMbS@u_!ISs37M*Cj%5r6jYzQfSCh?Ct3(dSahA6xxL~4|GoghG>>`)21b~& z3=Oc72^^Cruy9PAA;2N==RX)sp2#ddaX%*`!^DH)91h5W??Hke#W_5X1SiWg0!1e) zGKzETKo<4~3kL#)1^yz;om>wVZUhQPAPcVt3vUDpcOVIaT`~C~P;>*5=;Zs%f|Dhf o1W=XsbAtRa*#jt^fvmV6sJH?sXn-s@pNVsF29v<%4NMj+0Qs7eE&u=k delta 364 zcmdnM|AJSru_!ISs37M*2Ll8MOcYd~=pmqxDy0AxV1$ap532 zg3W7Lmwt5a%Luw`8X;yQ=%O*PQGDV94nY>r<189Si!*Iwboso6f2t<{#V0E;atMY_ zsnBGduNo&k$1~O>`Qe2MkVFDdqEPdcjat*B{dO76%_kSkWcUG+SOAo;_m#;@Xm_l> z_CIV#TU6$@gvkdP#V0=i%0w>dTG2Ucam2*OHvOM-SQvJIWfYh=1iwzTW$_M(6^_4X z6sG@NZp{yn4A5N?3QrGb?)?~3dEpwwUk3mFgTdqpOcD~cFPdW;(Y diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test index 67db5c98ef32..eda63203a304 100644 --- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -5,13 +5,15 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. // 2 * 8 binary ID sizes // + 2 * 20 binary IDs (of size 20) @@ -23,8 +25,11 @@ RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\20\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -51,14 +56,18 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t.profraw +RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test index 2394431e94de..38b838e0d100 100644 --- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -9,6 +9,9 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Check for a corrupted size being too large past the end of the file. RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test index 06f418d0235d..c967e850dbe3 100644 --- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -5,20 +5,25 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -35,7 +40,9 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index b718cf0fd8e9..e1e33824bf2f 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -5,20 +5,26 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -35,8 +41,10 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Make NumCounters = 0 so that we get "number of counters is zero" error message RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test index 38e40334a6a6..3c23bc7dd0f7 100644 --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -5,20 +5,25 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -38,10 +43,12 @@ RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw // Octal '\11' is 9 in decimal: this should push CounterOffset to 1. As there are two counters, // the profile reader should error out. RUN: printf '\11\0\6\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Counter Section diff --git a/llvm/test/tools/llvm-profdata/mcdc-bitmap.test b/llvm/test/tools/llvm-profdata/mcdc-bitmap.test new file mode 100644 index 000000000000..a7b1b5df8c30 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/mcdc-bitmap.test @@ -0,0 +1,201 @@ +# Test MC/DC bitmap reading and merging. + +# Merge as profdata. +RUN: split-file %s %t +RUN: llvm-profdata merge %t/mcdc-1.proftext %t/mcdc-2.proftext -o %t.profdata +RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC +# Merge as proftext. +RUN: llvm-profdata merge %t/mcdc-1.proftext %t/mcdc-2.proftext -o %t.proftext +RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC + +MCDC: # Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: # Bitmap Byte Values: +MCDC-NEXT: a +MCDC: # Num Bitmap Bytes: +MCDC-NEXT: $2 +MCDC-NEXT: # Bitmap Byte Values: +MCDC-NEXT: 0x29 +MCDC-NEXT: 0x0 + +# Merge as profdata. +RUN: llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-4.proftext -o %t.profdata +RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC2 +# Merge as proftext. +RUN: llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-4.proftext -o %t.proftext +RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC2 + +MCDC2: # Num Bitmap Bytes: +MCDC2-NEXT: $8 +MCDC2-NEXT: # Bitmap Byte Values: +MCDC2-NEXT: 0x1 +MCDC2-NEXT: 0x2 +MCDC2-NEXT: 0x3 +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xe +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xa + +# Incompatible size mismatch. +RUN: llvm-profdata merge %t/mcdc-2.proftext %t/mcdc-4.proftext -o %t.profdata 2>&1 | FileCheck %s --check-prefix=MCDC3 +# Merge as proftext +RUN: llvm-profdata merge %t/mcdc-2.proftext %t/mcdc-4.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC3 + +MCDC3: function bitmap size change detected (bitmap size mismatch) + +# Invalid number of bitmap bytes. +RUN: not llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-err0.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC4 + +MCDC4: malformed instrumentation profile data: number of bitmap bytes is not a valid integer + +# Invalid bitmap byte. +RUN: not llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-err1.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC5 + +MCDC5: malformed instrumentation profile data: bitmap byte is not a valid integer + +;--- mcdc-1.proftext +main +# Func Hash: +702755447896 +# Num Counters: +4 +# Counter Values: +1 +0 +1 +0 +# Num Bitmask Bytes: +$1 +# Bitmask Byte Values: +2 +;--- mcdc-2.proftext +main +# Func Hash: +702755447896 +# Num Counters: +4 +# Counter Values: +1 +1 +1 +1 +# Num Bitmask Bytes: +$1 +# Bitmask Byte Values: +8 + + +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$0x2 +# Bitmask Byte Values: +0x29 +0x0 +;--- mcdc-3.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8 +# Bitmask Byte Values: +0x0 +0x2 +0x3 +0xf +0xf +0xa +0xc +0x2 +;--- mcdc-4.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$ 8 +# Bitmask Byte Values: +1 +2 +3 +4 +5 +6 +7 +8 +;--- mcdc-err0.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8.9 +# Bitmask Byte Values: +1 +2 +3 +4 +5 +6 +7 +8 +;--- mcdc-err1.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8 +# Bitmask Byte Values: +1 +2 +3 +4 +5.4 +6 +7 +8 diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test index 171b5cc60878..4a5c42843ff4 100644 --- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test index 24f3f563e968..389646d64b1c 100644 --- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -8,6 +8,9 @@ RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test index c8e862009ef0..fbd31d044382 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,37 +1,46 @@ RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\0\0\0\11' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\0\1\0\0\0' >> %t +RUN: printf '\0\0\0\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\1\0\0\0' >> %t +RUN: printf '\3\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\3' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t -RUN: printf '\0\xff\xff\xe0' >> %t +RUN: printf '\0\xff\xff\xd8' >> %t +RUN: printf '\2\xff\xff\xd3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\2' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\1' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -48,3 +57,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test index 523ff1ceb480..bb899c5fdb55 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,37 +1,46 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\10\0\0\0\0\0\0\0' >> %t +RUN: printf '\11\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\0\0\0\2\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t +RUN: printf '\0\0\0\3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\3\0\0\0' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xe0\xff\xff\0' >> %t +RUN: printf '\xd8\xff\xff\0' >> %t +RUN: printf '\xd3\xff\xff\2' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\2\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\1\0\0\0' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -48,3 +57,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test index b2b8b31dafbf..8fcadb6a0dd2 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,35 +1,44 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\0\0\0\11' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t +RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t +RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\02' >> %t -RUN: printf '\0\0\0\1\0\3\xff\xd8' >> %t +RUN: printf '\0\0\0\1\0\3\xff\xc8' >> %t +RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\02\0\0\0\0' >> %t +RUN: printf '\0\0\0\1\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -46,3 +55,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test index 4e95798bc0af..0aa8b38f6926 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,35 +1,44 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\10\0\0\0\0\0\0\0' >> %t +RUN: printf '\11\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t +RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\4\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t +RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t +RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t +RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t +RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -46,3 +55,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test index 8d46c91e2732..f4a9aa8e1bbc 100644 --- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,12 +1,15 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -15,20 +18,25 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw @@ -37,7 +45,9 @@ RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw -- Gitee From 8208f4fec232123f3d2daaf7c17d5f7a38dd5647 Mon Sep 17 00:00:00 2001 From: Hans Wennborg Date: Thu, 21 Sep 2023 12:04:40 +0200 Subject: [PATCH 04/28] Revert "[InstrProf][compiler-rt] Enable MC/DC Support in LLVM Source-based Code Coverage (1/3)" This seems to cause Clang to crash, see comments on the code review. Reverting until the problem can be investigated. > Part 1 of 3. This includes the LLVM back-end processing and profile > reading/writing components. compiler-rt changes are included. > > Differential Revision: https://reviews.llvm.org/D138846 This reverts commit a50486fd736ab2fe03fcacaf8b98876db77217a7. --- .../CodeGen/coverage-profile-raw-version.c | 4 +- compiler-rt/include/profile/InstrProfData.inc | 22 +- compiler-rt/lib/profile/InstrProfiling.c | 4 - compiler-rt/lib/profile/InstrProfiling.h | 25 +- .../lib/profile/InstrProfilingBuffer.c | 42 +- compiler-rt/lib/profile/InstrProfilingFile.c | 50 +-- .../lib/profile/InstrProfilingInternal.h | 8 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 33 +- .../profile/InstrProfilingPlatformDarwin.c | 9 - .../lib/profile/InstrProfilingPlatformLinux.c | 10 - .../lib/profile/InstrProfilingPlatformOther.c | 4 - .../profile/InstrProfilingPlatformWindows.c | 7 - .../lib/profile/InstrProfilingWriter.c | 18 +- .../profile/instrprof-write-buffer-internal.c | 21 +- llvm/docs/LangRef.rst | 138 ------- llvm/include/llvm/IR/IntrinsicInst.h | 94 +---- llvm/include/llvm/IR/Intrinsics.td | 15 - .../ProfileData/Coverage/CoverageMapping.h | 4 +- llvm/include/llvm/ProfileData/InstrProf.h | 21 +- .../llvm/ProfileData/InstrProfData.inc | 22 +- .../llvm/ProfileData/InstrProfReader.h | 9 - .../Instrumentation/InstrProfiling.h | 46 +-- .../SelectionDAG/SelectionDAGBuilder.cpp | 6 - llvm/lib/IR/IntrinsicInst.cpp | 4 +- .../Coverage/CoverageMappingReader.cpp | 4 - llvm/lib/ProfileData/InstrProf.cpp | 23 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 4 - llvm/lib/ProfileData/InstrProfReader.cpp | 109 +---- llvm/lib/ProfileData/InstrProfWriter.cpp | 19 - .../Instrumentation/InstrProfiling.cpp | 379 ++++-------------- .../Instrumentation/InstrProfiling/mcdc.ll | 53 --- .../Transforms/PGOProfile/comdat_internal.ll | 4 +- .../tools/llvm-profdata/Inputs/basic.profraw | Bin 192 -> 152 bytes .../llvm-profdata/Inputs/c-general.profraw | Bin 2016 -> 1800 bytes .../llvm-profdata/Inputs/compressed.profraw | Bin 1968 -> 1768 bytes .../llvm-profdata/binary-ids-padding.test | 13 +- .../llvm-profdata/large-binary-id-size.test | 5 +- ...alformed-not-space-for-another-header.test | 9 +- .../malformed-num-counters-zero.test | 10 +- .../malformed-ptr-to-counter-array.test | 9 +- .../test/tools/llvm-profdata/mcdc-bitmap.test | 201 ---------- .../misaligned-binary-ids-size.test | 2 +- .../mismatched-raw-profile-header.test | 3 - .../tools/llvm-profdata/raw-32-bits-be.test | 28 +- .../tools/llvm-profdata/raw-32-bits-le.test | 28 +- .../tools/llvm-profdata/raw-64-bits-be.test | 24 +- .../tools/llvm-profdata/raw-64-bits-le.test | 24 +- .../tools/llvm-profdata/raw-two-profiles.test | 14 +- 48 files changed, 171 insertions(+), 1410 deletions(-) delete mode 100644 llvm/test/Instrumentation/InstrProfiling/mcdc.ll delete mode 100644 llvm/test/tools/llvm-profdata/mcdc-bitmap.test diff --git a/clang/test/CodeGen/coverage-profile-raw-version.c b/clang/test/CodeGen/coverage-profile-raw-version.c index bb30fd8c1c70..749dce50298f 100644 --- a/clang/test/CodeGen/coverage-profile-raw-version.c +++ b/clang/test/CodeGen/coverage-profile-raw-version.c @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -debug-info-kind=standalone -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -debug-info-kind=standalone -mllvm -debug-info-correlate -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s --check-prefix=DEBUG_INFO -// CHECK: @__llvm_profile_raw_version = {{.*}}constant i64 9 -// DEBUG_INFO: @__llvm_profile_raw_version = {{.*}}constant i64 576460752303423497 +// CHECK: @__llvm_profile_raw_version = {{.*}}constant i64 8 +// DEBUG_INFO: @__llvm_profile_raw_version = {{.*}}constant i64 576460752303423496 int main() { return 0; diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index fad14576c442..4456bf1ab176 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -76,7 +76,6 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) -INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -88,9 +87,7 @@ INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ -INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -135,13 +132,9 @@ INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) -INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) -INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) -INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, - (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -274,9 +267,6 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ - INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ - INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -656,11 +646,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 9 +#define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 11 +#define INSTR_PROF_INDEX_VERSION 10 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 6 +#define INSTR_PROF_COVMAP_VERSION 5 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -697,7 +687,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts -#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -709,7 +698,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" -#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -721,7 +709,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF -#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -736,7 +723,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) -#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c index da04d8ebdec9..0dd5ff5ae633 100644 --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -60,10 +60,6 @@ COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0; memset(I, ResetValue, E - I); - I = __llvm_profile_begin_bitmap(); - E = __llvm_profile_end_bitmap(); - memset(I, 0x0, E - I); - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const __llvm_profile_data *DI; diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index e143149fca82..4433d7bd4887 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -88,8 +88,6 @@ const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); -char *__llvm_profile_begin_bitmap(void); -char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); uint32_t *__llvm_profile_begin_orderfile(); @@ -103,11 +101,11 @@ void __llvm_profile_reset_counters(void); /*! * \brief Merge profile data from buffer. * - * Read profile data from buffer \p Profile and merge with in-process profile - * counters and bitmaps. The client is expected to have checked or already - * know the profile data in the buffer matches the in-process counter - * structure before calling it. Returns 0 (success) if the profile data is - * valid. Upon reading invalid/corrupted profile data, returns 1 (failure). + * Read profile data form buffer \p Profile and merge with in-process profile + * counters. The client is expected to have checked or already knows the profile + * data in the buffer matches the in-process counter structure before calling + * it. Returns 0 (success) if the profile data is valid. Upon reading + * invalid/corrupted profile data, returns 1 (failure). */ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); @@ -115,8 +113,8 @@ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); * * Returns 0 (success) if the profile data in buffer \p Profile with size * \p Size was generated by the same binary and therefore matches - * structurally the in-process counters and bitmaps. If the profile data in - * buffer is not compatible, the interface returns 1 (failure). + * structurally the in-process counters. If the profile data in buffer is + * not compatible, the interface returns 1 (failure). */ int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size); @@ -278,10 +276,6 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End); /*! \brief Get the size of the profile counters section in bytes. */ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); -/*! \brief Get the number of bytes in the profile bitmap section. */ -uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, - const char *End); - /* ! \brief Given the sizes of the data and counter information, return the * number of padding bytes before and after the counters, and after the names, * in the raw profile. @@ -292,9 +286,8 @@ uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, * needed to achieve that. */ void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, - uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, - uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmap, + uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterNames); /*! diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index c7217b2dfef8..61ac5d9c0285 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -43,14 +43,11 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); - const char *BitmapBegin = __llvm_profile_begin_bitmap(); - const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return __llvm_profile_get_size_for_buffer_internal( - DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, - NamesBegin, NamesEnd); + DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd); } COMPILER_RT_VISIBILITY @@ -86,12 +83,6 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { __llvm_profile_counter_entry_size(); } -COMPILER_RT_VISIBILITY -uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, - const char *End) { - return (End - Begin); -} - /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { @@ -111,16 +102,13 @@ static int needsCounterPadding(void) { COMPILER_RT_VISIBILITY void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, - uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, - uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes, + uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterNames) { if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = __llvm_profile_get_num_padding_bytes(CountersSize); - *PaddingBytesAfterBitmapBytes = - __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); return; } @@ -130,37 +118,31 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); - *PaddingBytesAfterBitmapBytes = - calculateBytesNeededToPageAlign(NumBitmapBytes); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) { + const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, + const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); - const uint64_t NumBitmapBytes = - __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; + PaddingBytesAfterNames; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + - PaddingBytesAfterCounters + NumBitmapBytes + - PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames; + PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames; } COMPILER_RT_VISIBILITY @@ -178,11 +160,9 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, - const char *NamesBegin, const char *NamesEnd) { + const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin, - NamesEnd, 0); + CountersEnd, 0, NamesBegin, NamesEnd, 0); } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 49fef96578c8..2bd6a49ce065 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -108,18 +108,14 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); - const char *BitmapBegin = __llvm_profile_begin_bitmap(); - const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); - uint64_t NumBitmapBytes = - __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); - /* Check that the counter, bitmap, and data sections in this image are + /* Check that the counter and data sections in this image are * page-aligned. */ unsigned PageSize = getpagesize(); if ((intptr_t)CountersBegin % PageSize != 0) { @@ -127,11 +123,6 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { CountersBegin, PageSize); return 1; } - if ((intptr_t)BitmapBegin % PageSize != 0) { - PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n", - BitmapBegin, PageSize); - return 1; - } if ((intptr_t)DataBegin % PageSize != 0) { PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", DataBegin, PageSize); @@ -141,11 +132,10 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Determine how much padding is needed before/after the counters and * after the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; + PaddingBytesAfterNames; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; uint64_t FileOffsetToCounters = CurrentFileOffset + @@ -165,31 +155,6 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { FileOffsetToCounters); return 1; } - - /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() - * will fail with EINVAL. */ - if (NumBitmapBytes == 0) - return 0; - - uint64_t PageAlignedBitmapLength = - NumBitmapBytes + PaddingBytesAfterBitmapBytes; - uint64_t FileOffsetToBitmap = - CurrentFileOffset + sizeof(__llvm_profile_header) + DataSize + - PaddingBytesBeforeCounters + CountersSize + PaddingBytesAfterCounters; - void *BitmapMmap = - mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap); - if (BitmapMmap != BitmapBegin) { - PROF_ERR( - "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" - " - BitmapBegin: %p\n" - " - PageAlignedBitmapLength: %" PRIu64 "\n" - " - Fileno: %d\n" - " - FileOffsetToBitmap: %" PRIu64 "\n", - strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno, - FileOffsetToBitmap); - return 1; - } return 0; } #elif defined(__ELF__) || defined(_WIN32) @@ -232,8 +197,6 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); - const char *BitmapBegin = __llvm_profile_begin_bitmap(); - const char *BitmapEnd = __llvm_profile_end_bitmap(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); /* Get the file size. */ uint64_t FileSize = 0; @@ -255,11 +218,6 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); - - /* BIAS MODE not supported yet for Bitmap (MCDC). */ - - /* Return the memory allocated for counters to OS. */ - lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); return 0; } #else diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h index 03ed67fcfa76..360165e32ab3 100644 --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -21,8 +21,8 @@ */ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, + const char *NamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -36,8 +36,7 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, - const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); /*! * The data structure describing the data to be written by the @@ -154,7 +153,6 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, - const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite); diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 520fb561e4d3..088ce71308ee 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -67,9 +67,6 @@ int __llvm_profile_check_compatibility(const char *ProfileData, Header->NumCounters != __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), __llvm_profile_end_counters()) || - Header->NumBitmapBytes != - __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), - __llvm_profile_end_bitmap()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()) || Header->ValueKindLast != IPVK_Last) @@ -78,8 +75,7 @@ int __llvm_profile_check_compatibility(const char *ProfileData, if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + - Header->NumCounters * __llvm_profile_counter_entry_size() + - Header->NumBitmapBytes) + Header->NumCounters * __llvm_profile_counter_entry_size()) return 1; for (SrcData = SrcDataStart, @@ -87,8 +83,7 @@ int __llvm_profile_check_compatibility(const char *ProfileData, SrcData < SrcDataEnd; ++SrcData, ++DstData) { if (SrcData->NameRef != DstData->NameRef || SrcData->FuncHash != DstData->FuncHash || - SrcData->NumCounters != DstData->NumCounters || - SrcData->NumBitmapBytes != DstData->NumBitmapBytes) + SrcData->NumCounters != DstData->NumCounters) return 1; } @@ -120,11 +115,9 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, char *SrcCountersStart, *DstCounter; const char *SrcCountersEnd, *SrcCounter; - const char *SrcBitmapStart; const char *SrcNameStart; const char *SrcValueProfDataStart, *SrcValueProfData; uintptr_t CountersDelta = Header->CountersDelta; - uintptr_t BitmapDelta = Header->BitmapDelta; SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + @@ -133,12 +126,11 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, SrcCountersStart = (char *)SrcDataEnd; SrcCountersEnd = SrcCountersStart + Header->NumCounters * __llvm_profile_counter_entry_size(); - SrcBitmapStart = SrcCountersEnd; - SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; + SrcNameStart = SrcCountersEnd; SrcValueProfDataStart = SrcNameStart + Header->NamesSize + __llvm_profile_get_num_padding_bytes(Header->NamesSize); - if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) + if (SrcNameStart < SrcCountersStart) return 1; // Merge counters when there is no data section and debug info correlation is @@ -172,8 +164,6 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // extend CounterPtr to get the original value. char *DstCounters = (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); - char *DstBitmap = - (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); unsigned NVK = 0; // SrcData is a serialized representation of the memory image. We need to @@ -203,21 +193,6 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, } } - const char *SrcBitmap = - SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); - // BitmapDelta also needs to be decreased as we advance to the next data - // record. - BitmapDelta -= sizeof(*SrcData); - unsigned NB = SrcData->NumBitmapBytes; - // NumBitmapBytes may legitimately be 0. Just keep going. - if (NB != 0) { - if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) - return 1; - // Merge Src and Dst Bitmap bytes by simply ORing them together. - for (unsigned I = 0; I < NB; I++) - DstBitmap[I] |= SrcBitmap[I]; - } - /* Now merge value profile data. */ if (!VPMergeHook) continue; diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c index 2154d242a817..d9f2a113f5b0 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -31,11 +31,6 @@ extern char COMPILER_RT_VISIBILITY extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY -extern char - BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME); -COMPILER_RT_VISIBILITY -extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); -COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -61,10 +56,6 @@ char *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &CountersEnd; } COMPILER_RT_VISIBILITY -char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } -COMPILER_RT_VISIBILITY -char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } -COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index d0c42462e5e3..2cce0a4b2c48 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -35,8 +35,6 @@ #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) -#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) -#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) #define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) #define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) @@ -50,8 +48,6 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; -extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; -extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -78,12 +74,6 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } -COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) { - return &PROF_BITS_START; -} -COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { - return &PROF_BITS_STOP; -} COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &PROF_ORDERFILE_START; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index 5319ca813b43..c7b6e842c9fa 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -88,10 +88,6 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return CountersLast; } -COMPILER_RT_VISIBILITY -char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; } -COMPILER_RT_VISIBILITY -char *__llvm_profile_end_bitmap(void) { return BitmapLast; } /* TODO: correctly set up OrderFileFirst. */ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 9dbd702865fd..dd576b2f8357 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -14,7 +14,6 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ #pragma comment(linker, "/MERGE:.lprfc=.data") -#pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") @@ -31,8 +30,6 @@ #pragma section(".lprfd$Z", read, write) #pragma section(".lprfc$A", read, write) #pragma section(".lprfc$Z", read, write) -#pragma section(".lprfb$A", read, write) -#pragma section(".lprfb$Z", read, write) #pragma section(".lorderfile$A", read, write) #pragma section(".lprfnd$A", read, write) #pragma section(".lprfnd$Z", read, write) @@ -46,8 +43,6 @@ const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; char COMPILER_RT_SECTION(".lprfc$A") CountersStart; char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; -char COMPILER_RT_SECTION(".lprfb$A") BitmapStart; -char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd; uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; @@ -63,8 +58,6 @@ const char *__llvm_profile_end_names(void) { return &NamesEnd; } char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } char *__llvm_profile_end_counters(void) { return &CountersEnd; } -char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } -char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 3b61f3def9f6..1e22398a4c0f 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -246,20 +246,17 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); - const char *BitmapBegin = __llvm_profile_begin_bitmap(); - const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, - CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, - NamesBegin, NamesEnd, SkipNameDataWrite); + CountersEnd, VPDataReader, NamesBegin, NamesEnd, + SkipNameDataWrite); } COMPILER_RT_VISIBILITY int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, - const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { int DebugInfoCorrelate = @@ -274,8 +271,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumCounters = __llvm_profile_get_num_counters(CountersBegin, CountersEnd); - const uint64_t NumBitmapBytes = - __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; /* Create the header. */ @@ -284,11 +279,11 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; + PaddingBytesAfterNames; __llvm_profile_get_padding_sizes_for_counters( - DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, + DataSectionSize, CountersSectionSize, NamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + &PaddingBytesAfterNames); { /* Initialize header structure. */ @@ -300,7 +295,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, * CountersDelta to match. */ #ifdef _WIN64 Header.CountersDelta = (uint32_t)Header.CountersDelta; - Header.BitmapDelta = (uint32_t)Header.BitmapDelta; #endif /* The data and names sections are omitted in lightweight mode. */ @@ -325,8 +319,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, - {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, - {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c index d9670f739ca9..7b96c6d91c33 100644 --- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -25,18 +25,17 @@ const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); -char *__llvm_profile_begin_bitmap(void); -char *__llvm_profile_end_bitmap(void); uint64_t __llvm_profile_get_size_for_buffer_internal( const void *DataBegin, const void *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, - const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); -int __llvm_profile_write_buffer_internal( - char *Buffer, const void *DataBegin, const void *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); +int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin, + const void *DataEnd, + const char *CountersBegin, + const char *CountersEnd, + const char *NamesBegin, + const char *NamesEnd); void __llvm_profile_set_dumped(void); @@ -44,14 +43,12 @@ int main(int argc, const char *argv[]) { uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal( __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), - __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); char *buf = malloc(bufsize); - int ret = __llvm_profile_write_buffer_internal( - buf, __llvm_profile_begin_data(), __llvm_profile_end_data(), + int ret = __llvm_profile_write_buffer_internal(buf, + __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), - __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); if (ret != 0) { diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index e9caee30b1a3..2f4d01c4cddd 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -13820,144 +13820,6 @@ pass will generate the appropriate data structures and replace the ``llvm.instrprof.value.profile`` intrinsic with the call to the profile runtime library with proper arguments. -'``llvm.instrprof.mcdc.parameters``' Intrinsic -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Syntax: -""""""" - -:: - - declare void @llvm.instrprof.mcdc.parameters(ptr , i64 , - i32 ) - -Overview: -""""""""" - -The '``llvm.instrprof.mcdc.parameters``' intrinsic is used to initiate MC/DC -code coverage instrumentation for a function. - -Arguments: -"""""""""" - -The first argument is a pointer to a global variable containing the -name of the entity being instrumented. This should generally be the -(mangled) function name for a set of counters. - -The second argument is a hash value that can be used by the consumer -of the profile data to detect changes to the instrumented source. - -The third argument is the number of bitmap bytes required by the function to -record the number of test vectors executed for each boolean expression. - -Semantics: -"""""""""" - -This intrinsic represents basic MC/DC parameters initiating one or more MC/DC -instrumentation sequences in a function. It will cause the ``-instrprof`` pass -to generate the appropriate data structures and the code to instrument MC/DC -test vectors in a format that can be written out by a compiler runtime and -consumed via the ``llvm-profdata`` tool. - -'``llvm.instrprof.mcdc.condbitmap.update``' Intrinsic -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Syntax: -""""""" - -:: - - declare void @llvm.instrprof.mcdc.condbitmap.update(ptr , i64 , - i32 , - ptr , - i1 ) - -Overview: -""""""""" - -The '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic is used to track -MC/DC condition evaluation for each condition in a boolean expression. - -Arguments: -"""""""""" - -The first argument is a pointer to a global variable containing the -name of the entity being instrumented. This should generally be the -(mangled) function name for a set of counters. - -The second argument is a hash value that can be used by the consumer -of the profile data to detect changes to the instrumented source. - -The third argument is an ID of a condition to track. This value is used as a -bit index into the condition bitmap. - -The fourth argument is the address of the condition bitmap. - -The fifth argument is the boolean value representing the evaluation of the -condition (true or false) - -Semantics: -"""""""""" - -This intrinsic represents the update of a condition bitmap that is local to a -function and will cause the ``-instrprof`` pass to generate the code to -instrument the control flow around each condition in a boolean expression. The -ID of each condition corresponds to a bit index in the condition bitmap which -is set based on the evaluation of the condition. - -'``llvm.instrprof.mcdc.tvbitmap.update``' Intrinsic -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Syntax: -""""""" - -:: - - declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr , i64 , - i32 ) - i32 , - ptr ) - -Overview: -""""""""" - -The '``llvm.instrprof.mcdc.tvbitmap.update``' intrinsic is used to track MC/DC -test vector execution after each boolean expression has been fully executed. -The overall value of the condition bitmap, after it has been successively -updated using the '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic with -the true or false evaluation of each condition, uniquely identifies an executed -MC/DC test vector and is used as a bit index into the global test vector -bitmap. - -Arguments: -"""""""""" - -The first argument is a pointer to a global variable containing the -name of the entity being instrumented. This should generally be the -(mangled) function name for a set of counters. - -The second argument is a hash value that can be used by the consumer -of the profile data to detect changes to the instrumented source. - -The third argument is the number of bitmap bytes required by the function to -record the number of test vectors executed for each boolean expression. - -The fourth argument is the byte index into the global test vector bitmap -corresponding to the function. - -The fifth argument is the address of the condition bitmap, which contains a -value representing an executed MC/DC test vector. It is loaded and used as the -bit index of the test vector bitmap. - -Semantics: -"""""""""" - -This intrinsic represents the final operation of an MC/DC instrumentation -sequence and will cause the ``-instrprof`` pass to generate the code to -instrument an update of a function's global test vector bitmap to indicate that -a test vector has been executed. The global test vector bitmap can be consumed -by the ``llvm-profdata`` and ``llvm-cov`` tools. - '``llvm.thread.pointer``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index f04b5ae152f2..62bd833198f0 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1399,11 +1399,6 @@ public: ConstantInt *getHash() const { return cast(const_cast(getArgOperand(1))); } -}; - -/// A base class for all instrprof counter intrinsics. -class InstrProfCntrInstBase : public InstrProfInstBase { -public: // The number of counters for the instrumented function. ConstantInt *getNumCounters() const; // The index of the counter that this instruction acts on. @@ -1411,7 +1406,7 @@ public: }; /// This represents the llvm.instrprof.cover intrinsic. -class InstrProfCoverInst : public InstrProfCntrInstBase { +class InstrProfCoverInst : public InstrProfInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_cover; @@ -1422,7 +1417,7 @@ public: }; /// This represents the llvm.instrprof.increment intrinsic. -class InstrProfIncrementInst : public InstrProfCntrInstBase { +class InstrProfIncrementInst : public InstrProfInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_increment || @@ -1446,7 +1441,7 @@ public: }; /// This represents the llvm.instrprof.timestamp intrinsic. -class InstrProfTimestampInst : public InstrProfCntrInstBase { +class InstrProfTimestampInst : public InstrProfInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_timestamp; @@ -1457,7 +1452,7 @@ public: }; /// This represents the llvm.instrprof.value.profile intrinsic. -class InstrProfValueProfileInst : public InstrProfCntrInstBase { +class InstrProfValueProfileInst : public InstrProfInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_value_profile; @@ -1480,87 +1475,6 @@ public: } }; -/// A base class for instrprof mcdc intrinsics that require global bitmap bytes. -class InstrProfMCDCBitmapInstBase : public InstrProfInstBase { -public: - static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters || - I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; - } - static bool classof(const Value *V) { - return isa(V) && classof(cast(V)); - } - - /// \return The number of bytes used for the MCDC bitmaps for the instrumented - /// function. - ConstantInt *getNumBitmapBytes() const { - return cast(const_cast(getArgOperand(2))); - } -}; - -/// This represents the llvm.instrprof.mcdc.parameters intrinsic. -class InstrProfMCDCBitmapParameters : public InstrProfMCDCBitmapInstBase { -public: - static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters; - } - static bool classof(const Value *V) { - return isa(V) && classof(cast(V)); - } -}; - -/// This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic. -class InstrProfMCDCTVBitmapUpdate : public InstrProfMCDCBitmapInstBase { -public: - static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; - } - static bool classof(const Value *V) { - return isa(V) && classof(cast(V)); - } - - /// \return The index of the TestVector Bitmap upon which this intrinsic - /// acts. - ConstantInt *getBitmapIndex() const { - return cast(const_cast(getArgOperand(3))); - } - - /// \return The address of the corresponding condition bitmap containing - /// the index of the TestVector to update within the TestVector Bitmap. - Value *getMCDCCondBitmapAddr() const { - return cast(const_cast(getArgOperand(4))); - } -}; - -/// This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic. -/// It does not pertain to global bitmap updates or parameters and so doesn't -/// inherit from InstrProfMCDCBitmapInstBase. -class InstrProfMCDCCondBitmapUpdate : public InstrProfInstBase { -public: - static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_condbitmap_update; - } - static bool classof(const Value *V) { - return isa(V) && classof(cast(V)); - } - - /// \return The ID of the condition to update. - ConstantInt *getCondID() const { - return cast(const_cast(getArgOperand(2))); - } - - /// \return The address of the corresponding condition bitmap. - Value *getMCDCCondBitmapAddr() const { - return cast(const_cast(getArgOperand(3))); - } - - /// \return The boolean value to set in the condition bitmap for the - /// corresponding condition ID. This represents how the condition evaluated. - Value *getCondBool() const { - return cast(const_cast(getArgOperand(4))); - } -}; - class PseudoProbeInst : public IntrinsicInst { public: static bool classof(const IntrinsicInst *I) { diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index a47d94452db4..20a8fa419465 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -909,21 +909,6 @@ def int_instrprof_value_profile : Intrinsic<[], llvm_i64_ty, llvm_i32_ty, llvm_i32_ty]>; -// A parameter configuration for instrumentation based MCDC profiling. -def int_instrprof_mcdc_parameters : Intrinsic<[], - [llvm_ptr_ty, llvm_i64_ty, - llvm_i32_ty]>; - -// A test vector bitmap update for instrumentation based MCDC profiling. -def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[], - [llvm_ptr_ty, llvm_i64_ty, - llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty]>; - -// A condition bitmap value update for instrumentation based MCDC profiling. -def int_instrprof_mcdc_condbitmap_update : Intrinsic<[], - [llvm_ptr_ty, llvm_i64_ty, - llvm_i32_ty, llvm_ptr_ty, llvm_i1_ty]>; - def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty]>; def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>; diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 12498630190d..3c8f940ba97b 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -1023,9 +1023,7 @@ enum CovMapVersion { // Compilation directory is stored separately and combined with relative // filenames to produce an absolute file path. Version6 = 5, - // Branch regions extended and Decision Regions added for MC/DC. - Version7 = 6, - // The current version is Version7. + // The current version is Version6. CurrentVersion = INSTR_PROF_COVMAP_VERSION }; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 2a780104b177..f9096b461572 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -96,9 +96,6 @@ inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } /// Return the name prefix of profile counter variables. inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } -/// Return the name prefix of profile bitmap variables. -inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; } - /// Return the name prefix of value profile variables. inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } @@ -338,7 +335,6 @@ enum class instrprof_error { invalid_prof, hash_mismatch, count_mismatch, - bitmap_mismatch, counter_overflow, value_site_count_mismatch, compress_failed, @@ -694,23 +690,18 @@ struct InstrProfValueSiteRecord { /// Profiling information for a single function. struct InstrProfRecord { std::vector Counts; - std::vector BitmapBytes; InstrProfRecord() = default; InstrProfRecord(std::vector Counts) : Counts(std::move(Counts)) {} - InstrProfRecord(std::vector Counts, - std::vector BitmapBytes) - : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {} InstrProfRecord(InstrProfRecord &&) = default; InstrProfRecord(const InstrProfRecord &RHS) - : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes), + : Counts(RHS.Counts), ValueData(RHS.ValueData ? std::make_unique(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { Counts = RHS.Counts; - BitmapBytes = RHS.BitmapBytes; if (!RHS.ValueData) { ValueData = nullptr; return *this; @@ -889,11 +880,6 @@ struct NamedInstrProfRecord : InstrProfRecord { NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} - NamedInstrProfRecord(StringRef Name, uint64_t Hash, - std::vector Counts, - std::vector BitmapBytes) - : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name), - Hash(Hash) {} static bool hasCSFlagInHash(uint64_t FuncHash) { return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); @@ -1029,9 +1015,7 @@ enum ProfVersion { Version9 = 9, // An additional (optional) temporal profile traces section is added. Version10 = 10, - // An additional field is used for bitmap bytes. - Version11 = 11, - // The current version is 11. + // The current version is 10. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1169,7 +1153,6 @@ namespace RawInstrProf { // Version 6: Added binary id. // Version 7: Reorder binary id and include version in signature. // Version 8: Use relative counter pointer. -// Version 9: Added relative bitmap bytes pointer and count used by MC/DC. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index fad14576c442..4456bf1ab176 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -76,7 +76,6 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) -INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -88,9 +87,7 @@ INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ -INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -135,13 +132,9 @@ INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) -INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) -INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) -INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, - (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -274,9 +267,6 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ - INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ - INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -656,11 +646,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 9 +#define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 11 +#define INSTR_PROF_INDEX_VERSION 10 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 6 +#define INSTR_PROF_COVMAP_VERSION 5 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -697,7 +687,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts -#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -709,7 +698,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" -#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -721,7 +709,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF -#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -736,7 +723,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) -#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 51d86ee0bce3..74e921e10c47 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -323,14 +323,11 @@ private: // the variant types of the profile. uint64_t Version; uint64_t CountersDelta; - uint64_t BitmapDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; const char *CountersStart; const char *CountersEnd; - const char *BitmapStart; - const char *BitmapEnd; const char *NamesStart; const char *NamesEnd; // After value profile is all read, this pointer points to @@ -432,7 +429,6 @@ private: Error readName(NamedInstrProfRecord &Record); Error readFuncHash(NamedInstrProfRecord &Record); Error readRawCounts(InstrProfRecord &Record); - Error readRawBitmapBytes(InstrProfRecord &Record); Error readValueProfilingData(InstrProfRecord &Record); bool atEnd() const { return Data == DataEnd; } @@ -445,7 +441,6 @@ private: // As we advance to the next record, we maintain the correct CountersDelta // with respect to the next record. CountersDelta -= sizeof(*Data); - BitmapDelta -= sizeof(*Data); } Data++; ValueDataStart += CurValueDataSize; @@ -737,10 +732,6 @@ public: Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts); - /// Fill Bitmap Bytes with the profile data for the given function name. - Error getFunctionBitmapBytes(StringRef FuncName, uint64_t FuncHash, - std::vector &BitmapBytes); - /// Return the maximum of all known function counts. /// \c UseCS indicates whether to use the context-sensitive count. uint64_t getMaximumFunctionCount(bool UseCS) { diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index d8f3e75087ac..cb0c055dcb74 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -50,7 +50,6 @@ private: uint32_t NumValueSites[IPVK_Last + 1]; GlobalVariable *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; - GlobalVariable *RegionBitmaps = nullptr; PerFunctionProfileData() { memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); @@ -106,59 +105,20 @@ private: /// Force emitting of name vars for unused functions. void lowerCoverageData(GlobalVariable *CoverageNamesVar); - /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction - /// using the index represented by the a temp value into a bitmap. - void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); - - /// Replace instrprof.mcdc.temp.update with a shift and or instruction using - /// the corresponding condition ID. - void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); - /// Compute the address of the counter value that this profiling instruction /// acts on. - Value *getCounterAddress(InstrProfCntrInstBase *I); + Value *getCounterAddress(InstrProfInstBase *I); /// Get the region counters for an increment, creating them if necessary. /// /// If the counter array doesn't yet exist, the profile data variables /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); + GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc); /// Create the region counters. - GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, - StringRef Name, + GlobalVariable *createRegionCounters(InstrProfInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage); - /// Compute the address of the test vector bitmap that this profiling - /// instruction acts on. - Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); - - /// Get the region bitmaps for an increment, creating them if necessary. - /// - /// If the bitmap array doesn't yet exist, the profile data variables - /// referring to them will also be created. - GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); - - /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with - /// an MC/DC Decision region. The number of bytes required is indicated by - /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called - /// as part of setupProfileSection() and is conceptually very similar to - /// what is done for profile data counters in createRegionCounters(). - GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, - StringRef Name, - GlobalValue::LinkageTypes Linkage); - - /// Set Comdat property of GV, if required. - void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName); - - /// Setup the sections into which counters and bitmaps are allocated. - GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, - InstrProfSectKind IPSK); - - /// Create INSTR_PROF_DATA variable for counters and bitmaps. - void createDataVariable(InstrProfCntrInstBase *Inc, - InstrProfMCDCBitmapParameters *Update); - /// Emit the section with compressed function names. void emitNameData(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 6e11bd28a683..20c37eb4cb11 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7153,12 +7153,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, llvm_unreachable("instrprof failed to lower a timestamp"); case Intrinsic::instrprof_value_profile: llvm_unreachable("instrprof failed to lower a value profiling call"); - case Intrinsic::instrprof_mcdc_parameters: - llvm_unreachable("instrprof failed to lower mcdc parameters"); - case Intrinsic::instrprof_mcdc_tvbitmap_update: - llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update"); - case Intrinsic::instrprof_mcdc_condbitmap_update: - llvm_unreachable("instrprof failed to lower an mcdc condbitmap update"); case Intrinsic::localescape: { MachineFunction &MF = DAG.getMachineFunction(); const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index 4270247ef0f5..36d56699c64e 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -270,13 +270,13 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef NameTable, return -1; } -ConstantInt *InstrProfCntrInstBase::getNumCounters() const { +ConstantInt *InstrProfInstBase::getNumCounters() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("InstrProfValueProfileInst does not have counters!"); return cast(const_cast(getArgOperand(2))); } -ConstantInt *InstrProfCntrInstBase::getIndex() const { +ConstantInt *InstrProfInstBase::getIndex() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()"); return cast(const_cast(getArgOperand(3))); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index cd3970c5151d..05737323314a 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -757,7 +757,6 @@ Expected> CovMapFuncRecordReader::get( case CovMapVersion::Version4: case CovMapVersion::Version5: case CovMapVersion::Version6: - case CovMapVersion::Version7: // Decompress the name data. if (Error E = P.create(P.getNameData())) return std::move(E); @@ -776,9 +775,6 @@ Expected> CovMapFuncRecordReader::get( else if (Version == CovMapVersion::Version6) return std::make_unique>(P, R, D, F); - else if (Version == CovMapVersion::Version7) - return std::make_unique>(P, R, D, F); } llvm_unreachable("Unsupported version"); } diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 7e2ee660978a..835dd697bc7b 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -136,9 +136,6 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::count_mismatch: OS << "function basic block count change detected (counter mismatch)"; break; - case instrprof_error::bitmap_mismatch: - OS << "function bitmap size change detected (bitmap size mismatch)"; - break; case instrprof_error::counter_overflow: OS << "counter overflow"; break; @@ -807,18 +804,6 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, Warn(instrprof_error::counter_overflow); } - // If the number of bitmap bytes doesn't match we either have bad data - // or a hash collision. - if (BitmapBytes.size() != Other.BitmapBytes.size()) { - Warn(instrprof_error::bitmap_mismatch); - return; - } - - // Bitmap bytes are merged by simply ORing them together. - for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) { - BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I]; - } - for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) mergeValueProfData(Kind, Other, Weight, Warn); } @@ -1491,11 +1476,9 @@ Expected
Header::readFromBuffer(const unsigned char *Buffer) { // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version11, + IndexedInstrProf::ProfVersion::CurrentVersion == Version10, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); - case 11ull: - [[fallthrough]]; case 10ull: H.TemporalProfTracesOffset = read(Buffer, offsetOf(&Header::TemporalProfTracesOffset)); @@ -1519,12 +1502,10 @@ size_t Header::size() const { // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); - case 11ull: - [[fallthrough]]; case 10ull: return offsetOf(&Header::TemporalProfTracesOffset) + sizeof(Header::TemporalProfTracesOffset); diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index c917394520df..c822d81f8bef 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -207,15 +207,11 @@ void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, // In this mode, CounterPtr actually stores the section relative address // of the counter. maybeSwap(CounterOffset), - // TODO: MC/DC is not yet supported. - /*BitmapOffset=*/maybeSwap(0), maybeSwap(FunctionPtr), // TODO: Value profiling is not yet supported. /*ValuesPtr=*/maybeSwap(0), maybeSwap(NumCounters), /*NumValueSites=*/{maybeSwap(0), maybeSwap(0)}, - // TODO: MC/DC is not yet supported. - /*NumBitmapBytes=*/maybeSwap(0), }); NamesVec.push_back(FunctionName.str()); } diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index e4ef1b52d415..db20441b712c 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -433,29 +433,6 @@ Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { Record.Counts.push_back(Count); } - // Bitmap byte information is indicated with special character. - if (Line->startswith("$")) { - Record.BitmapBytes.clear(); - // Read the number of bitmap bytes. - uint64_t NumBitmapBytes; - if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes)) - return error(instrprof_error::malformed, - "number of bitmap bytes is not a valid integer"); - if (NumBitmapBytes != 0) { - // Read each bitmap and fill our internal storage with the values. - Record.BitmapBytes.reserve(NumBitmapBytes); - for (uint8_t I = 0; I < NumBitmapBytes; ++I) { - if (Line.is_at_end()) - return error(instrprof_error::truncated); - uint8_t BitmapByte; - if ((Line++)->getAsInteger(0, BitmapByte)) - return error(instrprof_error::malformed, - "bitmap byte is not a valid integer"); - Record.BitmapBytes.push_back(BitmapByte); - } - } - } - // Check if value profile data exists and read it if so. if (Error E = readValueProfileData(Record)) return error(std::move(E)); @@ -572,14 +549,11 @@ Error RawInstrProfReader::readHeader( return error(instrprof_error::bad_header); CountersDelta = swap(Header.CountersDelta); - BitmapDelta = swap(Header.BitmapDelta); NamesDelta = swap(Header.NamesDelta); auto NumData = swap(Header.NumData); auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize(); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); - auto NumBitmapBytes = swap(Header.NumBitmapBytes); - auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes); auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); @@ -589,10 +563,8 @@ Error RawInstrProfReader::readHeader( // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize; ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters; - ptrdiff_t BitmapOffset = - CountersOffset + CountersSize + PaddingBytesAfterCounters; ptrdiff_t NamesOffset = - BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes; + CountersOffset + CountersSize + PaddingBytesAfterCounters; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; auto *Start = reinterpret_cast(&Header); @@ -621,8 +593,6 @@ Error RawInstrProfReader::readHeader( reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); CountersStart = Start + CountersOffset; CountersEnd = CountersStart + CountersSize; - BitmapStart = Start + BitmapOffset; - BitmapEnd = BitmapStart + NumBitmapBytes; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); @@ -713,49 +683,6 @@ Error RawInstrProfReader::readRawCounts( return success(); } -template -Error RawInstrProfReader::readRawBitmapBytes(InstrProfRecord &Record) { - uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes); - - Record.BitmapBytes.clear(); - Record.BitmapBytes.reserve(NumBitmapBytes); - - // It's possible MCDC is either not enabled or only used for some functions - // and not others. So if we record 0 bytes, just move on. - if (NumBitmapBytes == 0) - return success(); - - // BitmapDelta decreases as we advance to the next data record. - ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta; - if (BitmapOffset < 0) - return error( - instrprof_error::malformed, - ("bitmap offset " + Twine(BitmapOffset) + " is negative").str()); - - if (BitmapOffset >= BitmapEnd - BitmapStart) - return error(instrprof_error::malformed, - ("bitmap offset " + Twine(BitmapOffset) + - " is greater than the maximum bitmap offset " + - Twine(BitmapEnd - BitmapStart - 1)) - .str()); - - uint64_t MaxNumBitmapBytes = - (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t); - if (NumBitmapBytes > MaxNumBitmapBytes) - return error(instrprof_error::malformed, - ("number of bitmap bytes " + Twine(NumBitmapBytes) + - " is greater than the maximum number of bitmap bytes " + - Twine(MaxNumBitmapBytes)) - .str()); - - for (uint32_t I = 0; I < NumBitmapBytes; I++) { - const char *Ptr = BitmapStart + BitmapOffset + I; - Record.BitmapBytes.push_back(swap(*Ptr)); - } - - return success(); -} - template Error RawInstrProfReader::readValueProfilingData( InstrProfRecord &Record) { @@ -806,10 +733,6 @@ Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) if (Error E = readRawCounts(Record)) return error(std::move(E)); - // Read raw bitmap bytes and set Record. - if (Error E = readRawBitmapBytes(Record)) - return error(std::move(E)); - // Read value data and set Record. if (Error E = readValueProfilingData(Record)) return error(std::move(E)); @@ -871,7 +794,6 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, DataBuffer.clear(); std::vector CounterBuffer; - std::vector BitmapByteBuffer; const unsigned char *End = D + N; while (D < End) { @@ -897,24 +819,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); - // Read bitmap bytes for GET_VERSION(FormatVersion) > 8. - if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version8) { - uint64_t BitmapBytes = 0; - if (D + sizeof(uint64_t) > End) - return data_type(); - BitmapBytes = endian::readNext(D); - // Read bitmap byte values. - if (D + BitmapBytes * sizeof(uint8_t) > End) - return data_type(); - BitmapByteBuffer.clear(); - BitmapByteBuffer.reserve(BitmapBytes); - for (uint64_t J = 0; J < BitmapBytes; ++J) - BitmapByteBuffer.push_back(static_cast( - endian::readNext(D))); - } - - DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer), - std::move(BitmapByteBuffer)); + DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && @@ -1414,16 +1319,6 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, return success(); } -Error IndexedInstrProfReader::getFunctionBitmapBytes( - StringRef FuncName, uint64_t FuncHash, std::vector &BitmapBytes) { - Expected Record = getInstrProfRecord(FuncName, FuncHash); - if (Error E = Record.takeError()) - return error(std::move(E)); - - BitmapBytes = Record.get().BitmapBytes; - return success(); -} - Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { ArrayRef Data; diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index d8b8571bd5c9..b74d5c3862d8 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -131,8 +131,6 @@ public: M += sizeof(uint64_t); // The function hash M += sizeof(uint64_t); // The size of the Counts vector M += ProfRecord.Counts.size() * sizeof(uint64_t); - M += sizeof(uint64_t); // The size of the Bitmap vector - M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t); // Value data M += ValueProfData::getSize(ProfileData.second); @@ -162,10 +160,6 @@ public: for (uint64_t I : ProfRecord.Counts) LE.write(I); - LE.write(ProfRecord.BitmapBytes.size()); - for (uint64_t I : ProfRecord.BitmapBytes) - LE.write(I); - // Write value data std::unique_ptr VDataPtr = ValueProfData::serializeFrom(ProfileData.second); @@ -386,8 +380,6 @@ bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { const InstrProfRecord &IPR = Func.second; if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) return true; - if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; })) - return true; } return false; } @@ -711,17 +703,6 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, for (uint64_t Count : Func.Counts) OS << Count << "\n"; - if (Func.BitmapBytes.size() > 0) { - OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n"; - OS << "# Bitmap Byte Values:\n"; - for (uint8_t Byte : Func.BitmapBytes) { - OS << "0x"; - OS.write_hex(Byte); - OS << "\n"; - } - OS << "\n"; - } - uint32_t NumValueKinds = Func.getNumValueKinds(); if (!NumValueKinds) { OS << "\n"; diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index ce287551eed6..343554241da3 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -430,15 +430,6 @@ bool InstrProfiling::lowerIntrinsics(Function *F) { } else if (auto *IPVP = dyn_cast(&Instr)) { lowerValueProfileInst(IPVP); MadeChange = true; - } else if (auto *IPMP = dyn_cast(&Instr)) { - IPMP->eraseFromParent(); - MadeChange = true; - } else if (auto *IPBU = dyn_cast(&Instr)) { - lowerMCDCTestVectorBitmapUpdate(IPBU); - MadeChange = true; - } else if (auto *IPTU = dyn_cast(&Instr)) { - lowerMCDCCondBitmapUpdate(IPTU); - MadeChange = true; } } } @@ -553,33 +544,19 @@ bool InstrProfiling::run( // the instrumented function. This is counting the number of instrumented // target value sites to enter it as field in the profile data variable. for (Function &F : M) { - InstrProfCntrInstBase *FirstProfInst = nullptr; - InstrProfMCDCBitmapParameters *FirstProfMCDCParams = nullptr; - for (BasicBlock &BB : F) { - for (auto I = BB.begin(), E = BB.end(); I != E; I++) { + InstrProfInstBase *FirstProfInst = nullptr; + for (BasicBlock &BB : F) + for (auto I = BB.begin(), E = BB.end(); I != E; I++) if (auto *Ind = dyn_cast(I)) computeNumValueSiteCounts(Ind); - else { - if (FirstProfInst == nullptr && - (isa(I) || isa(I))) - FirstProfInst = dyn_cast(I); - if (FirstProfMCDCParams == nullptr) - FirstProfMCDCParams = dyn_cast(I); - } - } - } - - // If the MCDCBitmapParameters intrinsic was seen, create the bitmaps. - if (FirstProfMCDCParams != nullptr) { - static_cast(getOrCreateRegionBitmaps(FirstProfMCDCParams)); - } + else if (FirstProfInst == nullptr && + (isa(I) || isa(I))) + FirstProfInst = dyn_cast(I); - // Use a profile intrinsic to create the region counters and data variable. - // Also create the data variable based on the MCDCParams. - if (FirstProfInst != nullptr) { + // Value profiling intrinsic lowering requires per-function profile data + // variable to be created first. + if (FirstProfInst != nullptr) static_cast(getOrCreateRegionCounters(FirstProfInst)); - createDataVariable(FirstProfInst, FirstProfMCDCParams); - } } for (Function &F : M) @@ -693,7 +670,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { Ind->eraseFromParent(); } -Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { +Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { auto *Counters = getOrCreateRegionCounters(I); IRBuilder<> Builder(I); @@ -733,25 +710,6 @@ Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { return Builder.CreateIntToPtr(Add, Addr->getType()); } -Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { - auto *Bitmaps = getOrCreateRegionBitmaps(I); - IRBuilder<> Builder(I); - - auto *Addr = Builder.CreateConstInBoundsGEP2_32( - Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue()); - - if (isRuntimeCounterRelocationEnabled()) { - LLVMContext &Ctx = M->getContext(); - Ctx.diagnose(DiagnosticInfoPGOProfile( - M->getName().data(), - Twine("Runtime counter relocation is presently not supported for MC/DC " - "bitmaps."), - DS_Warning)); - } - - return Addr; -} - void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) { auto *Addr = getCounterAddress(CoverInstruction); IRBuilder<> Builder(CoverInstruction); @@ -811,86 +769,6 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { CoverageNamesVar->eraseFromParent(); } -void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( - InstrProfMCDCTVBitmapUpdate *Update) { - IRBuilder<> Builder(Update); - auto *Int8Ty = Type::getInt8Ty(M->getContext()); - auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext()); - auto *Int32Ty = Type::getInt32Ty(M->getContext()); - auto *Int64Ty = Type::getInt64Ty(M->getContext()); - auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); - auto *BitmapAddr = getBitmapAddress(Update); - - // Load Temp Val. - // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 - auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); - - // Calculate byte offset using div8. - // %1 = lshr i32 %mcdc.temp, 3 - auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3); - - // Add byte offset to section base byte address. - // %2 = zext i32 %1 to i64 - // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2 - auto *BitmapByteAddr = - Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty), - Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty)); - - // Convert to a pointer. - // %4 = inttoptr i32 %3 to ptr - BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy); - - // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8) - // %5 = and i32 %mcdc.temp, 7 - // %6 = trunc i32 %5 to i8 - auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty); - - // Shift bit offset left to form a bitmap. - // %7 = shl i8 1, %6 - auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet); - - // Load profile bitmap byte. - // %mcdc.bits = load i8, ptr %4, align 1 - auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits"); - - // Perform logical OR of profile bitmap byte and shifted bit offset. - // %8 = or i8 %mcdc.bits, %7 - auto *Result = Builder.CreateOr(Bitmap, ShiftedVal); - - // Store the updated profile bitmap byte. - // store i8 %8, ptr %3, align 1 - Builder.CreateStore(Result, BitmapByteAddr); - Update->eraseFromParent(); -} - -void InstrProfiling::lowerMCDCCondBitmapUpdate( - InstrProfMCDCCondBitmapUpdate *Update) { - IRBuilder<> Builder(Update); - auto *Int32Ty = Type::getInt32Ty(M->getContext()); - auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); - - // Load the MCDC temporary value from the stack. - // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 - auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); - - // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits. - // %1 = zext i1 %tobool to i32 - auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty); - - // Shift the boolean value left (by the condition's ID) to form a bitmap. - // %2 = shl i32 %1, getCondID()> - auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID()); - - // Perform logical OR of the bitmap against the loaded MCDC temporary value. - // %3 = or i32 %mcdc.temp, %2 - auto *Result = Builder.CreateOr(Temp, ShiftedVal); - - // Store the updated temporary value back to the stack. - // store i32 %3, ptr %mcdc.addr, align 4 - Builder.CreateStore(Result, MCDCCondBitmapAddr); - Update->eraseFromParent(); -} - /// Get the name of a profiling variable for a particular function. static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed) { @@ -1046,31 +924,37 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { return true; } -void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, - StringRef VarName) { - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); - bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); - - if (!UseComdat) - return; - - StringRef GroupName = - TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName; - Comdat *C = M->getOrInsertComdat(GroupName); - if (!NeedComdat) - C->setSelectionKind(Comdat::NoDeduplicate); - GV->setComdat(C); - // COFF doesn't allow the comdat group leader to have private linkage, so - // upgrade private linkage to internal linkage to produce a symbol table - // entry. - if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) - GV->setLinkage(GlobalValue::InternalLinkage); +GlobalVariable * +InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name, + GlobalValue::LinkageTypes Linkage) { + uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); + auto &Ctx = M->getContext(); + GlobalVariable *GV; + if (isa(Inc)) { + auto *CounterTy = Type::getInt8Ty(Ctx); + auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); + // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. + std::vector InitialValues(NumCounters, + Constant::getAllOnesValue(CounterTy)); + GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, + ConstantArray::get(CounterArrTy, InitialValues), + Name); + GV->setAlignment(Align(1)); + } else { + auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + GV = new GlobalVariable(*M, CounterTy, false, Linkage, + Constant::getNullValue(CounterTy), Name); + GV->setAlignment(Align(8)); + } + return GV; } -GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, - InstrProfSectKind IPSK) { +GlobalVariable * +InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionCounters) + return PD.RegionCounters; // Match the linkage and visibility of the name global. Function *Fn = Inc->getParent()->getParent(); @@ -1109,100 +993,42 @@ GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, // nodeduplicate COMDAT which is lowered to a zero-flag section group. This // allows -z start-stop-gc to discard the entire group when the function is // discarded. + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); bool Renamed; - GlobalVariable *Ptr; - StringRef VarPrefix; - std::string VarName; - if (IPSK == IPSK_cnts) { - VarPrefix = getInstrProfCountersVarPrefix(); - VarName = getVarName(Inc, VarPrefix, Renamed); - InstrProfCntrInstBase *CntrIncrement = dyn_cast(Inc); - Ptr = createRegionCounters(CntrIncrement, VarName, Linkage); - } else if (IPSK == IPSK_bitmap) { - VarPrefix = getInstrProfBitmapVarPrefix(); - VarName = getVarName(Inc, VarPrefix, Renamed); - InstrProfMCDCBitmapInstBase *BitmapUpdate = - dyn_cast(Inc); - Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage); - } else { - llvm_unreachable("Profile Section must be for Counters or Bitmaps"); - } - - Ptr->setVisibility(Visibility); - // Put the counters and bitmaps in their own sections so linkers can - // remove unneeded sections. - Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat())); - Ptr->setLinkage(Linkage); - maybeSetComdat(Ptr, Fn, VarName); - return Ptr; -} - -GlobalVariable * -InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, - StringRef Name, - GlobalValue::LinkageTypes Linkage) { - uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); - auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes); - auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage, - Constant::getNullValue(BitmapTy), Name); - GV->setAlignment(Align(1)); - return GV; -} - -GlobalVariable * -InstrProfiling::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) { - GlobalVariable *NamePtr = Inc->getName(); - auto &PD = ProfileDataMap[NamePtr]; - if (PD.RegionBitmaps) - return PD.RegionBitmaps; - - // If RegionBitmaps doesn't already exist, create it by first setting up - // the corresponding profile section. - auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap); - PD.RegionBitmaps = BitmapPtr; - return PD.RegionBitmaps; -} + std::string CntsVarName = + getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); + std::string DataVarName = + getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); + auto MaybeSetComdat = [&](GlobalVariable *GV) { + bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); + if (UseComdat) { + StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode + ? GV->getName() + : CntsVarName; + Comdat *C = M->getOrInsertComdat(GroupName); + if (!NeedComdat) + C->setSelectionKind(Comdat::NoDeduplicate); + GV->setComdat(C); + // COFF doesn't allow the comdat group leader to have private linkage, so + // upgrade private linkage to internal linkage to produce a symbol table + // entry. + if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) + GV->setLinkage(GlobalValue::InternalLinkage); + } + }; -GlobalVariable * -InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, - GlobalValue::LinkageTypes Linkage) { uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto &Ctx = M->getContext(); - GlobalVariable *GV; - if (isa(Inc)) { - auto *CounterTy = Type::getInt8Ty(Ctx); - auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); - // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. - std::vector InitialValues(NumCounters, - Constant::getAllOnesValue(CounterTy)); - GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, - ConstantArray::get(CounterArrTy, InitialValues), - Name); - GV->setAlignment(Align(1)); - } else { - auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); - GV = new GlobalVariable(*M, CounterTy, false, Linkage, - Constant::getNullValue(CounterTy), Name); - GV->setAlignment(Align(8)); - } - return GV; -} - -GlobalVariable * -InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { - GlobalVariable *NamePtr = Inc->getName(); - auto &PD = ProfileDataMap[NamePtr]; - if (PD.RegionCounters) - return PD.RegionCounters; + LLVMContext &Ctx = M->getContext(); - // If RegionCounters doesn't already exist, create it by first setting up - // the corresponding profile section. - auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts); + auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage); + CounterPtr->setVisibility(Visibility); + CounterPtr->setSection( + getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); + CounterPtr->setLinkage(Linkage); + MaybeSetComdat(CounterPtr); PD.RegionCounters = CounterPtr; - if (DebugInfoCorrelate) { - LLVMContext &Ctx = M->getContext(); - Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { @@ -1240,50 +1066,8 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); } - - // Mark the counter variable as used so that it isn't optimized out. - CompilerUsedVars.push_back(PD.RegionCounters); } - return PD.RegionCounters; -} - -void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, - InstrProfMCDCBitmapParameters *Params) { - // When debug information is correlated to profile data, a data variable - // is not needed. - if (DebugInfoCorrelate) - return; - - GlobalVariable *NamePtr = Inc->getName(); - auto &PD = ProfileDataMap[NamePtr]; - - LLVMContext &Ctx = M->getContext(); - - Function *Fn = Inc->getParent()->getParent(); - GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); - GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); - - // Due to the limitation of binder as of 2021/09/28, the duplicate weak - // symbols in the same csect won't be discarded. When there are duplicate weak - // symbols, we can NOT guarantee that the relocations get resolved to the - // intended weak symbol, so we can not ensure the correctness of the relative - // CounterPtr, so we have to use private linkage for counter and data symbols. - if (TT.isOSBinFormatXCOFF()) { - Linkage = GlobalValue::PrivateLinkage; - Visibility = GlobalValue::DefaultVisibility; - } - - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); - bool Renamed; - - // The Data Variable section is anchored to profile counters. - std::string CntsVarName = - getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); - std::string DataVarName = - getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); - auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for // the current function. @@ -1301,17 +1085,16 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); - maybeSetComdat(ValuesVar, Fn, CntsVarName); + MaybeSetComdat(ValuesVar); ValuesPtrExpr = ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } - uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto *CounterPtr = PD.RegionCounters; - - uint64_t NumBitmapBytes = 0; - if (Params != nullptr) - NumBitmapBytes = Params->getNumBitmapBytes()->getZExtValue(); + if (DebugInfoCorrelate) { + // Mark the counter variable as used so that it isn't optimized out. + CompilerUsedVars.push_back(PD.RegionCounters); + return PD.RegionCounters; + } // Create data variable. auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); @@ -1354,16 +1137,6 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); - // Bitmaps are relative to the same data variable as profile counters. - GlobalVariable *BitmapPtr = PD.RegionBitmaps; - Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); - - if (BitmapPtr != nullptr) { - RelativeBitmapPtr = - ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), - ConstantExpr::getPtrToInt(Data, IntPtrTy)); - } - Constant *DataVals[] = { #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, #include "llvm/ProfileData/InstrProfData.inc" @@ -1373,7 +1146,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, Data->setVisibility(Visibility); Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); - maybeSetComdat(Data, Fn, CntsVarName); + MaybeSetComdat(Data); PD.DataVar = Data; @@ -1385,6 +1158,8 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, NamePtr->setLinkage(GlobalValue::PrivateLinkage); // Collect the referenced names to be used by emitNameData. ReferencedNames.push_back(NamePtr); + + return PD.RegionCounters; } void InstrProfiling::emitVNodes() { diff --git a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll deleted file mode 100644 index fccb026c25bf..000000000000 --- a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll +++ /dev/null @@ -1,53 +0,0 @@ -; Check that MC/DC intrinsics are properly lowered -; RUN: opt < %s -passes=instrprof -S | FileCheck %s -; RUN: opt < %s -passes=instrprof -runtime-counter-relocation -S 2>&1 | FileCheck %s --check-prefix RELOC - -; RELOC: Runtime counter relocation is presently not supported for MC/DC bitmaps - -target triple = "x86_64-unknown-linux-gnu" - -@__profn_test = private constant [4 x i8] c"test" - -; CHECK: @__profbm_test = private global [1 x i8] zeroinitializer, section "__llvm_prf_bits", comdat, align 1 - -define dso_local void @test(i32 noundef %A) { -entry: - %A.addr = alloca i32, align 4 - %mcdc.addr = alloca i32, align 4 - call void @llvm.instrprof.cover(ptr @__profn_test, i64 99278, i32 5, i32 0) - ; CHECK: store i8 0, ptr @__profc_test, align 1 - - call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 99278, i32 1) - store i32 0, ptr %mcdc.addr, align 4 - %0 = load i32, ptr %A.addr, align 4 - %tobool = icmp ne i32 %0, 0 - - call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 99278, i32 0, ptr %mcdc.addr, i1 %tobool) - ; CHECK: %mcdc.temp = load i32, ptr %mcdc.addr, align 4 - ; CHECK-NEXT: %1 = zext i1 %tobool to i32 - ; CHECK-NEXT: %2 = shl i32 %1, 0 - ; CHECK-NEXT: %3 = or i32 %mcdc.temp, %2 - ; CHECK-NEXT: store i32 %3, ptr %mcdc.addr, align 4 - - call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 99278, i32 1, i32 0, ptr %mcdc.addr) - ; CHECK: %mcdc.temp1 = load i32, ptr %mcdc.addr, align 4 - ; CHECK-NEXT: %4 = lshr i32 %mcdc.temp1, 3 - ; CHECK-NEXT: %5 = zext i32 %4 to i64 - ; CHECK-NEXT: %6 = add i64 ptrtoint (ptr @__profbm_test to i64), %5 - ; CHECK-NEXT: %7 = inttoptr i64 %6 to ptr - ; CHECK-NEXT: %8 = and i32 %mcdc.temp1, 7 - ; CHECK-NEXT: %9 = trunc i32 %8 to i8 - ; CHECK-NEXT: %10 = shl i8 1, %9 - ; CHECK-NEXT: %mcdc.bits = load i8, ptr %7, align 1 - ; CHECK-NEXT: %11 = or i8 %mcdc.bits, %10 - ; CHECK-NEXT: store i8 %11, ptr %7, align 1 - ret void -} - -declare void @llvm.instrprof.cover(ptr, i64, i32, i32) - -declare void @llvm.instrprof.mcdc.parameters(ptr, i64, i32) - -declare void @llvm.instrprof.mcdc.condbitmap.update(ptr, i64, i32, ptr, i1) - -declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr, i64, i32, i32, ptr) diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll index 8c6942c0f527..1c44a274f3c0 100644 --- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll +++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll @@ -13,9 +13,9 @@ $foo = comdat any ; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; CHECK-NOT: __profn__stdin__foo ; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 -; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null +; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null ; CHECK-NOT: @foo -; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 +; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm ; CHECK: @llvm.compiler.used diff --git a/llvm/test/tools/llvm-profdata/Inputs/basic.profraw b/llvm/test/tools/llvm-profdata/Inputs/basic.profraw index 1b284b84fad6dd7f9407b1c3b99cb178af0e09c6..ad88759398c6020f4ab8a5606258e69d98e36687 100644 GIT binary patch delta 71 zcmX@WID?V1u_!ISs37M*$3#y3i5>!MPiq_)7#Jp|DQPSa2C)!;5z2(oEDX7cnRyHh E0CJEFxc~qF delta 63 zcmbQicz}_!u_!ISs37M*=R{6@Muv%+@_Y__aks4+{{I)d(5P~>exjd}00R^}C$QVgXRX-d83sq200i+W)W}ZBd!q5+)yH6rcP6 zC=1puPjh&2EJ delta 308 zcmeC+d%(}xSd^AuRFLzZb0Vky_8T_ z2Map_g$4d1%$=MM7A^z|M<5H&2MaF*3U?q0gIzKCB2aV#lIY~~%z~3Sm;_Lj)^mdV hG1&qro`I~m9;i42C}@Bz*w4f{If6-G^8%&}764*Dk$V6D diff --git a/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw b/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw index 9966729d92ddc33bf89eeb3fee87215bbabbbef1..134b78f7af5b760dc3af7422c1bf7661f4bae14a 100644 GIT binary patch delta 364 zcmdnM|AJSru_!ISs37M*2Ll8MOcYd~=pmqxDy0AxV1$ap532 zg3W7Lmwt5a%Luw`8X;yQ=%O*PQGDV94nY>r<189Si!*Iwboso6f2t<{#V0E;atMY_ zsnBGduNo&k$1~O>`Qe2MkVFDdqEPdcjat*B{dO76%_kSkWcUG+SOAo;_m#;@Xm_l> z_CIV#TU6$@gvkdP#V0=i%0w>dTG2Ucam2*OHvOM-SQvJIWfYh=1iwzTW$_M(6^_4X z6sG@NZp{yn4A5N?3QrGb?)?~3dEpwwUk3mFgTdqpOcD~cFPdW;(Y delta 326 zcmaFCyMbS@u_!ISs37M*Cj%5r6jYzQfSCh?Ct3(dSahA6xxL~4|GoghG>>`)21b~& z3=Oc72^^Cruy9PAA;2N==RX)sp2#ddaX%*`!^DH)91h5W??Hke#W_5X1SiWg0!1e) zGKzETKo<4~3kL#)1^yz;om>wVZUhQPAPcVt3vUDpcOVIaT`~C~P;>*5=;Zs%f|Dhf o1W=XsbAtRa*#jt^fvmV6sJH?sXn-s@pNVsF29v<%4NMj+0Qs7eE&u=k diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test index eda63203a304..67db5c98ef32 100644 --- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -5,15 +5,13 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) -// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) -// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. // 2 * 8 binary ID sizes // + 2 * 20 binary IDs (of size 20) @@ -25,11 +23,8 @@ RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\20\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -56,18 +51,14 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test index 38b838e0d100..2394431e94de 100644 --- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -9,9 +9,6 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Check for a corrupted size being too large past the end of the file. RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test index c967e850dbe3..06f418d0235d 100644 --- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -5,25 +5,20 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) -// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) -// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -40,9 +35,7 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index e1e33824bf2f..b718cf0fd8e9 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -5,26 +5,20 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) -// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) -// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -41,10 +35,8 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Make NumCounters = 0 so that we get "number of counters is zero" error message RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test index 3c23bc7dd0f7..38e40334a6a6 100644 --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -5,25 +5,20 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) -// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) -// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -43,12 +38,10 @@ RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw // Octal '\11' is 9 in decimal: this should push CounterOffset to 1. As there are two counters, // the profile reader should error out. RUN: printf '\11\0\6\0\1\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Counter Section diff --git a/llvm/test/tools/llvm-profdata/mcdc-bitmap.test b/llvm/test/tools/llvm-profdata/mcdc-bitmap.test deleted file mode 100644 index a7b1b5df8c30..000000000000 --- a/llvm/test/tools/llvm-profdata/mcdc-bitmap.test +++ /dev/null @@ -1,201 +0,0 @@ -# Test MC/DC bitmap reading and merging. - -# Merge as profdata. -RUN: split-file %s %t -RUN: llvm-profdata merge %t/mcdc-1.proftext %t/mcdc-2.proftext -o %t.profdata -RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC -# Merge as proftext. -RUN: llvm-profdata merge %t/mcdc-1.proftext %t/mcdc-2.proftext -o %t.proftext -RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC - -MCDC: # Num Bitmap Bytes: -MCDC-NEXT: $1 -MCDC-NEXT: # Bitmap Byte Values: -MCDC-NEXT: a -MCDC: # Num Bitmap Bytes: -MCDC-NEXT: $2 -MCDC-NEXT: # Bitmap Byte Values: -MCDC-NEXT: 0x29 -MCDC-NEXT: 0x0 - -# Merge as profdata. -RUN: llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-4.proftext -o %t.profdata -RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC2 -# Merge as proftext. -RUN: llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-4.proftext -o %t.proftext -RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC2 - -MCDC2: # Num Bitmap Bytes: -MCDC2-NEXT: $8 -MCDC2-NEXT: # Bitmap Byte Values: -MCDC2-NEXT: 0x1 -MCDC2-NEXT: 0x2 -MCDC2-NEXT: 0x3 -MCDC2-NEXT: 0xf -MCDC2-NEXT: 0xf -MCDC2-NEXT: 0xe -MCDC2-NEXT: 0xf -MCDC2-NEXT: 0xa - -# Incompatible size mismatch. -RUN: llvm-profdata merge %t/mcdc-2.proftext %t/mcdc-4.proftext -o %t.profdata 2>&1 | FileCheck %s --check-prefix=MCDC3 -# Merge as proftext -RUN: llvm-profdata merge %t/mcdc-2.proftext %t/mcdc-4.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC3 - -MCDC3: function bitmap size change detected (bitmap size mismatch) - -# Invalid number of bitmap bytes. -RUN: not llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-err0.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC4 - -MCDC4: malformed instrumentation profile data: number of bitmap bytes is not a valid integer - -# Invalid bitmap byte. -RUN: not llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-err1.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC5 - -MCDC5: malformed instrumentation profile data: bitmap byte is not a valid integer - -;--- mcdc-1.proftext -main -# Func Hash: -702755447896 -# Num Counters: -4 -# Counter Values: -1 -0 -1 -0 -# Num Bitmask Bytes: -$1 -# Bitmask Byte Values: -2 -;--- mcdc-2.proftext -main -# Func Hash: -702755447896 -# Num Counters: -4 -# Counter Values: -1 -1 -1 -1 -# Num Bitmask Bytes: -$1 -# Bitmask Byte Values: -8 - - -test3 -# Func Hash: -15288018065 -# Num Counters: -6 -# Counter Values: -4 -2 -1 -0 -0 -2 -# Num Bitmask Bytes: -$0x2 -# Bitmask Byte Values: -0x29 -0x0 -;--- mcdc-3.proftext -test3 -# Func Hash: -15288018065 -# Num Counters: -6 -# Counter Values: -4 -2 -1 -0 -0 -2 -# Num Bitmask Bytes: -$8 -# Bitmask Byte Values: -0x0 -0x2 -0x3 -0xf -0xf -0xa -0xc -0x2 -;--- mcdc-4.proftext -test3 -# Func Hash: -15288018065 -# Num Counters: -6 -# Counter Values: -4 -2 -1 -0 -0 -2 -# Num Bitmask Bytes: -$ 8 -# Bitmask Byte Values: -1 -2 -3 -4 -5 -6 -7 -8 -;--- mcdc-err0.proftext -test3 -# Func Hash: -15288018065 -# Num Counters: -6 -# Counter Values: -4 -2 -1 -0 -0 -2 -# Num Bitmask Bytes: -$8.9 -# Bitmask Byte Values: -1 -2 -3 -4 -5 -6 -7 -8 -;--- mcdc-err1.proftext -test3 -# Func Hash: -15288018065 -# Num Counters: -6 -# Counter Values: -4 -2 -1 -0 -0 -2 -# Num Bitmask Bytes: -$8 -# Bitmask Byte Values: -1 -2 -3 -4 -5.4 -6 -7 -8 diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test index 4a5c42843ff4..171b5cc60878 100644 --- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test index 389646d64b1c..24f3f563e968 100644 --- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -8,9 +8,6 @@ RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test index fbd31d044382..c8e862009ef0 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,46 +1,37 @@ RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\11' >> %t +RUN: printf '\0\0\0\0\0\0\0\10' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\4' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\0\1\0\0\0' >> %t -RUN: printf '\0\0\0\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\3\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0\0\0\0\3' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t -RUN: printf '\0\xff\xff\xd8' >> %t -RUN: printf '\2\xff\xff\xd3' >> %t +RUN: printf '\0\xff\xff\xe0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\2' >> %t -RUN: printf '\0\0\0\0\0\0\0\1' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t -RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s -RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -57,14 +48,3 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 - -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $3 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $1 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test index bb899c5fdb55..523ff1ceb480 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,46 +1,37 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\11\0\0\0\0\0\0\0' >> %t +RUN: printf '\10\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\4\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t -RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\0\0\0\2\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0\3\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xd8\xff\xff\0' >> %t -RUN: printf '\xd3\xff\xff\2' >> %t +RUN: printf '\xe0\xff\xff\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\2\0\0\0' >> %t -RUN: printf '\0\0\0\0\1\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t -RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s -RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -57,14 +48,3 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 - -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $3 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $1 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test index 8fcadb6a0dd2..b2b8b31dafbf 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,44 +1,35 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\11' >> %t +RUN: printf '\0\0\0\0\0\0\0\10' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\4' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t -RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t -RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t -RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\02' >> %t -RUN: printf '\0\0\0\1\0\3\xff\xc8' >> %t -RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t +RUN: printf '\0\0\0\1\0\3\xff\xd8' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\02\0\0\0\0' >> %t -RUN: printf '\0\0\0\1\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t -RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s -RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -55,14 +46,3 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 - -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $3 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $1 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test index 0aa8b38f6926..4e95798bc0af 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,44 +1,35 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\11\0\0\0\0\0\0\0' >> %t +RUN: printf '\10\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\4\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t -RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\4\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t -RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t -RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t -RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t +RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t -RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s -RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -55,14 +46,3 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 - -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $3 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC-NEXT: 55 -MCDC: Num Bitmap Bytes: -MCDC-NEXT: $1 -MCDC-NEXT: Bitmap Byte Values: -MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test index f4a9aa8e1bbc..8d46c91e2732 100644 --- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,15 +1,12 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -18,25 +15,20 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw @@ -45,9 +37,7 @@ RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw -- Gitee From c96755af4833645135f8bb8e34b4041e505c9b4a Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 23 Sep 2023 13:00:46 -0700 Subject: [PATCH 05/28] [ProfileData] Use llvm::byteswap instead of sys::getSwappedBytes (NFC) --- llvm/include/llvm/ProfileData/InstrProfCorrelator.h | 2 +- llvm/include/llvm/ProfileData/InstrProfReader.h | 2 +- llvm/lib/ProfileData/InstrProfReader.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index 2e26a21e8839..9533ad9e703a 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -148,7 +148,7 @@ private: // Byte-swap the value if necessary. template T maybeSwap(T Value) const { - return Ctx->ShouldSwapBytes ? sys::getSwappedBytes(Value) : Value; + return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; } }; diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 74e921e10c47..faf91e017756 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -409,7 +409,7 @@ private: Error readHeader(const RawInstrProf::Header &Header); template IntT swap(IntT Int) const { - return ShouldSwapBytes ? sys::getSwappedBytes(Int) : Int; + return ShouldSwapBytes ? llvm::byteswap(Int) : Int; } support::endianness getDataEndianness() const { diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index db20441b712c..9375678f8046 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -471,7 +471,7 @@ bool RawInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { uint64_t Magic = *reinterpret_cast(DataBuffer.getBufferStart()); return RawInstrProf::getMagic() == Magic || - sys::getSwappedBytes(RawInstrProf::getMagic()) == Magic; + llvm::byteswap(RawInstrProf::getMagic()) == Magic; } template -- Gitee From e52da311c81225d76b0e4ef456f469f438d4ed49 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Tue, 3 Oct 2023 10:15:22 -0400 Subject: [PATCH 06/28] [Profile] Use upper 32 bits of profile version for profile variants. (#67695) Currently all upper 8 bits are reserved for different profile variants. We need more bits for new mods in the future. Context: https://discourse.llvm.org/t/how-to-add-a-new-mode-to-llvm-raw-profile-version/73688 --- compiler-rt/include/profile/InstrProfData.inc | 9 ++++----- llvm/include/llvm/ProfileData/InstrProfData.inc | 9 ++++----- llvm/include/llvm/ProfileData/InstrProfReader.h | 4 ++-- llvm/lib/ProfileData/InstrProfReader.cpp | 2 +- .../llvm-profdata/mismatched-raw-profile-header.test | 4 ++-- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 4456bf1ab176..8ba7e186d4fb 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -644,7 +644,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 -/* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ @@ -652,9 +651,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 5 -/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the - * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation +/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the + * version for other variants of profile. We set the 8th most significant bit + * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. * The 59th bit indicates whether to use debug info to correlate profiles. @@ -663,7 +662,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * The 62nd bit indicates whether memory profile information is present. * The 63rd bit indicates if this is a temporal profile. */ -#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 4456bf1ab176..8ba7e186d4fb 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -644,7 +644,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 -/* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ @@ -652,9 +651,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 5 -/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the - * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation +/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the + * version for other variants of profile. We set the 8th most significant bit + * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. * The 59th bit indicates whether to use debug info to correlate profiles. @@ -663,7 +662,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * The 62nd bit indicates whether memory profile information is present. * The 63rd bit indicates if this is a temporal profile. */ -#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index faf91e017756..17194e2aa4c7 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -318,8 +318,8 @@ private: /// A list of timestamps paired with a function name reference. std::vector> TemporalProfTimestamps; bool ShouldSwapBytes; - // The value of the version field of the raw profile data header. The lower 56 - // bits specifies the format version and the most significant 8 bits specify + // The value of the version field of the raw profile data header. The lower 32 + // bits specifies the format version and the most significant 32 bits specify // the variant types of the profile. uint64_t Version; uint64_t CountersDelta; diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 9375678f8046..125184d40208 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -38,7 +38,7 @@ using namespace llvm; -// Extracts the variant information from the top 8 bits in the version and +// Extracts the variant information from the top 32 bits in the version and // returns an enum specifying the variants present. static InstrProfKind getProfileKindFromVersion(uint64_t Version) { InstrProfKind ProfileKind = InstrProfKind::Unknown; diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test index 24f3f563e968..c0072bcbde1b 100644 --- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -1,7 +1,7 @@ // Magic RUN: printf '\377lprofr\201' > %t // Version -RUN: printf '\0\01\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\10\0\0\10' >> %t // The rest of the header needs to be there to prevent a broken header error. RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t @@ -15,5 +15,5 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s -CHECK: raw profile version mismatch: Profile uses raw profile format version = 281474976710664; expected version = {{[0-9]+}} +CHECK: raw profile version mismatch: Profile uses raw profile format version = 134217736; expected version = {{[0-9]+}} CHECK-NEXT: PLEASE update this tool to version in the raw profile, or regenerate raw profile with expected version. -- Gitee From 6be0242c21c1f32a5731d7ed71652102f7c02624 Mon Sep 17 00:00:00 2001 From: Alan Phipps Date: Thu, 21 Sep 2023 13:07:31 -0500 Subject: [PATCH 07/28] Reland "[InstrProf][compiler-rt] Enable MC/DC Support in LLVM Source-based Code Coverage (1/3)" Part 1 of 3. This includes the LLVM back-end processing and profile reading/writing components. compiler-rt changes are included. Differential Revision: https://reviews.llvm.org/D138846 --- clang/lib/Driver/ToolChains/Darwin.cpp | 4 +- clang/test/Driver/darwin-ld.c | 2 +- compiler-rt/include/profile/InstrProfData.inc | 22 +- compiler-rt/lib/profile/InstrProfiling.c | 4 + compiler-rt/lib/profile/InstrProfiling.h | 25 +- .../lib/profile/InstrProfilingBuffer.c | 42 +- compiler-rt/lib/profile/InstrProfilingFile.c | 50 ++- .../lib/profile/InstrProfilingInternal.h | 8 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 33 +- .../lib/profile/InstrProfilingPlatformAIX.c | 7 +- .../profile/InstrProfilingPlatformDarwin.c | 9 + .../lib/profile/InstrProfilingPlatformLinux.c | 10 + .../lib/profile/InstrProfilingPlatformOther.c | 4 + .../profile/InstrProfilingPlatformWindows.c | 7 + .../lib/profile/InstrProfilingWriter.c | 18 +- .../profile/instrprof-write-buffer-internal.c | 21 +- llvm/docs/LangRef.rst | 138 +++++++ llvm/include/llvm/IR/IntrinsicInst.h | 94 ++++- llvm/include/llvm/IR/Intrinsics.td | 15 + .../ProfileData/Coverage/CoverageMapping.h | 4 +- llvm/include/llvm/ProfileData/InstrProf.h | 21 +- .../llvm/ProfileData/InstrProfData.inc | 22 +- .../llvm/ProfileData/InstrProfReader.h | 9 + .../Instrumentation/InstrProfiling.h | 46 ++- .../SelectionDAG/SelectionDAGBuilder.cpp | 6 + llvm/lib/IR/IntrinsicInst.cpp | 4 +- .../Coverage/CoverageMappingReader.cpp | 4 + llvm/lib/ProfileData/InstrProf.cpp | 23 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 4 + llvm/lib/ProfileData/InstrProfReader.cpp | 111 ++++- llvm/lib/ProfileData/InstrProfWriter.cpp | 19 + .../Instrumentation/InstrProfiling.cpp | 379 ++++++++++++++---- .../Instrumentation/InstrProfiling/mcdc.ll | 53 +++ .../Transforms/PGOProfile/comdat_internal.ll | 4 +- .../tools/llvm-profdata/Inputs/basic.profraw | Bin 152 -> 192 bytes .../llvm-profdata/Inputs/c-general.profraw | Bin 1800 -> 2016 bytes .../llvm-profdata/Inputs/compat.profdata.v10 | Bin 0 -> 872 bytes .../llvm-profdata/Inputs/compressed.profraw | Bin 1768 -> 1968 bytes .../llvm-profdata/binary-ids-padding.test | 13 +- llvm/test/tools/llvm-profdata/compat.proftext | 23 ++ .../llvm-profdata/large-binary-id-size.test | 5 +- ...alformed-not-space-for-another-header.test | 9 +- .../malformed-num-counters-zero.test | 10 +- .../malformed-ptr-to-counter-array.test | 9 +- .../test/tools/llvm-profdata/mcdc-bitmap.test | 201 ++++++++++ .../misaligned-binary-ids-size.test | 2 +- .../mismatched-raw-profile-header.test | 3 + .../tools/llvm-profdata/raw-32-bits-be.test | 28 +- .../tools/llvm-profdata/raw-32-bits-le.test | 28 +- .../tools/llvm-profdata/raw-64-bits-be.test | 24 +- .../tools/llvm-profdata/raw-64-bits-le.test | 24 +- .../tools/llvm-profdata/raw-two-profiles.test | 14 +- 52 files changed, 1441 insertions(+), 174 deletions(-) create mode 100644 llvm/test/Instrumentation/InstrProfiling/mcdc.ll create mode 100644 llvm/test/tools/llvm-profdata/Inputs/compat.profdata.v10 create mode 100644 llvm/test/tools/llvm-profdata/mcdc-bitmap.test diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp index 3b8e4d7e133a..ffc8a5eea184 100644 --- a/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/clang/lib/Driver/ToolChains/Darwin.cpp @@ -1405,7 +1405,7 @@ void Darwin::addProfileRTLibs(const ArgList &Args, addExportedSymbol(CmdArgs, "_reset_fn_list"); } - // Align __llvm_prf_{cnts,data} sections to the maximum expected page + // Align __llvm_prf_{cnts,bits,data} sections to the maximum expected page // alignment. This allows profile counters to be mmap()'d to disk. Note that // it's not enough to just page-align __llvm_prf_cnts: the following section // must also be page-aligned so that its data is not clobbered by mmap(). @@ -1415,7 +1415,7 @@ void Darwin::addProfileRTLibs(const ArgList &Args, // extra alignment also allows the same binary to be used with/without sync // enabled. if (!ForGCOV) { - for (auto IPSK : {llvm::IPSK_cnts, llvm::IPSK_data}) { + for (auto IPSK : {llvm::IPSK_cnts, llvm::IPSK_bitmap, llvm::IPSK_data}) { addSectalignToPage( Args, CmdArgs, "__DATA", llvm::getInstrProfSectionName(IPSK, llvm::Triple::MachO, diff --git a/clang/test/Driver/darwin-ld.c b/clang/test/Driver/darwin-ld.c index 13ec69b2db1a..4e2a408aa742 100644 --- a/clang/test/Driver/darwin-ld.c +++ b/clang/test/Driver/darwin-ld.c @@ -336,7 +336,7 @@ // RUN: FileCheck -check-prefix=PROFILE_SECTALIGN %s < %t.log // RUN: %clang -target arm64-apple-ios12 -fprofile-instr-generate -### %t.o 2> %t.log // RUN: FileCheck -check-prefix=PROFILE_SECTALIGN %s < %t.log -// PROFILE_SECTALIGN: "-sectalign" "__DATA" "__llvm_prf_cnts" "0x4000" "-sectalign" "__DATA" "__llvm_prf_data" "0x4000" +// PROFILE_SECTALIGN: "-sectalign" "__DATA" "__llvm_prf_cnts" "0x4000" "-sectalign" "__DATA" "__llvm_prf_bits" "0x4000" "-sectalign" "__DATA" "__llvm_prf_data" "0x4000" // RUN: %clang -target x86_64-apple-darwin12 -fprofile-instr-generate --coverage -### %t.o 2> %t.log // RUN: FileCheck -check-prefix=NO_PROFILE_EXPORT %s < %t.log diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 8ba7e186d4fb..1cf83011206a 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -76,6 +76,7 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -87,7 +88,9 @@ INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -132,9 +135,13 @@ INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -267,6 +274,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -645,11 +655,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 8 +#define INSTR_PROF_RAW_VERSION 9 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 10 +#define INSTR_PROF_INDEX_VERSION 11 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 5 +#define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the * version for other variants of profile. We set the 8th most significant bit @@ -686,6 +696,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -697,6 +708,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -708,6 +720,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -722,6 +735,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c index 0dd5ff5ae633..da04d8ebdec9 100644 --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -60,6 +60,10 @@ COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0; memset(I, ResetValue, E - I); + I = __llvm_profile_begin_bitmap(); + E = __llvm_profile_end_bitmap(); + memset(I, 0x0, E - I); + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const __llvm_profile_data *DI; diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 4433d7bd4887..e143149fca82 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -88,6 +88,8 @@ const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); uint32_t *__llvm_profile_begin_orderfile(); @@ -101,11 +103,11 @@ void __llvm_profile_reset_counters(void); /*! * \brief Merge profile data from buffer. * - * Read profile data form buffer \p Profile and merge with in-process profile - * counters. The client is expected to have checked or already knows the profile - * data in the buffer matches the in-process counter structure before calling - * it. Returns 0 (success) if the profile data is valid. Upon reading - * invalid/corrupted profile data, returns 1 (failure). + * Read profile data from buffer \p Profile and merge with in-process profile + * counters and bitmaps. The client is expected to have checked or already + * know the profile data in the buffer matches the in-process counter + * structure before calling it. Returns 0 (success) if the profile data is + * valid. Upon reading invalid/corrupted profile data, returns 1 (failure). */ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); @@ -113,8 +115,8 @@ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); * * Returns 0 (success) if the profile data in buffer \p Profile with size * \p Size was generated by the same binary and therefore matches - * structurally the in-process counters. If the profile data in buffer is - * not compatible, the interface returns 1 (failure). + * structurally the in-process counters and bitmaps. If the profile data in + * buffer is not compatible, the interface returns 1 (failure). */ int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size); @@ -276,6 +278,10 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End); /*! \brief Get the size of the profile counters section in bytes. */ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); +/*! \brief Get the number of bytes in the profile bitmap section. */ +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End); + /* ! \brief Given the sizes of the data and counter information, return the * number of padding bytes before and after the counters, and after the names, * in the raw profile. @@ -286,8 +292,9 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); * needed to achieve that. */ void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, - uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, + uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames); /*! diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 61ac5d9c0285..c7217b2dfef8 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -43,11 +43,14 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return __llvm_profile_get_size_for_buffer_internal( - DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd); + DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, + NamesBegin, NamesEnd); } COMPILER_RT_VISIBILITY @@ -83,6 +86,12 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { __llvm_profile_counter_entry_size(); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End) { + return (End - Begin); +} + /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { @@ -102,13 +111,16 @@ static int needsCounterPadding(void) { COMPILER_RT_VISIBILITY void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, - uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, + uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames) { if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = __llvm_profile_get_num_padding_bytes(CountersSize); + *PaddingBytesAfterBitmapBytes = + __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); return; } @@ -118,31 +130,37 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); + *PaddingBytesAfterBitmapBytes = + calculateBytesNeededToPageAlign(NumBitmapBytes); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, - const char *NamesEnd) { + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, + &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + - PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames; + PaddingBytesAfterCounters + NumBitmapBytes + + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames; } COMPILER_RT_VISIBILITY @@ -160,9 +178,11 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) { + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, 0, NamesBegin, NamesEnd, 0); + CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin, + NamesEnd, 0); } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 2bd6a49ce065..49fef96578c8 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -108,14 +108,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); - /* Check that the counter and data sections in this image are + /* Check that the counter, bitmap, and data sections in this image are * page-aligned. */ unsigned PageSize = getpagesize(); if ((intptr_t)CountersBegin % PageSize != 0) { @@ -123,6 +127,11 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { CountersBegin, PageSize); return 1; } + if ((intptr_t)BitmapBegin % PageSize != 0) { + PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n", + BitmapBegin, PageSize); + return 1; + } if ((intptr_t)DataBegin % PageSize != 0) { PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", DataBegin, PageSize); @@ -132,10 +141,11 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Determine how much padding is needed before/after the counters and * after the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, + &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; uint64_t FileOffsetToCounters = CurrentFileOffset + @@ -155,6 +165,31 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { FileOffsetToCounters); return 1; } + + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + uint64_t PageAlignedBitmapLength = + NumBitmapBytes + PaddingBytesAfterBitmapBytes; + uint64_t FileOffsetToBitmap = + CurrentFileOffset + sizeof(__llvm_profile_header) + DataSize + + PaddingBytesBeforeCounters + CountersSize + PaddingBytesAfterCounters; + void *BitmapMmap = + mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap); + if (BitmapMmap != BitmapBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - BitmapBegin: %p\n" + " - PageAlignedBitmapLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToBitmap: %" PRIu64 "\n", + strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno, + FileOffsetToBitmap); + return 1; + } return 0; } #elif defined(__ELF__) || defined(_WIN32) @@ -197,6 +232,8 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); /* Get the file size. */ uint64_t FileSize = 0; @@ -218,6 +255,11 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + + /* BIAS MODE not supported yet for Bitmap (MCDC). */ + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); return 0; } #else diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h index 360165e32ab3..03ed67fcfa76 100644 --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -21,8 +21,8 @@ */ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, - const char *NamesEnd); + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -36,7 +36,8 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); /*! * The data structure describing the data to be written by the @@ -153,6 +154,7 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite); diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 088ce71308ee..520fb561e4d3 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -67,6 +67,9 @@ int __llvm_profile_check_compatibility(const char *ProfileData, Header->NumCounters != __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), __llvm_profile_end_counters()) || + Header->NumBitmapBytes != + __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), + __llvm_profile_end_bitmap()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()) || Header->ValueKindLast != IPVK_Last) @@ -75,7 +78,8 @@ int __llvm_profile_check_compatibility(const char *ProfileData, if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + - Header->NumCounters * __llvm_profile_counter_entry_size()) + Header->NumCounters * __llvm_profile_counter_entry_size() + + Header->NumBitmapBytes) return 1; for (SrcData = SrcDataStart, @@ -83,7 +87,8 @@ int __llvm_profile_check_compatibility(const char *ProfileData, SrcData < SrcDataEnd; ++SrcData, ++DstData) { if (SrcData->NameRef != DstData->NameRef || SrcData->FuncHash != DstData->FuncHash || - SrcData->NumCounters != DstData->NumCounters) + SrcData->NumCounters != DstData->NumCounters || + SrcData->NumBitmapBytes != DstData->NumBitmapBytes) return 1; } @@ -115,9 +120,11 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, char *SrcCountersStart, *DstCounter; const char *SrcCountersEnd, *SrcCounter; + const char *SrcBitmapStart; const char *SrcNameStart; const char *SrcValueProfDataStart, *SrcValueProfData; uintptr_t CountersDelta = Header->CountersDelta; + uintptr_t BitmapDelta = Header->BitmapDelta; SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + @@ -126,11 +133,12 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, SrcCountersStart = (char *)SrcDataEnd; SrcCountersEnd = SrcCountersStart + Header->NumCounters * __llvm_profile_counter_entry_size(); - SrcNameStart = SrcCountersEnd; + SrcBitmapStart = SrcCountersEnd; + SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; SrcValueProfDataStart = SrcNameStart + Header->NamesSize + __llvm_profile_get_num_padding_bytes(Header->NamesSize); - if (SrcNameStart < SrcCountersStart) + if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; // Merge counters when there is no data section and debug info correlation is @@ -164,6 +172,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // extend CounterPtr to get the original value. char *DstCounters = (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); + char *DstBitmap = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); unsigned NVK = 0; // SrcData is a serialized representation of the memory image. We need to @@ -193,6 +203,21 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, } } + const char *SrcBitmap = + SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); + // BitmapDelta also needs to be decreased as we advance to the next data + // record. + BitmapDelta -= sizeof(*SrcData); + unsigned NB = SrcData->NumBitmapBytes; + // NumBitmapBytes may legitimately be 0. Just keep going. + if (NB != 0) { + if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) + return 1; + // Merge Src and Dst Bitmap bytes by simply ORing them together. + for (unsigned I = 0; I < NB; I++) + DstBitmap[I] |= SrcBitmap[I]; + } + /* Now merge value profile data. */ if (!VPMergeHook) continue; diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c b/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c index 63219da18ae3..9f46a98d78ac 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c @@ -187,6 +187,8 @@ void __llvm_profile_register_names_function(void *NamesStart, // define these zero length variables in each of the above 4 sections. static int dummy_cnts[0] COMPILER_RT_SECTION( COMPILER_RT_SEG INSTR_PROF_CNTS_SECT_NAME); +static int dummy_bits[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_BITS_SECT_NAME); static int dummy_data[0] COMPILER_RT_SECTION( COMPILER_RT_SEG INSTR_PROF_DATA_SECT_NAME); static const int dummy_name[0] COMPILER_RT_SECTION( @@ -202,8 +204,9 @@ static int dummy_vnds[0] COMPILER_RT_SECTION( #pragma GCC diagnostic ignored "-Wcast-qual" #endif COMPILER_RT_VISIBILITY -void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_data, - (void *)&dummy_name, (void *)&dummy_vnds}; +void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits, + (void *)&dummy_data, (void *)&dummy_name, + (void *)&dummy_vnds}; #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c index d9f2a113f5b0..2154d242a817 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -31,6 +31,11 @@ extern char COMPILER_RT_VISIBILITY extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY +extern char + BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -56,6 +61,10 @@ char *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &CountersEnd; } COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index 2cce0a4b2c48..d0c42462e5e3 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -35,6 +35,8 @@ #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) +#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) #define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) #define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) @@ -48,6 +50,8 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -74,6 +78,12 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) { + return &PROF_BITS_START; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { + return &PROF_BITS_STOP; +} COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &PROF_ORDERFILE_START; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index c7b6e842c9fa..5319ca813b43 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -88,6 +88,10 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return CountersLast; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return BitmapLast; } /* TODO: correctly set up OrderFileFirst. */ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index dd576b2f8357..9dbd702865fd 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -14,6 +14,7 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ #pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") @@ -30,6 +31,8 @@ #pragma section(".lprfd$Z", read, write) #pragma section(".lprfc$A", read, write) #pragma section(".lprfc$Z", read, write) +#pragma section(".lprfb$A", read, write) +#pragma section(".lprfb$Z", read, write) #pragma section(".lorderfile$A", read, write) #pragma section(".lprfnd$A", read, write) #pragma section(".lprfnd$Z", read, write) @@ -43,6 +46,8 @@ const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; char COMPILER_RT_SECTION(".lprfc$A") CountersStart; char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfb$A") BitmapStart; +char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd; uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; @@ -58,6 +63,8 @@ const char *__llvm_profile_end_names(void) { return &NamesEnd; } char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } char *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 1e22398a4c0f..3b61f3def9f6 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -246,17 +246,20 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, - CountersEnd, VPDataReader, NamesBegin, NamesEnd, - SkipNameDataWrite); + CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, + NamesBegin, NamesEnd, SkipNameDataWrite); } COMPILER_RT_VISIBILITY int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { int DebugInfoCorrelate = @@ -271,6 +274,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumCounters = __llvm_profile_get_num_counters(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; /* Create the header. */ @@ -279,11 +284,11 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSectionSize, CountersSectionSize, NamesSize, + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterNames); + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); { /* Initialize header structure. */ @@ -295,6 +300,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, * CountersDelta to match. */ #ifdef _WIN64 Header.CountersDelta = (uint32_t)Header.CountersDelta; + Header.BitmapDelta = (uint32_t)Header.BitmapDelta; #endif /* The data and names sections are omitted in lightweight mode. */ @@ -319,6 +325,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c index 7b96c6d91c33..d9670f739ca9 100644 --- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -25,17 +25,18 @@ const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); uint64_t __llvm_profile_get_size_for_buffer_internal( const void *DataBegin, const void *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); -int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin, - const void *DataEnd, - const char *CountersBegin, - const char *CountersEnd, - const char *NamesBegin, - const char *NamesEnd); +int __llvm_profile_write_buffer_internal( + char *Buffer, const void *DataBegin, const void *DataEnd, + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); void __llvm_profile_set_dumped(void); @@ -43,12 +44,14 @@ int main(int argc, const char *argv[]) { uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal( __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), + __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); char *buf = malloc(bufsize); - int ret = __llvm_profile_write_buffer_internal(buf, - __llvm_profile_begin_data(), __llvm_profile_end_data(), + int ret = __llvm_profile_write_buffer_internal( + buf, __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), + __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); if (ret != 0) { diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 2f4d01c4cddd..e9caee30b1a3 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -13820,6 +13820,144 @@ pass will generate the appropriate data structures and replace the ``llvm.instrprof.value.profile`` intrinsic with the call to the profile runtime library with proper arguments. +'``llvm.instrprof.mcdc.parameters``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.parameters(ptr , i64 , + i32 ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.parameters``' intrinsic is used to initiate MC/DC +code coverage instrumentation for a function. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is the number of bitmap bytes required by the function to +record the number of test vectors executed for each boolean expression. + +Semantics: +"""""""""" + +This intrinsic represents basic MC/DC parameters initiating one or more MC/DC +instrumentation sequences in a function. It will cause the ``-instrprof`` pass +to generate the appropriate data structures and the code to instrument MC/DC +test vectors in a format that can be written out by a compiler runtime and +consumed via the ``llvm-profdata`` tool. + +'``llvm.instrprof.mcdc.condbitmap.update``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.condbitmap.update(ptr , i64 , + i32 , + ptr , + i1 ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic is used to track +MC/DC condition evaluation for each condition in a boolean expression. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is an ID of a condition to track. This value is used as a +bit index into the condition bitmap. + +The fourth argument is the address of the condition bitmap. + +The fifth argument is the boolean value representing the evaluation of the +condition (true or false) + +Semantics: +"""""""""" + +This intrinsic represents the update of a condition bitmap that is local to a +function and will cause the ``-instrprof`` pass to generate the code to +instrument the control flow around each condition in a boolean expression. The +ID of each condition corresponds to a bit index in the condition bitmap which +is set based on the evaluation of the condition. + +'``llvm.instrprof.mcdc.tvbitmap.update``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr , i64 , + i32 ) + i32 , + ptr ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.tvbitmap.update``' intrinsic is used to track MC/DC +test vector execution after each boolean expression has been fully executed. +The overall value of the condition bitmap, after it has been successively +updated using the '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic with +the true or false evaluation of each condition, uniquely identifies an executed +MC/DC test vector and is used as a bit index into the global test vector +bitmap. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is the number of bitmap bytes required by the function to +record the number of test vectors executed for each boolean expression. + +The fourth argument is the byte index into the global test vector bitmap +corresponding to the function. + +The fifth argument is the address of the condition bitmap, which contains a +value representing an executed MC/DC test vector. It is loaded and used as the +bit index of the test vector bitmap. + +Semantics: +"""""""""" + +This intrinsic represents the final operation of an MC/DC instrumentation +sequence and will cause the ``-instrprof`` pass to generate the code to +instrument an update of a function's global test vector bitmap to indicate that +a test vector has been executed. The global test vector bitmap can be consumed +by the ``llvm-profdata`` and ``llvm-cov`` tools. + '``llvm.thread.pointer``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 62bd833198f0..f04b5ae152f2 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1399,6 +1399,11 @@ public: ConstantInt *getHash() const { return cast(const_cast(getArgOperand(1))); } +}; + +/// A base class for all instrprof counter intrinsics. +class InstrProfCntrInstBase : public InstrProfInstBase { +public: // The number of counters for the instrumented function. ConstantInt *getNumCounters() const; // The index of the counter that this instruction acts on. @@ -1406,7 +1411,7 @@ public: }; /// This represents the llvm.instrprof.cover intrinsic. -class InstrProfCoverInst : public InstrProfInstBase { +class InstrProfCoverInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_cover; @@ -1417,7 +1422,7 @@ public: }; /// This represents the llvm.instrprof.increment intrinsic. -class InstrProfIncrementInst : public InstrProfInstBase { +class InstrProfIncrementInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_increment || @@ -1441,7 +1446,7 @@ public: }; /// This represents the llvm.instrprof.timestamp intrinsic. -class InstrProfTimestampInst : public InstrProfInstBase { +class InstrProfTimestampInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_timestamp; @@ -1452,7 +1457,7 @@ public: }; /// This represents the llvm.instrprof.value.profile intrinsic. -class InstrProfValueProfileInst : public InstrProfInstBase { +class InstrProfValueProfileInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_value_profile; @@ -1475,6 +1480,87 @@ public: } }; +/// A base class for instrprof mcdc intrinsics that require global bitmap bytes. +class InstrProfMCDCBitmapInstBase : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters || + I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + /// \return The number of bytes used for the MCDC bitmaps for the instrumented + /// function. + ConstantInt *getNumBitmapBytes() const { + return cast(const_cast(getArgOperand(2))); + } +}; + +/// This represents the llvm.instrprof.mcdc.parameters intrinsic. +class InstrProfMCDCBitmapParameters : public InstrProfMCDCBitmapInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + +/// This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic. +class InstrProfMCDCTVBitmapUpdate : public InstrProfMCDCBitmapInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + /// \return The index of the TestVector Bitmap upon which this intrinsic + /// acts. + ConstantInt *getBitmapIndex() const { + return cast(const_cast(getArgOperand(3))); + } + + /// \return The address of the corresponding condition bitmap containing + /// the index of the TestVector to update within the TestVector Bitmap. + Value *getMCDCCondBitmapAddr() const { + return cast(const_cast(getArgOperand(4))); + } +}; + +/// This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic. +/// It does not pertain to global bitmap updates or parameters and so doesn't +/// inherit from InstrProfMCDCBitmapInstBase. +class InstrProfMCDCCondBitmapUpdate : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_condbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + /// \return The ID of the condition to update. + ConstantInt *getCondID() const { + return cast(const_cast(getArgOperand(2))); + } + + /// \return The address of the corresponding condition bitmap. + Value *getMCDCCondBitmapAddr() const { + return cast(const_cast(getArgOperand(3))); + } + + /// \return The boolean value to set in the condition bitmap for the + /// corresponding condition ID. This represents how the condition evaluated. + Value *getCondBool() const { + return cast(const_cast(getArgOperand(4))); + } +}; + class PseudoProbeInst : public IntrinsicInst { public: static bool classof(const IntrinsicInst *I) { diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 20a8fa419465..a47d94452db4 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -909,6 +909,21 @@ def int_instrprof_value_profile : Intrinsic<[], llvm_i64_ty, llvm_i32_ty, llvm_i32_ty]>; +// A parameter configuration for instrumentation based MCDC profiling. +def int_instrprof_mcdc_parameters : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty]>; + +// A test vector bitmap update for instrumentation based MCDC profiling. +def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty]>; + +// A condition bitmap value update for instrumentation based MCDC profiling. +def int_instrprof_mcdc_condbitmap_update : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_ptr_ty, llvm_i1_ty]>; + def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty]>; def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>; diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 3c8f940ba97b..12498630190d 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -1023,7 +1023,9 @@ enum CovMapVersion { // Compilation directory is stored separately and combined with relative // filenames to produce an absolute file path. Version6 = 5, - // The current version is Version6. + // Branch regions extended and Decision Regions added for MC/DC. + Version7 = 6, + // The current version is Version7. CurrentVersion = INSTR_PROF_COVMAP_VERSION }; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index f9096b461572..2a780104b177 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -96,6 +96,9 @@ inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } /// Return the name prefix of profile counter variables. inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } +/// Return the name prefix of profile bitmap variables. +inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; } + /// Return the name prefix of value profile variables. inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } @@ -335,6 +338,7 @@ enum class instrprof_error { invalid_prof, hash_mismatch, count_mismatch, + bitmap_mismatch, counter_overflow, value_site_count_mismatch, compress_failed, @@ -690,18 +694,23 @@ struct InstrProfValueSiteRecord { /// Profiling information for a single function. struct InstrProfRecord { std::vector Counts; + std::vector BitmapBytes; InstrProfRecord() = default; InstrProfRecord(std::vector Counts) : Counts(std::move(Counts)) {} + InstrProfRecord(std::vector Counts, + std::vector BitmapBytes) + : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {} InstrProfRecord(InstrProfRecord &&) = default; InstrProfRecord(const InstrProfRecord &RHS) - : Counts(RHS.Counts), + : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes), ValueData(RHS.ValueData ? std::make_unique(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { Counts = RHS.Counts; + BitmapBytes = RHS.BitmapBytes; if (!RHS.ValueData) { ValueData = nullptr; return *this; @@ -880,6 +889,11 @@ struct NamedInstrProfRecord : InstrProfRecord { NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} + NamedInstrProfRecord(StringRef Name, uint64_t Hash, + std::vector Counts, + std::vector BitmapBytes) + : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name), + Hash(Hash) {} static bool hasCSFlagInHash(uint64_t FuncHash) { return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); @@ -1015,7 +1029,9 @@ enum ProfVersion { Version9 = 9, // An additional (optional) temporal profile traces section is added. Version10 = 10, - // The current version is 10. + // An additional field is used for bitmap bytes. + Version11 = 11, + // The current version is 11. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1153,6 +1169,7 @@ namespace RawInstrProf { // Version 6: Added binary id. // Version 7: Reorder binary id and include version in signature. // Version 8: Use relative counter pointer. +// Version 9: Added relative bitmap bytes pointer and count used by MC/DC. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 8ba7e186d4fb..1cf83011206a 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -76,6 +76,7 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -87,7 +88,9 @@ INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -132,9 +135,13 @@ INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -267,6 +274,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -645,11 +655,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 8 +#define INSTR_PROF_RAW_VERSION 9 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 10 +#define INSTR_PROF_INDEX_VERSION 11 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 5 +#define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the * version for other variants of profile. We set the 8th most significant bit @@ -686,6 +696,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -697,6 +708,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -708,6 +720,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -722,6 +735,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 17194e2aa4c7..ff53df8af68a 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -323,11 +323,14 @@ private: // the variant types of the profile. uint64_t Version; uint64_t CountersDelta; + uint64_t BitmapDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; const char *CountersStart; const char *CountersEnd; + const char *BitmapStart; + const char *BitmapEnd; const char *NamesStart; const char *NamesEnd; // After value profile is all read, this pointer points to @@ -429,6 +432,7 @@ private: Error readName(NamedInstrProfRecord &Record); Error readFuncHash(NamedInstrProfRecord &Record); Error readRawCounts(InstrProfRecord &Record); + Error readRawBitmapBytes(InstrProfRecord &Record); Error readValueProfilingData(InstrProfRecord &Record); bool atEnd() const { return Data == DataEnd; } @@ -441,6 +445,7 @@ private: // As we advance to the next record, we maintain the correct CountersDelta // with respect to the next record. CountersDelta -= sizeof(*Data); + BitmapDelta -= sizeof(*Data); } Data++; ValueDataStart += CurValueDataSize; @@ -732,6 +737,10 @@ public: Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts); + /// Fill Bitmap Bytes with the profile data for the given function name. + Error getFunctionBitmapBytes(StringRef FuncName, uint64_t FuncHash, + std::vector &BitmapBytes); + /// Return the maximum of all known function counts. /// \c UseCS indicates whether to use the context-sensitive count. uint64_t getMaximumFunctionCount(bool UseCS) { diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index cb0c055dcb74..d8f3e75087ac 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -50,6 +50,7 @@ private: uint32_t NumValueSites[IPVK_Last + 1]; GlobalVariable *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; + GlobalVariable *RegionBitmaps = nullptr; PerFunctionProfileData() { memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); @@ -105,20 +106,59 @@ private: /// Force emitting of name vars for unused functions. void lowerCoverageData(GlobalVariable *CoverageNamesVar); + /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction + /// using the index represented by the a temp value into a bitmap. + void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); + + /// Replace instrprof.mcdc.temp.update with a shift and or instruction using + /// the corresponding condition ID. + void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); + /// Compute the address of the counter value that this profiling instruction /// acts on. - Value *getCounterAddress(InstrProfInstBase *I); + Value *getCounterAddress(InstrProfCntrInstBase *I); /// Get the region counters for an increment, creating them if necessary. /// /// If the counter array doesn't yet exist, the profile data variables /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc); + GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); /// Create the region counters. - GlobalVariable *createRegionCounters(InstrProfInstBase *Inc, StringRef Name, + GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, + StringRef Name, GlobalValue::LinkageTypes Linkage); + /// Compute the address of the test vector bitmap that this profiling + /// instruction acts on. + Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); + + /// Get the region bitmaps for an increment, creating them if necessary. + /// + /// If the bitmap array doesn't yet exist, the profile data variables + /// referring to them will also be created. + GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); + + /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with + /// an MC/DC Decision region. The number of bytes required is indicated by + /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called + /// as part of setupProfileSection() and is conceptually very similar to + /// what is done for profile data counters in createRegionCounters(). + GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage); + + /// Set Comdat property of GV, if required. + void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName); + + /// Setup the sections into which counters and bitmaps are allocated. + GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK); + + /// Create INSTR_PROF_DATA variable for counters and bitmaps. + void createDataVariable(InstrProfCntrInstBase *Inc, + InstrProfMCDCBitmapParameters *Update); + /// Emit the section with compressed function names. void emitNameData(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 20c37eb4cb11..6e11bd28a683 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7153,6 +7153,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, llvm_unreachable("instrprof failed to lower a timestamp"); case Intrinsic::instrprof_value_profile: llvm_unreachable("instrprof failed to lower a value profiling call"); + case Intrinsic::instrprof_mcdc_parameters: + llvm_unreachable("instrprof failed to lower mcdc parameters"); + case Intrinsic::instrprof_mcdc_tvbitmap_update: + llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update"); + case Intrinsic::instrprof_mcdc_condbitmap_update: + llvm_unreachable("instrprof failed to lower an mcdc condbitmap update"); case Intrinsic::localescape: { MachineFunction &MF = DAG.getMachineFunction(); const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index 36d56699c64e..4270247ef0f5 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -270,13 +270,13 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef NameTable, return -1; } -ConstantInt *InstrProfInstBase::getNumCounters() const { +ConstantInt *InstrProfCntrInstBase::getNumCounters() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("InstrProfValueProfileInst does not have counters!"); return cast(const_cast(getArgOperand(2))); } -ConstantInt *InstrProfInstBase::getIndex() const { +ConstantInt *InstrProfCntrInstBase::getIndex() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()"); return cast(const_cast(getArgOperand(3))); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 05737323314a..cd3970c5151d 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -757,6 +757,7 @@ Expected> CovMapFuncRecordReader::get( case CovMapVersion::Version4: case CovMapVersion::Version5: case CovMapVersion::Version6: + case CovMapVersion::Version7: // Decompress the name data. if (Error E = P.create(P.getNameData())) return std::move(E); @@ -775,6 +776,9 @@ Expected> CovMapFuncRecordReader::get( else if (Version == CovMapVersion::Version6) return std::make_unique>(P, R, D, F); + else if (Version == CovMapVersion::Version7) + return std::make_unique>(P, R, D, F); } llvm_unreachable("Unsupported version"); } diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 835dd697bc7b..7e2ee660978a 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -136,6 +136,9 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::count_mismatch: OS << "function basic block count change detected (counter mismatch)"; break; + case instrprof_error::bitmap_mismatch: + OS << "function bitmap size change detected (bitmap size mismatch)"; + break; case instrprof_error::counter_overflow: OS << "counter overflow"; break; @@ -804,6 +807,18 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, Warn(instrprof_error::counter_overflow); } + // If the number of bitmap bytes doesn't match we either have bad data + // or a hash collision. + if (BitmapBytes.size() != Other.BitmapBytes.size()) { + Warn(instrprof_error::bitmap_mismatch); + return; + } + + // Bitmap bytes are merged by simply ORing them together. + for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) { + BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I]; + } + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) mergeValueProfData(Kind, Other, Weight, Warn); } @@ -1476,9 +1491,11 @@ Expected
Header::readFromBuffer(const unsigned char *Buffer) { // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version10, + IndexedInstrProf::ProfVersion::CurrentVersion == Version11, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); + case 11ull: + [[fallthrough]]; case 10ull: H.TemporalProfTracesOffset = read(Buffer, offsetOf(&Header::TemporalProfTracesOffset)); @@ -1502,10 +1519,12 @@ size_t Header::size() const { // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); + case 11ull: + [[fallthrough]]; case 10ull: return offsetOf(&Header::TemporalProfTracesOffset) + sizeof(Header::TemporalProfTracesOffset); diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index c822d81f8bef..c917394520df 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -207,11 +207,15 @@ void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, // In this mode, CounterPtr actually stores the section relative address // of the counter. maybeSwap(CounterOffset), + // TODO: MC/DC is not yet supported. + /*BitmapOffset=*/maybeSwap(0), maybeSwap(FunctionPtr), // TODO: Value profiling is not yet supported. /*ValuesPtr=*/maybeSwap(0), maybeSwap(NumCounters), /*NumValueSites=*/{maybeSwap(0), maybeSwap(0)}, + // TODO: MC/DC is not yet supported. + /*NumBitmapBytes=*/maybeSwap(0), }); NamesVec.push_back(FunctionName.str()); } diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 125184d40208..983a0c3e6c6a 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -433,6 +433,29 @@ Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { Record.Counts.push_back(Count); } + // Bitmap byte information is indicated with special character. + if (Line->startswith("$")) { + Record.BitmapBytes.clear(); + // Read the number of bitmap bytes. + uint64_t NumBitmapBytes; + if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes)) + return error(instrprof_error::malformed, + "number of bitmap bytes is not a valid integer"); + if (NumBitmapBytes != 0) { + // Read each bitmap and fill our internal storage with the values. + Record.BitmapBytes.reserve(NumBitmapBytes); + for (uint8_t I = 0; I < NumBitmapBytes; ++I) { + if (Line.is_at_end()) + return error(instrprof_error::truncated); + uint8_t BitmapByte; + if ((Line++)->getAsInteger(0, BitmapByte)) + return error(instrprof_error::malformed, + "bitmap byte is not a valid integer"); + Record.BitmapBytes.push_back(BitmapByte); + } + } + } + // Check if value profile data exists and read it if so. if (Error E = readValueProfileData(Record)) return error(std::move(E)); @@ -549,11 +572,14 @@ Error RawInstrProfReader::readHeader( return error(instrprof_error::bad_header); CountersDelta = swap(Header.CountersDelta); + BitmapDelta = swap(Header.BitmapDelta); NamesDelta = swap(Header.NamesDelta); auto NumData = swap(Header.NumData); auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize(); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); + auto NumBitmapBytes = swap(Header.NumBitmapBytes); + auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes); auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); @@ -563,8 +589,10 @@ Error RawInstrProfReader::readHeader( // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize; ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters; - ptrdiff_t NamesOffset = + ptrdiff_t BitmapOffset = CountersOffset + CountersSize + PaddingBytesAfterCounters; + ptrdiff_t NamesOffset = + BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; auto *Start = reinterpret_cast(&Header); @@ -593,6 +621,8 @@ Error RawInstrProfReader::readHeader( reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); CountersStart = Start + CountersOffset; CountersEnd = CountersStart + CountersSize; + BitmapStart = Start + BitmapOffset; + BitmapEnd = BitmapStart + NumBitmapBytes; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); @@ -683,6 +713,49 @@ Error RawInstrProfReader::readRawCounts( return success(); } +template +Error RawInstrProfReader::readRawBitmapBytes(InstrProfRecord &Record) { + uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes); + + Record.BitmapBytes.clear(); + Record.BitmapBytes.reserve(NumBitmapBytes); + + // It's possible MCDC is either not enabled or only used for some functions + // and not others. So if we record 0 bytes, just move on. + if (NumBitmapBytes == 0) + return success(); + + // BitmapDelta decreases as we advance to the next data record. + ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta; + if (BitmapOffset < 0) + return error( + instrprof_error::malformed, + ("bitmap offset " + Twine(BitmapOffset) + " is negative").str()); + + if (BitmapOffset >= BitmapEnd - BitmapStart) + return error(instrprof_error::malformed, + ("bitmap offset " + Twine(BitmapOffset) + + " is greater than the maximum bitmap offset " + + Twine(BitmapEnd - BitmapStart - 1)) + .str()); + + uint64_t MaxNumBitmapBytes = + (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t); + if (NumBitmapBytes > MaxNumBitmapBytes) + return error(instrprof_error::malformed, + ("number of bitmap bytes " + Twine(NumBitmapBytes) + + " is greater than the maximum number of bitmap bytes " + + Twine(MaxNumBitmapBytes)) + .str()); + + for (uint32_t I = 0; I < NumBitmapBytes; I++) { + const char *Ptr = BitmapStart + BitmapOffset + I; + Record.BitmapBytes.push_back(swap(*Ptr)); + } + + return success(); +} + template Error RawInstrProfReader::readValueProfilingData( InstrProfRecord &Record) { @@ -733,6 +806,10 @@ Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) if (Error E = readRawCounts(Record)) return error(std::move(E)); + // Read raw bitmap bytes and set Record. + if (Error E = readRawBitmapBytes(Record)) + return error(std::move(E)); + // Read value data and set Record. if (Error E = readValueProfilingData(Record)) return error(std::move(E)); @@ -794,6 +871,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, DataBuffer.clear(); std::vector CounterBuffer; + std::vector BitmapByteBuffer; const unsigned char *End = D + N; while (D < End) { @@ -819,7 +897,26 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); - DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); + // Read bitmap bytes for GET_VERSION(FormatVersion) > 10. + if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version10) { + uint64_t BitmapBytes = 0; + if (D + sizeof(uint64_t) > End) + return data_type(); + BitmapBytes = + endian::readNext(D); + // Read bitmap byte values. + if (D + BitmapBytes * sizeof(uint8_t) > End) + return data_type(); + BitmapByteBuffer.clear(); + BitmapByteBuffer.reserve(BitmapBytes); + for (uint64_t J = 0; J < BitmapBytes; ++J) + BitmapByteBuffer.push_back(static_cast( + endian::readNext( + D))); + } + + DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer), + std::move(BitmapByteBuffer)); // Read value profiling data. if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && @@ -1319,6 +1416,16 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, return success(); } +Error IndexedInstrProfReader::getFunctionBitmapBytes( + StringRef FuncName, uint64_t FuncHash, std::vector &BitmapBytes) { + Expected Record = getInstrProfRecord(FuncName, FuncHash); + if (Error E = Record.takeError()) + return error(std::move(E)); + + BitmapBytes = Record.get().BitmapBytes; + return success(); +} + Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { ArrayRef Data; diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index b74d5c3862d8..d8b8571bd5c9 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -131,6 +131,8 @@ public: M += sizeof(uint64_t); // The function hash M += sizeof(uint64_t); // The size of the Counts vector M += ProfRecord.Counts.size() * sizeof(uint64_t); + M += sizeof(uint64_t); // The size of the Bitmap vector + M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t); // Value data M += ValueProfData::getSize(ProfileData.second); @@ -160,6 +162,10 @@ public: for (uint64_t I : ProfRecord.Counts) LE.write(I); + LE.write(ProfRecord.BitmapBytes.size()); + for (uint64_t I : ProfRecord.BitmapBytes) + LE.write(I); + // Write value data std::unique_ptr VDataPtr = ValueProfData::serializeFrom(ProfileData.second); @@ -380,6 +386,8 @@ bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { const InstrProfRecord &IPR = Func.second; if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) return true; + if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; })) + return true; } return false; } @@ -703,6 +711,17 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, for (uint64_t Count : Func.Counts) OS << Count << "\n"; + if (Func.BitmapBytes.size() > 0) { + OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n"; + OS << "# Bitmap Byte Values:\n"; + for (uint8_t Byte : Func.BitmapBytes) { + OS << "0x"; + OS.write_hex(Byte); + OS << "\n"; + } + OS << "\n"; + } + uint32_t NumValueKinds = Func.getNumValueKinds(); if (!NumValueKinds) { OS << "\n"; diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 343554241da3..ce287551eed6 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -430,6 +430,15 @@ bool InstrProfiling::lowerIntrinsics(Function *F) { } else if (auto *IPVP = dyn_cast(&Instr)) { lowerValueProfileInst(IPVP); MadeChange = true; + } else if (auto *IPMP = dyn_cast(&Instr)) { + IPMP->eraseFromParent(); + MadeChange = true; + } else if (auto *IPBU = dyn_cast(&Instr)) { + lowerMCDCTestVectorBitmapUpdate(IPBU); + MadeChange = true; + } else if (auto *IPTU = dyn_cast(&Instr)) { + lowerMCDCCondBitmapUpdate(IPTU); + MadeChange = true; } } } @@ -544,19 +553,33 @@ bool InstrProfiling::run( // the instrumented function. This is counting the number of instrumented // target value sites to enter it as field in the profile data variable. for (Function &F : M) { - InstrProfInstBase *FirstProfInst = nullptr; - for (BasicBlock &BB : F) - for (auto I = BB.begin(), E = BB.end(); I != E; I++) + InstrProfCntrInstBase *FirstProfInst = nullptr; + InstrProfMCDCBitmapParameters *FirstProfMCDCParams = nullptr; + for (BasicBlock &BB : F) { + for (auto I = BB.begin(), E = BB.end(); I != E; I++) { if (auto *Ind = dyn_cast(I)) computeNumValueSiteCounts(Ind); - else if (FirstProfInst == nullptr && - (isa(I) || isa(I))) - FirstProfInst = dyn_cast(I); + else { + if (FirstProfInst == nullptr && + (isa(I) || isa(I))) + FirstProfInst = dyn_cast(I); + if (FirstProfMCDCParams == nullptr) + FirstProfMCDCParams = dyn_cast(I); + } + } + } + + // If the MCDCBitmapParameters intrinsic was seen, create the bitmaps. + if (FirstProfMCDCParams != nullptr) { + static_cast(getOrCreateRegionBitmaps(FirstProfMCDCParams)); + } - // Value profiling intrinsic lowering requires per-function profile data - // variable to be created first. - if (FirstProfInst != nullptr) + // Use a profile intrinsic to create the region counters and data variable. + // Also create the data variable based on the MCDCParams. + if (FirstProfInst != nullptr) { static_cast(getOrCreateRegionCounters(FirstProfInst)); + createDataVariable(FirstProfInst, FirstProfMCDCParams); + } } for (Function &F : M) @@ -670,7 +693,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { Ind->eraseFromParent(); } -Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { +Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { auto *Counters = getOrCreateRegionCounters(I); IRBuilder<> Builder(I); @@ -710,6 +733,25 @@ Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { return Builder.CreateIntToPtr(Add, Addr->getType()); } +Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { + auto *Bitmaps = getOrCreateRegionBitmaps(I); + IRBuilder<> Builder(I); + + auto *Addr = Builder.CreateConstInBoundsGEP2_32( + Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue()); + + if (isRuntimeCounterRelocationEnabled()) { + LLVMContext &Ctx = M->getContext(); + Ctx.diagnose(DiagnosticInfoPGOProfile( + M->getName().data(), + Twine("Runtime counter relocation is presently not supported for MC/DC " + "bitmaps."), + DS_Warning)); + } + + return Addr; +} + void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) { auto *Addr = getCounterAddress(CoverInstruction); IRBuilder<> Builder(CoverInstruction); @@ -769,6 +811,86 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { CoverageNamesVar->eraseFromParent(); } +void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( + InstrProfMCDCTVBitmapUpdate *Update) { + IRBuilder<> Builder(Update); + auto *Int8Ty = Type::getInt8Ty(M->getContext()); + auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); + auto *BitmapAddr = getBitmapAddress(Update); + + // Load Temp Val. + // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); + + // Calculate byte offset using div8. + // %1 = lshr i32 %mcdc.temp, 3 + auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3); + + // Add byte offset to section base byte address. + // %2 = zext i32 %1 to i64 + // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2 + auto *BitmapByteAddr = + Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty), + Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty)); + + // Convert to a pointer. + // %4 = inttoptr i32 %3 to ptr + BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy); + + // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8) + // %5 = and i32 %mcdc.temp, 7 + // %6 = trunc i32 %5 to i8 + auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty); + + // Shift bit offset left to form a bitmap. + // %7 = shl i8 1, %6 + auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet); + + // Load profile bitmap byte. + // %mcdc.bits = load i8, ptr %4, align 1 + auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits"); + + // Perform logical OR of profile bitmap byte and shifted bit offset. + // %8 = or i8 %mcdc.bits, %7 + auto *Result = Builder.CreateOr(Bitmap, ShiftedVal); + + // Store the updated profile bitmap byte. + // store i8 %8, ptr %3, align 1 + Builder.CreateStore(Result, BitmapByteAddr); + Update->eraseFromParent(); +} + +void InstrProfiling::lowerMCDCCondBitmapUpdate( + InstrProfMCDCCondBitmapUpdate *Update) { + IRBuilder<> Builder(Update); + auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); + + // Load the MCDC temporary value from the stack. + // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); + + // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits. + // %1 = zext i1 %tobool to i32 + auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty); + + // Shift the boolean value left (by the condition's ID) to form a bitmap. + // %2 = shl i32 %1, getCondID()> + auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID()); + + // Perform logical OR of the bitmap against the loaded MCDC temporary value. + // %3 = or i32 %mcdc.temp, %2 + auto *Result = Builder.CreateOr(Temp, ShiftedVal); + + // Store the updated temporary value back to the stack. + // store i32 %3, ptr %mcdc.addr, align 4 + Builder.CreateStore(Result, MCDCCondBitmapAddr); + Update->eraseFromParent(); +} + /// Get the name of a profiling variable for a particular function. static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed) { @@ -924,37 +1046,31 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { return true; } -GlobalVariable * -InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name, - GlobalValue::LinkageTypes Linkage) { - uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto &Ctx = M->getContext(); - GlobalVariable *GV; - if (isa(Inc)) { - auto *CounterTy = Type::getInt8Ty(Ctx); - auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); - // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. - std::vector InitialValues(NumCounters, - Constant::getAllOnesValue(CounterTy)); - GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, - ConstantArray::get(CounterArrTy, InitialValues), - Name); - GV->setAlignment(Align(1)); - } else { - auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); - GV = new GlobalVariable(*M, CounterTy, false, Linkage, - Constant::getNullValue(CounterTy), Name); - GV->setAlignment(Align(8)); - } - return GV; +void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, + StringRef VarName) { + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); + + if (!UseComdat) + return; + + StringRef GroupName = + TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName; + Comdat *C = M->getOrInsertComdat(GroupName); + if (!NeedComdat) + C->setSelectionKind(Comdat::NoDeduplicate); + GV->setComdat(C); + // COFF doesn't allow the comdat group leader to have private linkage, so + // upgrade private linkage to internal linkage to produce a symbol table + // entry. + if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) + GV->setLinkage(GlobalValue::InternalLinkage); } -GlobalVariable * -InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { +GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK) { GlobalVariable *NamePtr = Inc->getName(); - auto &PD = ProfileDataMap[NamePtr]; - if (PD.RegionCounters) - return PD.RegionCounters; // Match the linkage and visibility of the name global. Function *Fn = Inc->getParent()->getParent(); @@ -993,42 +1109,100 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { // nodeduplicate COMDAT which is lowered to a zero-flag section group. This // allows -z start-stop-gc to discard the entire group when the function is // discarded. - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); bool Renamed; - std::string CntsVarName = - getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); - std::string DataVarName = - getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); - auto MaybeSetComdat = [&](GlobalVariable *GV) { - bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); - if (UseComdat) { - StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode - ? GV->getName() - : CntsVarName; - Comdat *C = M->getOrInsertComdat(GroupName); - if (!NeedComdat) - C->setSelectionKind(Comdat::NoDeduplicate); - GV->setComdat(C); - // COFF doesn't allow the comdat group leader to have private linkage, so - // upgrade private linkage to internal linkage to produce a symbol table - // entry. - if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) - GV->setLinkage(GlobalValue::InternalLinkage); - } - }; + GlobalVariable *Ptr; + StringRef VarPrefix; + std::string VarName; + if (IPSK == IPSK_cnts) { + VarPrefix = getInstrProfCountersVarPrefix(); + VarName = getVarName(Inc, VarPrefix, Renamed); + InstrProfCntrInstBase *CntrIncrement = dyn_cast(Inc); + Ptr = createRegionCounters(CntrIncrement, VarName, Linkage); + } else if (IPSK == IPSK_bitmap) { + VarPrefix = getInstrProfBitmapVarPrefix(); + VarName = getVarName(Inc, VarPrefix, Renamed); + InstrProfMCDCBitmapInstBase *BitmapUpdate = + dyn_cast(Inc); + Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage); + } else { + llvm_unreachable("Profile Section must be for Counters or Bitmaps"); + } + + Ptr->setVisibility(Visibility); + // Put the counters and bitmaps in their own sections so linkers can + // remove unneeded sections. + Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat())); + Ptr->setLinkage(Linkage); + maybeSetComdat(Ptr, Fn, VarName); + return Ptr; +} + +GlobalVariable * +InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage) { + uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); + auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes); + auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage, + Constant::getNullValue(BitmapTy), Name); + GV->setAlignment(Align(1)); + return GV; +} + +GlobalVariable * +InstrProfiling::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) { + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionBitmaps) + return PD.RegionBitmaps; + + // If RegionBitmaps doesn't already exist, create it by first setting up + // the corresponding profile section. + auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap); + PD.RegionBitmaps = BitmapPtr; + return PD.RegionBitmaps; +} +GlobalVariable * +InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, + GlobalValue::LinkageTypes Linkage) { uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - LLVMContext &Ctx = M->getContext(); + auto &Ctx = M->getContext(); + GlobalVariable *GV; + if (isa(Inc)) { + auto *CounterTy = Type::getInt8Ty(Ctx); + auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); + // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. + std::vector InitialValues(NumCounters, + Constant::getAllOnesValue(CounterTy)); + GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, + ConstantArray::get(CounterArrTy, InitialValues), + Name); + GV->setAlignment(Align(1)); + } else { + auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + GV = new GlobalVariable(*M, CounterTy, false, Linkage, + Constant::getNullValue(CounterTy), Name); + GV->setAlignment(Align(8)); + } + return GV; +} + +GlobalVariable * +InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionCounters) + return PD.RegionCounters; - auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage); - CounterPtr->setVisibility(Visibility); - CounterPtr->setSection( - getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); - CounterPtr->setLinkage(Linkage); - MaybeSetComdat(CounterPtr); + // If RegionCounters doesn't already exist, create it by first setting up + // the corresponding profile section. + auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts); PD.RegionCounters = CounterPtr; + if (DebugInfoCorrelate) { + LLVMContext &Ctx = M->getContext(); + Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { @@ -1066,8 +1240,50 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); } + + // Mark the counter variable as used so that it isn't optimized out. + CompilerUsedVars.push_back(PD.RegionCounters); } + return PD.RegionCounters; +} + +void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, + InstrProfMCDCBitmapParameters *Params) { + // When debug information is correlated to profile data, a data variable + // is not needed. + if (DebugInfoCorrelate) + return; + + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + + LLVMContext &Ctx = M->getContext(); + + Function *Fn = Inc->getParent()->getParent(); + GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); + GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); + + // Due to the limitation of binder as of 2021/09/28, the duplicate weak + // symbols in the same csect won't be discarded. When there are duplicate weak + // symbols, we can NOT guarantee that the relocations get resolved to the + // intended weak symbol, so we can not ensure the correctness of the relative + // CounterPtr, so we have to use private linkage for counter and data symbols. + if (TT.isOSBinFormatXCOFF()) { + Linkage = GlobalValue::PrivateLinkage; + Visibility = GlobalValue::DefaultVisibility; + } + + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool Renamed; + + // The Data Variable section is anchored to profile counters. + std::string CntsVarName = + getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); + std::string DataVarName = + getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); + auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for // the current function. @@ -1085,16 +1301,17 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); - MaybeSetComdat(ValuesVar); + maybeSetComdat(ValuesVar, Fn, CntsVarName); ValuesPtrExpr = ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } - if (DebugInfoCorrelate) { - // Mark the counter variable as used so that it isn't optimized out. - CompilerUsedVars.push_back(PD.RegionCounters); - return PD.RegionCounters; - } + uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); + auto *CounterPtr = PD.RegionCounters; + + uint64_t NumBitmapBytes = 0; + if (Params != nullptr) + NumBitmapBytes = Params->getNumBitmapBytes()->getZExtValue(); // Create data variable. auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); @@ -1137,6 +1354,16 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); + // Bitmaps are relative to the same data variable as profile counters. + GlobalVariable *BitmapPtr = PD.RegionBitmaps; + Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); + + if (BitmapPtr != nullptr) { + RelativeBitmapPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + ConstantExpr::getPtrToInt(Data, IntPtrTy)); + } + Constant *DataVals[] = { #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, #include "llvm/ProfileData/InstrProfData.inc" @@ -1146,7 +1373,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { Data->setVisibility(Visibility); Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); - MaybeSetComdat(Data); + maybeSetComdat(Data, Fn, CntsVarName); PD.DataVar = Data; @@ -1158,8 +1385,6 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { NamePtr->setLinkage(GlobalValue::PrivateLinkage); // Collect the referenced names to be used by emitNameData. ReferencedNames.push_back(NamePtr); - - return PD.RegionCounters; } void InstrProfiling::emitVNodes() { diff --git a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll new file mode 100644 index 000000000000..fccb026c25bf --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll @@ -0,0 +1,53 @@ +; Check that MC/DC intrinsics are properly lowered +; RUN: opt < %s -passes=instrprof -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -runtime-counter-relocation -S 2>&1 | FileCheck %s --check-prefix RELOC + +; RELOC: Runtime counter relocation is presently not supported for MC/DC bitmaps + +target triple = "x86_64-unknown-linux-gnu" + +@__profn_test = private constant [4 x i8] c"test" + +; CHECK: @__profbm_test = private global [1 x i8] zeroinitializer, section "__llvm_prf_bits", comdat, align 1 + +define dso_local void @test(i32 noundef %A) { +entry: + %A.addr = alloca i32, align 4 + %mcdc.addr = alloca i32, align 4 + call void @llvm.instrprof.cover(ptr @__profn_test, i64 99278, i32 5, i32 0) + ; CHECK: store i8 0, ptr @__profc_test, align 1 + + call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 99278, i32 1) + store i32 0, ptr %mcdc.addr, align 4 + %0 = load i32, ptr %A.addr, align 4 + %tobool = icmp ne i32 %0, 0 + + call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 99278, i32 0, ptr %mcdc.addr, i1 %tobool) + ; CHECK: %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + ; CHECK-NEXT: %1 = zext i1 %tobool to i32 + ; CHECK-NEXT: %2 = shl i32 %1, 0 + ; CHECK-NEXT: %3 = or i32 %mcdc.temp, %2 + ; CHECK-NEXT: store i32 %3, ptr %mcdc.addr, align 4 + + call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 99278, i32 1, i32 0, ptr %mcdc.addr) + ; CHECK: %mcdc.temp1 = load i32, ptr %mcdc.addr, align 4 + ; CHECK-NEXT: %4 = lshr i32 %mcdc.temp1, 3 + ; CHECK-NEXT: %5 = zext i32 %4 to i64 + ; CHECK-NEXT: %6 = add i64 ptrtoint (ptr @__profbm_test to i64), %5 + ; CHECK-NEXT: %7 = inttoptr i64 %6 to ptr + ; CHECK-NEXT: %8 = and i32 %mcdc.temp1, 7 + ; CHECK-NEXT: %9 = trunc i32 %8 to i8 + ; CHECK-NEXT: %10 = shl i8 1, %9 + ; CHECK-NEXT: %mcdc.bits = load i8, ptr %7, align 1 + ; CHECK-NEXT: %11 = or i8 %mcdc.bits, %10 + ; CHECK-NEXT: store i8 %11, ptr %7, align 1 + ret void +} + +declare void @llvm.instrprof.cover(ptr, i64, i32, i32) + +declare void @llvm.instrprof.mcdc.parameters(ptr, i64, i32) + +declare void @llvm.instrprof.mcdc.condbitmap.update(ptr, i64, i32, ptr, i1) + +declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr, i64, i32, i32, ptr) diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll index 1c44a274f3c0..8c6942c0f527 100644 --- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll +++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll @@ -13,9 +13,9 @@ $foo = comdat any ; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; CHECK-NOT: __profn__stdin__foo ; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 -; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null +; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null ; CHECK-NOT: @foo -; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 +; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm ; CHECK: @llvm.compiler.used diff --git a/llvm/test/tools/llvm-profdata/Inputs/basic.profraw b/llvm/test/tools/llvm-profdata/Inputs/basic.profraw index ad88759398c6020f4ab8a5606258e69d98e36687..1b284b84fad6dd7f9407b1c3b99cb178af0e09c6 100644 GIT binary patch delta 63 zcmbQicz}_!u_!ISs37M*=R{6@Muv%+@_Y__aks4+{{I)d(5P~>exjd}00R^}C!MPiq_)7#Jp|DQPSa2C)!;5z2(oEDX7cnRyHh E0CJEFxc~qF diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw index bc8fc5db1cb154d98ca962e84313463e3298cb92..9cd225587c92511e99f3497ce1d5f47c6fc5f0af 100644 GIT binary patch delta 308 zcmeC+d%(}xSd^AuRFLzZb0Vky_8T_ z2Map_g$4d1%$=MM7A^z|M<5H&2MaF*3U?q0gIzKCB2aV#lIY~~%z~3Sm;_Lj)^mdV hG1&qro`I~m9;i42C}@Bz*w4f{If6-G^8%&}764*Dk$V6D delta 330 zcmaFB-@(V(Sd^AuRFLzZV$QVgXRX-d83sq200i+W)W}ZBd!q5+)yH6rcP6 zC=1puPjh&2EJ diff --git a/llvm/test/tools/llvm-profdata/Inputs/compat.profdata.v10 b/llvm/test/tools/llvm-profdata/Inputs/compat.profdata.v10 new file mode 100644 index 0000000000000000000000000000000000000000..c331e30b48ff5d3be2efe4636d0c9fee56e764b5 GIT binary patch literal 872 zcmeyLQ&5zjmf6U~fE@@hqlzb>@!6o#0#KR>O2d@>u3-d=z-Xv61B@@A4iSOzA*vY| z7PO)2gNZxvLQF>`)21b~& z3=Oc72^^Cruy9PAA;2N==RX)sp2#ddaX%*`!^DH)91h5W??Hke#W_5X1SiWg0!1e) zGKzETKo<4~3kL#)1^yz;om>wVZUhQPAPcVt3vUDpcOVIaT`~C~P;>*5=;Zs%f|Dhf o1W=XsbAtRa*#jt^fvmV6sJH?sXn-s@pNVsF29v<%4NMj+0Qs7eE&u=k delta 364 zcmdnM|AJSru_!ISs37M*2Ll8MOcYd~=pmqxDy0AxV1$ap532 zg3W7Lmwt5a%Luw`8X;yQ=%O*PQGDV94nY>r<189Si!*Iwboso6f2t<{#V0E;atMY_ zsnBGduNo&k$1~O>`Qe2MkVFDdqEPdcjat*B{dO76%_kSkWcUG+SOAo;_m#;@Xm_l> z_CIV#TU6$@gvkdP#V0=i%0w>dTG2Ucam2*OHvOM-SQvJIWfYh=1iwzTW$_M(6^_4X z6sG@NZp{yn4A5N?3QrGb?)?~3dEpwwUk3mFgTdqpOcD~cFPdW;(Y diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test index 67db5c98ef32..eda63203a304 100644 --- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -5,13 +5,15 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. // 2 * 8 binary ID sizes // + 2 * 20 binary IDs (of size 20) @@ -23,8 +25,11 @@ RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\20\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -51,14 +56,18 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t.profraw +RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/compat.proftext b/llvm/test/tools/llvm-profdata/compat.proftext index 73321cc5e66d..111fd4198819 100644 --- a/llvm/test/tools/llvm-profdata/compat.proftext +++ b/llvm/test/tools/llvm-profdata/compat.proftext @@ -87,3 +87,26 @@ large_numbers # FORMATV4: Total functions: 3 # FORMATV4: Maximum function count: 2305843009213693952 # FORMATV4: Maximum internal block count: 1152921504606846976 + +# RUN: llvm-profdata show %S/Inputs/compat.profdata.v10 -all-functions --counts | FileCheck %s -check-prefix=FORMATV10 + +# FORMATV10: Counters: +# FORMATV10: large_numbers: +# FORMATV10: Hash: 0x3fffffffffffffff +# FORMATV10: Counters: 6 +# FORMATV10: Function count: 2305843009213693952 +# FORMATV10: Block counts: [1152921504606846976, 576460752303423488, 288230376151711744, 144115188075855872, 72057594037927936] +# FORMATV10: name with spaces: +# FORMATV10: Hash: 0x0000000000000400 +# FORMATV10: Counters: 2 +# FORMATV10: Function count: 0 +# FORMATV10: Block counts: [0] +# FORMATV10: function_count_only: +# FORMATV10: Hash: 0x0000000000000000 +# FORMATV10: Counters: 1 +# FORMATV10: Function count: 97531 +# FORMATV10: Block counts: [] +# FORMATV10: Functions shown: 3 +# FORMATV10: Total functions: 3 +# FORMATV10: Maximum function count: 2305843009213693952 +# FORMATV10: Maximum internal block count: 1152921504606846976 diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test index 2394431e94de..38b838e0d100 100644 --- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -9,6 +9,9 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Check for a corrupted size being too large past the end of the file. RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test index 06f418d0235d..c967e850dbe3 100644 --- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -5,20 +5,25 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -35,7 +40,9 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index b718cf0fd8e9..e1e33824bf2f 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -5,20 +5,26 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -35,8 +41,10 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Make NumCounters = 0 so that we get "number of counters is zero" error message RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test index 38e40334a6a6..3c23bc7dd0f7 100644 --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -5,20 +5,25 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -38,10 +43,12 @@ RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw // Octal '\11' is 9 in decimal: this should push CounterOffset to 1. As there are two counters, // the profile reader should error out. RUN: printf '\11\0\6\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Counter Section diff --git a/llvm/test/tools/llvm-profdata/mcdc-bitmap.test b/llvm/test/tools/llvm-profdata/mcdc-bitmap.test new file mode 100644 index 000000000000..a7b1b5df8c30 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/mcdc-bitmap.test @@ -0,0 +1,201 @@ +# Test MC/DC bitmap reading and merging. + +# Merge as profdata. +RUN: split-file %s %t +RUN: llvm-profdata merge %t/mcdc-1.proftext %t/mcdc-2.proftext -o %t.profdata +RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC +# Merge as proftext. +RUN: llvm-profdata merge %t/mcdc-1.proftext %t/mcdc-2.proftext -o %t.proftext +RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC + +MCDC: # Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: # Bitmap Byte Values: +MCDC-NEXT: a +MCDC: # Num Bitmap Bytes: +MCDC-NEXT: $2 +MCDC-NEXT: # Bitmap Byte Values: +MCDC-NEXT: 0x29 +MCDC-NEXT: 0x0 + +# Merge as profdata. +RUN: llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-4.proftext -o %t.profdata +RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC2 +# Merge as proftext. +RUN: llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-4.proftext -o %t.proftext +RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC2 + +MCDC2: # Num Bitmap Bytes: +MCDC2-NEXT: $8 +MCDC2-NEXT: # Bitmap Byte Values: +MCDC2-NEXT: 0x1 +MCDC2-NEXT: 0x2 +MCDC2-NEXT: 0x3 +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xe +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xa + +# Incompatible size mismatch. +RUN: llvm-profdata merge %t/mcdc-2.proftext %t/mcdc-4.proftext -o %t.profdata 2>&1 | FileCheck %s --check-prefix=MCDC3 +# Merge as proftext +RUN: llvm-profdata merge %t/mcdc-2.proftext %t/mcdc-4.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC3 + +MCDC3: function bitmap size change detected (bitmap size mismatch) + +# Invalid number of bitmap bytes. +RUN: not llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-err0.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC4 + +MCDC4: malformed instrumentation profile data: number of bitmap bytes is not a valid integer + +# Invalid bitmap byte. +RUN: not llvm-profdata merge %t/mcdc-3.proftext %t/mcdc-err1.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC5 + +MCDC5: malformed instrumentation profile data: bitmap byte is not a valid integer + +;--- mcdc-1.proftext +main +# Func Hash: +702755447896 +# Num Counters: +4 +# Counter Values: +1 +0 +1 +0 +# Num Bitmask Bytes: +$1 +# Bitmask Byte Values: +2 +;--- mcdc-2.proftext +main +# Func Hash: +702755447896 +# Num Counters: +4 +# Counter Values: +1 +1 +1 +1 +# Num Bitmask Bytes: +$1 +# Bitmask Byte Values: +8 + + +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$0x2 +# Bitmask Byte Values: +0x29 +0x0 +;--- mcdc-3.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8 +# Bitmask Byte Values: +0x0 +0x2 +0x3 +0xf +0xf +0xa +0xc +0x2 +;--- mcdc-4.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$ 8 +# Bitmask Byte Values: +1 +2 +3 +4 +5 +6 +7 +8 +;--- mcdc-err0.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8.9 +# Bitmask Byte Values: +1 +2 +3 +4 +5 +6 +7 +8 +;--- mcdc-err1.proftext +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8 +# Bitmask Byte Values: +1 +2 +3 +4 +5.4 +6 +7 +8 diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test index 171b5cc60878..4a5c42843ff4 100644 --- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test index c0072bcbde1b..2a92575ee340 100644 --- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -8,6 +8,9 @@ RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test index c8e862009ef0..fbd31d044382 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,37 +1,46 @@ RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\0\0\0\11' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\0\1\0\0\0' >> %t +RUN: printf '\0\0\0\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\1\0\0\0' >> %t +RUN: printf '\3\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\3' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t -RUN: printf '\0\xff\xff\xe0' >> %t +RUN: printf '\0\xff\xff\xd8' >> %t +RUN: printf '\2\xff\xff\xd3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\2' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\1' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -48,3 +57,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test index 523ff1ceb480..bb899c5fdb55 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,37 +1,46 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\10\0\0\0\0\0\0\0' >> %t +RUN: printf '\11\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\0\0\0\2\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t +RUN: printf '\0\0\0\3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\3\0\0\0' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xe0\xff\xff\0' >> %t +RUN: printf '\xd8\xff\xff\0' >> %t +RUN: printf '\xd3\xff\xff\2' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\2\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\1\0\0\0' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -48,3 +57,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test index b2b8b31dafbf..8fcadb6a0dd2 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,35 +1,44 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\0\0\0\11' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t +RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t +RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\02' >> %t -RUN: printf '\0\0\0\1\0\3\xff\xd8' >> %t +RUN: printf '\0\0\0\1\0\3\xff\xc8' >> %t +RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\02\0\0\0\0' >> %t +RUN: printf '\0\0\0\1\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -46,3 +55,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test index 4e95798bc0af..0aa8b38f6926 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,35 +1,44 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\10\0\0\0\0\0\0\0' >> %t +RUN: printf '\11\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t +RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\4\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t +RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t +RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t +RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t +RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -46,3 +55,14 @@ CHECK: Functions shown: 2 CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test index 8d46c91e2732..f4a9aa8e1bbc 100644 --- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,12 +1,15 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -15,20 +18,25 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw @@ -37,7 +45,9 @@ RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw -- Gitee From 6827d06d1a2d2ef654ceab8a06bea4967a0fcfb5 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Tue, 31 Oct 2023 16:40:51 -0400 Subject: [PATCH 08/28] [llvm-profdata] Emit warning when counter value is greater than 2^56. (#69513) Fixes #65416 --- llvm/include/llvm/ProfileData/InstrProf.h | 3 +- .../llvm/ProfileData/InstrProfReader.h | 17 +++++-- llvm/lib/ProfileData/InstrProf.cpp | 3 ++ llvm/lib/ProfileData/InstrProfReader.cpp | 21 ++++++--- .../malformed-num-counters-zero.test | 38 +++++++++++++-- llvm/tools/llvm-profdata/llvm-profdata.cpp | 46 +++++++++++++++---- 6 files changed, 103 insertions(+), 25 deletions(-) diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 2a780104b177..71431aabf577 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -345,7 +345,8 @@ enum class instrprof_error { uncompress_failed, empty_raw_profile, zlib_unavailable, - raw_profile_version_mismatch + raw_profile_version_mismatch, + counter_value_too_large, }; /// An ordered list of functions identified by their NameRef found in diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index ff53df8af68a..b8a6c4a453b6 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -202,11 +202,13 @@ public: /// instrprof file. static Expected> create(const Twine &Path, vfs::FileSystem &FS, - const InstrProfCorrelator *Correlator = nullptr); + const InstrProfCorrelator *Correlator = nullptr, + std::function Warn = nullptr); static Expected> create(std::unique_ptr Buffer, - const InstrProfCorrelator *Correlator = nullptr); + const InstrProfCorrelator *Correlator = nullptr, + std::function Warn = nullptr); /// \param Weight for raw profiles use this as the temporal profile trace /// weight @@ -344,12 +346,19 @@ private: /// Start address of binary id length and data pairs. const uint8_t *BinaryIdsStart; + std::function Warn; + + /// Maxium counter value 2^56. + static const uint64_t MaxCounterValue = (1ULL << 56); + public: RawInstrProfReader(std::unique_ptr DataBuffer, - const InstrProfCorrelator *Correlator) + const InstrProfCorrelator *Correlator, + std::function Warn) : DataBuffer(std::move(DataBuffer)), Correlator(dyn_cast_or_null>( - Correlator)) {} + Correlator)), + Warn(Warn) {} RawInstrProfReader(const RawInstrProfReader &) = delete; RawInstrProfReader &operator=(const RawInstrProfReader &) = delete; diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 7e2ee660978a..beaa175adb5c 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -161,6 +161,9 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::raw_profile_version_mismatch: OS << "raw profile version mismatch"; break; + case instrprof_error::counter_value_too_large: + OS << "excessively large counter value suggests corrupted profile data"; + break; } // If optional error message is not empty, append it to the message. diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 983a0c3e6c6a..412335ff24d1 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -166,17 +166,20 @@ static Error printBinaryIdsInternal(raw_ostream &OS, Expected> InstrProfReader::create(const Twine &Path, vfs::FileSystem &FS, - const InstrProfCorrelator *Correlator) { + const InstrProfCorrelator *Correlator, + std::function Warn) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path, FS); if (Error E = BufferOrError.takeError()) return std::move(E); - return InstrProfReader::create(std::move(BufferOrError.get()), Correlator); + return InstrProfReader::create(std::move(BufferOrError.get()), Correlator, + Warn); } Expected> InstrProfReader::create(std::unique_ptr Buffer, - const InstrProfCorrelator *Correlator) { + const InstrProfCorrelator *Correlator, + std::function Warn) { if (Buffer->getBufferSize() == 0) return make_error(instrprof_error::empty_raw_profile); @@ -185,9 +188,9 @@ InstrProfReader::create(std::unique_ptr Buffer, if (IndexedInstrProfReader::hasFormat(*Buffer)) Result.reset(new IndexedInstrProfReader(std::move(Buffer))); else if (RawInstrProfReader64::hasFormat(*Buffer)) - Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator)); + Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator, Warn)); else if (RawInstrProfReader32::hasFormat(*Buffer)) - Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator)); + Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator, Warn)); else if (TextInstrProfReader::hasFormat(*Buffer)) Result.reset(new TextInstrProfReader(std::move(Buffer))); else @@ -705,8 +708,12 @@ Error RawInstrProfReader::readRawCounts( // A value of zero signifies the block is covered. Record.Counts.push_back(*Ptr == 0 ? 1 : 0); } else { - const auto *CounterValue = reinterpret_cast(Ptr); - Record.Counts.push_back(swap(*CounterValue)); + uint64_t CounterValue = swap(*reinterpret_cast(Ptr)); + if (CounterValue > MaxCounterValue && Warn) + Warn(make_error( + instrprof_error::counter_value_too_large, Twine(CounterValue))); + + Record.Counts.push_back(CounterValue); } } diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index e1e33824bf2f..2e747f81a6bf 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -21,7 +21,6 @@ RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -42,6 +41,11 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw + +// Make two copies for another test. +RUN: cp %t.profraw %t-bad.profraw +RUN: cp %t.profraw %t-good.profraw + // Make NumCounters = 0 so that we get "number of counters is zero" error message RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -49,5 +53,33 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw -RUN: not llvm-profdata show %t.profraw 2>&1 | FileCheck %s -CHECK: malformed instrumentation profile data: number of counters is zero +RUN: not llvm-profdata show %t.profraw 2>&1 | FileCheck %s --check-prefix=ZERO +ZERO: malformed instrumentation profile data: number of counters is zero + +// Test a counter value greater than 2^56. +RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bad.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bad.profraw +// Counter value is 72057594037927937 +RUN: printf '\1\0\0\0\0\0\0\1' >> %t-bad.profraw +RUN: printf '\3\0foo\0\0\0' >> %t-bad.profraw + +RUN: printf '\1\0\0\0\0\0\0\0' >> %t-good.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-good.profraw +// Counter value is 72057594037927937 +RUN: printf '\1\0\0\0\0\0\0\0' >> %t-good.profraw +RUN: printf '\3\0foo\0\0\0' >> %t-good.profraw + +// llvm-profdata fails if there is a warning for any input file under default failure mode (any). +RUN: not llvm-profdata merge %t-bad.profraw %t-good.profraw -o %t.profdata 2>&1 | FileCheck %s --check-prefix=ANY +ANY: {{.*}} excessively large counter value suggests corrupted profile data: 72057594037927937 + +// -failure-mode=all only fails if there is a warning for every input file. +RUN: not llvm-profdata merge %t-bad.profraw -failure-mode=all -o %t.profdata 2>&1 | FileCheck %s --check-prefix=ALL-ERR +ALL-ERR: {{.*}} excessively large counter value suggests corrupted profile data: 72057594037927937 + +RUN: llvm-profdata merge %t-bad.profraw %t-good.profraw -failure-mode=all -o %t.profdata 2>&1 | FileCheck %s --check-prefix=ALL-WARN +ALL-WARN: {{.*}} excessively large counter value suggests corrupted profile data: 72057594037927937 + +// -failure-mode=warn does not fail at all. It only prints warnings. +RUN: llvm-profdata merge %t-bad.profraw -failure-mode=warn -o %t.profdata 2>&1 | FileCheck %s --check-prefix=WARN +WARN: {{.*}} excessively large counter value suggests corrupted profile data: 72057594037927937 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 9da5368f5a42..378025afc2b9 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -114,8 +114,8 @@ static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { namespace { enum ProfileKinds { instr, sample, memory }; -enum FailureMode { failIfAnyAreInvalid, failIfAllAreInvalid }; -} +enum FailureMode { warnOnly, failIfAnyAreInvalid, failIfAllAreInvalid }; +} // namespace static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC, StringRef Whence = "") { @@ -306,7 +306,22 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, } auto FS = vfs::getRealFileSystem(); - auto ReaderOrErr = InstrProfReader::create(Input.Filename, *FS, Correlator); + // TODO: This only saves the first non-fatal error from InstrProfReader, and + // then added to WriterContext::Errors. However, this is not extensible, if + // we have more non-fatal errors from InstrProfReader in the future. How + // should this interact with different -failure-mode? + std::optional> ReaderWarning; + auto Warn = [&](Error E) { + if (ReaderWarning) { + consumeError(std::move(E)); + return; + } + // Only show the first time an error occurs in this file. + auto [ErrCode, Msg] = InstrProfError::take(std::move(E)); + ReaderWarning = {make_error(ErrCode, Msg), Filename}; + }; + auto ReaderOrErr = + InstrProfReader::create(Input.Filename, *FS, Correlator, Warn); if (Error E = ReaderOrErr.takeError()) { // Skip the empty profiles by returning silently. auto [ErrCode, Msg] = InstrProfError::take(std::move(E)); @@ -354,14 +369,23 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, Traces, Reader->getTemporalProfTraceStreamSize()); } if (Reader->hasError()) { - if (Error E = Reader->getError()) + if (Error E = Reader->getError()) { WC->Errors.emplace_back(std::move(E), Filename); + return; + } } std::vector BinaryIds; - if (Error E = Reader->readBinaryIds(BinaryIds)) + if (Error E = Reader->readBinaryIds(BinaryIds)) { WC->Errors.emplace_back(std::move(E), Filename); + return; + } WC->Writer.addBinaryIds(BinaryIds); + + if (ReaderWarning) { + WC->Errors.emplace_back(std::move(ReaderWarning->first), + ReaderWarning->second); + } } /// Merge the \p Src writer context into \p Dst. @@ -484,7 +508,7 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, warn(toString(std::move(ErrorPair.first)), ErrorPair.second); } } - if (NumErrors == Inputs.size() || + if ((NumErrors == Inputs.size() && FailMode == failIfAllAreInvalid) || (NumErrors > 0 && FailMode == failIfAnyAreInvalid)) exitWithError("no profile can be merged"); @@ -1193,10 +1217,12 @@ static int merge_main(int argc, const char *argv[]) { "GCC encoding (only meaningful for -sample)"))); cl::opt FailureMode( "failure-mode", cl::init(failIfAnyAreInvalid), cl::desc("Failure mode:"), - cl::values(clEnumValN(failIfAnyAreInvalid, "any", - "Fail if any profile is invalid."), - clEnumValN(failIfAllAreInvalid, "all", - "Fail only if all profiles are invalid."))); + cl::values( + clEnumValN(warnOnly, "warn", "Do not fail and just print warnings."), + clEnumValN(failIfAnyAreInvalid, "any", + "Fail if any profile is invalid."), + clEnumValN(failIfAllAreInvalid, "all", + "Fail only if all profiles are invalid."))); cl::opt OutputSparse("sparse", cl::init(false), cl::desc("Generate a sparse profile (only meaningful for -instr)")); cl::opt NumThreads( -- Gitee From 3fd4ed80e85d8beae3c1a4e224d5dde7fd1046e3 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 1 Nov 2023 14:16:43 -0400 Subject: [PATCH 09/28] [Profile] Refactor profile correlation. (#70856) Refactor some code from https://github.com/llvm/llvm-project/pull/69493. --- compiler-rt/lib/profile/InstrProfiling.c | 4 + compiler-rt/lib/profile/InstrProfiling.h | 6 ++ .../lib/profile/InstrProfilingBuffer.c | 11 ++ compiler-rt/lib/profile/InstrProfilingMerge.c | 13 +-- .../lib/profile/InstrProfilingWriter.c | 21 ++-- .../instrprof-show-debug-info-correlation.c | 2 +- .../llvm/ProfileData/InstrProfCorrelator.h | 15 ++- .../llvm/ProfileData/InstrProfReader.h | 2 + llvm/lib/ProfileData/InstrProfCorrelator.cpp | 102 +++++++++++------- llvm/lib/ProfileData/InstrProfReader.cpp | 4 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 9 +- 11 files changed, 122 insertions(+), 67 deletions(-) diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c index da04d8ebdec9..7d69e37815c9 100644 --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -89,3 +89,7 @@ COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { } lprofSetProfileDumped(0); } + +inline int hasCorrelation() { + return (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; +} diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index e143149fca82..b8104af5f12b 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -261,6 +261,9 @@ uint64_t __llvm_profile_get_magic(void); /*! \brief Get the version of the file format. */ uint64_t __llvm_profile_get_version(void); +/*! \brief If the binary is compiled with profile correlation. */ +int hasCorrelation(); + /*! \brief Get the number of entries in the profile data section. */ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, const __llvm_profile_data *End); @@ -282,6 +285,9 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, const char *End); +/*! \brief Get the size of the profile name section in bytes. */ +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); + /* ! \brief Given the sizes of the data and counter information, return the * number of padding bytes before and after the counters, and after the names, * in the raw profile. diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index c7217b2dfef8..69965142d978 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -56,6 +56,8 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { + if (hasCorrelation()) + return 0; intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / sizeof(__llvm_profile_data); @@ -64,6 +66,8 @@ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { + if (hasCorrelation()) + return 0; return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); } @@ -92,6 +96,13 @@ uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, return (End - Begin); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) { + if (hasCorrelation()) + return 0; + return End - Begin; +} + /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 520fb561e4d3..da93c0e33c5b 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -70,8 +70,9 @@ int __llvm_profile_check_compatibility(const char *ProfileData, Header->NumBitmapBytes != __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap()) || - Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - - __llvm_profile_begin_names()) || + Header->NamesSize != + __llvm_profile_get_name_size(__llvm_profile_begin_names(), + __llvm_profile_end_names()) || Header->ValueKindLast != IPVK_Last) return 1; @@ -141,13 +142,9 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; - // Merge counters when there is no data section and debug info correlation is + // Merge counters by iterating the entire counter section when correlation is // enabled. - if (Header->NumData == 0) { - if (!(__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE)) { - PROF_ERR("%s\n", "Missing profile data section."); - return 1; - } + if (hasCorrelation()) { for (SrcCounter = SrcCountersStart, DstCounter = __llvm_profile_begin_counters(); SrcCounter < SrcCountersEnd;) { diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 3b61f3def9f6..d35ee6b20504 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -262,21 +262,19 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { - int DebugInfoCorrelate = - (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; + int ProfileCorrelation = hasCorrelation(); /* Calculate size of sections. */ const uint64_t DataSectionSize = - DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd); - const uint64_t NumData = - DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd); + __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd); const uint64_t CountersSectionSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumCounters = __llvm_profile_get_num_counters(CountersBegin, CountersEnd); const uint64_t NumBitmapBytes = __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); - const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; + const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); /* Create the header. */ __llvm_profile_header Header; @@ -304,7 +302,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, #endif /* The data and names sections are omitted in lightweight mode. */ - if (DebugInfoCorrelate) { + if (ProfileCorrelation) { Header.CountersDelta = 0; Header.NamesDelta = 0; } @@ -320,21 +318,22 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Write the profile data. */ ProfDataIOVec IOVecData[] = { - {DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize, + {ProfileCorrelation ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, - {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, + {(SkipNameDataWrite || ProfileCorrelation) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) return -1; - /* Value profiling is not yet supported in continuous mode. */ - if (__llvm_profile_is_continuous_mode_enabled()) + /* Value profiling is not yet supported in continuous mode and profile + * correlation mode. */ + if (__llvm_profile_is_continuous_mode_enabled() || ProfileCorrelation) return 0; return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); diff --git a/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c b/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c index 226d678aca34..245dc7989104 100644 --- a/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c +++ b/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c @@ -4,7 +4,7 @@ // RUN: %clang_pgogen -o %t.no.dbg -mllvm --debug-info-correlate -mllvm --disable-vp=true %s // RUN: not llvm-profdata show --debug-info=%t.no.dbg 2>&1 | FileCheck %s --check-prefix NO-DBG -// NO-DBG: unable to correlate profile: could not find any profile metadata in debug info +// NO-DBG: unable to correlate profile: could not find any profile data metadata in correlated file // YAML: Probes: // YAML: - Function Name: a diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index 9533ad9e703a..cc08bda37b35 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -31,8 +31,11 @@ class ObjectFile; /// to their functions. class InstrProfCorrelator { public: + /// Indicate which kind correlator to use. + enum ProfCorrelatorKind { NONE, DEBUG_INFO }; + static llvm::Expected> - get(StringRef DebugInfoFilename); + get(StringRef Filename, ProfCorrelatorKind FileKind); /// Construct a ProfileData vector used to correlate raw instrumentation data /// to their functions. @@ -102,7 +105,7 @@ protected: private: static llvm::Expected> - get(std::unique_ptr Buffer); + get(std::unique_ptr Buffer, ProfCorrelatorKind FileKind); const InstrProfCorrelatorKind Kind; }; @@ -126,7 +129,7 @@ public: static llvm::Expected>> get(std::unique_ptr Ctx, - const object::ObjectFile &Obj); + const object::ObjectFile &Obj, ProfCorrelatorKind FileKind); protected: std::vector> Data; @@ -135,7 +138,9 @@ protected: virtual void correlateProfileDataImpl( InstrProfCorrelator::CorrelationData *Data = nullptr) = 0; - Error dumpYaml(raw_ostream &OS) override; + virtual Error correlateProfileNameImpl() = 0; + + Error dumpYaml(int MaxWarnings, raw_ostream &OS) override; void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset, IntPtrT FunctionPtr, uint32_t NumCounters); @@ -199,6 +204,8 @@ private: /// \endcode void correlateProfileDataImpl( InstrProfCorrelator::CorrelationData *Data = nullptr) override; + + Error correlateProfileNameImpl() override; }; } // end namespace llvm diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index b8a6c4a453b6..e3fe3dc2b216 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -386,6 +386,8 @@ public: return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; } + bool useCorrelate() const { return useDebugInfoCorrelate(); } + bool hasSingleByteCoverage() const override { return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; } diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index c917394520df..3c77d7bf9e8b 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -22,15 +22,20 @@ using namespace llvm; -/// Get the __llvm_prf_cnts section. -Expected getCountersSection(const object::ObjectFile &Obj) { +/// Get profile section. +Expected getInstrProfSection(const object::ObjectFile &Obj, + InstrProfSectKind IPSK) { + Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); + std::string ExpectedSectionName = + getInstrProfSectionName(IPSK, ObjFormat, + /*AddSegmentInfo=*/false); for (auto &Section : Obj.sections()) if (auto SectionName = Section.getName()) - if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) + if (SectionName.get() == ExpectedSectionName) return Section; return make_error( instrprof_error::unable_to_correlate_profile, - "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")"); + "could not find section (" + Twine(ExpectedSectionName) + ")"); } const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; @@ -40,7 +45,7 @@ const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; llvm::Expected> InstrProfCorrelator::Context::get(std::unique_ptr Buffer, const object::ObjectFile &Obj) { - auto CountersSection = getCountersSection(Obj); + auto CountersSection = getInstrProfSection(Obj, IPSK_cnts); if (auto Err = CountersSection.takeError()) return std::move(Err); auto C = std::make_unique(); @@ -52,30 +57,32 @@ InstrProfCorrelator::Context::get(std::unique_ptr Buffer, } llvm::Expected> -InstrProfCorrelator::get(StringRef DebugInfoFilename) { - auto DsymObjectsOrErr = - object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); - if (auto Err = DsymObjectsOrErr.takeError()) - return std::move(Err); - if (!DsymObjectsOrErr->empty()) { - // TODO: Enable profile correlation when there are multiple objects in a - // dSYM bundle. - if (DsymObjectsOrErr->size() > 1) - return make_error( - instrprof_error::unable_to_correlate_profile, - "using multiple objects is not yet supported"); - DebugInfoFilename = *DsymObjectsOrErr->begin(); +InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) { + if (FileKind == DEBUG_INFO) { + auto DsymObjectsOrErr = + object::MachOObjectFile::findDsymObjectMembers(Filename); + if (auto Err = DsymObjectsOrErr.takeError()) + return std::move(Err); + if (!DsymObjectsOrErr->empty()) { + // TODO: Enable profile correlation when there are multiple objects in a + // dSYM bundle. + if (DsymObjectsOrErr->size() > 1) + return make_error( + instrprof_error::unable_to_correlate_profile, + "using multiple objects is not yet supported"); + Filename = *DsymObjectsOrErr->begin(); + } } - auto BufferOrErr = - errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); + auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); if (auto Err = BufferOrErr.takeError()) return std::move(Err); - return get(std::move(*BufferOrErr)); + return get(std::move(*BufferOrErr), FileKind); } llvm::Expected> -InstrProfCorrelator::get(std::unique_ptr Buffer) { +InstrProfCorrelator::get(std::unique_ptr Buffer, + ProfCorrelatorKind FileKind) { auto BinOrErr = object::createBinary(*Buffer); if (auto Err = BinOrErr.takeError()) return std::move(Err); @@ -86,9 +93,11 @@ InstrProfCorrelator::get(std::unique_ptr Buffer) { return std::move(Err); auto T = Obj->makeTriple(); if (T.isArch64Bit()) - return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj); + return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj, + FileKind); if (T.isArch32Bit()) - return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj); + return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj, + FileKind); } return make_error( instrprof_error::unable_to_correlate_profile, "not an object file"); @@ -130,29 +139,33 @@ template llvm::Expected>> InstrProfCorrelatorImpl::get( std::unique_ptr Ctx, - const object::ObjectFile &Obj) { - if (Obj.isELF() || Obj.isMachO()) { - auto DICtx = DWARFContext::create(Obj); - return std::make_unique>(std::move(DICtx), - std::move(Ctx)); + const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) { + if (FileKind == DEBUG_INFO) { + if (Obj.isELF() || Obj.isMachO()) { + auto DICtx = DWARFContext::create(Obj); + return std::make_unique>( + std::move(DICtx), std::move(Ctx)); + } + return make_error( + instrprof_error::unable_to_correlate_profile, + "unsupported debug info format (only DWARF is supported)"); } return make_error( instrprof_error::unable_to_correlate_profile, - "unsupported debug info format (only DWARF is supported)"); + "unsupported correlation file type (only DWARF is supported)"); } template Error InstrProfCorrelatorImpl::correlateProfileData() { assert(Data.empty() && Names.empty() && NamesVec.empty()); - correlateProfileDataImpl(); - if (Data.empty() || NamesVec.empty()) + correlateProfileDataImpl(MaxWarnings); + if (this->Data.empty()) return make_error( instrprof_error::unable_to_correlate_profile, - "could not find any profile metadata in debug info"); - auto Result = - collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names); - CounterOffsets.clear(); - NamesVec.clear(); + "could not find any profile data metadata in correlated file"); + Error Result = correlateProfileNameImpl(); + this->CounterOffsets.clear(); + this->NamesVec.clear(); return Result; } @@ -186,7 +199,7 @@ Error InstrProfCorrelatorImpl::dumpYaml(raw_ostream &OS) { if (Data.Probes.empty()) return make_error( instrprof_error::unable_to_correlate_profile, - "could not find any profile metadata in debug info"); + "could not find any profile data metadata in debug info"); yaml::Output YamlOS(OS); YamlOS << Data; return Error::success(); @@ -353,3 +366,16 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( for (const auto &Entry : CU->dies()) maybeAddProbe(DWARFDie(CU.get(), &Entry)); } + +template +Error DwarfInstrProfCorrelator::correlateProfileNameImpl() { + if (this->NamesVec.empty()) { + return make_error( + instrprof_error::unable_to_correlate_profile, + "could not find any profile name metadata in debug info"); + } + auto Result = + collectGlobalObjectNameStrings(this->NamesVec, + /*doCompression=*/false, this->Names); + return Result; +} diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 412335ff24d1..a86ff67d01a1 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -565,9 +565,9 @@ Error RawInstrProfReader::readHeader( "\nPLEASE update this tool to version in the raw profile, or " "regenerate raw profile with expected version.") .str()); - if (useDebugInfoCorrelate() && !Correlator) + if (useCorrelate() && !Correlator) return error(instrprof_error::missing_debug_info_for_correlation); - if (!useDebugInfoCorrelate() && Correlator) + if (!useCorrelate() && Correlator) return error(instrprof_error::unexpected_debug_info_for_correlation); BinaryIdsSize = swap(Header.BinaryIdsSize); diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 378025afc2b9..dd3b6e1723f4 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -442,8 +442,9 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, std::unique_ptr Correlator; if (!DebugInfoFilename.empty()) { - if (auto Err = - InstrProfCorrelator::get(DebugInfoFilename).moveInto(Correlator)) + if (auto Err = InstrProfCorrelator::get(DebugInfoFilename, + InstrProfCorrelator::DEBUG_INFO) + .moveInto(Correlator)) exitWithError(std::move(Err), DebugInfoFilename); if (auto Err = Correlator->correlateProfileData()) exitWithError(std::move(Err), DebugInfoFilename); @@ -2907,7 +2908,9 @@ static int showDebugInfoCorrelation(const std::string &Filename, if (SFormat == ShowFormat::Json) exitWithError("JSON output is not supported for debug info correlation"); std::unique_ptr Correlator; - if (auto Err = InstrProfCorrelator::get(Filename).moveInto(Correlator)) + if (auto Err = + InstrProfCorrelator::get(Filename, InstrProfCorrelator::DEBUG_INFO) + .moveInto(Correlator)) exitWithError(std::move(Err), Filename); if (SFormat == ShowFormat::Yaml) { if (auto Err = Correlator->dumpYaml(OS)) -- Gitee From f202ece446ffb5eaaaf861fb7c01ef8933dbb041 Mon Sep 17 00:00:00 2001 From: Ellis Hoag Date: Mon, 20 Nov 2023 09:25:33 -0600 Subject: [PATCH 10/28] [InstrProf] Add pgo use block coverage test (#72443) Back in https://reviews.llvm.org/D124490 we added a block coverage mode that instruments a subset of basic blocks using single byte counters to get coverage for the whole function. This commit adds a test to make sure that we correctly assign branch weights based on the coverage profile. I noticed this test was missing after seeing that we had no coverage on `PGOUseFunc::populateCoverage()` https://lab.llvm.org/coverage/coverage-reports/coverage/Users/buildslave/jenkins/workspace/coverage/llvm-project/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp.html#L1383 --- llvm/lib/ProfileData/InstrProfReader.cpp | 2 + llvm/lib/ProfileData/InstrProfWriter.cpp | 2 + .../PGOProfile/Inputs/coverage.proftext | 54 +++++++++++++++++++ llvm/test/Transforms/PGOProfile/coverage.ll | 34 +++++++++--- 4 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/coverage.proftext diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index a86ff67d01a1..55da7fdfa554 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -270,6 +270,8 @@ Error TextInstrProfReader::readHeader() { ProfileKind |= InstrProfKind::FunctionEntryInstrumentation; else if (Str.equals_insensitive("not_entry_first")) ProfileKind &= ~InstrProfKind::FunctionEntryInstrumentation; + else if (Str.equals_insensitive("single_byte_coverage")) + ProfileKind |= InstrProfKind::SingleByteCoverage; else if (Str.equals_insensitive("temporal_prof_traces")) { ProfileKind |= InstrProfKind::TemporalProfile; if (auto Err = readTemporalProfTraceData()) diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index d8b8571bd5c9..75c04493b52f 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -762,6 +762,8 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { if (static_cast(ProfileKind & InstrProfKind::FunctionEntryInstrumentation)) OS << "# Always instrument the function entry block\n:entry_first\n"; + if (static_cast(ProfileKind & InstrProfKind::SingleByteCoverage)) + OS << "# Instrument block coverage\n:single_byte_coverage\n"; InstrProfSymtab Symtab; using FuncPair = detail::DenseMapPair; diff --git a/llvm/test/Transforms/PGOProfile/Inputs/coverage.proftext b/llvm/test/Transforms/PGOProfile/Inputs/coverage.proftext new file mode 100644 index 000000000000..229530ba4140 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/Inputs/coverage.proftext @@ -0,0 +1,54 @@ +:ir +:single_byte_coverage + +foo +# Func Hash: +848064302753700500 +# Num Counters: +2 +# Counter Values: +3 +4 + + +bar +# Func Hash: +848064302952419074 +# Num Counters: +2 +# Counter Values: +2 +0 + + +goo +# Func Hash: +1106497858086895615 +# Num Counters: +1 +# Counter Values: +5 + + +loop +# Func Hash: +92940490389974880 +# Num Counters: +2 +# Counter Values: +1 +1 + + +hoo +# Func Hash: +1073332642652768409 +# Num Counters: +6 +# Counter Values: +1 +0 +1 +1 +0 +0 diff --git a/llvm/test/Transforms/PGOProfile/coverage.ll b/llvm/test/Transforms/PGOProfile/coverage.ll index d636be14a2cc..1b7658a2ea9e 100644 --- a/llvm/test/Transforms/PGOProfile/coverage.ll +++ b/llvm/test/Transforms/PGOProfile/coverage.ll @@ -1,14 +1,20 @@ -; RUN: opt < %s -passes=pgo-instr-gen -pgo-function-entry-coverage -S | FileCheck %s --implicit-check-not="instrprof.cover" --check-prefixes=CHECK,ENTRY -; RUN: opt < %s -passes=pgo-instr-gen -pgo-block-coverage -S | FileCheck %s --implicit-check-not="instrprof.cover" --check-prefixes=CHECK,BLOCK +; RUN: opt < %s -passes=pgo-instr-gen -pgo-function-entry-coverage -S | FileCheck %s --implicit-check-not="instrprof.cover" --check-prefixes=CHECK,GEN,ENTRY +; RUN: opt < %s -passes=pgo-instr-gen -pgo-block-coverage -S | FileCheck %s --implicit-check-not="instrprof.cover" --check-prefixes=CHECK,GEN,BLOCK + +; RUN: llvm-profdata merge %S/Inputs/coverage.proftext -o %t.profdata +; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=CHECK,USE target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; CHECK-LABEL: @foo() +; USE-SAME: !prof ![[HOT:[0-9]+]] define void @foo() { ; CHECK-LABEL: entry: entry: ; ENTRY: call void @llvm.instrprof.cover({{.*}}) %c = call i1 @choice() br i1 %c, label %if.then, label %if.else + ; USE: br i1 %c, label %if.then, label %if.else, !prof ![[WEIGHTS0:[0-9]+]] ; CHECK-LABEL: if.then: if.then: @@ -25,12 +31,15 @@ if.end: ret void } +; CHECK-LABEL: @bar() +; USE-SAME: !prof ![[HOT:[0-9]+]] define void @bar() { ; CHECK-LABEL: entry: entry: ; ENTRY: call void @llvm.instrprof.cover({{.*}}) %c = call i1 @choice() br i1 %c, label %if.then, label %if.end + ; USE: br i1 %c, label %if.then, label %if.end, !prof ![[WEIGHTS1:[0-9]+]] ; CHECK-LABEL: if.then: if.then: @@ -43,24 +52,29 @@ if.end: ret void } +; CHECK-LABEL: @goo() +; USE-SAME: !prof ![[HOT:[0-9]+]] define void @goo() { ; CHECK-LABEL: entry: entry: - ; CHECK: call void @llvm.instrprof.cover({{.*}}) + ; GEN: call void @llvm.instrprof.cover({{.*}}) ret void } +; CHECK-LABEL: @loop() +; USE-SAME: !prof ![[HOT:[0-9]+]] define void @loop() { ; CHECK-LABEL: entry: entry: - ; CHECK: call void @llvm.instrprof.cover({{.*}}) + ; GEN: call void @llvm.instrprof.cover({{.*}}) br label %while while: ; BLOCK: call void @llvm.instrprof.cover({{.*}}) br label %while } -; Function Attrs: noinline nounwind ssp uwtable +; CHECK-LABEL: @hoo( +; USE-SAME: !prof ![[HOT:[0-9]+]] define void @hoo(i32 %a) #0 { ; CHECK-LABEL: entry: entry: @@ -72,6 +86,7 @@ entry: %rem = srem i32 %0, 2 %cmp = icmp eq i32 %rem, 0 br i1 %cmp, label %if.then, label %if.else + ; USE: br i1 %cmp, label %if.then, label %if.else, !prof ![[WEIGHTS1]] ; CHECK-LABEL: if.then: if.then: ; preds = %entry @@ -94,6 +109,7 @@ for.cond: ; preds = %for.inc, %if.end %2 = load i32, i32* %a.addr, align 4 %cmp1 = icmp slt i32 %1, %2 br i1 %cmp1, label %for.body, label %for.end + ; USE: br i1 %cmp1, label %for.body, label %for.end, !prof ![[WEIGHTS1]] ; CHECK-LABEL: for.body: for.body: ; preds = %for.cond @@ -101,6 +117,7 @@ for.body: ; preds = %for.cond %rem2 = srem i32 %3, 3 %cmp3 = icmp eq i32 %rem2, 0 br i1 %cmp3, label %if.then4, label %if.else5 + ; USE: br i1 %cmp3, label %if.then4, label %if.else5, !prof ![[WEIGHTS0]] ; CHECK-LABEL: if.then4: if.then4: ; preds = %for.body @@ -113,6 +130,7 @@ if.else5: ; preds = %for.body %rem6 = srem i32 %4, 1001 %cmp7 = icmp eq i32 %rem6, 0 br i1 %cmp7, label %if.then8, label %if.end9 + ; USE: br i1 %cmp7, label %if.then8, label %if.end9, !prof ![[WEIGHTS1]] ; CHECK-LABEL: if.then8: if.then8: ; preds = %if.else5 @@ -147,4 +165,8 @@ return: ; preds = %for.end, %if.then8 declare i1 @choice() -; CHECK: declare void @llvm.instrprof.cover({{.*}}) +; GEN: declare void @llvm.instrprof.cover({{.*}}) + +; USE-DAG: ![[HOT]] = !{!"function_entry_count", i64 10000} +; USE-DAG: ![[WEIGHTS0]] = !{!"branch_weights", i32 1, i32 1} +; USE-DAG: ![[WEIGHTS1]] = !{!"branch_weights", i32 1, i32 0} -- Gitee From 4dfe6cb7aefbc87d23e5fd7bd086e9c06fe21da0 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Mon, 20 Nov 2023 14:25:24 -0500 Subject: [PATCH 11/28] =?UTF-8?q?[llvm-profdata]=20Fix=20binary=20ids=20wi?= =?UTF-8?q?th=20multiple=20raw=20profiles=20in=20a=20single=E2=80=A6=20(#7?= =?UTF-8?q?2740)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Save binary ids when iterating through `RawInstrProfReader`. Fixes #72699. --- compiler-rt/test/profile/Linux/binary-id.c | 41 ++++++++++++- .../llvm/ProfileData/InstrProfReader.h | 6 +- llvm/lib/ProfileData/InstrProfReader.cpp | 59 +++++++++---------- 3 files changed, 68 insertions(+), 38 deletions(-) diff --git a/compiler-rt/test/profile/Linux/binary-id.c b/compiler-rt/test/profile/Linux/binary-id.c index 61b8ed926849..9bff9004e97a 100644 --- a/compiler-rt/test/profile/Linux/binary-id.c +++ b/compiler-rt/test/profile/Linux/binary-id.c @@ -17,16 +17,35 @@ // RUN: llvm-profdata show --binary-ids %t.profdir/default_*.profraw > %t.profraw.out // RUN: FileCheck %s --check-prefix=BINARY-ID-MERGE-PROF < %t.profraw.out -// RUN: llvm-profdata merge -o %t.profdata %t.profraw %t.profraw +// RUN: llvm-profdata merge -o %t.profdata %t.profdir/default_*.profraw // RUN: llvm-profdata show --binary-ids %t.profdata > %t.profdata.out -// RUN: FileCheck %s --check-prefix=BINARY-ID-INDEXED-PROF < %t.profraw.out +// RUN: FileCheck %s --check-prefix=BINARY-ID-INDEXED-PROF < %t.profdata.out +// Test raw profiles with shared libraries. +// RUN: split-file %s %t.dir +// RUN: %clang_profgen -Wl,--build-id -fpic -shared -O2 %t.dir/foo.c -o %t.dir/libfoo.so +// RUN: %clang_profgen -Wl,--build-id -fpic -shared -O2 %t.dir/bar.c -o %t.dir/libbar.so +// RUN: %clang_profgen -Wl,--build-id -O2 %t.dir/main.c %t.dir/libfoo.so %t.dir/libbar.so -o %t +// RUN: env LLVM_PROFILE_FILE=%t.profraw LD_LIBRARY_PATH=%t.dir %run %t +// RUN: llvm-profdata show --binary-ids %t.profraw > %t.profraw.out +// RUN: llvm-profdata merge -o %t.profdata %t.profraw +// RUN: FileCheck %s --check-prefix=BINARY-ID-SHARE-RAW-PROF < %t.profraw.out + +// RUN: llvm-profdata merge -o %t.profdata %t.profraw +// RUN: llvm-profdata show --binary-ids %t.profdata > %t.profdata.out +// RUN: FileCheck %s --check-prefix=BINARY-ID-SHARE-INDEXED-PROF < %t.profraw.out + +//--- foo.c void foo() { } +//--- bar.c void bar() { } +//--- main.c +void foo(); +void bar(); int main() { foo(); bar(); @@ -59,3 +78,21 @@ int main() { // BINARY-ID-INDEXED-PROF-NEXT: Maximum internal block count: 0 // BINARY-ID-INDEXED-PROF-NEXT: Binary IDs: // BINARY-ID-INDEXED-PROF-NEXT: {{[0-9a-f]+}} + +// BINARY-ID-SHARE-RAW-PROF: Instrumentation level: Front-end +// BINARY-ID-SHARE-RAW-PROF-NEXT: Total functions: 3 +// BINARY-ID-SHARE-RAW-PROF-NEXT: Maximum function count: 1 +// BINARY-ID-SHARE-RAW-PROF-NEXT: Maximum internal block count: 0 +// BINARY-ID-SHARE-RAW-PROF-NEXT: Binary IDs: +// BINARY-ID-SHARE-RAW-PROF-NEXT: {{[0-9a-f]+}} +// BINARY-ID-SHARE-RAW-PROF-NEXT: {{[0-9a-f]+}} +// BINARY-ID-SHARE-RAW-PROF-NEXT: {{[0-9a-f]+}} + +// BINARY-ID-SHARE-INDEXED-PROF: Instrumentation level: Front-end +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: Total functions: 3 +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: Maximum function count: 1 +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: Maximum internal block count: 0 +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: Binary IDs: +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: {{[0-9a-f]+}} +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: {{[0-9a-f]+}} +// BINARY-ID-SHARE-INDEXED-PROF-NEXT: {{[0-9a-f]+}} diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index e3fe3dc2b216..76671ec8be91 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -340,11 +340,7 @@ private: const uint8_t *ValueDataStart; uint32_t ValueKindLast; uint32_t CurValueDataSize; - - /// Total size of binary ids. - uint64_t BinaryIdsSize{0}; - /// Start address of binary id length and data pairs. - const uint8_t *BinaryIdsStart; + std::vector BinaryIds; std::function Warn; diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 55da7fdfa554..6f88e31754c3 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -141,27 +141,15 @@ readBinaryIdsInternal(const MemoryBuffer &DataBuffer, return Error::success(); } -static Error printBinaryIdsInternal(raw_ostream &OS, - const MemoryBuffer &DataBuffer, - uint64_t BinaryIdsSize, - const uint8_t *BinaryIdsStart, - llvm::support::endianness Endian) { - if (BinaryIdsSize == 0) - return Error::success(); - - std::vector BinaryIds; - if (Error E = readBinaryIdsInternal(DataBuffer, BinaryIdsSize, BinaryIdsStart, - BinaryIds, Endian)) - return E; - +static void +printBinaryIdsInternal(raw_ostream &OS, + std::vector &BinaryIds) { OS << "Binary IDs: \n"; for (auto BI : BinaryIds) { for (uint64_t I = 0; I < BI.size(); I++) OS << format("%02x", BI[I]); OS << "\n"; } - - return Error::success(); } Expected> @@ -572,9 +560,20 @@ Error RawInstrProfReader::readHeader( if (!useCorrelate() && Correlator) return error(instrprof_error::unexpected_debug_info_for_correlation); - BinaryIdsSize = swap(Header.BinaryIdsSize); - if (BinaryIdsSize % sizeof(uint64_t)) + uint64_t BinaryIdSize = swap(Header.BinaryIdsSize); + // Binary id start just after the header if exists. + const uint8_t *BinaryIdStart = + reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); + const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize; + const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); + if (BinaryIdSize % sizeof(uint64_t) || BinaryIdEnd > BufferEnd) return error(instrprof_error::bad_header); + if (BinaryIdSize != 0) { + if (Error Err = + readBinaryIdsInternal(*DataBuffer, BinaryIdSize, BinaryIdStart, + BinaryIds, getDataEndianness())) + return Err; + } CountersDelta = swap(Header.CountersDelta); BitmapDelta = swap(Header.BitmapDelta); @@ -592,7 +591,7 @@ Error RawInstrProfReader::readHeader( auto PaddingSize = getNumPaddingBytes(NamesSize); // Profile data starts after profile header and binary ids if exist. - ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize; + ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize; ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters; ptrdiff_t BitmapOffset = CountersOffset + CountersSize + PaddingBytesAfterCounters; @@ -621,19 +620,12 @@ Error RawInstrProfReader::readHeader( NamesEnd = NamesStart + NamesSize; } - // Binary ids start just after the header. - BinaryIdsStart = - reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); CountersStart = Start + CountersOffset; CountersEnd = CountersStart + CountersSize; BitmapStart = Start + BitmapOffset; BitmapEnd = BitmapStart + NumBitmapBytes; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); - const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); - if (BinaryIdsStart + BinaryIdsSize > BufferEnd) - return error(instrprof_error::bad_header); - std::unique_ptr NewSymtab = std::make_unique(); if (Error E = createSymtab(*NewSymtab)) return E; @@ -831,14 +823,16 @@ Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) template Error RawInstrProfReader::readBinaryIds( std::vector &BinaryIds) { - return readBinaryIdsInternal(*DataBuffer, BinaryIdsSize, BinaryIdsStart, - BinaryIds, getDataEndianness()); + BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(), + this->BinaryIds.end()); + return Error::success(); } template Error RawInstrProfReader::printBinaryIds(raw_ostream &OS) { - return printBinaryIdsInternal(OS, *DataBuffer, BinaryIdsSize, BinaryIdsStart, - getDataEndianness()); + if (!BinaryIds.empty()) + printBinaryIdsInternal(OS, BinaryIds); + return Error::success(); } namespace llvm { @@ -1457,8 +1451,11 @@ Error IndexedInstrProfReader::readBinaryIds( } Error IndexedInstrProfReader::printBinaryIds(raw_ostream &OS) { - return printBinaryIdsInternal(OS, *DataBuffer, BinaryIdsSize, BinaryIdsStart, - llvm::support::little); + std::vector BinaryIds; + if (Error E = readBinaryIds(BinaryIds)) + return E; + printBinaryIdsInternal(OS, BinaryIds); + return Error::success(); } void InstrProfReader::accumulateCounts(CountSumOrPercent &Sum, bool IsCS) { -- Gitee From 1aaf3d9a9e4f08de7c8676f18242abddfa318874 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Thu, 14 Dec 2023 14:16:38 -0500 Subject: [PATCH 12/28] [Profile] Add binary profile correlation for code coverage. (#69493) Since we don't need the metadata sections at runtime, we can somehow offload them from memory at runtime. Initially, I explored [debug info correlation](https://discourse.llvm.org/t/instrprofiling-lightweight-instrumentation/59113), which is used for PGO with value profiling disabled. However, it currently only works with DWARF and it's be hard to add such artificial debug info for every function in to CodeView which is used on Windows. So, offloading profile metadata sections at runtime seems to be a platform independent option. The idea is to use new section names for profile name and data sections and mark them as metadata sections. Under this mode, the new sections are non-SHF_ALLOC in ELF. So, they are not loaded into memory at runtime and can be stripped away as a post-linking step. After the process exits, the generated raw profiles will contains only headers + counters. llvm-profdata can be used correlate raw profiles with the unstripped binary to generate indexed profile. For chromium base_unittests with code coverage on linux, the binary size overhead due to instrumentation reduced from 64M to 38.8M (39.4%) and the raw profile files size reduce from 128M to 68M (46.9%) ``` $ bloaty out/cov/base_unittests.stripped -- out/no-cov/base_unittests.stripped FILE SIZE VM SIZE -------------- -------------- +121% +30.4Mi +121% +30.4Mi .text [NEW] +14.6Mi [NEW] +14.6Mi __llvm_prf_data [NEW] +10.6Mi [NEW] +10.6Mi __llvm_prf_names [NEW] +5.86Mi [NEW] +5.86Mi __llvm_prf_cnts +95% +1.75Mi +95% +1.75Mi .eh_frame +108% +400Ki +108% +400Ki .eh_frame_hdr +9.5% +211Ki +9.5% +211Ki .rela.dyn +9.2% +95.0Ki +9.2% +95.0Ki .data.rel.ro +5.0% +87.3Ki +5.0% +87.3Ki .rodata [ = ] 0 +13% +47.0Ki .bss +40% +1.78Ki +40% +1.78Ki .got +12% +1.49Ki +12% +1.49Ki .gcc_except_table [ = ] 0 +65% +1.23Ki .relro_padding +62% +1.20Ki [ = ] 0 [Unmapped] +13% +448 +19% +448 .init_array +8.8% +192 [ = ] 0 [ELF Section Headers] +0.0% +136 +0.0% +80 [7 Others] +0.1% +96 +0.1% +96 .dynsym +1.2% +96 +1.2% +96 .rela.plt +1.5% +80 +1.2% +64 .plt [ = ] 0 -99.2% -3.68Ki [LOAD #5 [RW]] +195% +64.0Mi +194% +64.0Mi TOTAL $ bloaty out/cov-cor/base_unittests.stripped -- out/no-cov/base_unittests.stripped FILE SIZE VM SIZE -------------- -------------- +121% +30.4Mi +121% +30.4Mi .text [NEW] +5.86Mi [NEW] +5.86Mi __llvm_prf_cnts +95% +1.75Mi +95% +1.75Mi .eh_frame +108% +400Ki +108% +400Ki .eh_frame_hdr +9.5% +211Ki +9.5% +211Ki .rela.dyn +9.2% +95.0Ki +9.2% +95.0Ki .data.rel.ro +5.0% +87.3Ki +5.0% +87.3Ki .rodata [ = ] 0 +13% +47.0Ki .bss +40% +1.78Ki +40% +1.78Ki .got +12% +1.49Ki +12% +1.49Ki .gcc_except_table +13% +448 +19% +448 .init_array +0.1% +96 +0.1% +96 .dynsym +1.2% +96 +1.2% +96 .rela.plt +1.2% +64 +1.2% +64 .plt +2.9% +64 [ = ] 0 [ELF Section Headers] +0.0% +40 +0.0% +40 .data +1.2% +32 +1.2% +32 .got.plt +0.0% +24 +0.0% +8 [5 Others] [ = ] 0 -22.9% -872 [LOAD #5 [RW]] -74.5% -1.44Ki [ = ] 0 [Unmapped] [ = ] 0 -76.5% -1.45Ki .relro_padding +118% +38.8Mi +117% +38.8Mi TOTAL ``` A few things to note: 1. llvm-profdata doesn't support filter raw profiles by binary id yet, so when a raw profile doesn't belongs to the binary being digested by llvm-profdata, merging will fail. Once this is implemented, llvm-profdata should be able to only merge raw profiles with the same binary id as the binary and discard the rest (with mismatched/missing binary id). The workflow I have in mind is to have scripts invoke llvm-profdata to get all binary ids for all raw profiles, and selectively choose the raw pnrofiles with matching binary id and the binary to llvm-profdata for merging. 2. Note: In COFF, currently they are still loaded into memory but not used. I didn't do it in this patch because I noticed that `.lcovmap` and `.lcovfunc` are loaded into memory. A separate patch will address it. 3. This should works with PGO when value profiling is disabled as debug info correlation currently doing, though I haven't tested this yet. --- clang/lib/CodeGen/BackendUtil.cpp | 16 +- compiler-rt/include/profile/InstrProfData.inc | 17 + .../profile/InstrProfilingPlatformWindows.c | 3 +- compiler-rt/test/CMakeLists.txt | 5 +- .../test/profile/instrprof-binary-correlate.c | 46 +++ llvm/docs/CommandGuide/llvm-profdata.rst | 15 +- llvm/include/llvm/ProfileData/InstrProf.h | 4 +- .../llvm/ProfileData/InstrProfCorrelator.h | 55 ++- .../llvm/ProfileData/InstrProfData.inc | 17 + .../llvm/ProfileData/InstrProfReader.h | 9 - .../CodeGen/TargetLoweringObjectFileImpl.cpp | 4 + .../Coverage/CoverageMappingReader.cpp | 63 ++-- llvm/lib/ProfileData/InstrProf.cpp | 8 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 128 ++++++- llvm/lib/ProfileData/InstrProfReader.cpp | 9 +- .../Instrumentation/InstrProfiling.cpp | 75 ++-- .../Instrumentation/PGOInstrumentation.cpp | 4 +- .../InstrProfiling/coverage.ll | 8 + llvm/tools/llvm-profdata/llvm-profdata.cpp | 351 +++++++++++++++++- 19 files changed, 719 insertions(+), 118 deletions(-) create mode 100644 compiler-rt/test/profile/instrprof-binary-correlate.c diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index d19e6a99cdfd..d6064ecabcee 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -43,6 +43,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" @@ -103,13 +104,22 @@ extern cl::opt DebugInfoCorrelate; static cl::opt ClSanitizeOnOptimizerEarlyEP( "sanitizer-early-opt-ep", cl::Optional, cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); -} + +extern cl::opt ProfileCorrelate; + +// Re-link builtin bitcodes after optimization +cl::opt ClRelinkBuiltinBitcodePostop( + "relink-builtin-bitcode-postop", cl::Optional, + cl::desc("Re-link builtin bitcodes after optimization."), cl::init(false)); +} // namespace llvm namespace { // Default filename used for profile generation. std::string getDefaultProfileGenName() { - return DebugInfoCorrelate ? "default_%p.proflite" : "default_%m.profraw"; + return DebugInfoCorrelate || ProfileCorrelate != InstrProfCorrelator::NONE + ? "default_%m.proflite" + : "default_%m.profraw"; } class EmitAssemblyHelper { @@ -202,7 +212,7 @@ public: void EmitAssembly(BackendAction Action, std::unique_ptr OS); }; -} +} // namespace static SanitizerCoverageOptions getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) { diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 1cf83011206a..4a538e592c89 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -701,6 +707,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. @@ -713,6 +721,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 @@ -729,6 +742,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ @@ -744,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 9dbd702865fd..9070b8a606eb 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -13,13 +13,14 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ -#pragma comment(linker, "/MERGE:.lprfc=.data") #pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find * after the fact. + * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled, + * llvm-cov must be able to find after the fact. */ /* Allocate read-only section bounds. */ diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index bc37c85a140a..c6a4c61ff487 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -34,8 +34,9 @@ if(NOT ANDROID) if(NOT COMPILER_RT_STANDALONE_BUILD AND NOT LLVM_RUNTIMES_BUILD) # Use LLVM utils and Clang from the same build tree. list(APPEND SANITIZER_COMMON_LIT_TEST_DEPS - clang clang-resource-headers FileCheck count not llvm-config llvm-nm llvm-objdump - llvm-readelf llvm-readobj llvm-size llvm-symbolizer compiler-rt-headers sancov split-file) + clang clang-resource-headers FileCheck count not llvm-config llvm-nm + llvm-objdump llvm-readelf llvm-readobj llvm-size llvm-symbolizer + compiler-rt-headers sancov split-file llvm-strip) if (WIN32) list(APPEND SANITIZER_COMMON_LIT_TEST_DEPS KillTheDoctor) endif() diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c new file mode 100644 index 000000000000..8f421014cf5c --- /dev/null +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -0,0 +1,46 @@ +// REQUIRES: linux || windows +// Default +// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw +// RUN: llvm-cov report --instr-profile=%t.normal.profdata %t.normal > %t.normal.report +// RUN: llvm-cov show --instr-profile=%t.normal.profdata %t.normal > %t.normal.show + +// With -profile-correlate=binary flag +// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe +// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw +// RUN: llvm-cov report --instr-profile=%t-1.profdata %t-1.exe > %t-1.report +// RUN: llvm-cov show --instr-profile=%t-1.profdata %t-1.exe > %t-1.show +// RUN: diff %t.normal.profdata %t-1.profdata +// RUN: diff %t.normal.report %t-1.report +// RUN: diff %t.normal.show %t-1.show + +// Strip above binary and run +// RUN: llvm-strip %t-1.exe -o %t-2.exe +// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe +// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw +// RUN: llvm-cov report --instr-profile=%t-2.profdata %t-1.exe > %t-2.report +// RUN: llvm-cov show --instr-profile=%t-2.profdata %t-1.exe > %t-2.show +// RUN: diff %t.normal.profdata %t-2.profdata +// RUN: diff %t.normal.report %t-2.report +// RUN: diff %t.normal.show %t-2.show + +// Online merging. +// RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal +// RUN: env LLVM_PROFILE_FILE=%t-4.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.merged.profdata %t-3.profraw %t-4.profraw +// RUN: llvm-cov report --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.report +// RUN: llvm-cov show --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.show + +// RUN: rm -rf %t.profdir && mkdir %t.profdir +// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe +// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe +// RUN: llvm-profdata merge -o %t-4.profdata --binary-file=%t-1.exe %t.profdir +// RUN: llvm-cov report --instr-profile=%t-4.profdata %t-1.exe > %t-4.report +// RUN: llvm-cov show --instr-profile=%t-4.profdata %t-1.exe > %t-4.show +// RUN: diff %t.normal.merged.profdata %t-4.profdata +// RUN: diff %t.normal.merged.report %t-4.report +// RUN: diff %t.normal.merged.show %t-4.show + +// TODO: After adding support for binary ID, test binaries with different binary IDs. diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst index be42733ca140..f5e3c13ffbc8 100644 --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -195,8 +195,14 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-debug-info-correlate`` was used for instrumentation, use this option - to correlate the raw profile. + When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used + for instrumentation, use this option to correlate the raw profile. + +.. option:: --binary-file= + + Specify the executable that contains profile data and profile name sections for + the raw profile. When ``-profile-correlate=binary`` was used for + instrumentation, use this option to correlate the raw profile. .. option:: --temporal-profile-trace-reservoir-size @@ -346,8 +352,9 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-debug-info-correlate`` was used for instrumentation, use this option - to show the correlated functions from the raw profile. + When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used + for instrumentation, use this option to show the correlated functions from the + raw profile. .. option:: --covered diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 71431aabf577..1d4049ece54e 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -331,8 +331,8 @@ enum class instrprof_error { too_large, truncated, malformed, - missing_debug_info_for_correlation, - unexpected_debug_info_for_correlation, + missing_correlation_info, + unexpected_correlation_info, unable_to_correlate_profile, unknown_function, invalid_prof, diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index cc08bda37b35..2f9be48f2833 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -5,8 +5,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// This file defines InstrProfCorrelator used to generate PGO profiles from -// raw profile data and debug info. +// This file defines InstrProfCorrelator used to generate PGO/coverage profiles +// from raw profile data and debug info/binary file. //===----------------------------------------------------------------------===// #ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H @@ -31,8 +31,9 @@ class ObjectFile; /// to their functions. class InstrProfCorrelator { public: - /// Indicate which kind correlator to use. - enum ProfCorrelatorKind { NONE, DEBUG_INFO }; + /// Indicate if we should use the debug info or profile metadata sections to + /// correlate. + enum ProfCorrelatorKind { NONE, DEBUG_INFO, BINARY }; static llvm::Expected> get(StringRef Filename, ProfCorrelatorKind FileKind); @@ -69,11 +70,18 @@ public: protected: struct Context { static llvm::Expected> - get(std::unique_ptr Buffer, const object::ObjectFile &Obj); + get(std::unique_ptr Buffer, const object::ObjectFile &Obj, + ProfCorrelatorKind FileKind); std::unique_ptr Buffer; /// The address range of the __llvm_prf_cnts section. uint64_t CountersSectionStart; uint64_t CountersSectionEnd; + /// The pointer points to start/end of profile data/name sections if + /// FileKind is Binary. + const char *DataStart; + const char *DataEnd; + const char *NameStart; + size_t NameSize; /// True if target and host have different endian orders. bool ShouldSwapBytes; }; @@ -142,19 +150,20 @@ protected: Error dumpYaml(int MaxWarnings, raw_ostream &OS) override; - void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset, - IntPtrT FunctionPtr, uint32_t NumCounters); + void addDataProbe(uint64_t FunctionName, uint64_t CFGHash, + IntPtrT CounterOffset, IntPtrT FunctionPtr, + uint32_t NumCounters); + + // Byte-swap the value if necessary. + template T maybeSwap(T Value) const { + return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; + } private: InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind, std::unique_ptr Ctx) : InstrProfCorrelator(Kind, std::move(Ctx)){}; llvm::DenseSet CounterOffsets; - - // Byte-swap the value if necessary. - template T maybeSwap(T Value) const { - return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; - } }; /// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes @@ -208,6 +217,28 @@ private: Error correlateProfileNameImpl() override; }; +/// BinaryInstrProfCorrelator - A child of InstrProfCorrelatorImpl that +/// takes an object file as input to correlate profiles. +template +class BinaryInstrProfCorrelator : public InstrProfCorrelatorImpl { +public: + BinaryInstrProfCorrelator(std::unique_ptr Ctx) + : InstrProfCorrelatorImpl(std::move(Ctx)) {} + + /// Return a pointer to the names string that this class constructs. + const char *getNamesPointer() const { return this->Ctx.NameStart; } + + /// Return the number of bytes in the names string. + size_t getNamesSize() const { return this->Ctx.NameSize; } + +private: + void correlateProfileDataImpl( + int MaxWarnings, + InstrProfCorrelator::CorrelationData *Data = nullptr) override; + + Error correlateProfileNameImpl() override; +}; + } // end namespace llvm #endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 1cf83011206a..4a538e592c89 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -701,6 +707,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. @@ -713,6 +721,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 @@ -729,6 +742,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ @@ -744,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 76671ec8be91..3c9958a75c30 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -123,9 +123,6 @@ public: virtual bool instrEntryBBEnabled() const = 0; - /// Return true if we must provide debug info to create PGO profiles. - virtual bool useDebugInfoCorrelate() const { return false; } - /// Return true if the profile has single byte counters representing coverage. virtual bool hasSingleByteCoverage() const = 0; @@ -378,12 +375,6 @@ public: return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; } - bool useDebugInfoCorrelate() const override { - return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; - } - - bool useCorrelate() const { return useDebugInfoCorrelate(); } - bool hasSingleByteCoverage() const override { return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; } diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index c336c933ffa8..ccff862a6a4b 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -479,6 +479,10 @@ static SectionKind getELFKindForNamedSection(StringRef Name, SectionKind K) { /*AddSegmentInfo=*/false) || Name == getInstrProfSectionName(IPSK_covfun, Triple::ELF, /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(IPSK_covdata, Triple::ELF, + /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(IPSK_covname, Triple::ELF, + /*AddSegmentInfo=*/false) || Name == ".llvmbc" || Name == ".llvmcmd") return SectionKind::getMetadata(); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index cd3970c5151d..80f3e787ef43 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -454,9 +454,13 @@ Error InstrProfSymtab::create(SectionRef &Section) { // If this is a linked PE/COFF file, then we have to skip over the null byte // that is allocated in the .lprfn$A section in the LLVM profiling runtime. + // If the name section is .lprfcovnames, it doesn't have the null byte at the + // beginning. const ObjectFile *Obj = Section.getObject(); if (isa(Obj) && !Obj->isRelocatableObject()) - Data = Data.drop_front(1); + if (Expected NameOrErr = Section.getName()) + if (*NameOrErr != getInstrProfSectionName(IPSK_covname, Triple::COFF)) + Data = Data.drop_front(1); return Error::success(); } @@ -930,10 +934,13 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) { BytesInAddress, Endian, CompilationDir); } -/// Find all sections that match \p Name. There may be more than one if comdats -/// are in use, e.g. for the __llvm_covfun section on ELF. -static Expected> lookupSections(ObjectFile &OF, - StringRef Name) { +/// Find all sections that match \p IPSK name. There may be more than one if +/// comdats are in use, e.g. for the __llvm_covfun section on ELF. +static Expected> +lookupSections(ObjectFile &OF, InstrProfSectKind IPSK) { + auto ObjFormat = OF.getTripleObjectFormat(); + auto Name = + getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); // On COFF, the object file section name may end in "$M". This tells the // linker to sort these sections between "$A" and "$Z". The linker removes the // dollar and everything after it in the final binary. Do the same to match. @@ -948,8 +955,13 @@ static Expected> lookupSections(ObjectFile &OF, Expected NameOrErr = Section.getName(); if (!NameOrErr) return NameOrErr.takeError(); - if (stripSuffix(*NameOrErr) == Name) + if (stripSuffix(*NameOrErr) == Name) { + // COFF profile name section contains two null bytes indicating the + // start/end of the section. If its size is 2 bytes, it's empty. + if (IsCOFF && IPSK == IPSK_name && Section.getSize() == 2) + continue; Sections.push_back(Section); + } } if (Sections.empty()) return make_error(coveragemap_error::no_data_found); @@ -985,15 +997,27 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, : support::endianness::big; // Look for the sections that we are interested in. - auto ObjFormat = OF->getTripleObjectFormat(); - auto NamesSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_name, ObjFormat, - /*AddSegmentInfo=*/false)); - if (auto E = NamesSection.takeError()) + InstrProfSymtab ProfileNames; + std::vector NamesSectionRefs; + // If IPSK_name is not found, fallback to search for IPK_covname, which is + // used when binary correlation is enabled. + auto NamesSection = lookupSections(*OF, IPSK_name); + if (auto E = NamesSection.takeError()) { + consumeError(std::move(E)); + NamesSection = lookupSections(*OF, IPSK_covname); + if (auto E = NamesSection.takeError()) + return std::move(E); + } + NamesSectionRefs = *NamesSection; + + if (NamesSectionRefs.size() != 1) + return make_error( + coveragemap_error::malformed, + "the size of coverage mapping section is not one"); + if (Error E = ProfileNames.create(NamesSectionRefs.back())) return std::move(E); - auto CoverageSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_covmap, ObjFormat, - /*AddSegmentInfo=*/false)); + + auto CoverageSection = lookupSections(*OF, IPSK_covmap); if (auto E = CoverageSection.takeError()) return std::move(E); std::vector CoverageSectionRefs = *CoverageSection; @@ -1004,17 +1028,8 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, return CoverageMappingOrErr.takeError(); StringRef CoverageMapping = CoverageMappingOrErr.get(); - InstrProfSymtab ProfileNames; - std::vector NamesSectionRefs = *NamesSection; - if (NamesSectionRefs.size() != 1) - return make_error(coveragemap_error::malformed); - if (Error E = ProfileNames.create(NamesSectionRefs.back())) - return std::move(E); - // Look for the coverage records section (Version4 only). - auto CoverageRecordsSections = - lookupSections(*OF, getInstrProfSectionName(IPSK_covfun, ObjFormat, - /*AddSegmentInfo=*/false)); + auto CoverageRecordsSections = lookupSections(*OF, IPSK_covfun); BinaryCoverageReader::FuncRecordsStorage FuncRecords; if (auto E = CoverageRecordsSections.takeError()) { diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index beaa175adb5c..b4e789533695 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -113,11 +113,11 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::malformed: OS << "malformed instrumentation profile data"; break; - case instrprof_error::missing_debug_info_for_correlation: - OS << "debug info for correlation is required"; + case instrprof_error::missing_correlation_info: + OS << "debug info/binary for correlation is required"; break; - case instrprof_error::unexpected_debug_info_for_correlation: - OS << "debug info for correlation is not necessary"; + case instrprof_error::unexpected_correlation_info: + OS << "debug info/binary for correlation is not necessary"; break; case instrprof_error::unable_to_correlate_profile: OS << "unable to correlate profile"; diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 3c77d7bf9e8b..6fdac036b165 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -25,14 +25,22 @@ using namespace llvm; /// Get profile section. Expected getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) { + // On COFF, the getInstrProfSectionName returns the section names may followed + // by "$M". The linker removes the dollar and everything after it in the final + // binary. Do the same to match. Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); + auto StripSuffix = [ObjFormat](StringRef N) { + return ObjFormat == Triple::COFF ? N.split('$').first : N; + }; std::string ExpectedSectionName = getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); - for (auto &Section : Obj.sections()) + ExpectedSectionName = StripSuffix(ExpectedSectionName); + for (auto &Section : Obj.sections()) { if (auto SectionName = Section.getName()) - if (SectionName.get() == ExpectedSectionName) + if (*SectionName == ExpectedSectionName) return Section; + } return make_error( instrprof_error::unable_to_correlate_profile, "could not find section (" + Twine(ExpectedSectionName) + ")"); @@ -44,14 +52,38 @@ const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; llvm::Expected> InstrProfCorrelator::Context::get(std::unique_ptr Buffer, - const object::ObjectFile &Obj) { + const object::ObjectFile &Obj, + ProfCorrelatorKind FileKind) { + auto C = std::make_unique(); auto CountersSection = getInstrProfSection(Obj, IPSK_cnts); if (auto Err = CountersSection.takeError()) return std::move(Err); - auto C = std::make_unique(); + if (FileKind == InstrProfCorrelator::BINARY) { + auto DataSection = getInstrProfSection(Obj, IPSK_covdata); + if (auto Err = DataSection.takeError()) + return std::move(Err); + auto DataOrErr = DataSection->getContents(); + if (!DataOrErr) + return DataOrErr.takeError(); + auto NameSection = getInstrProfSection(Obj, IPSK_covname); + if (auto Err = NameSection.takeError()) + return std::move(Err); + auto NameOrErr = NameSection->getContents(); + if (!NameOrErr) + return NameOrErr.takeError(); + C->DataStart = DataOrErr->data(); + C->DataEnd = DataOrErr->data() + DataOrErr->size(); + C->NameStart = NameOrErr->data(); + C->NameSize = NameOrErr->size(); + } C->Buffer = std::move(Buffer); C->CountersSectionStart = CountersSection->getAddress(); C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); + // In COFF object file, there's a null byte at the beginning of the counter + // section which doesn't exist in raw profile. + if (Obj.getTripleObjectFormat() == Triple::COFF) + ++C->CountersSectionStart; + C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; return Expected>(std::move(C)); } @@ -72,12 +104,23 @@ InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) { "using multiple objects is not yet supported"); Filename = *DsymObjectsOrErr->begin(); } + auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); + if (auto Err = BufferOrErr.takeError()) + return std::move(Err); + + return get(std::move(*BufferOrErr), FileKind); } - auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); - if (auto Err = BufferOrErr.takeError()) - return std::move(Err); + if (FileKind == BINARY) { + auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); + if (auto Err = BufferOrErr.takeError()) + return std::move(Err); - return get(std::move(*BufferOrErr), FileKind); + return get(std::move(*BufferOrErr), FileKind); + } + return make_error( + instrprof_error::unable_to_correlate_profile, + "unsupported correlation kind (only DWARF debug info and Binary format " + "(ELF/COFF) are supported)"); } llvm::Expected> @@ -88,7 +131,7 @@ InstrProfCorrelator::get(std::unique_ptr Buffer, return std::move(Err); if (auto *Obj = dyn_cast(BinOrErr->get())) { - auto CtxOrErr = Context::get(std::move(Buffer), *Obj); + auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind); if (auto Err = CtxOrErr.takeError()) return std::move(Err); auto T = Obj->makeTriple(); @@ -150,9 +193,11 @@ InstrProfCorrelatorImpl::get( instrprof_error::unable_to_correlate_profile, "unsupported debug info format (only DWARF is supported)"); } + if (Obj.isELF() || Obj.isCOFF()) + return std::make_unique>(std::move(Ctx)); return make_error( instrprof_error::unable_to_correlate_profile, - "unsupported correlation file type (only DWARF is supported)"); + "unsupported binary format (only ELF and COFF are supported)"); } template @@ -206,16 +251,16 @@ Error InstrProfCorrelatorImpl::dumpYaml(raw_ostream &OS) { } template -void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, - uint64_t CFGHash, - IntPtrT CounterOffset, - IntPtrT FunctionPtr, - uint32_t NumCounters) { +void InstrProfCorrelatorImpl::addDataProbe(uint64_t NameRef, + uint64_t CFGHash, + IntPtrT CounterOffset, + IntPtrT FunctionPtr, + uint32_t NumCounters) { // Check if a probe was already added for this counter offset. if (!CounterOffsets.insert(CounterOffset).second) return; Data.push_back({ - maybeSwap(IndexedInstrProf::ComputeHash(FunctionName)), + maybeSwap(NameRef), maybeSwap(CFGHash), // In this mode, CounterPtr actually stores the section relative address // of the counter. @@ -230,7 +275,6 @@ void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, // TODO: MC/DC is not yet supported. /*NumBitmapBytes=*/maybeSwap(0), }); - NamesVec.push_back(FunctionName.str()); } template @@ -338,6 +382,8 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( << "\n"); LLVM_DEBUG(Die.dump(dbgs())); } + // In debug info correlation mode, the CounterPtr is an absolute address of + // the counter, but it's expected to be relative later when iterating Data. IntPtrT CounterOffset = *CounterPtr - CountersStart; if (Data) { InstrProfCorrelator::Probe P; @@ -355,8 +401,9 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( P.LineNumber = LineNumber; Data->Probes.push_back(P); } else { - this->addProbe(*FunctionName, *CFGHash, CounterOffset, - FunctionPtr.value_or(0), *NumCounters); + this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash, + CounterOffset, FunctionPtr.value_or(0), *NumCounters); + this->NamesVec.push_back(*FunctionName); } }; for (auto &CU : DICtx->normal_units()) @@ -379,3 +426,46 @@ Error DwarfInstrProfCorrelator::correlateProfileNameImpl() { /*doCompression=*/false, this->Names); return Result; } + +template +void BinaryInstrProfCorrelator::correlateProfileDataImpl( + int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) { + using RawProfData = RawInstrProf::ProfileData; + bool UnlimitedWarnings = (MaxWarnings == 0); + // -N suppressed warnings means we can emit up to N (unsuppressed) warnings + int NumSuppressedWarnings = -MaxWarnings; + + const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart; + const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd; + // We need to use < here because the last data record may have no padding. + for (const RawProfData *I = DataStart; I < DataEnd; ++I) { + uint64_t CounterPtr = this->template maybeSwap(I->CounterPtr); + uint64_t CountersStart = this->Ctx->CountersSectionStart; + uint64_t CountersEnd = this->Ctx->CountersSectionEnd; + if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) { + if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { + WithColor::warning() + << format("CounterPtr out of range for function: Actual=0x%x " + "Expected=[0x%x, 0x%x) at data offset=0x%x\n", + CounterPtr, CountersStart, CountersEnd, + (I - DataStart) * sizeof(RawProfData)); + } + } + // In binary correlation mode, the CounterPtr is an absolute address of the + // counter, but it's expected to be relative later when iterating Data. + IntPtrT CounterOffset = CounterPtr - CountersStart; + this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset, + I->FunctionPointer, I->NumCounters); + } +} + +template +Error BinaryInstrProfCorrelator::correlateProfileNameImpl() { + if (this->Ctx->NameSize == 0) { + return make_error( + instrprof_error::unable_to_correlate_profile, + "could not find any profile data metadata in object file"); + } + this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize); + return Error::success(); +} diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 6f88e31754c3..5d0271204ce2 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -555,10 +555,6 @@ Error RawInstrProfReader::readHeader( "\nPLEASE update this tool to version in the raw profile, or " "regenerate raw profile with expected version.") .str()); - if (useCorrelate() && !Correlator) - return error(instrprof_error::missing_debug_info_for_correlation); - if (!useCorrelate() && Correlator) - return error(instrprof_error::unexpected_debug_info_for_correlation); uint64_t BinaryIdSize = swap(Header.BinaryIdsSize); // Binary id start just after the header if exists. @@ -606,8 +602,9 @@ Error RawInstrProfReader::readHeader( if (Correlator) { // These sizes in the raw file are zero because we constructed them in the // Correlator. - assert(DataSize == 0 && NamesSize == 0); - assert(CountersDelta == 0 && NamesDelta == 0); + if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 && + NamesDelta == 0)) + return error(instrprof_error::unexpected_correlation_info); Data = Correlator->getDataPointer(); DataEnd = Data + Correlator->getDataSize(); NamesStart = Correlator->getNamesPointer(); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index ce287551eed6..9e6af7d940c2 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -59,10 +59,24 @@ using namespace llvm; #define DEBUG_TYPE "instrprof" namespace llvm { -cl::opt - DebugInfoCorrelate("debug-info-correlate", - cl::desc("Use debug info to correlate profiles."), - cl::init(false)); +// TODO: Remove -debug-info-correlate in next LLVM release, in favor of +// -profile-correlate=debug-info. +cl::opt DebugInfoCorrelate( + "debug-info-correlate", + cl::desc("Use debug info to correlate profiles. (Deprecated, use " + "-profile-correlate=debug-info)"), + cl::init(false)); + +cl::opt ProfileCorrelate( + "profile-correlate", + cl::desc("Use debug info or binary file to correlate profiles."), + cl::init(InstrProfCorrelator::NONE), + cl::values(clEnumValN(InstrProfCorrelator::NONE, "", + "No profile correlation"), + clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", + "Use debug info to correlate"), + clEnumValN(InstrProfCorrelator::BINARY, "binary", + "Use binary to correlate"))); } // namespace llvm namespace { @@ -648,7 +662,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { // in lightweight mode. We need to move the value profile pointer to the // Counter struct to get this working. assert( - !DebugInfoCorrelate && + !DebugInfoCorrelate && ProfileCorrelate == InstrProfCorrelator::NONE && "Value profiling is not yet supported with lightweight instrumentation"); GlobalVariable *Name = Ind->getName(); auto It = ProfileDataMap.find(Name); @@ -1079,8 +1093,9 @@ GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, // Use internal rather than private linkage so the counter variable shows up // in the symbol table when using debug info for correlation. - if (DebugInfoCorrelate && TT.isOSBinFormatMachO() && - Linkage == GlobalValue::PrivateLinkage) + if ((DebugInfoCorrelate || + ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) && + TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage) Linkage = GlobalValue::InternalLinkage; // Due to the limitation of binder as of 2021/09/28, the duplicate weak @@ -1200,8 +1215,9 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts); PD.RegionCounters = CounterPtr; - if (DebugInfoCorrelate) { - LLVMContext &Ctx = M->getContext(); + if (DebugInfoCorrelate || + ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) { + LLVMContext &Ctx = M.getContext(); Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); @@ -1252,7 +1268,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, InstrProfMCDCBitmapParameters *Params) { // When debug information is correlated to profile data, a data variable // is not needed. - if (DebugInfoCorrelate) + if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) return; GlobalVariable *NamePtr = Inc->getName(); @@ -1347,21 +1363,29 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, Visibility = GlobalValue::DefaultVisibility; } auto *Data = - new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName); - // Reference the counter variable with a label difference (link-time - // constant). - auto *RelativeCounterPtr = - ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), - ConstantExpr::getPtrToInt(Data, IntPtrTy)); - - // Bitmaps are relative to the same data variable as profile counters. + new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName); + Constant *RelativeCounterPtr; GlobalVariable *BitmapPtr = PD.RegionBitmaps; Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); - - if (BitmapPtr != nullptr) { - RelativeBitmapPtr = - ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + InstrProfSectKind DataSectionKind; + // With binary profile correlation, profile data is not loaded into memory. + // profile data must reference profile counter with an absolute relocation. + if (ProfileCorrelate == InstrProfCorrelator::BINARY) { + DataSectionKind = IPSK_covdata; + RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy); + if (BitmapPtr != nullptr) + RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy); + } else { + // Reference the counter variable with a label difference (link-time + // constant). + DataSectionKind = IPSK_data; + RelativeCounterPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); + if (BitmapPtr != nullptr) + RelativeBitmapPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + ConstantExpr::getPtrToInt(Data, IntPtrTy)); } Constant *DataVals[] = { @@ -1371,7 +1395,8 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, Data->setInitializer(ConstantStruct::get(DataTy, DataVals)); Data->setVisibility(Visibility); - Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); + Data->setSection( + getInstrProfSectionName(DataSectionKind, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); maybeSetComdat(Data, Fn, CntsVarName); @@ -1457,7 +1482,9 @@ void InstrProfiling::emitNameData() { getInstrProfNamesVarName()); NamesSize = CompressedNameStr.size(); NamesVar->setSection( - getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); + ProfileCorrelate == InstrProfCorrelator::BINARY + ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat()) + : getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); // On COFF, it's important to reduce the alignment down to 1 to prevent the // linker from inserting padding before the start of the names section or // between names entries. diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 43969427175c..cd439818db80 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -327,7 +327,7 @@ extern cl::opt PGOViewCounts; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; -extern cl::opt DebugInfoCorrelate; +extern cl::opt ProfileCorrelate; } // namespace llvm static cl::opt @@ -382,7 +382,7 @@ static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) { ProfileVersion |= VARIANT_MASK_CSIR_PROF; if (PGOInstrumentEntry) ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; - if (DebugInfoCorrelate) + if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) ProfileVersion |= VARIANT_MASK_DBG_CORRELATE; if (PGOFunctionEntryCoverage) ProfileVersion |= diff --git a/llvm/test/Instrumentation/InstrProfiling/coverage.ll b/llvm/test/Instrumentation/InstrProfiling/coverage.ll index 1401d8f620b3..bbf895ea4b34 100644 --- a/llvm/test/Instrumentation/InstrProfiling/coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/coverage.ll @@ -1,11 +1,19 @@ ; RUN: opt < %s -passes=instrprof -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -profile-correlate=binary -S | FileCheck %s --check-prefix=BINARY target triple = "aarch64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 +; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) +; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), @__profn_bar = private constant [3 x i8] c"bar" ; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 +; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) +; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), + +; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names" +; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames" define void @_Z3foov() { call void @llvm.instrprof.cover(ptr @__profn_foo, i64 12345678, i32 1, i32 0) diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index dd3b6e1723f4..761e7d1a3e73 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -46,6 +46,7 @@ #include using namespace llvm; +using ProfCorrelatorKind = InstrProfCorrelator::ProfCorrelatorKind; // We use this string to indicate that there are // multiple static functions map to the same name. @@ -62,6 +63,331 @@ enum ProfileFormat { enum class ShowFormat { Text, Json, Yaml }; +// Common options. +cl::opt OutputFilename("output", cl::value_desc("output"), + cl::init("-"), cl::desc("Output file"), + cl::sub(ShowSubcommand), + cl::sub(OrderSubcommand), + cl::sub(OverlapSubcommand), + cl::sub(MergeSubcommand)); +// NOTE: cl::alias must not have cl::sub(), since aliased option's cl::sub() +// will be used. llvm::cl::alias::done() method asserts this condition. +cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), + cl::aliasopt(OutputFilename)); + +// Options common to at least two commands. +cl::opt ProfileKind( + cl::desc("Profile kind:"), cl::sub(MergeSubcommand), + cl::sub(OverlapSubcommand), cl::init(instr), + cl::values(clEnumVal(instr, "Instrumentation profile (default)"), + clEnumVal(sample, "Sample profile"))); +cl::opt Filename(cl::Positional, cl::desc(""), + cl::sub(ShowSubcommand), + cl::sub(OrderSubcommand)); +cl::opt MaxDbgCorrelationWarnings( + "max-debug-info-correlation-warnings", + cl::desc("The maximum number of warnings to emit when correlating " + "profile from debug info (0 = no limit)"), + cl::sub(MergeSubcommand), cl::sub(ShowSubcommand), cl::init(5)); +cl::opt ProfiledBinary( + "profiled-binary", cl::init(""), + cl::desc("Path to binary from which the profile was collected."), + cl::sub(ShowSubcommand), cl::sub(MergeSubcommand)); +cl::opt DebugInfoFilename( + "debug-info", cl::init(""), + cl::desc( + "For show, read and extract profile metadata from debug info and show " + "the functions it found. For merge, use the provided debug info to " + "correlate the raw profile."), + cl::sub(ShowSubcommand), cl::sub(MergeSubcommand)); +cl::opt + BinaryFilename("binary-file", cl::init(""), + cl::desc("For merge, use the provided unstripped bianry to " + "correlate the raw profile."), + cl::sub(MergeSubcommand)); +cl::opt FuncNameFilter( + "function", + cl::desc("Details for matching functions. For overlapping CSSPGO, this " + "takes a function name with calling context."), + cl::sub(ShowSubcommand), cl::sub(OverlapSubcommand)); + +// TODO: Consider creating a template class (e.g., MergeOption, ShowOption) to +// factor out the common cl::sub in cl::opt constructor for subcommand-specific +// options. + +// Options specific to merge subcommand. +cl::list InputFilenames(cl::Positional, cl::sub(MergeSubcommand), + cl::desc("")); +cl::list WeightedInputFilenames("weighted-input", + cl::sub(MergeSubcommand), + cl::desc(",")); +cl::opt OutputFormat( + cl::desc("Format of output profile"), cl::sub(MergeSubcommand), + cl::init(PF_Ext_Binary), + cl::values(clEnumValN(PF_Binary, "binary", "Binary encoding"), + clEnumValN(PF_Ext_Binary, "extbinary", + "Extensible binary encoding " + "(default)"), + clEnumValN(PF_Text, "text", "Text encoding"), + clEnumValN(PF_GCC, "gcc", + "GCC encoding (only meaningful for -sample)"))); +cl::opt + InputFilenamesFile("input-files", cl::init(""), cl::sub(MergeSubcommand), + cl::desc("Path to file containing newline-separated " + "[,] entries")); +cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"), + cl::aliasopt(InputFilenamesFile)); +cl::opt DumpInputFileList( + "dump-input-file-list", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("Dump the list of input files and their weights, then exit")); +cl::opt RemappingFile("remapping-file", cl::value_desc("file"), + cl::sub(MergeSubcommand), + cl::desc("Symbol remapping file")); +cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"), + cl::aliasopt(RemappingFile)); +cl::opt + UseMD5("use-md5", cl::init(false), cl::Hidden, + cl::desc("Choose to use MD5 to represent string in name table (only " + "meaningful for -extbinary)"), + cl::sub(MergeSubcommand)); +cl::opt CompressAllSections( + "compress-all-sections", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("Compress all sections when writing the profile (only " + "meaningful for -extbinary)")); +cl::opt SampleMergeColdContext( + "sample-merge-cold-context", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc( + "Merge context sample profiles whose count is below cold threshold")); +cl::opt SampleTrimColdContext( + "sample-trim-cold-context", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc( + "Trim context sample profiles whose count is below cold threshold")); +cl::opt SampleColdContextFrameDepth( + "sample-frame-depth-for-cold-context", cl::init(1), + cl::sub(MergeSubcommand), + cl::desc("Keep the last K frames while merging cold profile. 1 means the " + "context-less base profile")); +cl::opt OutputSizeLimit( + "output-size-limit", cl::init(0), cl::Hidden, cl::sub(MergeSubcommand), + cl::desc("Trim cold functions until profile size is below specified " + "limit in bytes. This uses a heursitic and functions may be " + "excessively trimmed")); +cl::opt GenPartialProfile( + "gen-partial-profile", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("Generate a partial profile (only meaningful for -extbinary)")); +cl::opt SupplInstrWithSample( + "supplement-instr-with-sample", cl::init(""), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("Supplement an instr profile with sample profile, to correct " + "the profile unrepresentativeness issue. The sample " + "profile is the input of the flag. Output will be in instr " + "format (The flag only works with -instr)")); +cl::opt ZeroCounterThreshold( + "zero-counter-threshold", cl::init(0.7), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("For the function which is cold in instr profile but hot in " + "sample profile, if the ratio of the number of zero counters " + "divided by the total number of counters is above the " + "threshold, the profile of the function will be regarded as " + "being harmful for performance and will be dropped.")); +cl::opt SupplMinSizeThreshold( + "suppl-min-size-threshold", cl::init(10), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("If the size of a function is smaller than the threshold, " + "assume it can be inlined by PGO early inliner and it won't " + "be adjusted based on sample profile.")); +cl::opt InstrProfColdThreshold( + "instr-prof-cold-threshold", cl::init(0), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("User specified cold threshold for instr profile which will " + "override the cold threshold got from profile summary. ")); +// WARNING: This reservoir size value is propagated to any input indexed +// profiles for simplicity. Changing this value between invocations could +// result in sample bias. +cl::opt TemporalProfTraceReservoirSize( + "temporal-profile-trace-reservoir-size", cl::init(100), + cl::sub(MergeSubcommand), + cl::desc("The maximum number of stored temporal profile traces (default: " + "100)")); +cl::opt TemporalProfMaxTraceLength( + "temporal-profile-max-trace-length", cl::init(10000), + cl::sub(MergeSubcommand), + cl::desc("The maximum length of a single temporal profile trace " + "(default: 10000)")); + +cl::opt + FailMode("failure-mode", cl::init(failIfAnyAreInvalid), + cl::desc("Failure mode:"), cl::sub(MergeSubcommand), + cl::values(clEnumValN(warnOnly, "warn", + "Do not fail and just print warnings."), + clEnumValN(failIfAnyAreInvalid, "any", + "Fail if any profile is invalid."), + clEnumValN(failIfAllAreInvalid, "all", + "Fail only if all profiles are invalid."))); + +cl::opt OutputSparse( + "sparse", cl::init(false), cl::sub(MergeSubcommand), + cl::desc("Generate a sparse profile (only meaningful for -instr)")); +cl::opt NumThreads( + "num-threads", cl::init(0), cl::sub(MergeSubcommand), + cl::desc("Number of merge threads to use (default: autodetect)")); +cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"), + cl::aliasopt(NumThreads)); + +cl::opt ProfileSymbolListFile( + "prof-sym-list", cl::init(""), cl::sub(MergeSubcommand), + cl::desc("Path to file containing the list of function symbols " + "used to populate profile symbol list")); + +cl::opt ProfileLayout( + "convert-sample-profile-layout", + cl::desc("Convert the generated profile to a profile with a new layout"), + cl::sub(MergeSubcommand), cl::init(SPL_None), + cl::values( + clEnumValN(SPL_Nest, "nest", + "Nested profile, the input should be CS flat profile"), + clEnumValN(SPL_Flat, "flat", + "Profile with nested inlinee flatten out"))); + +cl::opt DropProfileSymbolList( + "drop-profile-symbol-list", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("Drop the profile symbol list when merging AutoFDO profiles " + "(only meaningful for -sample)")); + +// Options specific to overlap subcommand. +cl::opt BaseFilename(cl::Positional, cl::Required, + cl::desc(""), + cl::sub(OverlapSubcommand)); +cl::opt TestFilename(cl::Positional, cl::Required, + cl::desc(""), + cl::sub(OverlapSubcommand)); + +cl::opt SimilarityCutoff( + "similarity-cutoff", cl::init(0), + cl::desc("For sample profiles, list function names (with calling context " + "for csspgo) for overlapped functions " + "with similarities below the cutoff (percentage times 10000)."), + cl::sub(OverlapSubcommand)); + +cl::opt IsCS( + "cs", cl::init(false), + cl::desc("For context sensitive PGO counts. Does not work with CSSPGO."), + cl::sub(OverlapSubcommand)); + +cl::opt OverlapValueCutoff( + "value-cutoff", cl::init(-1), + cl::desc( + "Function level overlap information for every function (with calling " + "context for csspgo) in test " + "profile with max count value greater then the parameter value"), + cl::sub(OverlapSubcommand)); + +// Options unique to show subcommand. +cl::opt ShowCounts("counts", cl::init(false), + cl::desc("Show counter values for shown functions"), + cl::sub(ShowSubcommand)); +cl::opt + SFormat("show-format", cl::init(ShowFormat::Text), + cl::desc("Emit output in the selected format if supported"), + cl::sub(ShowSubcommand), + cl::values(clEnumValN(ShowFormat::Text, "text", + "emit normal text output (default)"), + clEnumValN(ShowFormat::Json, "json", "emit JSON"), + clEnumValN(ShowFormat::Yaml, "yaml", "emit YAML"))); +// TODO: Consider replacing this with `--show-format=text-encoding`. +cl::opt + TextFormat("text", cl::init(false), + cl::desc("Show instr profile data in text dump format"), + cl::sub(ShowSubcommand)); +cl::opt + JsonFormat("json", + cl::desc("Show sample profile data in the JSON format " + "(deprecated, please use --show-format=json)"), + cl::sub(ShowSubcommand)); +cl::opt ShowIndirectCallTargets( + "ic-targets", cl::init(false), + cl::desc("Show indirect call site target values for shown functions"), + cl::sub(ShowSubcommand)); +cl::opt ShowMemOPSizes( + "memop-sizes", cl::init(false), + cl::desc("Show the profiled sizes of the memory intrinsic calls " + "for shown functions"), + cl::sub(ShowSubcommand)); +cl::opt ShowDetailedSummary("detailed-summary", cl::init(false), + cl::desc("Show detailed profile summary"), + cl::sub(ShowSubcommand)); +cl::list DetailedSummaryCutoffs( + cl::CommaSeparated, "detailed-summary-cutoffs", + cl::desc( + "Cutoff percentages (times 10000) for generating detailed summary"), + cl::value_desc("800000,901000,999999"), cl::sub(ShowSubcommand)); +cl::opt + ShowHotFuncList("hot-func-list", cl::init(false), + cl::desc("Show profile summary of a list of hot functions"), + cl::sub(ShowSubcommand)); +cl::opt ShowAllFunctions("all-functions", cl::init(false), + cl::desc("Details for each and every function"), + cl::sub(ShowSubcommand)); +cl::opt ShowCS("showcs", cl::init(false), + cl::desc("Show context sensitive counts"), + cl::sub(ShowSubcommand)); +cl::opt ShowProfileKind( + cl::desc("Profile kind supported by show:"), cl::sub(ShowSubcommand), + cl::init(instr), + cl::values(clEnumVal(instr, "Instrumentation profile (default)"), + clEnumVal(sample, "Sample profile"), + clEnumVal(memory, "MemProf memory access profile"))); +cl::opt TopNFunctions( + "topn", cl::init(0), + cl::desc("Show the list of functions with the largest internal counts"), + cl::sub(ShowSubcommand)); +cl::opt ShowValueCutoff( + "value-cutoff", cl::init(0), + cl::desc("Set the count value cutoff. Functions with the maximum count " + "less than this value will not be printed out. (Default is 0)"), + cl::sub(ShowSubcommand)); +cl::opt OnlyListBelow( + "list-below-cutoff", cl::init(false), + cl::desc("Only output names of functions whose max count values are " + "below the cutoff value"), + cl::sub(ShowSubcommand)); +cl::opt ShowProfileSymbolList( + "show-prof-sym-list", cl::init(false), + cl::desc("Show profile symbol list if it exists in the profile. "), + cl::sub(ShowSubcommand)); +cl::opt ShowSectionInfoOnly( + "show-sec-info-only", cl::init(false), + cl::desc("Show the information of each section in the sample profile. " + "The flag is only usable when the sample profile is in " + "extbinary format"), + cl::sub(ShowSubcommand)); +cl::opt ShowBinaryIds("binary-ids", cl::init(false), + cl::desc("Show binary ids in the profile. "), + cl::sub(ShowSubcommand)); +cl::opt ShowTemporalProfTraces( + "temporal-profile-traces", + cl::desc("Show temporal profile traces in the profile."), + cl::sub(ShowSubcommand)); + +cl::opt + ShowCovered("covered", cl::init(false), + cl::desc("Show only the functions that have been executed."), + cl::sub(ShowSubcommand)); + +cl::opt ShowProfileVersion("profile-version", cl::init(false), + cl::desc("Show profile version. "), + cl::sub(ShowSubcommand)); + +// We use this string to indicate that there are +// multiple static functions map to the same name. +const std::string DuplicateNameStr = "----"; + static void warn(Twine Message, std::string Whence = "", std::string Hint = "") { WithColor::warning(); @@ -440,14 +766,27 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, OutputFormat != PF_Text) exitWithError("unknown format is specified"); - std::unique_ptr Correlator; + // TODO: Maybe we should support correlation with mixture of different + // correlation modes(w/wo debug-info/object correlation). + if (!DebugInfoFilename.empty() && !BinaryFilename.empty()) + exitWithError("Expected only one of -debug-info, -binary-file"); + std::string CorrelateFilename; + ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE; if (!DebugInfoFilename.empty()) { - if (auto Err = InstrProfCorrelator::get(DebugInfoFilename, - InstrProfCorrelator::DEBUG_INFO) + CorrelateFilename = DebugInfoFilename; + CorrelateKind = ProfCorrelatorKind::DEBUG_INFO; + } else if (!BinaryFilename.empty()) { + CorrelateFilename = BinaryFilename; + CorrelateKind = ProfCorrelatorKind::BINARY; + } + + std::unique_ptr Correlator; + if (CorrelateKind != InstrProfCorrelator::NONE) { + if (auto Err = InstrProfCorrelator::get(CorrelateFilename, CorrelateKind) .moveInto(Correlator)) - exitWithError(std::move(Err), DebugInfoFilename); - if (auto Err = Correlator->correlateProfileData()) - exitWithError(std::move(Err), DebugInfoFilename); + exitWithError(std::move(Err), CorrelateFilename); + if (auto Err = Correlator->correlateProfileData(MaxDbgCorrelationWarnings)) + exitWithError(std::move(Err), CorrelateFilename); } std::mutex ErrorLock; -- Gitee From 8370b2437d9b77cb1dd013b1b2298ca2a659af46 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Tue, 2 Jan 2024 10:23:29 -0800 Subject: [PATCH 13/28] [RawProfReader]When constructing symbol table, read the MD5 of function name in the proper byte order (#76312) Before this patch, when the field `NameRef` is generated in little-endian systems and read back in big-endian systems, the information gets dropped. - The bug gets caught by a buildbot https://lab.llvm.org/buildbot/#/builders/94/builds/17931. In the error message (pasted below), two indirect call targets are not imported. ``` ; IMPORTS-DAG: Import _Z7callee1v ^ :1:1: note: scanning from here main.ll: Import _Z11global_funcv from lib.cc ^ :1:10: note: possible intended match here main.ll: Import _Z11global_funcv from lib.cc ^ Input file: Check file: /home/uweigand/sandbox/buildbot/clang-s390x-linux/llvm/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll -dump-input=help explains the following input dump. Input was: <<<<<< 1: main.ll: Import _Z11global_funcv from lib.cc dag:34'0 X~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ error: no match found dag:34'1 ? possible intended match ``` [This commit](https://github.com/llvm/llvm-project/commit/b3999246b1a93bd8e6dc7acac36a7d57fac96542#diff-b196b796c5a396c7cdf93b347fe47e2b29b72d0b7dd0e2b88abb964d376ee50e) gates the fix by flag and provide test data by creating big-endian profiles (rather than reading the little-endian data on a big-endian system that might require a VM). - [This](https://github.com/llvm/llvm-project/commit/b3999246b1a93bd8e6dc7acac36a7d57fac96542#diff-643176077ddbe537bd0a05d2a8a53bdff6339420a30e8511710bf232afdda8b9) is a hexdump of little-endian profile data, and [this](https://github.com/llvm/llvm-project/commit/b3999246b1a93bd8e6dc7acac36a7d57fac96542#diff-1736a3ee25dde02bba55d670df78988fdb227e5a85b94b8707cf182cf70b28f0) is the big-endian version of it. - The [README.md](https://github.com/llvm/llvm-project/commit/b3999246b1a93bd8e6dc7acac36a7d57fac96542#diff-6717b6a385de3ae60ab3aec9638af2a43b55adaf6784b6f0393ebe1a6639438b) shows the result of `llvm-profdata show -ic-targets` before and after the fix when the profile is in big-endian. --- llvm/lib/ProfileData/InstrProfReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 5d0271204ce2..8933b879ba32 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -538,7 +538,7 @@ Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { const IntPtrT FPtr = swap(I->FunctionPointer); if (!FPtr) continue; - Symtab.mapAddress(FPtr, I->NameRef); + Symtab.mapAddress(FPtr, swap(I->NameRef)); } return success(); } -- Gitee From a3e8d1315cfb0fd6aad59a9ceb2b4d04ff64d2e7 Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Mon, 5 Feb 2024 14:16:05 +0900 Subject: [PATCH 14/28] InstrProf::getFunctionBitmap: Fix BE hosts (#80608) --- llvm/lib/ProfileData/InstrProfReader.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 8933b879ba32..9b76b45bdce8 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1422,7 +1422,23 @@ Error IndexedInstrProfReader::getFunctionBitmapBytes( if (Error E = Record.takeError()) return error(std::move(E)); - BitmapBytes = Record.get().BitmapBytes; + const auto &BitmapBytes = Record.get().BitmapBytes; + size_t I = 0, E = BitmapBytes.size(); + Bitmap.resize(E * CHAR_BIT); + BitVector::apply( + [&](auto X) { + using XTy = decltype(X); + alignas(XTy) uint8_t W[sizeof(X)]; + size_t N = std::min(E - I, sizeof(W)); + std::memset(W, 0, sizeof(W)); + std::memcpy(W, &BitmapBytes[I], N); + I += N; + return support::endian::read(W); + }, + Bitmap, Bitmap); + assert(I == E); + return success(); } -- Gitee From 6d076b027e76a95755020aece5f547b745f83f50 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Tue, 27 Feb 2024 11:07:40 -0800 Subject: [PATCH 15/28] Reland "[TypeProf][InstrPGO] Introduce raw and instr profile format change for type profiling." (#82711) New change on top of [reviewed patch](https://github.com/llvm/llvm-project/pull/81691) are [in commits after this one](https://github.com/llvm/llvm-project/pull/82711/commits/d0757f46b3e3865b5f7c552bc0744309a363e0ac). Previous commits are restored from the remote branch with timestamps. 1. Fix build breakage for non-ELF platforms, by defining the missing functions {`__llvm_profile_begin_vtables`, `__llvm_profile_end_vtables`, `__llvm_profile_begin_vtabnames `, `__llvm_profile_end_vtabnames`} everywhere. * Tested on mac laptop (for darwins) and Windows. Specifically, functions in `InstrProfilingPlatformWindows.c` returns `NULL` to make it more explicit that type prof isn't supported; see comments for the reason. * For the rest (AIX, other), mostly follow existing examples (like this [one](https://github.com/llvm/llvm-project/commit/f95b2f1acf1171abb0d00089fd4c9238753847e3)) 2. Rename `__llvm_prf_vtabnames` -> `__llvm_prf_vns` for shorter section name, and make returned pointers [const](https://github.com/llvm/llvm-project/pull/82711/commits/a825d2a4ec00f07772a373091a702f149c3b0c34#diff-4de780ce726d76b7abc9d3353aef95013e7b21e7bda01be8940cc6574fb0b5ffR120-R121) **Original Description** * Raw profile format - Header: records the byte size of compressed vtable names, and the number of profiled vtable entries (call it `VTableProfData`). Header also records padded bytes of each section. - Payload: adds a section for compressed vtable names, and a section to store `VTableProfData`. Both sections are padded so the size is a multiple of 8. * Indexed profile format - Header: records the byte offset of compressed vtable names. - Payload: adds a section to store compressed vtable names. This section is used by `llvm-profdata` to show the list of vtables profiled for an instrumented site. [The originally reviewed patch](https://github.com/llvm/llvm-project/pull/66825) will have profile reader/write change and llvm-profdata change. - To ensure this PR has all the necessary profile format change along with profile version bump, created a copy of the originally reviewed patch in https://github.com/llvm/llvm-project/pull/80761. The copy doesn't have profile format change, but it has the set of tests which covers type profile generation, profile read and profile merge. Tests pass there. rfc in https://discourse.llvm.org/t/rfc-dynamic-type-profiling-and-optimizations-in-llvm/74600 --------- Co-authored-by: modiking --- compiler-rt/include/profile/InstrProfData.inc | 54 +++++++++- compiler-rt/lib/profile/InstrProfiling.h | 35 +++++-- .../lib/profile/InstrProfilingBuffer.c | 96 +++++++++++++++--- compiler-rt/lib/profile/InstrProfilingFile.c | 11 +- .../lib/profile/InstrProfilingInternal.h | 8 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 23 ++++- .../lib/profile/InstrProfilingPlatformAIX.c | 16 ++- .../profile/InstrProfilingPlatformDarwin.c | 23 +++++ .../lib/profile/InstrProfilingPlatformLinux.c | 21 ++++ .../lib/profile/InstrProfilingPlatformOther.c | 15 +++ .../profile/InstrProfilingPlatformWindows.c | 19 ++++ .../lib/profile/InstrProfilingWriter.c | 42 +++++--- .../profile/instrprof-write-buffer-internal.c | 6 +- llvm/include/llvm/ProfileData/InstrProf.h | 17 +++- .../llvm/ProfileData/InstrProfData.inc | 54 +++++++++- .../llvm/ProfileData/InstrProfReader.h | 13 +++ llvm/lib/ProfileData/InstrProf.cpp | 11 +- llvm/lib/ProfileData/InstrProfReader.cpp | 44 +++++++- llvm/lib/ProfileData/InstrProfWriter.cpp | 45 ++++++-- .../InstrProfiling/coverage.ll | 8 +- .../thinlto_indirect_call_promotion.profraw | Bin 0 -> 544 bytes .../Transforms/PGOProfile/comdat_internal.ll | 4 +- .../llvm-profdata/Inputs/c-general.profraw | Bin 2016 -> 2032 bytes .../llvm-profdata/Inputs/compressed.profraw | Bin 1968 -> 1984 bytes .../thinlto_indirect_call_promotion.profraw | Bin 0 -> 528 bytes .../llvm-profdata/binary-ids-padding.test | 6 +- .../llvm-profdata/large-binary-id-size.test | 4 +- ...alformed-not-space-for-another-header.test | 6 +- .../malformed-num-counters-zero.test | 6 +- .../malformed-ptr-to-counter-array.test | 6 +- .../misaligned-binary-ids-size.test | 4 +- .../mismatched-raw-profile-header.test | 2 + .../tools/llvm-profdata/raw-32-bits-be.test | 13 ++- .../tools/llvm-profdata/raw-32-bits-le.test | 12 ++- .../tools/llvm-profdata/raw-64-bits-be.test | 14 ++- .../tools/llvm-profdata/raw-64-bits-le.test | 14 ++- .../tools/llvm-profdata/raw-two-profiles.test | 8 +- 37 files changed, 557 insertions(+), 103 deletions(-) create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw create mode 100644 llvm/test/tools/llvm-profdata/Inputs/thinlto_indirect_call_promotion.profraw diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 4a538e592c89..ea5330254e29 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -94,6 +94,25 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ +/* For a virtual table object, record the name hash to associate profiled + * addresses with global variables, and record {starting address, size in bytes} + * to map the profiled virtual table (which usually have an offset from the + * starting address) back to a virtual table object. */ +#ifndef INSTR_PROF_VTABLE_DATA +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_VTABLE_DATA_DEFINED +#endif +INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ + VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + IndexedInstrProf::ComputeHash(PGOVTableName))) +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ + VTablePointer, VTableAddr) +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ + VTableSizeVal)) +#undef INSTR_PROF_VTABLE_DATA +/* INSTR_PROF_VTABLE_DATA end. */ /* This is an internal data structure used by value profiler. It * is defined here to allow serialization code sharing by LLVM @@ -143,6 +162,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ @@ -184,13 +205,26 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") +/* For virtual table address profiling, the address point of the virtual table + * (i.e., the address contained in objects pointing to a virtual table) are + * profiled. Note this may not be the address of the per C++ class virtual table + * object (e.g., there might be an offset). + * + * The profiled addresses are stored in raw profile, together with the following + * two types of information. + * 1. The (starting and ending) addresses of per C++ class virtual table objects. + * 2. The (compressed) virtual table object names. + * RawInstrProfReader converts profiled virtual table addresses to virtual table + * objects' MD5 hash. + */ +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") +VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -280,12 +314,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vname, \ + INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \ + INSTR_PROF_VNAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ INSTR_PROF_VNODES_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vtab, \ + INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \ + INSTR_PROF_VTAB_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") @@ -661,9 +701,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 9 +#define INSTR_PROF_RAW_VERSION 10 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 11 +#define INSTR_PROF_INDEX_VERSION 12 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 6 @@ -701,10 +741,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, than WIN32 */ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names +#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts #define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds +#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun #define INSTR_PROF_COVDATA_COMMON __llvm_covdata @@ -715,10 +757,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_VNAME_COFF ".lprfvn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" #define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_VTAB_COFF ".lprfvt$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" /* Since cov data and cov names sections are not allocated, we don't need to @@ -734,6 +778,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF +#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF +#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -751,6 +797,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) +#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON) +#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index b8104af5f12b..67b410876ef4 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -38,6 +38,12 @@ typedef struct ValueProfNode { #include "profile/InstrProfData.inc" } ValueProfNode; +typedef void *IntPtrT; +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) VTableProfData { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} VTableProfData; + /*! * \brief Return 1 if profile counters are continuously synced to the raw * profile via an mmap(). This is in contrast to the default mode, in which @@ -86,12 +92,16 @@ const __llvm_profile_data *__llvm_profile_begin_data(void); const __llvm_profile_data *__llvm_profile_end_data(void); const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); +const char *__llvm_profile_begin_vtabnames(void); +const char *__llvm_profile_end_vtabnames(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); char *__llvm_profile_begin_bitmap(void); char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); +const VTableProfData *__llvm_profile_begin_vtables(); +const VTableProfData *__llvm_profile_end_vtables(); uint32_t *__llvm_profile_begin_orderfile(); /*! @@ -288,20 +298,31 @@ uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, /*! \brief Get the size of the profile name section in bytes. */ uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); -/* ! \brief Given the sizes of the data and counter information, return the - * number of padding bytes before and after the counters, and after the names, - * in the raw profile. +/*! \brief Get the number of virtual table profile data entries */ +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End); + +/*! \brief Get the size of virtual table profile data in bytes. */ +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End); + +/* ! \brief Given the sizes of the data and counter information, computes the + * number of padding bytes before and after the counter section, as well as the + * number of padding bytes after other setions in the raw profile. + * Returns -1 upon errors and 0 upon success. Output parameters should be used + * iff return value is 0. * * Note: When mmap() mode is disabled, no padding bytes before/after counters * are needed. However, in mmap() mode, the counter section in the raw profile * must be page-aligned: this API computes the number of padding bytes * needed to achieve that. */ -void __llvm_profile_get_padding_sizes_for_counters( +int __llvm_profile_get_padding_sizes_for_counters( uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, - uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, - uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmap, - uint64_t *PaddingBytesAfterNames); + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames); /*! * \brief Set the flag that profile data has been dumped to the file. diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 69965142d978..f72ba709fc0e 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -47,10 +47,14 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); return __llvm_profile_get_size_for_buffer_internal( DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, - NamesBegin, NamesEnd); + NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd); } COMPILER_RT_VISIBILITY @@ -59,6 +63,15 @@ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, if (hasCorrelation()) return 0; intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + // `sizeof(__llvm_profile_data) - 1` is required in the numerator when + // [Begin, End] represents an inclusive range. + // For ELF, [Begin, End) represents the address of linker-inserted + // symbols `__start__` and `__stop_`. + // Thereby, `End` is one byte past the inclusive range, and + // `sizeof(__llvm_profile_data) - 1` is not necessary in the numerator to get + // the correct number of profile data. + // FIXME: Consider removing `sizeof(__llvm_profile_data) - 1` if this is true + // across platforms. return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / sizeof(__llvm_profile_data); } @@ -71,6 +84,26 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); } +// Counts the number of `VTableProfData` elements within the range of [Begin, +// End). Caller should guarantee that End points to one byte past the inclusive +// range. +// FIXME: Add a compiler-rt test to make sure the number of vtables in the +// raw profile is the same as the number of vtable elements in the instrumented +// binary. +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End) { + // Convert pointers to intptr_t to use integer arithmetic. + intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; + return (EndI - BeginI) / sizeof(VTableProfData); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End) { + return (intptr_t)(End) - (intptr_t)(Begin); +} + COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) return sizeof(uint8_t); @@ -121,11 +154,13 @@ static int needsCounterPadding(void) { } COMPILER_RT_VISIBILITY -void __llvm_profile_get_padding_sizes_for_counters( +int __llvm_profile_get_padding_sizes_for_counters( uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, - uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, - uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes, - uint64_t *PaddingBytesAfterNames) { + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) { + // Counter padding is needed only if continuous mode is enabled. if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = @@ -133,9 +168,19 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesAfterBitmapBytes = __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); - return; + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = + __llvm_profile_get_num_padding_bytes(VTableSize); + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize); + return 0; } + // Value profiling not supported in continuous mode at profile-write time. + // Return -1 to alert the incompatibility. + if (VTableSize != 0 || VNameSize != 0) + return -1; + // In continuous mode, the file offsets for headers and for the start of // counter sections need to be page-aligned. *PaddingBytesBeforeCounters = @@ -144,13 +189,22 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesAfterBitmapBytes = calculateBytesNeededToPageAlign(NumBitmapBytes); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); + // Set these two variables to zero to avoid uninitialized variables + // even if VTableSize and VNameSize are known to be zero. + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = 0; + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = 0; + return 0; } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) { + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); @@ -158,20 +212,29 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumBitmapBytes = __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + const uint64_t VTableSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNameSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */, + 0 /* VNameSize */, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + PaddingBytesAfterCounters + NumBitmapBytes + - PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames; + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames + + VTableSize + PaddingBytesAfterVTable + VNameSize + + PaddingBytesAfterVNames; } COMPILER_RT_VISIBILITY @@ -193,7 +256,10 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); - return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin, - NamesEnd, 0); + // Set virtual table arguments to NULL since they are not supported yet. + return lprofWriteDataImpl( + &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, + BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd, + /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL, + /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0); } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 49fef96578c8..6efee9fe2c1b 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -137,15 +137,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { DataBegin, PageSize); return 1; } + int Fileno = fileno(File); /* Determine how much padding is needed before/after the counters and * after the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0, + /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames, + &PaddingBytesAfterVTable, &PaddingBytesAfterVNames); uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; uint64_t FileOffsetToCounters = CurrentFileOffset + diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h index 03ed67fcfa76..d5bd0e41fb12 100644 --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -22,7 +22,9 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -156,7 +158,9 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, - const char *NamesEnd, int SkipNameDataWrite); + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite); /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index da93c0e33c5b..2dcf828fcf64 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -105,6 +105,26 @@ static uintptr_t signextIfWin64(void *V) { #endif } +// Skip names section, vtable profile data section and vtable names section +// for runtime profile merge. To merge runtime addresses from multiple +// profiles collected from the same instrumented binary, the binary should be +// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR +// disabled). In this set-up these three sections remain unchanged. +static uint64_t +getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { + const uint64_t VTableSectionSize = + Header->NumVTables * sizeof(VTableProfData); + const uint64_t PaddingBytesAfterVTableSection = + __llvm_profile_get_num_padding_bytes(VTableSectionSize); + const uint64_t VNamesSize = Header->VNamesSize; + const uint64_t PaddingBytesAfterVNamesSize = + __llvm_profile_get_num_padding_bytes(VNamesSize); + return Header->NamesSize + + __llvm_profile_get_num_padding_bytes(Header->NamesSize) + + VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize + + PaddingBytesAfterVNamesSize; +} + COMPILER_RT_VISIBILITY int __llvm_profile_merge_from_buffer(const char *ProfileData, uint64_t ProfileSize) { @@ -137,8 +157,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, SrcBitmapStart = SrcCountersEnd; SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; SrcValueProfDataStart = - SrcNameStart + Header->NamesSize + - __llvm_profile_get_num_padding_bytes(Header->NamesSize); + SrcNameStart + getDistanceFromCounterToValueProf(Header); if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c b/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c index 9f46a98d78ac..b9d51b698b41 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c @@ -175,7 +175,8 @@ void __llvm_profile_register_names_function(void *NamesStart, uint64_t NamesSize) {} // The __start_SECNAME and __stop_SECNAME symbols (for SECNAME \in -// {"__llvm_prf_cnts", "__llvm_prf_data", "__llvm_prf_name", "__llvm_prf_vnds"}) +// {"__llvm_prf_cnts", "__llvm_prf_data", "__llvm_prf_name", "__llvm_prf_vnds", +// "__llvm_prf_vns", "__llvm_prf_vtab"}) // are always live when linking on AIX, regardless if the .o's being linked // reference symbols from the profile library (for example when no files were // compiled with -fprofile-generate). That's because these symbols are kept @@ -195,6 +196,12 @@ static const int dummy_name[0] COMPILER_RT_SECTION( COMPILER_RT_SEG INSTR_PROF_NAME_SECT_NAME); static int dummy_vnds[0] COMPILER_RT_SECTION( COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); +static int dummy_orderfile[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_ORDERFILE_SECT_NAME); +static int dummy_vname[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNAME_SECT_NAME); +static int dummy_vtab[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VTAB_SECT_NAME); // To avoid GC'ing of the dummy variables by the linker, reference them in an // array and reference the array in the runtime registration code @@ -204,9 +211,10 @@ static int dummy_vnds[0] COMPILER_RT_SECTION( #pragma GCC diagnostic ignored "-Wcast-qual" #endif COMPILER_RT_VISIBILITY -void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits, - (void *)&dummy_data, (void *)&dummy_name, - (void *)&dummy_vnds}; +void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits, + (void *)&dummy_data, (void *)&dummy_name, + (void *)&dummy_vnds, (void *)&dummy_orderfile, + (void *)&dummy_vname, (void *)&dummy_vtab}; #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c index 2154d242a817..6adc7f328cbf 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -36,6 +36,17 @@ extern char COMPILER_RT_VISIBILITY extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); COMPILER_RT_VISIBILITY +extern VTableProfData + VTableProfStart __asm("section$start$__DATA$" INSTR_PROF_VTAB_SECT_NAME); +COMPILER_RT_VISIBILITY +extern VTableProfData + VTableProfEnd __asm("section$end$__DATA$" INSTR_PROF_VTAB_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + VNameStart __asm("section$start$__DATA$" INSTR_PROF_VNAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char VNameEnd __asm("section$end$__DATA$" INSTR_PROF_VNAME_SECT_NAME); +COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -65,6 +76,18 @@ char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } COMPILER_RT_VISIBILITY +const VTableProfData *__llvm_profile_begin_vtables(void) { + return &VTableProfStart; +} +COMPILER_RT_VISIBILITY +const VTableProfData *__llvm_profile_end_vtables(void) { + return &VTableProfEnd; +} +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_vtabnames(void) { return &VNameStart; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_vtabnames(void) { return &VNameEnd; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index d0c42462e5e3..761f21d6dbb8 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -33,8 +33,12 @@ #define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) #define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON) +#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON) +#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON) #define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) #define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) @@ -50,6 +54,10 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -72,6 +80,19 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &PROF_NAME_STOP; } +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) { + return &PROF_VNAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) { + return &PROF_VNAME_STOP; +} +COMPILER_RT_VISIBILITY const VTableProfData * +__llvm_profile_begin_vtables(void) { + return &PROF_VTABLE_START; +} +COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) { + return &PROF_VTABLE_STOP; +} COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return &PROF_CNTS_START; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index 5319ca813b43..aa79a5641cec 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -18,8 +18,12 @@ static const __llvm_profile_data *DataFirst = NULL; static const __llvm_profile_data *DataLast = NULL; +static const VTableProfData *VTableProfDataFirst = NULL; +static const VTableProfData *VTableProfDataLast = NULL; static const char *NamesFirst = NULL; static const char *NamesLast = NULL; +static const char *VNamesFirst = NULL; +static const char *VNamesLast = NULL; static char *CountersFirst = NULL; static char *CountersLast = NULL; static uint32_t *OrderFileFirst = NULL; @@ -80,11 +84,22 @@ COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; } COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; } +COMPILER_RT_VISIBILITY const VTableProfData * +__llvm_profile_begin_vtables(void) { + return VTableProfDataFirst; +} +COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) { + return VTableProfDataLast; +} COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { return NamesFirst; } COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return NamesLast; } COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_vtabnames(void) { return VNamesFirst; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_vtabnames(void) { return VNamesLast; } +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return CountersLast; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 9070b8a606eb..3b9b44eb6a98 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -6,6 +6,8 @@ |* \*===----------------------------------------------------------------------===*/ +#include + #include "InstrProfiling.h" #include "InstrProfilingInternal.h" @@ -59,9 +61,26 @@ const __llvm_profile_data *__llvm_profile_begin_data(void) { } const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } +// Type profiling isn't implemented under MSVC ABI, so return NULL (rather than +// implementing linker magic on Windows) to make it more explicit. To elaborate, +// the current type profiling implementation maps a profiled vtable address to a +// vtable variable through vtables mangled name. Under MSVC ABI, the variable +// name for vtables might not be the mangled name (see +// MicrosoftCXXABI::getAddrOfVTable in MicrosoftCXXABI.cpp for more details on +// how a vtable name is computed). Note the mangled name is still in the vtable +// IR (just not variable name) for mapping purpose, but more implementation work +// is required. +const VTableProfData *__llvm_profile_begin_vtables(void) { return NULL; } +const VTableProfData *__llvm_profile_end_vtables(void) { return NULL; } + const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } const char *__llvm_profile_end_names(void) { return &NamesEnd; } +// Type profiling isn't supported on Windows, so return NULl to make it more +// explicit. +const char *__llvm_profile_begin_vtabnames(void) { return NULL; } +const char *__llvm_profile_end_vtabnames(void) { return NULL; } + char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } char *__llvm_profile_end_counters(void) { return &CountersEnd; } char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index d35ee6b20504..3261408232b6 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -250,9 +250,14 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, - NamesBegin, NamesEnd, SkipNameDataWrite); + NamesBegin, NamesEnd, VTableBegin, VTableEnd, + VNamesBegin, VNamesEnd, SkipNameDataWrite); } COMPILER_RT_VISIBILITY int @@ -261,9 +266,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, - const char *NamesEnd, int SkipNameDataWrite) { - int ProfileCorrelation = hasCorrelation(); - + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite) { /* Calculate size of sections. */ const uint64_t DataSectionSize = __llvm_profile_get_data_size(DataBegin, DataEnd); @@ -275,6 +280,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const uint64_t NumBitmapBytes = __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); + const uint64_t NumVTables = + __llvm_profile_get_num_vtable(VTableBegin, VTableEnd); + const uint64_t VTableSectionSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNamesSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); /* Create the header. */ __llvm_profile_header Header; @@ -282,11 +293,15 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; - __llvm_profile_get_padding_sizes_for_counters( - DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + if (__llvm_profile_get_padding_sizes_for_counters( + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, + VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames) == -1) + return -1; { /* Initialize header structure. */ @@ -325,9 +340,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, - {(SkipNameDataWrite || ProfileCorrelation) ? NULL : NamesBegin, - sizeof(uint8_t), NamesSize, 0}, - {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; + {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}, + {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1}, + {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}}; if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) return -1; diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c index d9670f739ca9..2c1c29ac0c58 100644 --- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -31,7 +31,8 @@ char *__llvm_profile_end_bitmap(void); uint64_t __llvm_profile_get_size_for_buffer_internal( const void *DataBegin, const void *DataEnd, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, - const char *NamesBegin, const char *NamesEnd); + const char *NamesBegin, const char *NamesEnd, const void *VTableBegin, + const void *VTableEnd, const char *VNamesBegin, const char *VNamesEnd); int __llvm_profile_write_buffer_internal( char *Buffer, const void *DataBegin, const void *DataEnd, @@ -45,7 +46,8 @@ int main(int argc, const char *argv[]) { __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), - __llvm_profile_begin_names(), __llvm_profile_end_names()); + __llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL, + NULL, NULL); char *buf = malloc(bufsize); int ret = __llvm_profile_write_buffer_internal( diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 1d4049ece54e..8deab64c3905 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -823,6 +823,7 @@ private: struct ValueProfData { std::vector IndirectCallSites; std::vector MemOPSizes; + std::vector VTableTargets; }; std::unique_ptr ValueData; @@ -845,6 +846,8 @@ private: return ValueData->IndirectCallSites; case IPVK_MemOPSize: return ValueData->MemOPSizes; + case IPVK_VTableTarget: + return ValueData->VTableTargets; default: llvm_unreachable("Unknown value kind!"); } @@ -1032,7 +1035,9 @@ enum ProfVersion { Version10 = 10, // An additional field is used for bitmap bytes. Version11 = 11, - // The current version is 11. + // VTable profiling, + Version12 = 12, + // The current version is 12. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1052,6 +1057,7 @@ struct Header { uint64_t MemProfOffset; uint64_t BinaryIdOffset; uint64_t TemporalProfTracesOffset; + uint64_t VTableNamesOffset; // New fields should only be added at the end to ensure that the size // computation is correct. The methods below need to be updated to ensure that // the new field is read correctly. @@ -1188,8 +1194,13 @@ template <> inline uint64_t getMagic() { // It should also match the synthesized type in // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. template struct alignas(8) ProfileData { - #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; - #include "llvm/ProfileData/InstrProfData.inc" +#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" +}; + +template struct alignas(8) VTableProfileData { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" }; // File header structure of the LLVM profile data in raw format. diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 4a538e592c89..ea5330254e29 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -94,6 +94,25 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ +/* For a virtual table object, record the name hash to associate profiled + * addresses with global variables, and record {starting address, size in bytes} + * to map the profiled virtual table (which usually have an offset from the + * starting address) back to a virtual table object. */ +#ifndef INSTR_PROF_VTABLE_DATA +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_VTABLE_DATA_DEFINED +#endif +INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ + VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + IndexedInstrProf::ComputeHash(PGOVTableName))) +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ + VTablePointer, VTableAddr) +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ + VTableSizeVal)) +#undef INSTR_PROF_VTABLE_DATA +/* INSTR_PROF_VTABLE_DATA end. */ /* This is an internal data structure used by value profiler. It * is defined here to allow serialization code sharing by LLVM @@ -143,6 +162,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ @@ -184,13 +205,26 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") +/* For virtual table address profiling, the address point of the virtual table + * (i.e., the address contained in objects pointing to a virtual table) are + * profiled. Note this may not be the address of the per C++ class virtual table + * object (e.g., there might be an offset). + * + * The profiled addresses are stored in raw profile, together with the following + * two types of information. + * 1. The (starting and ending) addresses of per C++ class virtual table objects. + * 2. The (compressed) virtual table object names. + * RawInstrProfReader converts profiled virtual table addresses to virtual table + * objects' MD5 hash. + */ +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") +VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -280,12 +314,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vname, \ + INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \ + INSTR_PROF_VNAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ INSTR_PROF_VNODES_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vtab, \ + INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \ + INSTR_PROF_VTAB_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") @@ -661,9 +701,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 9 +#define INSTR_PROF_RAW_VERSION 10 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 11 +#define INSTR_PROF_INDEX_VERSION 12 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 6 @@ -701,10 +741,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, than WIN32 */ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names +#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts #define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds +#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun #define INSTR_PROF_COVDATA_COMMON __llvm_covdata @@ -715,10 +757,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_VNAME_COFF ".lprfvn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" #define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_VTAB_COFF ".lprfvt$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" /* Since cov data and cov names sections are not allocated, we don't need to @@ -734,6 +778,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF +#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF +#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -751,6 +797,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) +#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON) +#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 3c9958a75c30..fa0e54a5bd51 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -326,12 +326,16 @@ private: uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; + const RawInstrProf::VTableProfileData *VTableBegin = nullptr; + const RawInstrProf::VTableProfileData *VTableEnd = nullptr; const char *CountersStart; const char *CountersEnd; const char *BitmapStart; const char *BitmapEnd; const char *NamesStart; const char *NamesEnd; + const char *VNamesStart = nullptr; + const char *VNamesEnd = nullptr; // After value profile is all read, this pointer points to // the header of next profile data (if exists) const uint8_t *ValueDataStart; @@ -655,6 +659,15 @@ private: std::unique_ptr MemProfRecordTable; /// MemProf frame profile data on-disk indexed via frame id. std::unique_ptr MemProfFrameTable; + /// VTableNamePtr points to the beginning of compressed vtable names. + /// When a symtab is constructed from profiles by llvm-profdata, the list of + /// names could be decompressed based on `VTableNamePtr` and + /// `CompressedVTableNamesLen`. + /// A compiler that reads indexed profiles could construct symtab from module + /// IR so it doesn't need the decompressed names. + const char *VTableNamePtr = nullptr; + /// The length of compressed vtable names. + uint64_t CompressedVTableNamesLen = 0; /// Total size of binary ids. uint64_t BinaryIdsSize{0}; /// Start address of binary id length and data pairs. diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index b4e789533695..fd3ba3054835 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1494,9 +1494,12 @@ Expected
Header::readFromBuffer(const unsigned char *Buffer) { // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version11, + IndexedInstrProf::ProfVersion::CurrentVersion == Version12, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); + case 12ull: + H.VTableNamesOffset = read(Buffer, offsetOf(&Header::VTableNamesOffset)); + [[fallthrough]]; case 11ull: [[fallthrough]]; case 10ull: @@ -1522,10 +1525,14 @@ size_t Header::size() const { // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version12, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); + case 12ull: + return offsetOf(&Header::VTableNamesOffset) + + sizeof(Header::VTableNamesOffset); + [[fallthrough]]; case 11ull: [[fallthrough]]; case 10ull: diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 9b76b45bdce8..ea3a9adbb200 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -365,6 +365,11 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { return E; Value = IndexedInstrProf::ComputeHash(VD.first); } + } else if (ValueKind == IPVK_VTableTarget) { + if (InstrProfSymtab::isExternalSymbol(VD.first)) + Value = 0; + else + Value = IndexedInstrProf::ComputeHash(VD.first); } else { READ_NUM(VD.first, Value); } @@ -581,10 +586,17 @@ Error RawInstrProfReader::readHeader( auto NumBitmapBytes = swap(Header.NumBitmapBytes); auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes); auto NamesSize = swap(Header.NamesSize); + auto VTableNameSize = swap(Header.VNamesSize); + auto NumVTables = swap(Header.NumVTables); ValueKindLast = swap(Header.ValueKindLast); auto DataSize = NumData * sizeof(RawInstrProf::ProfileData); - auto PaddingSize = getNumPaddingBytes(NamesSize); + auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize); + auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize); + + auto VTableSectionSize = + NumVTables * sizeof(RawInstrProf::VTableProfileData); + auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize); // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize; @@ -593,7 +605,12 @@ Error RawInstrProfReader::readHeader( CountersOffset + CountersSize + PaddingBytesAfterCounters; ptrdiff_t NamesOffset = BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes; - ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; + ptrdiff_t VTableProfDataOffset = + NamesOffset + NamesSize + PaddingBytesAfterNames; + ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize + + PaddingBytesAfterVTableProfData; + ptrdiff_t ValueDataOffset = + VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames; auto *Start = reinterpret_cast(&Header); if (Start + ValueDataOffset > DataBuffer->getBufferEnd()) @@ -613,8 +630,14 @@ Error RawInstrProfReader::readHeader( Data = reinterpret_cast *>( Start + DataOffset); DataEnd = Data + NumData; + VTableBegin = + reinterpret_cast *>( + Start + VTableProfDataOffset); + VTableEnd = VTableBegin + NumVTables; NamesStart = Start + NamesOffset; NamesEnd = NamesStart + NamesSize; + VNamesStart = Start + VTableNameOffset; + VNamesEnd = VNamesStart + VTableNameSize; } CountersStart = Start + CountersOffset; @@ -1247,6 +1270,23 @@ Error IndexedInstrProfReader::readHeader() { "corrupted binary ids"); } + if (GET_VERSION(Header->formatVersion()) >= 12) { + uint64_t VTableNamesOffset = + endian::byte_swap( + Header->VTableNamesOffset); + const unsigned char *Ptr = Start + VTableNamesOffset; + + CompressedVTableNamesLen = + support::endian::readNext(Ptr); + + // Writer first writes the length of compressed string, and then the actual + // content. + VTableNamePtr = (const char *)Ptr; + if (VTableNamePtr > (const char *)DataBuffer->getBufferEnd()) + return make_error(instrprof_error::truncated); + } + if (GET_VERSION(Header->formatVersion()) >= 10 && Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) { uint64_t TemporalProfTracesOffset = diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 75c04493b52f..f2b6c34eb9a9 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -455,12 +455,11 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { Header.MemProfOffset = 0; Header.BinaryIdOffset = 0; Header.TemporalProfTracesOffset = 0; - int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t); + Header.VTableNamesOffset = 0; - // Only write out all the fields except 'HashOffset', 'MemProfOffset', - // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the - // offset of these fields to allow back patching later. - for (int I = 0; I < N - 4; I++) + // Only write out the first four fields. We need to remember the offset of the + // remaining fields to allow back patching later. + for (int I = 0; I < 4; I++) OS.write(reinterpret_cast(&Header)[I]); // Save the location of Header.HashOffset field in \c OS. @@ -484,6 +483,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { uint64_t TemporalProfTracesOffset = OS.tell(); OS.write(0); + uint64_t VTableNamesOffset = OS.tell(); + OS.write(0); + // Reserve space to write profile summary data. uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size(); uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); @@ -597,6 +599,31 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { OS.writeByte(0); } + uint64_t VTableNamesSectionStart = OS.tell(); + + // Use a dummy (and uncompressed) string as compressed vtable names and get + // the necessary profile format change in place for version 12. + // TODO: Store the list of vtable names in InstrProfWriter and use the + // real compressed name. + std::string CompressedVTableNames = "VTableNames"; + + uint64_t CompressedStringLen = CompressedVTableNames.length(); + + // Record the length of compressed string. + OS.write(CompressedStringLen); + + // Write the chars in compressed strings. + for (auto &c : CompressedVTableNames) + OS.writeByte(static_cast(c)); + + // Pad up to a multiple of 8. + // InstrProfReader would read bytes according to 'CompressedStringLen'. + uint64_t PaddedLength = alignTo(CompressedStringLen, 8); + + for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { + OS.writeByte(0); + } + uint64_t TemporalProfTracesSectionStart = 0; if (static_cast(ProfileKind & InstrProfKind::TemporalProfile)) { TemporalProfTracesSectionStart = OS.tell(); @@ -640,6 +667,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { // Patch the Header.TemporalProfTracesOffset (=0 for profiles without // traces). {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1}, + {VTableNamesOffset, &VTableNamesSectionStart, 1}, // Patch the summary data. {SummaryOffset, reinterpret_cast(TheSummary.get()), (int)(SummarySize / sizeof(uint64_t))}, @@ -692,7 +720,8 @@ Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) { std::unique_ptr VD = Func.getValueForSite(VK, S); DenseSet SeenValues; for (uint32_t I = 0; I < ND; I++) - if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second) + if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) && + !SeenValues.insert(VD[I].Value).second) return make_error(instrprof_error::invalid_prof); } } @@ -740,8 +769,8 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, OS << ND << "\n"; std::unique_ptr VD = Func.getValueForSite(VK, S); for (uint32_t I = 0; I < ND; I++) { - if (VK == IPVK_IndirectCallTarget) - OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":" + if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget) + OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":" << VD[I].Count << "\n"; else OS << VD[I].Value << ":" << VD[I].Count << "\n"; diff --git a/llvm/test/Instrumentation/InstrProfiling/coverage.ll b/llvm/test/Instrumentation/InstrProfiling/coverage.ll index bbf895ea4b34..08cbcaa962b7 100644 --- a/llvm/test/Instrumentation/InstrProfiling/coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/coverage.ll @@ -5,12 +5,12 @@ target triple = "aarch64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 -; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) -; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), +; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) +; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), @__profn_bar = private constant [3 x i8] c"bar" ; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 -; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) -; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), +; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) +; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), ; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names" ; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames" diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw new file mode 100644 index 0000000000000000000000000000000000000000..3daa98f937b691880ffff203c9426bfacddf749d GIT binary patch literal 544 zcmZoHO3N=Q$obF300xW@ih+Rz#(>i3d^BkWXQ;q~{}7OI>G2ckRsa8Qu9KF25{j;x zfq@AsB=~Bx!ru*(PXKkjzs%*!1=A1H5wHqXKiq*(1<^B&b9}Dq)WGzwfZ7k!pFlwW zRaFhE#6x-aLE412-D596(_aA9KSNIj><^IrFt>y3_;t1Sxmn88+dyq#_gkRpKS02K zn16852F4Y)y8Dg%{mz{9^flE?Kz`(e%(w!XuMI#TR diff --git a/llvm/test/tools/llvm-profdata/Inputs/thinlto_indirect_call_promotion.profraw b/llvm/test/tools/llvm-profdata/Inputs/thinlto_indirect_call_promotion.profraw new file mode 100644 index 0000000000000000000000000000000000000000..84707ba2070a92b8683010d9daaef747df35f9ac GIT binary patch literal 528 zcmZoHO3N=Q$obF700xW@ih+Rz#(>i3d^BkWXQ;q~{}8~jee0hktN#DrJkOIkI+TF{ zX0YI^%?f`vOg;fr_5L!KFBeQb%shva5cM!VOdpINJ<~YH=c-N(O#cd~eK7d|0{XA2 zYFH&6%DWHJCbaDydjXpM1gQQWo?dWwGr?0yS0{Tm3_5AzQ$ z+Q7KtR(HRVzu%dYp1!6!$!AXbT=MqY*4O{3u}gA_;W2kfsb$ZfsH;9ZvV7_@)#;23 r{WSu+S$HaLo%TI*hM9pynsFJ}wH81UW(Uaqj8G0Nd|-00@P_dLvBrhT literal 0 HcmV?d00001 diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test index eda63203a304..61881b69cfd5 100644 --- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. // 2 * 8 binary ID sizes // + 2 * 20 binary IDs (of size 20) @@ -32,6 +34,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Binary IDs - There are only two in this case that are 20 bytes. RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test index 38b838e0d100..316a9a4c9df4 100644 --- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Check for a corrupted size being too large past the end of the file. RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test index c967e850dbe3..8b686d5c50cb 100644 --- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -26,6 +28,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Data Section // diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index 2e747f81a6bf..089afad42062 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -26,6 +28,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Data Section // diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test index 3c23bc7dd0f7..e404ba4210cc 100644 --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -26,6 +28,8 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Data Section // diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test index 4a5c42843ff4..ee54bfb97856 100644 --- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -10,6 +10,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Binary IDs - There are only two in this case that are 20 bytes. RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test index 2a92575ee340..dfa163f1f343 100644 --- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -15,6 +15,8 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test index fbd31d044382..63782c8b94d4 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,5 +1,6 @@ +// Header RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\11' >> %t +RUN: printf '\0\0\0\0\0\0\0\12' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +13,8 @@ RUN: printf '\0\0\0\0\1\0\0\0' >> %t RUN: printf '\0\0\0\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t @@ -20,8 +23,8 @@ RUN: printf '\3\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0\0\0\0\3' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\3' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t @@ -30,8 +33,8 @@ RUN: printf '\2\xff\xff\xd3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\2' >> %t -RUN: printf '\0\0\0\0\0\0\0\1' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\1' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test index bb899c5fdb55..e9569bec1178 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,5 +1,5 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\11\0\0\0\0\0\0\0' >> %t +RUN: printf '\12\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\1\0\0\0\0' >> %t RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\0\0\0\2\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t @@ -20,8 +22,8 @@ RUN: printf '\0\0\0\3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0\3\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\3\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t @@ -30,8 +32,8 @@ RUN: printf '\xd3\xff\xff\2' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\2\0\0\0' >> %t -RUN: printf '\0\0\0\0\1\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test index 8fcadb6a0dd2..0bc579eec58a 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,5 +1,5 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\11' >> %t +RUN: printf '\0\0\0\0\0\0\0\12' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t @@ -19,8 +21,9 @@ RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\1\0\0\0\0' >> %t -RUN: printf '\0\0\0\3\0\0\0\0' >> %t +RUN: printf '\0\0\0\1' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\3' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\02' >> %t @@ -28,8 +31,9 @@ RUN: printf '\0\0\0\1\0\3\xff\xc8' >> %t RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\0\0\0\02\0\0\0\0' >> %t -RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\02' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\1' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test index 0aa8b38f6926..ca9ea54c3f01 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\11\0\0\0\0\0\0\0' >> %t +RUN: printf '\12\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +12,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\4\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t @@ -19,8 +21,9 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\1\0\0\0\0\0\0\0' >> %t -RUN: printf '\3\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\3\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t @@ -28,8 +31,9 @@ RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t -RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\1\0\0\0\0\0\0\0' >> %t +RUN: printf '\02\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test index f4a9aa8e1bbc..70a4210dea9f 100644 --- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -12,6 +12,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\254\275\030\333\114\302\370\134' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -26,7 +28,7 @@ RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw @@ -39,6 +41,8 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\067\265\035\031\112\165\023\344' >> %t-bar.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw -- Gitee From f666c635e4cf9eb781ab3814f4d15d131a3b7b7b Mon Sep 17 00:00:00 2001 From: Szymon Sobieszek Date: Wed, 6 Aug 2025 16:30:46 -0400 Subject: [PATCH 16/28] [DTP] Fixed build errors related to porting dynamic type profiling --- .../lib/profile/InstrProfilingWriter.c | 8 +- .../ProfileData/Coverage/CoverageMapping.h | 6 +- .../Coverage/CoverageMappingReader.h | 17 +- llvm/include/llvm/ProfileData/InstrProf.h | 197 ++++++++++-- .../llvm/ProfileData/InstrProfCorrelator.h | 8 +- .../llvm/ProfileData/InstrProfReader.h | 4 +- llvm/include/llvm/Support/Debug.h | 4 +- .../Instrumentation/PGOInstrumentation.h | 2 + .../Coverage/CoverageMappingReader.cpp | 35 ++- llvm/lib/ProfileData/InstrProf.cpp | 296 +++++++++++++++--- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 48 +-- llvm/lib/ProfileData/InstrProfReader.cpp | 19 +- llvm/lib/ProfileData/InstrProfWriter.cpp | 2 +- .../Instrumentation/InstrProfiling.cpp | 4 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 40 ++- 15 files changed, 551 insertions(+), 139 deletions(-) diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 3261408232b6..8816a7115551 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -317,7 +317,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, #endif /* The data and names sections are omitted in lightweight mode. */ - if (ProfileCorrelation) { + if (NumData == 0 && NamesSize == 0) { Header.CountersDelta = 0; Header.NamesDelta = 0; } @@ -333,8 +333,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Write the profile data. */ ProfDataIOVec IOVecData[] = { - {ProfileCorrelation ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize, - 0}, + {DataBegin, sizeof(uint8_t), DataSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, @@ -351,7 +350,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Value profiling is not yet supported in continuous mode and profile * correlation mode. */ - if (__llvm_profile_is_continuous_mode_enabled() || ProfileCorrelation) + if (__llvm_profile_is_continuous_mode_enabled() || + (NumData == 0 && NamesSize == 0)) return 0; return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 12498630190d..3e76ba7187f3 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -75,7 +75,8 @@ inline std::error_code make_error_code(coveragemap_error E) { class CoverageMapError : public ErrorInfo { public: - CoverageMapError(coveragemap_error Err) : Err(Err) { + CoverageMapError(coveragemap_error Err, const Twine &ErrStr = Twine()) + : Err(Err), Msg(ErrStr.str()) { assert(Err != coveragemap_error::success && "Not an error"); } @@ -93,6 +94,7 @@ public: private: coveragemap_error Err; + std::string Msg; }; /// A Counter is an abstract value that describes how to compute the @@ -810,7 +812,7 @@ template Error getFuncNameViaRef(const FuncRecordTy *Record, InstrProfSymtab &ProfileNames, StringRef &FuncName) { uint64_t NameRef = getFuncNameRef(Record); - FuncName = ProfileNames.getFuncName(NameRef); + FuncName = ProfileNames.getFuncOrVarName(NameRef); return Error::success(); } diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 326c1b0d3338..48e4b78c6238 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -184,7 +184,7 @@ public: private: std::vector Filenames; std::vector MappingRecords; - InstrProfSymtab ProfileNames; + std::unique_ptr ProfileNames; size_t CurrentRecord = 0; std::vector FunctionsFilenames; std::vector Expressions; @@ -195,8 +195,9 @@ private: // D69471, which can split up function records into multiple sections on ELF. FuncRecordsStorage FuncRecords; - BinaryCoverageReader(FuncRecordsStorage &&FuncRecords) - : FuncRecords(std::move(FuncRecords)) {} + BinaryCoverageReader(std::unique_ptr Symtab, + FuncRecordsStorage &&FuncRecords) + : ProfileNames(std::move(Symtab)), FuncRecords(std::move(FuncRecords)) {} public: BinaryCoverageReader(const BinaryCoverageReader &) = delete; @@ -209,12 +210,10 @@ public: SmallVectorImpl *BinaryIDs = nullptr); static Expected> - createCoverageReaderFromBuffer(StringRef Coverage, - FuncRecordsStorage &&FuncRecords, - InstrProfSymtab &&ProfileNames, - uint8_t BytesInAddress, - support::endianness Endian, - StringRef CompilationDir = ""); + createCoverageReaderFromBuffer( + StringRef Coverage, FuncRecordsStorage &&FuncRecords, + std::unique_ptr ProfileNamesPtr, uint8_t BytesInAddress, + support::endianness Endian, StringRef CompilationDir = ""); Error readNextRecord(CoverageMappingRecord &Record) override; }; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 8deab64c3905..0ae11402d310 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" @@ -223,6 +224,18 @@ StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar); StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName = ""); +/// Given a vector of strings (names of global objects like functions or, +/// virtual tables) \c NameStrs, the method generates a combined string \c +/// Result that is ready to be serialized. The \c result string is comprised of +/// three fields: The first field is the length of the uncompressed strings, and +/// the the second field is the length of the zlib-compressed string. Both +/// fields are encoded in ULEB128. If \c doCompress is false, the +/// third field is the uncompressed strings; otherwise it is the +/// compressed string. When the string compression is off, the +/// second field will have value zero. +Error collectGlobalObjectNameStrings(ArrayRef NameStrs, + bool doCompression, std::string &Result); + /// Given a vector of strings (function PGO names) \c NameStrs, the /// method generates a combined string \c Result that is ready to be /// serialized. The \c Result string is comprised of three fields: @@ -240,6 +253,9 @@ Error collectPGOFuncNameStrings(ArrayRef NameStrs, Error collectPGOFuncNameStrings(ArrayRef NameVars, std::string &Result, bool doCompression = true); +Error collectVTableStrings(ArrayRef VTables, + std::string &Result, bool doCompression); + /// \c NameStrings is a string composed of one of more sub-strings encoded in /// the format described above. The substrings are separated by 0 or more zero /// bytes. This method decodes the string and populates the \c Symtab. @@ -284,9 +300,13 @@ bool getValueProfDataFromInst(const Instruction &Inst, inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } +inline StringRef getPGONameMetadataName() { return "PGOName"; } + /// Return the PGOFuncName meta data associated with a function. MDNode *getPGOFuncNameMetadata(const Function &F); +std::string getPGOName(const GlobalVariable &V, bool InLTO = false); + /// Create the PGOFuncName meta data if PGOFuncName is different from /// function's raw name. This should only apply to internal linkage functions /// declared by users only. @@ -430,26 +450,59 @@ public: using AddrHashMap = std::vector>; private: + using AddrIntervalMap = + IntervalMap>; StringRef Data; uint64_t Address = 0; - // Unique name strings. + // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's + // going to be sorted) has unique MD5 keys in the first place. StringSet<> NameTab; + // Record the unique virtual table names. This is used by InstrProfWriter to + // write out an on-disk chained hash table of virtual table names. + // InstrProfWriter stores per function profile data (keyed by function names) + // so it doesn't use a StringSet for function names. + StringSet<> VTableNames; // A map from MD5 keys to function name strings. std::vector> MD5NameMap; // A map from MD5 keys to function define. We only populate this map // when build the Symtab from a Module. std::vector> MD5FuncMap; + // A map from MD5 to the global variable. This map is only populated when + // building the symtab from a module. Use separate container instances for + // `MD5FuncMap` and `MD5VTableMap`. + // TODO: Unify the container type and the lambda function 'mapName' inside + // add{Func,VTable}WithName. + DenseMap MD5VTableMap; // A map from function runtime address to function name MD5 hash. // This map is only populated and used by raw instr profile reader. AddrHashMap AddrToMD5Map; + + AddrIntervalMap::Allocator VTableAddrMapAllocator; + // This map is only populated and used by raw instr profile reader. + AddrIntervalMap VTableAddrMap; bool Sorted = false; - static StringRef getExternalSymbol() { - return "** External Symbol **"; - } + static StringRef getExternalSymbol() { return "** External Symbol **"; } + //Returns the canonical name of the given PGOName. In a canonical name, all + //suffixes that begins with "." except ".uniq." are stripped. + //FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. + static StringRef getCanonicalName(StringRef PGOName); + + // Add the function into the symbol table, by creating the following + // map entries: + // name-set = {PGOFuncName} union {getCanonicalName(PGOFuncName)} + // - In MD5NameMap: for name in name-set + // - In MD5FuncMap: for name in name-set Error addFuncWithName(Function &F, StringRef PGOFuncName); + // Add the vtable into the symbol table, by creating the following + // map entries: + // name-set = {PGOName} union {getCanonicalName(PGOName)} + // - In MD5NameMap: for name in name-set + // - In MD5VTableMap: for name in name-set + Error addVTableWithName(GlobalVariable &V, StringRef PGOVTableName); + // If the symtab is created by a series of calls to \c addFuncName, \c // finalizeSymtab needs to be called before looking up function names. // This is required because the underlying map is a vector (for space @@ -457,7 +510,14 @@ private: inline void finalizeSymtab(); public: - InstrProfSymtab() = default; + InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {} + + // Not copyable or movable. + // Consider std::unique_ptr for move. + InstrProfSymtab(const InstrProfSymtab &) = delete; + InstrProfSymtab &operator=(const InstrProfSymtab &) = delete; + InstrProfSymtab(InstrProfSymtab &&) = delete; + InstrProfSymtab &operator=(InstrProfSymtab &&) = delete; /// Create InstrProfSymtab from an object file section which /// contains function PGO names. When section may contain raw @@ -467,14 +527,24 @@ public: /// until before it is used. See also \c create(StringRef) method. Error create(object::SectionRef &Section); - /// This interface is used by reader of CoverageMapping test - /// format. - inline Error create(StringRef D, uint64_t BaseAddr); - /// \c NameStrings is a string composed of one of more sub-strings /// encoded in the format described in \c collectPGOFuncNameStrings. /// This method is a wrapper to \c readPGOFuncNameStrings method. - inline Error create(StringRef NameStrings); + Error create(StringRef NameStrings); + + /// Initialize symtab states with function names and vtable names. \c + /// FuncNameStrings is a string composed of one or more encoded function name + /// strings, and \c VTableNameStrings composes of one or more encoded vtable + /// names. This interface is solely used by raw profile reader. + Error create(StringRef FuncNameStrings, StringRef VTableNameStrings); + + /// Initialize 'this' with the set of vtable names encoded in + /// \c CompressedVTableNames. + Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames); + + /// This interface is used by reader of CoverageMapping test + /// format. + inline Error create(StringRef D, uint64_t BaseAddr); /// A wrapper interface to populate the PGO symtab with functions /// decls from module \c M. This interface is used by transformation @@ -486,35 +556,76 @@ public: /// \p IterRange. This interface is used by IndexedProfReader. template Error create(const NameIterRange &IterRange); - /// Update the symtab by adding \p FuncName to the table. This interface - /// is used by the raw and text profile readers. - Error addFuncName(StringRef FuncName) { - if (FuncName.empty()) + /// Create InstrProfSymtab from a set of function names and vtable + /// names iteratable from \p IterRange. This interface is used by + /// IndexedProfReader. + template + Error create(const FuncNameIterRange &FuncIterRange, + const VTableNameIterRange &VTableIterRange); + + // Map the MD5 of the symbol name to the name. + Error addSymbolName(StringRef SymbolName) { + if (SymbolName.empty()) return make_error(instrprof_error::malformed, - "function name is empty"); - auto Ins = NameTab.insert(FuncName); + "symbol name is empty"); + + // Insert into NameTab so that MD5NameMap (a vector that will be sorted) + // won't have duplicated entries in the first place. + auto Ins = NameTab.insert(SymbolName); if (Ins.second) { MD5NameMap.push_back(std::make_pair( - IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); + IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey())); Sorted = false; } return Error::success(); } + /// The method name is kept since there are many callers. + /// It just forwards to 'addSymbolName'. + Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); } + + /// Adds VTableName as a known symbol, and inserts it to a map that + /// tracks all vtable names. + Error addVTableName(StringRef VTableName) { + if (Error E = addSymbolName(VTableName)) + return E; + + // Record VTableName. InstrProfWriter uses this set. The comment around + // class member explains why. + VTableNames.insert(VTableName); + return Error::success(); + } + + const StringSet<> &getVTableNames() const { return VTableNames; } + /// Map a function address to its name's MD5 hash. This interface /// is only used by the raw profiler reader. void mapAddress(uint64_t Addr, uint64_t MD5Val) { AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); } + /// Map the address range (i.e., [start_address, end_address)) of a variable + /// to its names' MD5 hash. This interface is only used by the raw profile + /// reader. + void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) { + VTableAddrMap.insert(StartAddr, EndAddr, MD5Val); + } + /// Return a function's hash, or 0, if the function isn't in this SymTab. uint64_t getFunctionHashFromAddress(uint64_t Address); + /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab. + uint64_t getVTableHashFromAddress(uint64_t Address); + /// Return function's PGO name from the function name's symbol /// address in the object file. If an error occurs, return /// an empty string. StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize); + /// Return name of functions or global variables from the name's md5 hash + /// value. If not found, return an empty string. + inline StringRef getFuncOrVarName(uint64_t ValMD5Hash); + /// Return function's PGO name from the name's md5 hash value. /// If not found, return an empty string. inline StringRef getFuncName(uint64_t FuncMD5Hash); @@ -524,6 +635,11 @@ public: /// will be represented using the same StringRef value. inline StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash); + /// Just like getFuncOrVarName, except that it will return literal string + /// 'External Symbol' if the function or global variable is external to + /// this symbol table. + inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash); + /// True if Symbol is the value used to represent external symbols. static bool isExternalSymbol(const StringRef &Symbol) { return Symbol == InstrProfSymtab::getExternalSymbol(); @@ -532,6 +648,10 @@ public: /// Return function from the name's md5 hash. Return nullptr if not found. inline Function *getFunction(uint64_t FuncMD5Hash); + /// Return the global variable corresponding to md5 hash. Return nullptr if + /// not found. + inline GlobalVariable *getGlobalVariable(uint64_t MD5Hash); + /// Return the name section data. inline StringRef getNameData() const { return Data; } @@ -545,10 +665,6 @@ Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { return Error::success(); } -Error InstrProfSymtab::create(StringRef NameStrings) { - return readPGOFuncNameStrings(NameStrings, *this); -} - template Error InstrProfSymtab::create(const NameIterRange &IterRange) { for (auto Name : IterRange) @@ -559,6 +675,24 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) { return Error::success(); } +template +Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange, + const VTableNameIterRange &VTableIterRange) { + // Iterate elements by StringRef rather than by const reference. + // StringRef is small enough, so the loop is efficient whether + // element in the range is std::string or StringRef. + for (StringRef Name : FuncIterRange) + if (Error E = addFuncName(Name)) + return E; + + for (StringRef VTableName : VTableIterRange) + if (Error E = addVTableName(VTableName)) + return E; + + finalizeSymtab(); + return Error::success(); +} + void InstrProfSymtab::finalizeSymtab() { if (Sorted) return; @@ -570,6 +704,23 @@ void InstrProfSymtab::finalizeSymtab() { Sorted = true; } +StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) { + StringRef ret = getFuncOrVarName(MD5Hash); + if (ret.empty()) + return InstrProfSymtab::getExternalSymbol(); + return ret; +} + +StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) { + finalizeSymtab(); + auto Result = llvm::lower_bound(MD5NameMap, MD5Hash, + [](const std::pair &LHS, + uint64_t RHS) { return LHS.first < RHS; }); + if (Result != MD5NameMap.end() && Result->first == MD5Hash) + return Result->second; + return StringRef(); +} + StringRef InstrProfSymtab::getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash) { StringRef ret = getFuncName(FuncMD5Hash); if (ret.empty()) @@ -597,6 +748,10 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { return nullptr; } +GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) { + return MD5VTableMap.lookup(MD5Hash); +} + // To store the sums of profile count values, or the percentage of // the sums of the total count values. struct CountSumOrPercent { diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index 2f9be48f2833..fa4dc46dba90 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -40,10 +40,10 @@ public: /// Construct a ProfileData vector used to correlate raw instrumentation data /// to their functions. - virtual Error correlateProfileData() = 0; + virtual Error correlateProfileData(int MaxWarnings) = 0; /// Process debug info and dump the correlation data. - virtual Error dumpYaml(raw_ostream &OS) = 0; + virtual Error dumpYaml(int MaxWarnings, raw_ostream &OS) = 0; /// Return the number of ProfileData elements. std::optional getDataSize() const; @@ -142,8 +142,9 @@ public: protected: std::vector> Data; - Error correlateProfileData() override; + Error correlateProfileData(int MaxWarnings) override; virtual void correlateProfileDataImpl( + int MaxWarnings, InstrProfCorrelator::CorrelationData *Data = nullptr) = 0; virtual Error correlateProfileNameImpl() = 0; @@ -212,6 +213,7 @@ private: /// NULL /// \endcode void correlateProfileDataImpl( + int MaxWarnings, InstrProfCorrelator::CorrelationData *Data = nullptr) override; Error correlateProfileNameImpl() override; diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index fa0e54a5bd51..c15737f6c098 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -749,8 +749,8 @@ public: std::vector &Counts); /// Fill Bitmap Bytes with the profile data for the given function name. - Error getFunctionBitmapBytes(StringRef FuncName, uint64_t FuncHash, - std::vector &BitmapBytes); + Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, + BitVector &Bitmap); /// Return the maximum of all known function counts. /// \c UseCS indicates whether to use the context-sensitive count. diff --git a/llvm/include/llvm/Support/Debug.h b/llvm/include/llvm/Support/Debug.h index 5788ab3b2138..3e2f0d9b43fc 100644 --- a/llvm/include/llvm/Support/Debug.h +++ b/llvm/include/llvm/Support/Debug.h @@ -53,7 +53,7 @@ void setCurrentDebugType(const char *Type); void setCurrentDebugTypes(const char **Types, unsigned Count); /// DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug -/// information. In the '-debug' option is specified on the commandline, and if +/// information. If the '-debug' option is specified on the commandline, and if /// this is a debug build, then the code specified as the option to the macro /// will be executed. Otherwise it will not be. Example: /// @@ -92,7 +92,7 @@ extern bool EnableDebugBuffering; raw_ostream &dbgs(); // DEBUG macro - This macro should be used by passes to emit debug information. -// In the '-debug' option is specified on the commandline, and if this is a +// If the '-debug' option is specified on the commandline, and if this is a // debug build, then the code specified as the option to the macro will be // executed. Otherwise it will not be. Example: // diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h index c77d3214ed01..6d2ad3d75744 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -23,6 +23,8 @@ namespace llvm { +extern cl::opt DebugInfoCorrelate; + class Function; class Instruction; class Module; diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 80f3e787ef43..90797826e2be 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -835,33 +835,36 @@ static const char *TestingFormatMagic = "llvmcovmtestdata"; Expected> BinaryCoverageReader::createCoverageReaderFromBuffer( StringRef Coverage, FuncRecordsStorage &&FuncRecords, - InstrProfSymtab &&ProfileNames, uint8_t BytesInAddress, + std::unique_ptr ProfileNamesPtr, uint8_t BytesInAddress, support::endianness Endian, StringRef CompilationDir) { - std::unique_ptr Reader( - new BinaryCoverageReader(std::move(FuncRecords))); - Reader->ProfileNames = std::move(ProfileNames); + if (ProfileNamesPtr == nullptr) + return make_error(coveragemap_error::malformed, + "Caller must provide ProfileNames"); + std::unique_ptr Reader(new BinaryCoverageReader( + std::move(ProfileNamesPtr), std::move(FuncRecords))); + InstrProfSymtab &ProfileNames = *Reader->ProfileNames; StringRef FuncRecordsRef = Reader->FuncRecords->getBuffer(); if (BytesInAddress == 4 && Endian == support::endianness::little) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 4 && Endian == support::endianness::big) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 8 && Endian == support::endianness::little) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 8 && Endian == support::endianness::big) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else return make_error(coveragemap_error::malformed); @@ -890,8 +893,8 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) { Data = Data.substr(N); if (Data.size() < ProfileNamesSize) return make_error(coveragemap_error::malformed); - InstrProfSymtab ProfileNames; - if (Error E = ProfileNames.create(Data.substr(0, ProfileNamesSize), Address)) + auto ProfileNames = std::make_unique(); + if (Error E = ProfileNames->create(Data.substr(0, ProfileNamesSize), Address)) return std::move(E); Data = Data.substr(ProfileNamesSize); // Skip the padding bytes because coverage map data has an alignment of 8. @@ -997,7 +1000,7 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, : support::endianness::big; // Look for the sections that we are interested in. - InstrProfSymtab ProfileNames; + auto ProfileNames = std::make_unique(); std::vector NamesSectionRefs; // If IPSK_name is not found, fallback to search for IPK_covname, which is // used when binary correlation is enabled. @@ -1014,7 +1017,7 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, return make_error( coveragemap_error::malformed, "the size of coverage mapping section is not one"); - if (Error E = ProfileNames.create(NamesSectionRefs.back())) + if (Error E = ProfileNames->create(NamesSectionRefs.back())) return std::move(E); auto CoverageSection = lookupSections(*OF, IPSK_covmap); diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index fd3ba3054835..01147247513b 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -36,6 +36,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Compression.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" @@ -271,8 +272,8 @@ static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { return PathNameStr.substr(LastPos); } -static StringRef getStrippedSourceFileName(const Function &F) { - StringRef FileName(F.getParent()->getSourceFileName()); +static StringRef getStrippedSourceFileName(const GlobalObject &GO) { + StringRef FileName(GO.getParent()->getSourceFileName()); uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; if (StripLevel < StaticFuncStripDirNamePrefix) StripLevel = StaticFuncStripDirNamePrefix; @@ -281,6 +282,60 @@ static StringRef getStrippedSourceFileName(const Function &F) { return FileName; } +// The PGO name has the format [;] where ; is +// provided if linkage is local and is used to discriminate possibly identical +// mangled names. ";" is used because it is unlikely to be found in either +// or . +// +// Older compilers used getPGOFuncName() which has the format +// [:]. This caused trouble for Objective-C functions +// which commonly have :'s in their names. We still need to compute this name to +// lookup functions from profiles built by older compilers. +static std::string +getIRPGONameForGlobalObject(const GlobalObject &GO, + GlobalValue::LinkageTypes Linkage, + StringRef FileName) { + return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName); +} + +static std::optional lookupPGONameFromMetadata(MDNode *MD) { + if (MD != nullptr) { + StringRef S = cast(MD->getOperand(0))->getString(); + return S.str(); + } + return {}; +} + +// Returns the PGO object name. This function has some special handling +// when called in LTO optimization. The following only applies when calling in +// LTO passes (when \c InLTO is true): LTO's internalization privatizes many +// global linkage symbols. This happens after value profile annotation, but +// those internal linkage functions should not have a source prefix. +// Additionally, for ThinLTO mode, exported internal functions are promoted +// and renamed. We need to ensure that the original internal PGO name is +// used when computing the GUID that is compared against the profiled GUIDs. +// To differentiate compiler generated internal symbols from original ones, +// PGOFuncName meta data are created and attached to the original internal +// symbols in the value profile annotation step +// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta +// data, its original linkage must be non-internal. +static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, + MDNode *PGONameMetadata) { + if (!InLTO) { + auto FileName = getStrippedSourceFileName(GO); + return getIRPGONameForGlobalObject(GO, GO.getLinkage(), FileName); + } + + // In LTO mode (when InLTO is true), first check if there is a meta data. + if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata)) + return *IRPGOFuncName; + + // If there is no meta data, the function must be a global before the value + // profile annotation pass. Its current linkage may be internal if it is + // internalized in LTO mode. + return getIRPGONameForGlobalObject(GO, GlobalValue::ExternalLinkage, ""); +} + // The PGO name has the format [;] where ; is // provided if linkage is local and is the mangled function // name. The filepath is used to discriminate possibly identical function names. @@ -315,21 +370,10 @@ static std::optional lookupPGOFuncName(const Function &F) { return {}; } -// See getPGOFuncName() +// Returns the IRPGO function name and does special handling when called +// in LTO optimization. See the comments of `getIRPGOObjectName` for details. std::string getIRPGOFuncName(const Function &F, bool InLTO) { - if (!InLTO) { - auto FileName = getStrippedSourceFileName(F); - return getIRPGOFuncName(F, F.getLinkage(), FileName); - } - - // In LTO mode (when InLTO is true), first check if there is a meta data. - if (auto IRPGOFuncName = lookupPGOFuncName(F)) - return *IRPGOFuncName; - - // If there is no meta data, the function must be a global before the value - // profile annotation pass. Its current linkage may be internal if it is - // internalized in LTO mode. - return getIRPGOFuncName(F, GlobalValue::ExternalLinkage, ""); + return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F)); } // Return the PGOFuncName. This function has some special handling when called @@ -361,6 +405,13 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } +std::string getPGOName(const GlobalVariable &V, bool InLTO) { + // PGONameMetadata should be set by compiler at profile use time + // and read by symtab creation to look up symbols corresponding to + // a MD5 hash. + return getIRPGOObjectName(V, InLTO, V.getMetadata(getPGONameMetadataName())); +} + // See getIRPGOFuncName() for a discription of the format. std::pair getParsedIRPGOFuncName(StringRef IRPGOFuncName) { @@ -442,43 +493,162 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) { if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO))) return E; } + + SmallVector Types; + for (GlobalVariable &G : M.globals()) { + if (!G.hasName() || !G.hasMetadata(LLVMContext::MD_type)) + continue; + if (Error E = addVTableWithName(G, getPGOName(G, InLTO))) + return E; + } + Sorted = false; finalizeSymtab(); return Error::success(); } -Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { - if (Error E = addFuncName(PGOFuncName)) +Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable, + StringRef VTablePGOName) { + auto NameToGUIDMap = [&](StringRef Name) -> Error { + if (Error E = addSymbolName(Name)) + return E; + + bool Inserted = true; + std::tie(std::ignore, Inserted) = + MD5VTableMap.try_emplace(GlobalValue::getGUID(Name), &VTable); + return Error::success(); + }; + if (Error E = NameToGUIDMap(VTablePGOName)) return E; - MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); + + StringRef CanonicalName = getCanonicalName(VTablePGOName); + if (CanonicalName != VTablePGOName) + return NameToGUIDMap(CanonicalName); + + return Error::success(); +} + +/// \c NameStrings is a string composed of one of more possibly encoded +/// sub-strings. The substrings are separated by 0 or more zero bytes. This +/// method decodes the string and calls `NameCallback` for each substring. +static Error +readAndDecodeStrings(StringRef NameStrings, + std::function NameCallback) { + const uint8_t *P = NameStrings.bytes_begin(); + const uint8_t *EndP = NameStrings.bytes_end(); + while (P < EndP) { + uint32_t N; + uint64_t UncompressedSize = decodeULEB128(P, &N); + P += N; + uint64_t CompressedSize = decodeULEB128(P, &N); + P += N; + const bool IsCompressed = (CompressedSize != 0); + SmallVector UncompressedNameStrings; + StringRef NameStrings; + if (IsCompressed) { + if (!llvm::compression::zlib::isAvailable()) + return make_error(instrprof_error::zlib_unavailable); + + if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize), + UncompressedNameStrings, + UncompressedSize)) { + consumeError(std::move(E)); + return make_error(instrprof_error::uncompress_failed); + } + P += CompressedSize; + NameStrings = toStringRef(UncompressedNameStrings); + } else { + NameStrings = + StringRef(reinterpret_cast(P), UncompressedSize); + P += UncompressedSize; + } + // Now parse the name strings. + SmallVector Names; + NameStrings.split(Names, getInstrProfNameSeparator()); + for (StringRef &Name : Names) + if (Error E = NameCallback(Name)) + return E; + + while (P < EndP && *P == 0) + P++; + } + return Error::success(); +} + +Error InstrProfSymtab::create(StringRef NameStrings) { + return readAndDecodeStrings( + NameStrings, + std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1)); +} + +Error InstrProfSymtab::create(StringRef FuncNameStrings, + StringRef VTableNameStrings) { + if (Error E = readAndDecodeStrings(FuncNameStrings, + std::bind(&InstrProfSymtab::addFuncName, + this, std::placeholders::_1))) + return E; + + return readAndDecodeStrings( + VTableNameStrings, + std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); +} + +Error InstrProfSymtab::initVTableNamesFromCompressedStrings( + StringRef CompressedVTableStrings) { + return readAndDecodeStrings( + CompressedVTableStrings, + std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); +} + +StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) { // In ThinLTO, local function may have been promoted to global and have // suffix ".llvm." added to the function name. We need to add the // stripped function name to the symbol table so that we can find a match // from profile. // - // We may have other suffixes similar as ".llvm." which are needed to - // be stripped before the matching, but ".__uniq." suffix which is used - // to differentiate internal linkage functions in different modules - // should be kept. Now this is the only suffix with the pattern ".xxx" - // which is kept before matching. + // ".__uniq." suffix is used to differentiate internal linkage functions in + // different modules and should be kept. This is the only suffix with the + // pattern ".xxx" which is kept before matching, other suffixes similar as + // ".llvm." will be stripped. const std::string UniqSuffix = ".__uniq."; - auto pos = PGOFuncName.find(UniqSuffix); - // Search '.' after ".__uniq." if ".__uniq." exists, otherwise - // search '.' from the beginning. - if (pos != std::string::npos) - pos += UniqSuffix.length(); + size_t Pos = PGOName.find(UniqSuffix); + if (Pos != StringRef::npos) + Pos += UniqSuffix.length(); else - pos = 0; - pos = PGOFuncName.find('.', pos); - if (pos != std::string::npos && pos != 0) { - StringRef OtherFuncName = PGOFuncName.substr(0, pos); - if (Error E = addFuncName(OtherFuncName)) + Pos = 0; + + // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from + // the beginning. + Pos = PGOName.find('.', Pos); + if (Pos != StringRef::npos && Pos != 0) + return PGOName.substr(0, Pos); + + return PGOName; +} + +Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { + auto NameToGUIDMap = [&](StringRef Name) -> Error { + if (Error E = addFuncName(Name)) return E; - MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); - } + MD5FuncMap.emplace_back(Function::getGUID(Name), &F); + return Error::success(); + }; + if (Error E = NameToGUIDMap(PGOFuncName)) + return E; + + StringRef CanonicalFuncName = getCanonicalName(PGOFuncName); + if (CanonicalFuncName != PGOFuncName) + return NameToGUIDMap(CanonicalFuncName); + return Error::success(); } +uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) { + // Given a runtime address, look up the hash value in the interval map, and + // fallback to value 0 if a hash value is not found. + return VTableAddrMap.lookup(Address, 0); +} + uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { finalizeSymtab(); auto It = partition_point(AddrToMD5Map, [=](std::pair A) { @@ -500,6 +670,45 @@ void InstrProfSymtab::dumpNames(raw_ostream &OS) const { OS << S << '\n'; } +Error collectGlobalObjectNameStrings(ArrayRef NameStrs, + bool DoCompression, std::string &Result) { + assert(!NameStrs.empty() && "No name data to emit"); + + uint8_t Header[20], *P = Header; + std::string UncompressedNameStrings = + join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); + + assert( + StringRef(UncompressedNameStrings).count(getInstrProfNameSeparator()) == + (NameStrs.size() - 1) && + "PGO name is invalid (contains separator token)"); + + unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); + P += EncLen; + + auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { + EncLen = encodeULEB128(CompressedLen, P); + P += EncLen; + char *HeaderStr = reinterpret_cast(&Header[0]); + unsigned HeaderLen = P - &Header[0]; + Result.append(HeaderStr, HeaderLen); + Result += InputStr; + return Error::success(); + }; + + if (!DoCompression) { + return WriteStringToResult(0, UncompressedNameStrings); + } + + SmallVector CompressedNameStrings; + compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings), + CompressedNameStrings, + compression::zlib::BestSizeCompression); + + return WriteStringToResult(CompressedNameStrings.size(), + toStringRef(CompressedNameStrings)); +} + Error collectPGOFuncNameStrings(ArrayRef NameStrs, bool doCompression, std::string &Result) { assert(!NameStrs.empty() && "No name data to emit"); @@ -551,10 +760,20 @@ Error collectPGOFuncNameStrings(ArrayRef NameVars, for (auto *NameVar : NameVars) { NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar))); } - return collectPGOFuncNameStrings( + return collectGlobalObjectNameStrings( NameStrs, compression::zlib::isAvailable() && doCompression, Result); } +Error collectVTableStrings(ArrayRef VTables, + std::string &Result, bool DoCompression) { + std::vector VTableNameStrs; + for (auto *VTable : VTables) + VTableNameStrs.push_back(getPGOName(*VTable)); + return collectGlobalObjectNameStrings( + VTableNameStrs, compression::zlib::isAvailable() && DoCompression, + Result); +} + Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { const uint8_t *P = NameStrings.bytes_begin(); const uint8_t *EndP = NameStrings.bytes_end(); @@ -859,6 +1078,9 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, if (ValueKind == IPVK_IndirectCallTarget) return SymTab->getFunctionHashFromAddress(Value); + if (ValueKind == IPVK_VTableTarget) + return SymTab->getVTableHashFromAddress(Value); + return Value; } diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 6fdac036b165..fbd345156d85 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -201,7 +201,7 @@ InstrProfCorrelatorImpl::get( } template -Error InstrProfCorrelatorImpl::correlateProfileData() { +Error InstrProfCorrelatorImpl::correlateProfileData(int MaxWarnings) { assert(Data.empty() && Names.empty() && NamesVec.empty()); correlateProfileDataImpl(MaxWarnings); if (this->Data.empty()) @@ -238,9 +238,10 @@ template <> struct yaml::SequenceElementTraits { }; template -Error InstrProfCorrelatorImpl::dumpYaml(raw_ostream &OS) { +Error InstrProfCorrelatorImpl::dumpYaml(int MaxWarnings, + raw_ostream &OS) { InstrProfCorrelator::CorrelationData Data; - correlateProfileDataImpl(&Data); + correlateProfileDataImpl(MaxWarnings, &Data); if (Data.Probes.empty()) return make_error( instrprof_error::unable_to_correlate_profile, @@ -321,7 +322,10 @@ bool DwarfInstrProfCorrelator::isDIEOfProbe(const DWARFDie &Die) { template void DwarfInstrProfCorrelator::correlateProfileDataImpl( - InstrProfCorrelator::CorrelationData *Data) { + int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) { + bool UnlimitedWarnings = (MaxWarnings == 0); + // -N suppressed warnings means we can emit up to N (unsuppressed) warnings + int NumSuppressedWarnings = -MaxWarnings; auto maybeAddProbe = [&](DWARFDie Die) { if (!isDIEOfProbe(Die)) return; @@ -358,28 +362,30 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( } } if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { - LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " - << FunctionName << "\n\tCFGHash: " << CFGHash - << "\n\tCounterPtr: " << CounterPtr - << "\n\tNumCounters: " << NumCounters); - LLVM_DEBUG(Die.dump(dbgs())); + if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { + WithColor::warning() + << "Incomplete DIE for function " << FunctionName + << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr + << " NumCounters=" << NumCounters << "\n"; + LLVM_DEBUG(Die.dump(dbgs())); + } return; } uint64_t CountersStart = this->Ctx->CountersSectionStart; uint64_t CountersEnd = this->Ctx->CountersSectionEnd; if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { - LLVM_DEBUG( - dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " - << FunctionName << "\n\tExpected: [0x" - << Twine::utohexstr(CountersStart) << ", 0x" - << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" - << Twine::utohexstr(*CounterPtr)); - LLVM_DEBUG(Die.dump(dbgs())); + if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { + WithColor::warning() + << format("CounterPtr out of range for function %s: Actual=0x%x " + "Expected=[0x%x, 0x%x)\n", + *FunctionName, *CounterPtr, CountersStart, CountersEnd); + LLVM_DEBUG(Die.dump(dbgs())); + } return; } - if (!FunctionPtr) { - LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName - << "\n"); + if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) { + WithColor::warning() << format("Could not find address of function %s\n", + *FunctionName); LLVM_DEBUG(Die.dump(dbgs())); } // In debug info correlation mode, the CounterPtr is an absolute address of @@ -412,6 +418,10 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( for (auto &CU : DICtx->dwo_units()) for (const auto &Entry : CU->dies()) maybeAddProbe(DWARFDie(CU.get(), &Entry)); + + if (!UnlimitedWarnings && NumSuppressedWarnings > 0) + WithColor::warning() << format("Suppressed %d additional warnings\n", + NumSuppressedWarnings); } template diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index ea3a9adbb200..f9c9b9d75568 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -925,8 +925,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, uint64_t BitmapBytes = 0; if (D + sizeof(uint64_t) > End) return data_type(); - BitmapBytes = - endian::readNext(D); + BitmapBytes = endian::readNext(D); // Read bitmap byte values. if (D + BitmapBytes * sizeof(uint8_t) > End) return data_type(); @@ -934,8 +933,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, BitmapByteBuffer.reserve(BitmapBytes); for (uint64_t J = 0; J < BitmapBytes; ++J) BitmapByteBuffer.push_back(static_cast( - endian::readNext( - D))); + endian::readNext(D))); } DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer), @@ -1272,13 +1270,11 @@ Error IndexedInstrProfReader::readHeader() { if (GET_VERSION(Header->formatVersion()) >= 12) { uint64_t VTableNamesOffset = - endian::byte_swap( - Header->VTableNamesOffset); + endian::byte_swap(Header->VTableNamesOffset); const unsigned char *Ptr = Start + VTableNamesOffset; CompressedVTableNamesLen = - support::endian::readNext(Ptr); + support::endian::readNext(Ptr); // Writer first writes the length of compressed string, and then the actual // content. @@ -1456,8 +1452,9 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, return success(); } -Error IndexedInstrProfReader::getFunctionBitmapBytes( - StringRef FuncName, uint64_t FuncHash, std::vector &BitmapBytes) { +Error IndexedInstrProfReader::getFunctionBitmap(StringRef FuncName, + uint64_t FuncHash, + BitVector &Bitmap) { Expected Record = getInstrProfRecord(FuncName, FuncHash); if (Error E = Record.takeError()) return error(std::move(E)); @@ -1473,7 +1470,7 @@ Error IndexedInstrProfReader::getFunctionBitmapBytes( std::memset(W, 0, sizeof(W)); std::memcpy(W, &BitmapBytes[I], N); I += N; - return support::endian::read(W); }, Bitmap, Bitmap); diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index f2b6c34eb9a9..55e5c8e3e885 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -840,7 +840,7 @@ void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS, for (auto &Trace : TemporalProfTraces) { OS << "# Weight:\n" << Trace.Weight << "\n"; for (auto &NameRef : Trace.FunctionNameRefs) - OS << Symtab.getFuncName(NameRef) << ","; + OS << Symtab.getFuncOrVarName(NameRef) << ","; OS << "\n"; } OS << "\n"; diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 9e6af7d940c2..7131af6bc459 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -1217,7 +1217,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) { - LLVMContext &Ctx = M.getContext(); + LLVMContext &Ctx = M->getContext(); Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); @@ -1363,7 +1363,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, Visibility = GlobalValue::DefaultVisibility; } auto *Data = - new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName); + new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName); Constant *RelativeCounterPtr; GlobalVariable *BitmapPtr = PD.RegionBitmaps; Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 761e7d1a3e73..911f9ab06ca4 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -48,9 +48,33 @@ using namespace llvm; using ProfCorrelatorKind = InstrProfCorrelator::ProfCorrelatorKind; -// We use this string to indicate that there are -// multiple static functions map to the same name. -const std::string DuplicateNameStr = "----"; +// https://llvm.org/docs/CommandGuide/llvm-profdata.html has documentations +// on each subcommand. +cl::SubCommand ShowSubcommand( + "show", + "Takes a profile data file and displays the profiles. See detailed " + "documentation in " + "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-show"); +cl::SubCommand OrderSubcommand( + "order", + "Reads temporal profiling traces from a profile and outputs a function " + "order that reduces the number of page faults for those traces. See " + "detailed documentation in " + "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-order"); +cl::SubCommand OverlapSubcommand( + "overlap", + "Computes and displays the overlap between two profiles. See detailed " + "documentation in " + "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-overlap"); +cl::SubCommand MergeSubcommand( + "merge", + "Takes several profiles and merge them together. See detailed " + "documentation in " + "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge"); + +namespace { +enum ProfileKinds { instr, sample, memory }; +enum FailureMode { warnOnly, failIfAnyAreInvalid, failIfAllAreInvalid }; enum ProfileFormat { PF_None = 0, @@ -62,6 +86,7 @@ enum ProfileFormat { }; enum class ShowFormat { Text, Json, Yaml }; +} // namespace // Common options. cl::opt OutputFilename("output", cl::value_desc("output"), @@ -438,11 +463,6 @@ static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { exitWithError(EC.message(), std::string(Whence)); } -namespace { -enum ProfileKinds { instr, sample, memory }; -enum FailureMode { warnOnly, failIfAnyAreInvalid, failIfAllAreInvalid }; -} // namespace - static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC, StringRef Whence = "") { if (FailMode == failIfAnyAreInvalid) @@ -3252,12 +3272,12 @@ static int showDebugInfoCorrelation(const std::string &Filename, .moveInto(Correlator)) exitWithError(std::move(Err), Filename); if (SFormat == ShowFormat::Yaml) { - if (auto Err = Correlator->dumpYaml(OS)) + if (auto Err = Correlator->dumpYaml(MaxDbgCorrelationWarnings, OS)) exitWithError(std::move(Err), Filename); return 0; } - if (auto Err = Correlator->correlateProfileData()) + if (auto Err = Correlator->correlateProfileData(MaxDbgCorrelationWarnings)) exitWithError(std::move(Err), Filename); InstrProfSymtab Symtab; -- Gitee From c252844165449a7e7f1847ff006cec7e1e1e7139 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Thu, 28 Mar 2024 16:48:29 -0700 Subject: [PATCH 17/28] [docs][TypeProf]Update instrumentation file format document (#83309) This is a follow-up to the profile format change in https://github.com/llvm/llvm-project/pull/82711 --- llvm/docs/InstrProfileFormat.rst | 531 +++++++++++++++++++++++++++++++ 1 file changed, 531 insertions(+) create mode 100644 llvm/docs/InstrProfileFormat.rst diff --git a/llvm/docs/InstrProfileFormat.rst b/llvm/docs/InstrProfileFormat.rst new file mode 100644 index 000000000000..3b33c09f8c7a --- /dev/null +++ b/llvm/docs/InstrProfileFormat.rst @@ -0,0 +1,531 @@ +=================================== +Instrumentation Profile Format +=================================== + +.. contents:: + :local: + + +Overview +========= + +Clang supports two types of profiling via instrumentation [1]_: frontend-based +and IR-based, and both could support a variety of use cases [2]_ . +This document describes two binary serialization formats (raw and indexed) to +store instrumented profiles with a specific emphasis on IRPGO use case, in the +sense that when specific header fields and payload sections have different ways +of interpretation across use cases, the documentation is based on IRPGO. + +.. note:: + Frontend-generated profiles are used together with coverage mapping for + `source-based code coverage`_. The `coverage mapping format`_ is different from + profile format. + +.. _`source-based code coverage`: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html +.. _`coverage mapping format`: https://llvm.org/docs/CoverageMappingFormat.html + +Raw Profile Format +=================== + +The raw profile is generated by running the instrumented binary. The raw profile +data from an executable or a shared library [3]_ consists of a header and +multiple sections, with each section as a memory dump. The raw profile data needs +to be reasonably compact and fast to generate. + +There are no backward or forward version compatiblity guarantees for the raw profile +format. That is, compilers and tools `require`_ a specific raw profile version +to parse the profiles. + +.. _`require`: https://github.com/llvm/llvm-project/blob/bffdde8b8e5d9a76a47949cd0f574f3ce656e181/llvm/lib/ProfileData/InstrProfReader.cpp#L551-L558 + +To feed profiles back into compilers for an optimized build (e.g., via +``-fprofile-use`` for IR instrumentation), a raw profile must to be converted into +indexed format. + +General Storage Layout +----------------------- + +The storage layout of raw profile data format is illustrated below. Basically, +when the raw profile is read into an memory buffer, the actual byte offset of a +section is inferred from the section's order in the layout and size information +of all the sections ahead of it. + +:: + + +----+-----------------------+ + | | Magic | + | +-----------------------+ + | | Version | + | +-----------------------+ + H | Size Info for | + E | Section 1 | + A +-----------------------+ + D | Size Info for | + E | Section 2 | + R +-----------------------+ + | | ... | + | +-----------------------+ + | | Size Info for | + | | Section N | + +----+-----------------------+ + P | Section 1 | + A +-----------------------+ + Y | Section 2 | + L +-----------------------+ + O | ... | + A +-----------------------+ + D | Section N | + +----+-----------------------+ + + +.. note:: + Sections might be padded to meet specific alignment requirements. For + simplicity, header fields and data sections solely for padding purpose are + omitted in the data layout graph above and the rest of this document. + +Header +------- + +``Magic`` + Magic number encodes profile format (raw, indexed or text). For the raw format, + the magic number also encodes the endianness (big or little) and C pointer + size (4 or 8 bytes) of the platform on which the profile is generated. + + A factory method reads the magic number to construct reader properly and returns + error upon unrecognized format. Specifically, the factory method and raw profile + reader implementation make sure that a raw profile file could be read back on + a platform with the opposite endianness and/or the other C pointer size. + +``Version`` + The lower 32 bits specify the actual version and the most significant 32 bits + specify the variant types of the profile. IR-based instrumentation PGO and + context-sensitive IR-based instrumentation PGO are two variant types. + +``BinaryIdsSize`` + The byte size of `binary id`_ section. + +``NumData`` + The number of profile metadata. The byte size of `profile metadata`_ section + could be computed with this field. + +``NumCounter`` + The number of entries in the profile counter section. The byte size of `counter`_ + section could be computed with this field. + +``NumBitmapBytes`` + The number of bytes in the profile `bitmap`_ section. + +``NamesSize`` + The number of bytes in the name section. + +.. _`CountersDelta`: + +``CountersDelta`` + This field records the in-memory address difference between the `profile metadata`_ + and counter section in the instrumented binary, i.e., ``start(__llvm_prf_cnts) - start(__llvm_prf_data)``. + + It's used jointly with the `CounterPtr`_ field to compute the counter offset + relative to ``start(__llvm_prf_cnts)``. Check out calculation-of-counter-offset_ + for a visualized explanation. + + .. note:: + The ``__llvm_prf_data`` object file section might not be loaded into memory + when instrumented binary runs or might not get generated in the instrumented + binary in the first place. In those cases, ``CountersDelta`` is not used and + other mechanisms are used to match counters with instrumented code. See + `lightweight instrumentation`_ and `binary profile correlation`_ for examples. + +``BitmapDelta`` + This field records the in-memory address difference between the `profile metadata`_ + and bitmap section in the instrumented binary, i.e., ``start(__llvm_prf_bits) - start(__llvm_prf_data)``. + + It's used jointly with the `BitmapPtr`_ to find the bitmap of a profile data + record, in a similar way to how counters are referenced as explained by + calculation-of-counter-offset_ . + + Similar to `CountersDelta`_ field, this field may not be used in non-PGO variants + of profiles. + +``NamesDelta`` + Records the in-memory address of name section. Not used except for raw profile + reader error checking. + +``NumVTables`` + Records the number of instrumented vtable entries in the binary. Used for + `type profiling`_. + +``VNamesSize`` + Records the byte size in the virtual table names section. Used for `type profiling`_. + +``ValueKindLast`` + Records the number of value kinds. Macro `VALUE_PROF_KIND`_ defines the value + kinds with a description of the kind. + +.. _`VALUE_PROF_KIND`: https://github.com/llvm/llvm-project/blob/7e405eb722e40c79b7726201d0f76b5dab34ba0f/compiler-rt/include/profile/InstrProfData.inc#L184-L186 + +Payload Sections +------------------ + +Binary Ids +^^^^^^^^^^^ +Stores the binary ids of the instrumented binaries to associate binaries with +profiles for source code coverage. See `binary id`_ RFC for the design. + +.. _`profile metadata`: + +Profile Metadata +^^^^^^^^^^^^^^^^^^ + +This section stores the metadata to map counters and value profiles back to +instrumented code regions (e.g., LLVM IR for IRPGO). + +The in-memory representation of the metadata is `__llvm_profile_data`_. +Some fields are used to reference data from other sections in the profile. +The fields are documented as follows: + +.. _`__llvm_profile_data`: https://github.com/llvm/llvm-project/blob/7c3b67d2038cfb48a80299089f6a1308eee1df7f/compiler-rt/include/profile/InstrProfData.inc#L65-L95 + +``NameRef`` + The MD5 of the function's PGO name. PGO name has the format + ``[]`` where ```` and + ```` are provided for local-linkage functions to tell possibly + identical functions. + +.. _FuncHash: + +``FuncHash`` + A checksum of the function's IR, taking control flow graph and instrumented + value sites into accounts. See `computeCFGHash`_ for details. + +.. _`computeCFGHash`: https://github.com/llvm/llvm-project/blob/7c3b67d2038cfb48a80299089f6a1308eee1df7f/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp#L616-L685 + +.. _`CounterPtr`: + +``CounterPtr`` + The in-memory address difference between profile data and the start of corresponding + counters. Counter position is stored this way (as a link-time constant) to reduce + instrumented binary size compared with snapshotting the address of symbols directly. + See `commit a1532ed`_ for further information. + +.. _`commit a1532ed`: https://github.com/llvm/llvm-project/commit/a1532ed27582038e2d9588108ba0fe8237f01844 + + .. note:: + ``CounterPtr`` might represent a different value for non-IRPGO use case. For + example, for `binary profile correlation`_, it represents the absolute address of counter. + When in doubt, check source code. + +.. _`BitmapPtr`: + +``BitmapPtr`` + The in-memory address difference between profile data and the start address of + corresponding bitmap. + + .. note:: + Similar to `CounterPtr`_, this field may represent a different value for non-IRPGO use case. + +``FunctionPointer`` + Records the function address when instrumented binary runs. This is used to + map the profiled callee address of indirect calls to the ``NameRef`` during + conversion from raw to indexed profiles. + +``Values`` + Represents value profiles in a two dimensional array. The number of elements + in the first dimension is the number of instrumented value sites across all + kinds. Each element in the first dimension is the head of a linked list, and + the each element in the second dimension is linked list element, carrying + ```` as payload. This is used by compiler runtime when + writing out value profiles. + + .. note:: + Value profiling is supported by frontend and IR PGO instrumentation, + but it's not supported in all cases (e.g., `lightweight instrumentation`_). + +``NumCounters`` + The number of counters for the instrumented function. + +``NumValueSites`` + This is an array of counters, and each counter represents the number of + instrumented sites for a kind of value in the function. + +``NumBitmapBytes`` + The number of bitmap bytes for the function. + +.. _`counter`: + +Profile Counters +^^^^^^^^^^^^^^^^^ + +For PGO [4]_, the counters within an instrumented function of a specific `FuncHash`_ +are stored contiguously and in an order that is consistent with instrumentation points selection. + +.. _calculation-of-counter-offset: + +As mentioned above, the recorded counter offset is relative to the profile metadata. +So how are function counters located in the raw profile data? + +Basically, the profile reader iterates profile metadata (from the `profile metadata`_ +section) and makes use of the recorded relative distances, as illustrated below. + +:: + + + --> start(__llvm_prf_data) --> +---------------------+ ------------+ + | | Data 1 | | + | +---------------------+ =====|| | + | | Data 2 | || | + | +---------------------+ || | + | | ... | || | + Counter| +---------------------+ || | + Delta | | Data N | || | + | +---------------------+ || | CounterPtr1 + | || | + | CounterPtr2 || | + | || | + | || | + + --> start(__llvm_prf_cnts) --> +---------------------+ || | + | ... | || | + +---------------------+ -----||----+ + | Counter for | || + | Data 1 | || + +---------------------+ || + | ... | || + +---------------------+ =====|| + | Counter for | + | Data 2 | + +---------------------+ + | ... | + +---------------------+ + | Counter for | + | Data N | + +---------------------+ + + +In the graph, + +* The profile header records ``CounterDelta`` with the value as ``start(__llvm_prf_cnts) - start(__llvm_prf_data)``. + We will call it ``CounterDeltaInitVal`` below for convenience. +* For each profile data record ``ProfileDataN``, ``CounterPtr`` is recorded as + ``start(CounterN) - start(ProfileDataN)``, where ``ProfileDataN`` is the N-th + entry in ``__llvm_prf_data``, and ``CounterN`` represents the corresponding + profile counters. + +Each time the reader advances to the next data record, it `updates`_ ``CounterDelta`` +to minus the size of one ``ProfileData``. + +.. _`updates`: https://github.com/llvm/llvm-project/blob/17ff25a58ee4f29816d932fdb75f0d305718069f/llvm/include/llvm/ProfileData/InstrProfReader.h#L439-L444 + +For the counter corresponding to the first data record, the byte offset +relative to the start of the counter section is calculated as ``CounterPtr1 - CounterDeltaInitVal``. +When profile reader advances to the second data record, note ``CounterDelta`` +is updated to ``CounterDeltaInitVal - sizeof(ProfileData)``. +Thus the byte offset relative to the start of the counter section is calculated +as ``CounterPtr2 - (CounterDeltaInitVal - sizeof(ProfileData))``. + +.. _`bitmap`: + +Bitmap +^^^^^^^ +This section is used for source-based `Modified Condition/Decision Coverage`_ code coverage. Check out `Bitmap RFC`_ +for the design. + +.. _`Modified Condition/Decision Coverage`: https://en.wikipedia.org/wiki/Modified_condition/decision_coverage +.. _`Bitmap RFC`: https://discourse.llvm.org/t/rfc-source-based-mc-dc-code-coverage/59244 + +.. _`function names`: + +Names +^^^^^^ + +This section contains possibly compressed concatenated string of functions' PGO +names. If compressed, zlib library is used. + +Function names serve as keys in the PGO data hash table when raw profiles are +converted into indexed profiles. They are also crucial for ``llvm-profdata`` to +show the profiles in a human-readable way. + +Virtual Table Profile Data +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section is used for `type profiling`_. Each entry corresponds to one virtual +table and is defined by the following C++ struct + +.. code-block:: c++ + + struct VTableProfData { + // The start address of the vtable, collected at runtime. + uint64_t StartAddress; + // The byte size of the vtable. `StartAddress` and `ByteSize` specifies an address range to look up. + uint32_t ByteSize; + // The hash of vtable's (PGO) name + uint64_t MD5HashOfName; + }; + +At profile use time, the compiler looks up a profiled address in the sorted vtable +address ranges and maps the address to a specific vtable through hashed name. + +Virtual Table Names +^^^^^^^^^^^^^^^^^^^^ + +This section is similar to `function names`_ section above, except it contains the PGO +names of profiled virtual tables. It's a standalone section such that raw profile +readers could directly find each name set by accessing the corresponding profile +data section. + +This section is stored in raw profiles such that `llvm-profdata` could show the +profiles in a human-readable way. + +Value Profile Data +^^^^^^^^^^^^^^^^^^^^ + +This section contains the profile data for value profiling. + +The value profiles corresponding to a profile metadata are serialized contiguously +as one record, and value profile records are stored in the same order as the +respective profile data, such that a raw profile reader `advances`_ the pointer to +profile data and the pointer to value profile records simutaneously [5]_ to find +value profiles for a per function, per `FuncHash`_ profile data. + +.. _`advances`: https://github.com/llvm/llvm-project/blob/7e15fa9161eda7497a5d6abf0d951a1d12d86550/llvm/include/llvm/ProfileData/InstrProfReader.h#L456-L457 + +Indexed Profile Format +=========================== + +Indexed profiles are generated from ``llvm-profdata``. In the indexed profiles, +function data are organized as on-disk hash table such that compilers can +look up profile data for functions in an IR module. + +Compilers and tools must retain backward compatibility with indexed profiles. +That is, a tool or a compiler built at newer versions of code must understand +profiles generated by older tools or compilers. + +General Storage Layout +----------------------- + +The ASCII art depicts the general storage layout of indexed profiles. +Specifically, the indexed profile header describes the byte offset of individual +payload sections. + +:: + + +-----------------------+---+ + | Magic | | + +-----------------------+ | + | Version | | + +-----------------------+ | + | HashType | H + +-----------------------+ E + | Byte Offset | A + +------ | of section A | D + | +-----------------------+ E + | | Byte Of fset | R + +-----------| of section B | | + | | +-----------------------+ | + | | | ... | | + | | +-----------------------+ | + | | | Byte Offset | | + +---------------| of section Z | | + | | | +-----------------------+---+ + | | | | Profile Summary | | + | | | +-----------------------+ P + | | +------>| Section A | A + | | +-----------------------+ Y + | +---------->| Section B | L + | +-----------------------+ O + | | ... | A + | +-----------------------+ D + +-------------->| Section Z | | + +-----------------------+---+ + +.. note:: + + Profile summary section is at the beginning of payload. It's right after the + header so its position is implicitly known after reading the header. + +Header +-------- + +The `Header struct`_ is the source of truth and struct fields should explain +what's in the header. At a high level, `*Offset` fields record section byte +offsets, which are used by readers to locate interesting sections and skip +uninteresting ones. + +.. note:: + + To maintain backward compatibility of the indexed profiles, existing fields + shouldn't be deleted from struct definition; the field order shouldn't be + modified. New fields should be appended. + +.. _`Header struct`: https://github.com/llvm/llvm-project/blob/1a2960bab6381f2b288328e2371829b460ac020c/llvm/include/llvm/ProfileData/InstrProf.h#L1053-L1080 + + +Payload Sections +------------------ + +(CS) Profile Summary +^^^^^^^^^^^^^^^^^^^^^ +This section is right after profile header. It stores the serialized profile +summary. For context-sensitive IR-based instrumentation PGO, this section stores +an additional profile summary corresponding to the context-sensitive profiles. + +.. _`function data`: + +Function data +^^^^^^^^^^^^^^^^^^ +This section stores functions and their profiling data as an on-disk hash table. +Profile data for functions with the same name are grouped together and share one +hash table entry (the functions may come from different shared libraries for +instance). The profile data for them are organized as a sequence of key-value +pair where the key is `FuncHash`_, and the value is profiled information (represented +by `InstrProfRecord`_) for the function. + +.. _`InstrProfRecord`: https://github.com/llvm/llvm-project/blob/7e405eb722e40c79b7726201d0f76b5dab34ba0f/llvm/include/llvm/ProfileData/InstrProf.h#L693 + +MemProf Profile data +^^^^^^^^^^^^^^^^^^^^^^ +This section stores function's memory profiling data. See +`MemProf binary serialization format RFC`_ for the design. + +.. _`MemProf binary serialization format RFC`: https://lists.llvm.org/pipermail/llvm-dev/2021-September/153007.html + +Binary Ids +^^^^^^^^^^^^^^^^^^^^^^ +The section is used to carry on `binary id`_ information from raw profiles. + +Temporal Profile Traces +^^^^^^^^^^^^^^^^^^^^^^^^ +The section is used to carry on temporal profile information from raw profiles. +See `temporal profiling`_ for the design. + +Virtual Table Names +^^^^^^^^^^^^^^^^^^^^ +This section is used to store the names of vtables from raw profile in the indexed +profile. + +Unlike function names which are stored as keys of `function data`_ hash table, +vtable names need to be stored in a standalone section in indexed profiles. +This way, `llvm-profdata` could show the profiled vtable information in a +human-readable way. + +Profile Data Usage +======================================= + +``llvm-profdata`` is the command line tool to display and process instrumentation- +based profile data. For supported usages, check out `llvm-profdata documentation `_. + +.. [1] For usage, see https://clang.llvm.org/docs/UsersManual.html#profiling-with-instrumentation +.. [2] For example, IR-based instrumentation supports `lightweight instrumentation`_ + and `temporal profiling`_. Frontend instrumentation could support `single-byte counters`_. +.. [3] A raw profile file could contain the concatenation of multiple raw + profiles, for example, from an executable and its shared libraries. Raw + profile reader could parse all raw profiles from the file correctly. +.. [4] The counter section is used by a few variant types (like temporal + profiling) and might have different semantics there. +.. [5] The step size of data pointer is the ``sizeof(ProfileData)``, and the step + size of value profile pointer is calcuated based on the number of collected + values. + +.. _`lightweight instrumentation`: https://groups.google.com/g/llvm-dev/c/r03Z6JoN7d4 +.. _`temporal profiling`: https://discourse.llvm.org/t/rfc-temporal-profiling-extension-for-irpgo/68068 +.. _`single-byte counters`: https://discourse.llvm.org/t/rfc-single-byte-counters-for-source-based-code-coverage/75685 +.. _`binary profile correlation`: https://discourse.llvm.org/t/rfc-add-binary-profile-correlation-to-not-load-profile-metadata-sections-into-memory-at-runtime/74565 +.. _`binary id`: https://lists.llvm.org/pipermail/llvm-dev/2021-June/151154.html +.. _`type profiling`: https://discourse.llvm.org/t/rfc-dynamic-type-profiling-and-optimizations-in-llvm/74600 -- Gitee From 06834b1b35fbe32de8fc748c6e42ae7117efca01 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 1 Apr 2024 08:52:35 -0700 Subject: [PATCH 18/28] [InstrFDO][TypeProf] Implement binary instrumentation and profile read/write (#66825) (The profile format change is split into a standalone change into https://github.com/llvm/llvm-project/pull/81691) * For InstrFDO value profiling, implement instrumentation and lowering for virtual table address. * This is controlled by `-enable-vtable-value-profiling` and off by default. * When the option is on, raw profiles will carry serialized `VTableProfData` structs and compressed vtables as payloads. * Implement profile reader and writer support * Raw profile reader is used by `llvm-profdata` but not compiler. Raw profile reader will construct InstrProfSymtab with symbol names, and map profiled runtime address to vtable symbols. * Indexed profile reader is used by `llvm-profdata` and compiler. When initialized, the reader stores a pointer to the beginning of in-memory compressed vtable names and the length of string. When used in `llvm-profdata`, reader decompress the string to show symbols of a profiled site. When used in compiler, string decompression doesn't happen since IR is used to construct InstrProfSymtab. * Indexed profile writer collects the list of vtable names, and stores that to index profiles. * Text profile reader and writer support are added but mostly follow the implementation for indirect-call value type. * `llvm-profdata show -show-vtables ` is implemented. rfc in https://discourse.llvm.org/t/rfc-dynamic-type-profiling-and-optimizations-in-llvm/74600#pick-instrumentation-points-and-instrument-runtime-types-7 --- .../Linux/instrprof-vtable-value-prof.cpp | 142 ++++ .../llvm/Analysis/IndirectCallVisitor.h | 62 +- llvm/include/llvm/ProfileData/InstrProf.h | 40 +- .../llvm/ProfileData/InstrProfReader.h | 6 + .../llvm/ProfileData/InstrProfWriter.h | 4 + llvm/lib/ProfileData/InstrProf.cpp | 80 +- llvm/lib/ProfileData/InstrProfReader.cpp | 33 +- llvm/lib/ProfileData/InstrProfWriter.cpp | 33 +- .../Instrumentation/InstrProfiling.cpp | 334 +++++++- .../Instrumentation/PGOInstrumentation.cpp | 14 + .../Instrumentation/ValueProfilePlugins.inc | 35 +- .../PGOProfile/vtable_prof_unsupported.ll | 34 + .../Transforms/PGOProfile/vtable_profile.ll | 98 +++ .../Inputs/vtable-value-prof.proftext | 74 ++ .../llvm-profdata/vtable-value-prof.test | 83 ++ llvm/tools/llvm-profdata/llvm-profdata.cpp | 26 + llvm/unittests/ProfileData/InstrProfTest.cpp | 740 ++++++++++++------ 17 files changed, 1497 insertions(+), 341 deletions(-) create mode 100644 compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp create mode 100644 llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll create mode 100644 llvm/test/Transforms/PGOProfile/vtable_profile.ll create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext create mode 100644 llvm/test/tools/llvm-profdata/vtable-value-prof.test diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp new file mode 100644 index 000000000000..5c8426b40892 --- /dev/null +++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp @@ -0,0 +1,142 @@ +// REQUIRES: lld-available + +// RUN: %clangxx_pgogen -fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling %s -o %t-test +// RUN: env LLVM_PROFILE_FILE=%t-test.profraw %t-test + +// Show vtable profiles from raw profile. +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profraw | FileCheck %s --check-prefixes=COMMON,RAW + +// Generate indexed profile from raw profile and show the data. +// RUN: llvm-profdata merge %t-test.profraw -o %t-test.profdata +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED + +// Generate text profile from raw and indexed profiles respectively and show the data. +// RUN: llvm-profdata merge --text %t-test.profraw -o %t-raw.proftext +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-raw.proftext | FileCheck %s --check-prefix=ICTEXT +// RUN: llvm-profdata merge --text %t-test.profdata -o %t-indexed.proftext +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-indexed.proftext | FileCheck %s --check-prefix=ICTEXT + +// Generate indexed profile from text profiles and show the data +// RUN: llvm-profdata merge --binary %t-raw.proftext -o %t-text.profraw +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profraw | FileCheck %s --check-prefixes=COMMON,INDEXED +// RUN: llvm-profdata merge --binary %t-indexed.proftext -o %t-text.profdata +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED + +// COMMON: Counters: +// COMMON-NEXT: main: +// COMMON-NEXT: Hash: 0x0f9a16fe6d398548 +// COMMON-NEXT: Counters: 2 +// COMMON-NEXT: Indirect Call Site Count: 2 +// COMMON-NEXT: Number of instrumented vtables: 2 +// RAW: Indirect Target Results: +// RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) +// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) +// RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +// RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +// RAW-NEXT: VTable Results: +// RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) +// RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// INDEXED: Indirect Target Results: +// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) +// INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) +// INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +// INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +// INDEXED-NEXT: VTable Results: +// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +// INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) +// COMMON: Instrumentation level: IR entry_first = 0 +// COMMON-NEXT: Functions shown: 1 +// COMMON-NEXT: Total functions: 6 +// COMMON-NEXT: Maximum function count: 1000 +// COMMON-NEXT: Maximum internal block count: 250 +// COMMON-NEXT: Statistics for indirect call sites profile: +// COMMON-NEXT: Total number of sites: 2 +// COMMON-NEXT: Total number of sites with values: 2 +// COMMON-NEXT: Total number of profiled values: 4 +// COMMON-NEXT: Value sites histogram: +// COMMON-NEXT: NumTargets, SiteCount +// COMMON-NEXT: 2, 2 +// COMMON-NEXT: Statistics for vtable profile: +// COMMON-NEXT: Total number of sites: 2 +// COMMON-NEXT: Total number of sites with values: 2 +// COMMON-NEXT: Total number of profiled values: 4 +// COMMON-NEXT: Value sites histogram: +// COMMON-NEXT: NumTargets, SiteCount +// COMMON-NEXT: 2, 2 + +// ICTEXT: :ir +// ICTEXT: main +// ICTEXT: # Func Hash: +// ICTEXT: 1124236338992350536 +// ICTEXT: # Num Counters: +// ICTEXT: 2 +// ICTEXT: # Counter Values: +// ICTEXT: 1000 +// ICTEXT: 1 +// ICTEXT: # Num Value Kinds: +// ICTEXT: 2 +// ICTEXT: # ValueKind = IPVK_IndirectCallTarget: +// ICTEXT: 0 +// ICTEXT: # NumValueSites: +// ICTEXT: 2 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii:750 +// ICTEXT: _ZN8Derived15func1Eii:250 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii:750 +// ICTEXT: _ZN8Derived15func2Eii:250 +// ICTEXT: # ValueKind = IPVK_VTableTarget: +// ICTEXT: 2 +// ICTEXT: # NumValueSites: +// ICTEXT: 2 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 +// ICTEXT: _ZTV8Derived1:250 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 +// ICTEXT: _ZTV8Derived1:250 + +#include +#include +class Base { +public: + virtual int func1(int a, int b) = 0; + virtual int func2(int a, int b) = 0; +}; +class Derived1 : public Base { +public: + int func1(int a, int b) override { return a + b; } + + int func2(int a, int b) override { return a * b; } +}; +namespace { +class Derived2 : public Base { +public: + int func1(int a, int b) override { return a - b; } + + int func2(int a, int b) override { return a * (a - b); } +}; +} // namespace +__attribute__((noinline)) Base *createType(int a) { + Base *base = nullptr; + if (a % 4 == 0) + base = new Derived1(); + else + base = new Derived2(); + return base; +} +int main(int argc, char **argv) { + int sum = 0; + for (int i = 0; i < 1000; i++) { + int a = rand(); + int b = rand(); + Base *ptr = createType(i); + sum += ptr->func1(a, b) + ptr->func2(b, a); + } + printf("sum is %d\n", sum); + return 0; +} diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h index 0825e19ecd2d..50815f4e3e83 100644 --- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h +++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h @@ -16,23 +16,75 @@ #include namespace llvm { -// Visitor class that finds all indirect call. +// Visitor class that finds indirect calls or instructions that gives vtable +// value, depending on Type. struct PGOIndirectCallVisitor : public InstVisitor { + enum class InstructionType { + kIndirectCall = 0, + kVTableVal = 1, + }; std::vector IndirectCalls; - PGOIndirectCallVisitor() = default; + std::vector ProfiledAddresses; + PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {} void visitCallBase(CallBase &Call) { - if (Call.isIndirectCall()) + if (!Call.isIndirectCall()) + return; + + if (Type == InstructionType::kIndirectCall) { IndirectCalls.push_back(&Call); + return; + } + + assert(Type == InstructionType::kVTableVal && "Control flow guaranteed"); + + LoadInst *LI = dyn_cast(Call.getCalledOperand()); + // The code pattern to look for + // + // %vtable = load ptr, ptr %b + // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 + // %2 = load ptr, ptr %vfn + // %call = tail call i32 %2(ptr %b) + // + // %vtable is the vtable address value to profile, and + // %2 is the indirect call target address to profile. + if (LI != nullptr) { + Value *Ptr = LI->getPointerOperand(); + Value *VTablePtr = Ptr->stripInBoundsConstantOffsets(); + // This is a heuristic to find address feeding instructions. + // FIXME: Add support in the frontend so LLVM type intrinsics are + // emitted without LTO. This way, added intrinsics could filter + // non-vtable instructions and reduce instrumentation overhead. + // Since a non-vtable profiled address is not within the address + // range of vtable objects, it's stored as zero in indexed profiles. + // A pass that looks up symbol with an zero hash will (almost) always + // find nullptr and skip the actual transformation (e.g., comparison + // of symbols). So the performance overhead from non-vtable profiled + // address is negligible if exists at all. Comparing loaded address + // with symbol address guarantees correctness. + if (VTablePtr != nullptr && isa(VTablePtr)) + ProfiledAddresses.push_back(cast(VTablePtr)); + } } + +private: + InstructionType Type; }; -// Helper function that finds all indirect call sites. inline std::vector findIndirectCalls(Function &F) { - PGOIndirectCallVisitor ICV; + PGOIndirectCallVisitor ICV( + PGOIndirectCallVisitor::InstructionType::kIndirectCall); ICV.visit(F); return ICV.IndirectCalls; } + +inline std::vector findVTableAddrs(Function &F) { + PGOIndirectCallVisitor ICV( + PGOIndirectCallVisitor::InstructionType::kVTableVal); + ICV.visit(F); + return ICV.ProfiledAddresses; +} + } // namespace llvm #endif diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 0ae11402d310..33937ad6469d 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -91,6 +91,9 @@ inline StringRef getInstrProfValueProfMemOpFuncName() { /// Return the name prefix of variables containing instrumented function names. inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } +/// Return the name prefix of variables containing virtual table profile data. +inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; } + /// Return the name prefix of variables containing per-function control data. inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } @@ -108,9 +111,9 @@ inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } /// Return the name of the variable holding the strings (possibly compressed) /// of all function's PGO names. -inline StringRef getInstrProfNamesVarName() { - return "__llvm_prf_nm"; -} +inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; } + +inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; } /// Return the name of a covarage mapping variable (internal linkage) /// for each instrumented source module. Such variables are allocated @@ -142,7 +145,8 @@ inline StringRef getInstrProfRegFuncName() { return "__llvm_profile_register_function"; } -/// Return the name of the runtime interface that registers the PGO name strings. +/// Return the name of the runtime interface that registers the PGO name +/// strings. inline StringRef getInstrProfNamesRegFuncName() { return "__llvm_profile_register_names_function"; } @@ -254,12 +258,7 @@ Error collectPGOFuncNameStrings(ArrayRef NameVars, std::string &Result, bool doCompression = true); Error collectVTableStrings(ArrayRef VTables, - std::string &Result, bool doCompression); - -/// \c NameStrings is a string composed of one of more sub-strings encoded in -/// the format described above. The substrings are separated by 0 or more zero -/// bytes. This method decodes the string and populates the \c Symtab. -Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab); + std::string &Result, bool doCompression); /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being /// set in IR PGO compilation. @@ -314,7 +313,7 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); /// Check if we can use Comdat for profile variables. This will eliminate /// the duplicated profile variables for Comdat functions. -bool needsComdatForCounter(const Function &F, const Module &M); +bool needsComdatForCounter(const GlobalObject &GV, const Module &M); /// An enum describing the attributes of an instrumented profile. enum class InstrProfKind { @@ -457,13 +456,14 @@ private: // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's // going to be sorted) has unique MD5 keys in the first place. StringSet<> NameTab; - // Record the unique virtual table names. This is used by InstrProfWriter to + // Records the unique virtual table names. This is used by InstrProfWriter to // write out an on-disk chained hash table of virtual table names. // InstrProfWriter stores per function profile data (keyed by function names) // so it doesn't use a StringSet for function names. StringSet<> VTableNames; // A map from MD5 keys to function name strings. std::vector> MD5NameMap; + // A map from MD5 keys to function define. We only populate this map // when build the Symtab from a Module. std::vector> MD5FuncMap; @@ -529,7 +529,7 @@ public: /// \c NameStrings is a string composed of one of more sub-strings /// encoded in the format described in \c collectPGOFuncNameStrings. - /// This method is a wrapper to \c readPGOFuncNameStrings method. + /// This method is a wrapper to \c readAndDecodeStrings method. Error create(StringRef NameStrings); /// Initialize symtab states with function names and vtable names. \c @@ -554,18 +554,18 @@ public: /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. - template Error create(const NameIterRange &IterRange); + template + Error create(const NameIterRange &IterRange); /// Create InstrProfSymtab from a set of function names and vtable /// names iteratable from \p IterRange. This interface is used by /// IndexedProfReader. template Error create(const FuncNameIterRange &FuncIterRange, - const VTableNameIterRange &VTableIterRange); + const VTableNameIterRange &VTableIterRange); - // Map the MD5 of the symbol name to the name. Error addSymbolName(StringRef SymbolName) { - if (SymbolName.empty()) + if (SymbolName.empty()) return make_error(instrprof_error::malformed, "symbol name is empty"); @@ -589,7 +589,7 @@ public: Error addVTableName(StringRef VTableName) { if (Error E = addSymbolName(VTableName)) return E; - + // Record VTableName. InstrProfWriter uses this set. The comment around // class member explains why. VTableNames.insert(VTableName); @@ -677,7 +677,7 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) { template Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange, - const VTableNameIterRange &VTableIterRange) { + const VTableNameIterRange &VTableIterRange) { // Iterate elements by StringRef rather than by const reference. // StringRef is small enough, so the loop is efficient whether // element in the range is std::string or StringRef. @@ -1017,6 +1017,8 @@ private: return ValueData->IndirectCallSites; case IPVK_MemOPSize: return ValueData->MemOPSizes; + case IPVK_VTableTarget: + return ValueData->VTableTargets; default: llvm_unreachable("Unknown value kind!"); } diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index c15737f6c098..206254c569d1 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -625,6 +625,12 @@ public: InstrProfKind getProfileKind() const override; Error populateSymtab(InstrProfSymtab &Symtab) override { + // FIXME: the create method calls 'finalizeSymtab' and sorts a bunch of + // arrays/maps. Since there are other data sources other than 'HashTable' to + // populate a symtab, it might make sense to have something like this + // 1. Let each data source populate Symtab and init the arrays/maps without + // calling 'finalizeSymtab' + // 2. Call 'finalizeSymtab' once to get all arrays/maps sorted if needed. return Symtab.create(HashTable->keys()); } }; diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index e50705ee053e..c38bc621469e 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -64,6 +64,9 @@ private: // List of binary ids. std::vector BinaryIds; + // Read the vtable names from raw instr profile reader. + StringSet<> VTableNames; + // An enum describing the attributes of the profile. InstrProfKind ProfileKind = InstrProfKind::Unknown; // Use raw pointer here for the incomplete type object. @@ -85,6 +88,7 @@ public: void addRecord(NamedInstrProfRecord &&I, function_ref Warn) { addRecord(std::move(I), 1, Warn); } + void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); } /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the /// total number of temporal profiling traces the source has seen. diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 01147247513b..498d8708beaf 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -60,6 +60,8 @@ using namespace llvm; +#define DEBUG_TYPE "instrprof" + static cl::opt StaticFuncFullModulePrefix( "static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " @@ -222,6 +224,12 @@ cl::opt DoInstrProfNameCompression( "enable-name-compression", cl::desc("Enable name/filename string compression"), cl::init(true)); +cl::opt EnableVTableValueProfiling( + "enable-vtable-value-profiling", cl::init(false), + cl::desc("If true, the virtual table address will be instrumented to know " + "the types of a C++ pointer. The information is used in indirect " + "call promotion to do selective vtable-based comparison.")); + std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo) { @@ -412,13 +420,12 @@ std::string getPGOName(const GlobalVariable &V, bool InLTO) { return getIRPGOObjectName(V, InLTO, V.getMetadata(getPGONameMetadataName())); } -// See getIRPGOFuncName() for a discription of the format. -std::pair -getParsedIRPGOFuncName(StringRef IRPGOFuncName) { - auto [FileName, FuncName] = IRPGOFuncName.split(';'); - if (FuncName.empty()) - return std::make_pair(StringRef(), IRPGOFuncName); - return std::make_pair(FileName, FuncName); +// See getIRPGOObjectName() for a discription of the format. +std::pair getParsedIRPGOFuncName(StringRef IRPGOName) { + auto [FileName, MangledName] = IRPGOFuncName.split(GlobalIdentifierDelimiter); + if (MangledName.empty()) + return std::make_pair(StringRef(), IRPGOName); + return std::make_pair(FileName, MangledName); } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { @@ -582,10 +589,10 @@ Error InstrProfSymtab::create(StringRef NameStrings) { } Error InstrProfSymtab::create(StringRef FuncNameStrings, - StringRef VTableNameStrings) { + StringRef VTableNameStrings) { if (Error E = readAndDecodeStrings(FuncNameStrings, - std::bind(&InstrProfSymtab::addFuncName, - this, std::placeholders::_1))) + std::bind(&InstrProfSymtab::addFuncName, + this, std::placeholders::_1))) return E; return readAndDecodeStrings( @@ -774,48 +781,6 @@ Error collectVTableStrings(ArrayRef VTables, Result); } -Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { - const uint8_t *P = NameStrings.bytes_begin(); - const uint8_t *EndP = NameStrings.bytes_end(); - while (P < EndP) { - uint32_t N; - uint64_t UncompressedSize = decodeULEB128(P, &N); - P += N; - uint64_t CompressedSize = decodeULEB128(P, &N); - P += N; - bool isCompressed = (CompressedSize != 0); - SmallVector UncompressedNameStrings; - StringRef NameStrings; - if (isCompressed) { - if (!llvm::compression::zlib::isAvailable()) - return make_error(instrprof_error::zlib_unavailable); - - if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize), - UncompressedNameStrings, - UncompressedSize)) { - consumeError(std::move(E)); - return make_error(instrprof_error::uncompress_failed); - } - P += CompressedSize; - NameStrings = toStringRef(UncompressedNameStrings); - } else { - NameStrings = - StringRef(reinterpret_cast(P), UncompressedSize); - P += UncompressedSize; - } - // Now parse the name strings. - SmallVector Names; - NameStrings.split(Names, getInstrProfNameSeparator()); - for (StringRef &Name : Names) - if (Error E = Symtab.addFuncName(Name)) - return E; - - while (P < EndP && *P == 0) - P++; - } - return Error::success(); -} - void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { uint64_t FuncSum = 0; Sum.NumEntries += Counts.size(); @@ -1472,8 +1437,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { F.setMetadata(getPGOFuncNameMetadataName(), N); } -bool needsComdatForCounter(const Function &F, const Module &M) { - if (F.hasComdat()) +bool needsComdatForCounter(const GlobalObject &GO, const Module &M) { + if (GO.hasComdat()) return true; if (!Triple(M.getTargetTriple()).supportsCOMDAT()) @@ -1489,7 +1454,7 @@ bool needsComdatForCounter(const Function &F, const Module &M) { // available_externally functions will end up being duplicated in raw profile // data. This can result in distorted profile as the counts of those dups // will be accumulated by the profile merger. - GlobalValue::LinkageTypes Linkage = F.getLinkage(); + GlobalValue::LinkageTypes Linkage = GO.getLinkage(); if (Linkage != GlobalValue::ExternalWeakLinkage && Linkage != GlobalValue::AvailableExternallyLinkage) return false; @@ -1645,7 +1610,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) continue; - char ProfileKindName[20]; + char ProfileKindName[20] = {0}; switch (I) { case IPVK_IndirectCallTarget: strncpy(ProfileKindName, "IndirectCall", 19); @@ -1653,6 +1618,9 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { case IPVK_MemOPSize: strncpy(ProfileKindName, "MemOP", 19); break; + case IPVK_VTableTarget: + strncpy(ProfileKindName, "VTable", 19); + break; default: snprintf(ProfileKindName, 19, "VP[%d]", I); break; diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index f9c9b9d75568..413c41414eee 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -368,8 +368,11 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { } else if (ValueKind == IPVK_VTableTarget) { if (InstrProfSymtab::isExternalSymbol(VD.first)) Value = 0; - else + else { + if (Error E = Symtab->addVTableName(VD.first)) + return E; Value = IndexedInstrProf::ComputeHash(VD.first); + } } else { READ_NUM(VD.first, Value); } @@ -537,7 +540,8 @@ Error RawInstrProfReader::readNextHeader(const char *CurrentPos) { template Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { - if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart))) + if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart), + StringRef(VNamesStart, VNamesEnd - VNamesStart))) return error(std::move(E)); for (const RawInstrProf::ProfileData *I = Data; I != DataEnd; ++I) { const IntPtrT FPtr = swap(I->FunctionPointer); @@ -545,6 +549,21 @@ Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { continue; Symtab.mapAddress(FPtr, swap(I->NameRef)); } + + if (VTableBegin != nullptr && VTableEnd != nullptr) { + for (const RawInstrProf::VTableProfileData *I = VTableBegin; + I != VTableEnd; ++I) { + const IntPtrT VPtr = swap(I->VTablePointer); + if (!VPtr) + continue; + // Map both begin and end address to the name hash, since the instrumented + // address could be somewhere in the middle. + // VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks + // the end of vtable address. + Symtab.mapVTableAddress(VPtr, VPtr + swap(I->VTableSize), + swap(I->VTableNameHash)); + } + } return success(); } @@ -1336,7 +1355,15 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() { if (Symtab) return *Symtab; - std::unique_ptr NewSymtab = std::make_unique(); + auto NewSymtab = std::make_unique(); + + if (Error E = NewSymtab->initVTableNamesFromCompressedStrings( + StringRef(VTableNamePtr, CompressedVTableNamesLen))) { + auto [ErrCode, Msg] = InstrProfError::take(std::move(E)); + consumeError(error(ErrCode, Msg)); + } + + // finalizeSymtab is called inside populateSymtab. if (Error E = Index->populateSymtab(*NewSymtab)) { auto [ErrCode, Msg] = InstrProfError::take(std::move(E)); consumeError(error(ErrCode, Msg)); diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 55e5c8e3e885..5534bc220356 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -19,6 +19,7 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/MemProf.h" #include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" @@ -601,13 +602,19 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { uint64_t VTableNamesSectionStart = OS.tell(); - // Use a dummy (and uncompressed) string as compressed vtable names and get - // the necessary profile format change in place for version 12. - // TODO: Store the list of vtable names in InstrProfWriter and use the - // real compressed name. - std::string CompressedVTableNames = "VTableNames"; + if (!WritePrevVersion) { + std::vector VTableNameStrs; + for (StringRef VTableName : VTableNames.keys()) + VTableNameStrs.push_back(VTableName.str()); - uint64_t CompressedStringLen = CompressedVTableNames.length(); + std::string CompressedVTableNames; + if (!VTableNameStrs.empty()) + if (Error E = collectGlobalObjectNameStrings( + VTableNameStrs, compression::zlib::isAvailable(), + CompressedVTableNames)) + return E; + + const uint64_t CompressedStringLen = CompressedVTableNames.length(); // Record the length of compressed string. OS.write(CompressedStringLen); @@ -616,12 +623,12 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { for (auto &c : CompressedVTableNames) OS.writeByte(static_cast(c)); - // Pad up to a multiple of 8. - // InstrProfReader would read bytes according to 'CompressedStringLen'. - uint64_t PaddedLength = alignTo(CompressedStringLen, 8); + // Pad up to a multiple of 8. + // InstrProfReader could read bytes according to 'CompressedStringLen'. + const uint64_t PaddedLength = alignTo(CompressedStringLen, 8); - for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { - OS.writeByte(0); + for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) + OS.writeByte(0); } uint64_t TemporalProfTracesSectionStart = 0; @@ -808,6 +815,10 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { } } + for (const auto &VTableName : VTableNames) + if (Error E = Symtab.addVTableName(VTableName.getKey())) + return E; + if (static_cast(ProfileKind & InstrProfKind::TemporalProfile)) writeTextTemporalProfTraceData(OS, Symtab); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 7131af6bc459..6493ae974cce 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -59,6 +59,9 @@ using namespace llvm; #define DEBUG_TYPE "instrprof" namespace llvm { +// Command line option to enable vtable value profiling. Defined in +// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= +extern cl::opt EnableVTableValueProfiling; // TODO: Remove -debug-info-correlate in next LLVM release, in favor of // -profile-correlate=debug-info. cl::opt DebugInfoCorrelate( @@ -161,6 +164,190 @@ cl::opt SkipRetExitBlock( "skip-ret-exit-block", cl::init(true), cl::desc("Suppress counter promotion if exit blocks contain ret.")); +using LoadStorePair = std::pair; + +static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { + auto *MD = dyn_cast_or_null(M.getModuleFlag(Flag)); + if (!MD) + return 0; + + // If the flag is a ConstantAsMetadata, it should be an integer representable + // in 64-bits. + return cast(MD->getValue())->getZExtValue(); +} + +static bool enablesValueProfiling(const Module &M) { + return isIRPGOFlagSet(&M) || + getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0; +} + +// Conservatively returns true if value profiling is enabled. +static bool profDataReferencedByCode(const Module &M) { + return enablesValueProfiling(M); +} + +class InstrLowerer final { +public: + InstrLowerer(Module &M, const InstrProfOptions &Options, + std::function GetTLI, + bool IsCS) + : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS), + GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {} + + bool lower(); + +private: + Module &M; + const InstrProfOptions Options; + const Triple TT; + // Is this lowering for the context-sensitive instrumentation. + const bool IsCS; + + std::function GetTLI; + + const bool DataReferencedByCode; + + struct PerFunctionProfileData { + uint32_t NumValueSites[IPVK_Last + 1] = {}; + GlobalVariable *RegionCounters = nullptr; + GlobalVariable *DataVar = nullptr; + GlobalVariable *RegionBitmaps = nullptr; + uint32_t NumBitmapBytes = 0; + + PerFunctionProfileData() = default; + }; + DenseMap ProfileDataMap; + // Key is virtual table variable, value is 'VTableProfData' in the form of + // GlobalVariable. + DenseMap VTableDataMap; + /// If runtime relocation is enabled, this maps functions to the load + /// instruction that produces the profile relocation bias. + DenseMap FunctionToProfileBiasMap; + std::vector CompilerUsedVars; + std::vector UsedVars; + std::vector ReferencedNames; + // The list of virtual table variables of which the VTableProfData is + // collected. + std::vector ReferencedVTables; + GlobalVariable *NamesVar = nullptr; + size_t NamesSize = 0; + + // vector of counter load/store pairs to be register promoted. + std::vector PromotionCandidates; + + int64_t TotalCountersPromoted = 0; + + /// Lower instrumentation intrinsics in the function. Returns true if there + /// any lowering. + bool lowerIntrinsics(Function *F); + + /// Register-promote counter loads and stores in loops. + void promoteCounterLoadStores(Function *F); + + /// Returns true if relocating counters at runtime is enabled. + bool isRuntimeCounterRelocationEnabled() const; + + /// Returns true if profile counter update register promotion is enabled. + bool isCounterPromotionEnabled() const; + + /// Count the number of instrumented value sites for the function. + void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins); + + /// Replace instrprof.value.profile with a call to runtime library. + void lowerValueProfileInst(InstrProfValueProfileInst *Ins); + + /// Replace instrprof.cover with a store instruction to the coverage byte. + void lowerCover(InstrProfCoverInst *Inc); + + /// Replace instrprof.timestamp with a call to + /// INSTR_PROF_PROFILE_SET_TIMESTAMP. + void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction); + + /// Replace instrprof.increment with an increment of the appropriate value. + void lowerIncrement(InstrProfIncrementInst *Inc); + + /// Force emitting of name vars for unused functions. + void lowerCoverageData(GlobalVariable *CoverageNamesVar); + + /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction + /// using the index represented by the a temp value into a bitmap. + void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); + + /// Replace instrprof.mcdc.temp.update with a shift and or instruction using + /// the corresponding condition ID. + void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); + + /// Compute the address of the counter value that this profiling instruction + /// acts on. + Value *getCounterAddress(InstrProfCntrInstBase *I); + + /// Get the region counters for an increment, creating them if necessary. + /// + /// If the counter array doesn't yet exist, the profile data variables + /// referring to them will also be created. + GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); + + /// Create the region counters. + GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage); + + /// Compute the address of the test vector bitmap that this profiling + /// instruction acts on. + Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); + + /// Get the region bitmaps for an increment, creating them if necessary. + /// + /// If the bitmap array doesn't yet exist, the profile data variables + /// referring to them will also be created. + GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); + + /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with + /// an MC/DC Decision region. The number of bytes required is indicated by + /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called + /// as part of setupProfileSection() and is conceptually very similar to + /// what is done for profile data counters in createRegionCounters(). + GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage); + + /// Set Comdat property of GV, if required. + void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName); + + /// Setup the sections into which counters and bitmaps are allocated. + GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK); + + /// Create INSTR_PROF_DATA variable for counters and bitmaps. + void createDataVariable(InstrProfCntrInstBase *Inc); + + /// Get the counters for virtual table values, creating them if necessary. + void getOrCreateVTableProfData(GlobalVariable *GV); + + /// Emit the section with compressed function names. + void emitNameData(); + + /// Emit the section with compressed vtable names. + void emitVTableNames(); + + /// Emit value nodes section for value profiling. + void emitVNodes(); + + /// Emit runtime registration functions for each profile data variable. + void emitRegistration(); + + /// Emit the necessary plumbing to pull in the runtime initialization. + /// Returns true if a change was made. + bool emitRuntimeHook(); + + /// Add uses of our data variables and runtime hook. + void emitUses(); + + /// Create a static initializer for our data, on platforms that need it, + /// and for any profile output file that was specified. + void emitInitialization(); +}; + /// /// A helper class to promote one counter RMW operation in the loop /// into register update. @@ -596,6 +783,12 @@ bool InstrProfiling::run( } } + if (EnableVTableValueProfiling) + for (GlobalVariable &GV : M.globals()) + // Global variables with type metadata are virtual table variables. + if (GV.hasMetadata(LLVMContext::MD_type)) + getOrCreateVTableProfData(&GV); + for (Function &F : M) MadeChange |= lowerIntrinsics(&F); @@ -609,6 +802,7 @@ bool InstrProfiling::run( emitVNodes(); emitNameData(); + emitVTableNames(); // Emit runtime hook for the cases where the target does not unconditionally // require pulling in profile runtime, and coverage is enabled on code that is @@ -1060,10 +1254,13 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { return true; } -void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, - StringRef VarName) { - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); +void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, + StringRef CounterGroupName) { + // Place lowered global variables in a comdat group if the associated function + // or global variable is a COMDAT. This will make sure that only one copy of + // global variable (e.g. function counters) of the COMDAT function will be + // emitted after linking. + bool NeedComdat = needsComdatForCounter(*GO, M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); if (!UseComdat) @@ -1082,8 +1279,106 @@ void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, GV->setLinkage(GlobalValue::InternalLinkage); } -GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, - InstrProfSectKind IPSK) { +static inline bool shouldRecordVTableAddr(GlobalVariable *GV) { + if (!profDataReferencedByCode(*GV->getParent())) + return false; + + if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() && + !GV->hasAvailableExternallyLinkage()) + return true; + + // This avoids the profile data from referencing internal symbols in + // COMDAT. + if (GV->hasLocalLinkage() && GV->hasComdat()) + return false; + + return true; +} + +// FIXME: Introduce an internal alias like what's done for functions to reduce +// the number of relocation entries. +static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) { + auto *Int8PtrTy = PointerType::getUnqual(GV->getContext()); + + // Store a nullptr in __profvt_ if a real address shouldn't be used. + if (!shouldRecordVTableAddr(GV)) + return ConstantPointerNull::get(Int8PtrTy); + + return ConstantExpr::getBitCast(GV, Int8PtrTy); +} + +void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { + assert(!DebugInfoCorrelate && + "Value profiling is not supported with lightweight instrumentation"); + if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) + return; + + // Skip llvm internal global variable or __prof variables. + if (GV->getName().starts_with("llvm.") || + GV->getName().starts_with("__llvm") || + GV->getName().starts_with("__prof")) + return; + + // VTableProfData already created + auto It = VTableDataMap.find(GV); + if (It != VTableDataMap.end() && It->second) + return; + + GlobalValue::LinkageTypes Linkage = GV->getLinkage(); + GlobalValue::VisibilityTypes Visibility = GV->getVisibility(); + + // This is to keep consistent with per-function profile data + // for correctness. + if (TT.isOSBinFormatXCOFF()) { + Linkage = GlobalValue::InternalLinkage; + Visibility = GlobalValue::DefaultVisibility; + } + + LLVMContext &Ctx = M.getContext(); + Type *DataTypes[] = { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType, +#include "llvm/ProfileData/InstrProfData.inc" +#undef INSTR_PROF_VTABLE_DATA + }; + + auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes)); + + // Used by INSTR_PROF_VTABLE_DATA MACRO + Constant *VTableAddr = getVTableAddrForProfData(GV); + const std::string PGOVTableName = getPGOName(*GV); + // Record the length of the vtable. This is needed since vtable pointers + // loaded from C++ objects might be from the middle of a vtable definition. + uint32_t VTableSizeVal = + M.getDataLayout().getTypeAllocSize(GV->getValueType()); + + Constant *DataVals[] = { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init, +#include "llvm/ProfileData/InstrProfData.inc" +#undef INSTR_PROF_VTABLE_DATA + }; + + auto *Data = + new GlobalVariable(M, DataTy, /*constant=*/false, Linkage, + ConstantStruct::get(DataTy, DataVals), + getInstrProfVTableVarPrefix() + PGOVTableName); + + Data->setVisibility(Visibility); + Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat())); + Data->setAlignment(Align(8)); + + maybeSetComdat(Data, GV, Data->getName()); + + VTableDataMap[GV] = Data; + + ReferencedVTables.push_back(GV); + + // VTable is used by runtime but not referenced by other + // sections. Conservatively mark it linker retained. + UsedVars.push_back(Data); +} + +GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK) { GlobalVariable *NamePtr = Inc->getName(); // Match the linkage and visibility of the name global. @@ -1497,7 +1792,32 @@ void InstrProfiling::emitNameData() { NamePtr->eraseFromParent(); } -void InstrProfiling::emitRegistration() { +void InstrLowerer::emitVTableNames() { + if (!EnableVTableValueProfiling || ReferencedVTables.empty()) + return; + + // Collect the PGO names of referenced vtables and compress them. + std::string CompressedVTableNames; + if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames, + DoInstrProfNameCompression)) { + report_fatal_error(Twine(toString(std::move(E))), false); + } + + auto &Ctx = M.getContext(); + auto *VTableNamesVal = ConstantDataArray::getString( + Ctx, StringRef(CompressedVTableNames), false /* AddNull */); + GlobalVariable *VTableNamesVar = + new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */, + GlobalValue::PrivateLinkage, VTableNamesVal, + getInstrProfVTableNamesVarName()); + VTableNamesVar->setSection( + getInstrProfSectionName(IPSK_vname, TT.getObjectFormat())); + VTableNamesVar->setAlignment(Align(1)); + // Make VTableNames linker retained. + UsedVars.push_back(VTableNamesVar); +} + +void InstrLowerer::emitRegistration() { if (!needsRuntimeRegistrationOfSectionRange(TT)) return; diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index cd439818db80..c5a68f355941 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -327,6 +327,9 @@ extern cl::opt PGOViewCounts; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; +// Command line option to enable vtable value profiling. Defined in +// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= +extern cl::opt EnableVTableValueProfiling; extern cl::opt ProfileCorrelate; } // namespace llvm @@ -585,6 +588,8 @@ public: NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size(); NumOfPGOBB += MST.BBInfos.size(); ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget); + if (EnableVTableValueProfiling) + ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget); } else { NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size(); @@ -1801,6 +1806,15 @@ static bool InstrumentAllFunctions( // (before LTO/ThinLTO linking) to create these variables. if (!IsCS) createIRLevelProfileFlagVar(M, /*IsCS=*/false); + + Triple TT(M.getTargetTriple()); + LLVMContext &Ctx = M.getContext(); + if (!TT.isOSBinFormatELF() && EnableVTableValueProfiling) + Ctx.diagnose(DiagnosticInfoPGOProfile( + M.getName().data(), + Twine("VTable value profiling is presently not " + "supported for non-ELF object formats"), + DS_Warning)); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers); diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc index 3a129de1acd0..b47ef8523ea1 100644 --- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc +++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc @@ -90,9 +90,38 @@ public: } }; +///--------------------- VirtualTableValueProfilingPlugin -------------------- +class VTableProfilingPlugin { + Function &F; + +public: + static constexpr InstrProfValueKind Kind = IPVK_VTableTarget; + + VTableProfilingPlugin(Function &Fn, TargetLibraryInfo &TLI) : F(Fn) {} + + void run(std::vector &Candidates) { + std::vector Result = findVTableAddrs(F); + for (Instruction *I : Result) { + Instruction *InsertPt = I->getNextNonDebugInstruction(); + // When finding an insertion point, keep PHI and EH pad instructions + // before vp intrinsics. This is similar to + // `BasicBlock::getFirstInsertionPt`. + while (InsertPt && (dyn_cast(InsertPt) || InsertPt->isEHPad())) + InsertPt = InsertPt->getNextNonDebugInstruction(); + // Skip instrumentating the value if InsertPt is the last instruction. + // FIXME: Set InsertPt to the end of basic block to instrument the value + // if InsertPt is the last instruction. + if (InsertPt == nullptr) + continue; + + Instruction *AnnotatedInst = I; + Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst}); + } + } +}; + ///----------------------- Registration of the plugins ------------------------- /// For now, registering a plugin with the ValueProfileCollector is done by /// adding the plugin type to the VP_PLUGIN_LIST macro. -#define VP_PLUGIN_LIST \ - MemIntrinsicPlugin, \ - IndirectCallPromotionPlugin +#define VP_PLUGIN_LIST \ + MemIntrinsicPlugin, IndirectCallPromotionPlugin, VTableProfilingPlugin diff --git a/llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll b/llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll new file mode 100644 index 000000000000..f72a20fdc71a --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll @@ -0,0 +1,34 @@ +; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S 2>&1 | FileCheck %s + +; Test that unsupported warning is emitted for non-ELF object files. +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx14.0.0" + +; CHECK: warning: {{.*}} VTable value profiling is presently not supported for non-ELF object formats + +@_ZTV4Base = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN4Base4funcEi] }, !type !0, !type !1 +@_ZTV7Derived = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN7Derived4funcEi] }, !type !0, !type !1, !type !2, !type !3 + +@llvm.compiler.used = appending global [2 x ptr] [ptr @_ZTV4Base, ptr @_ZTV7Derived], section "llvm.metadata" + +define i32 @_Z4funci(i32 %a) { +entry: + %call = call ptr @_Z10createTypev() + %vtable = load ptr, ptr %call + %0 = call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS7Derived") + call void @llvm.assume(i1 %0) + %1 = load ptr, ptr %vtable + %call1 = call i32 %1(ptr %call, i32 %a) + ret i32 %call1 +} + +declare ptr @_Z10createTypev() +declare i1 @llvm.public.type.test(ptr, metadata) +declare void @llvm.assume(i1) +declare i32 @_ZN4Base4funcEi(ptr, i32) +declare i32 @_ZN7Derived4funcEi(ptr , i32) + +!0 = !{i64 16, !"_ZTS4Base"} +!1 = !{i64 16, !"_ZTSM4BaseFiiE.virtual"} +!2 = !{i64 16, !"_ZTS7Derived"} +!3 = !{i64 16, !"_ZTSM7DerivedFiiE.virtual"} diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll new file mode 100644 index 000000000000..a8440031e149 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll @@ -0,0 +1,98 @@ +; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S 2>&1 | FileCheck %s --check-prefix=GEN --implicit-check-not="VTable value profiling is presently not supported" +; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S 2>&1 | FileCheck %s --check-prefix=LOWER --implicit-check-not="VTable value profiling is presently not supported" + +; __llvm_prf_vnm stores zlib-compressed vtable names. +; REQUIRES: zlib + +source_filename = "vtable_local.ll" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; The test IR is generated based on the following C++ program. +; Base1 has external linkage and Base2 has local linkage. +; class Derived uses multiple inheritance so its virtual table +; global variable contains two vtables. func1 is loaded from +; the vtable compatible with class Base1, and func2 is loaded +; from the vtable compatible with class Base2. + +; class Base1 { +; public: +; virtual int func1(int a) ; +; }; +; +; namespace { +; class Base2 { +; public: +; __attribute__((noinline)) virtual int func2(int a) { +; return a; +; } +; }; +; } + +; class Derived : public Base1, public Base2 { +; public: +; Derived(int c) : v(c) {} +; private: +; int v; +; }; +; +; Derived* createType(); + +; int func(int a) { +; Derived* d = createType(); +; return d->func2(a) + d->func1(a); +; } + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@_ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !0, !type !3, !type !6, !type !8, !type !10 +@_ZTV5Base1 = available_externally constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei] }, !type !0 +@_ZTVN12_GLOBAL__N_15Base2E = internal constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !11, !type !8; !vcall_visibility !12 +@llvm.compiler.used = appending global [1 x ptr] [ptr @_ZTV5Base1], section "llvm.metadata" + +; GEN: __llvm_profile_raw_version = comdat any +; GEN: __llvm_profile_raw_version = hidden constant i64 72057594037927946, comdat +; GEN: __profn__Z4funci = private constant [8 x i8] c"_Z4funci" + +; LOWER: $__profvt__ZTV7Derived = comdat nodeduplicate +; LOWER: $"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = comdat nodeduplicate +; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8 +; LOWER: @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = internal global { i64, ptr, i32 } { i64 1419990121885302679, ptr @_ZTVN12_GLOBAL__N_15Base2E, i32 24 }, section "__llvm_prf_vtab", comdat, align 8 +; LOWER: @__llvm_prf_vnm = private constant [64 x i8] c"7>x\DA\8B\8F\0A\093wI-\CA,KMa,+IL\CAI\8D\CF\C9ON\CC\D1\CB\C9\B1\8E\07J\FA\19\1A\C5\BB\FB\F8;9\FA\C4\C7\FB\C5\1B\9A:%\16\A7\1A\B9\02\00\19:\12o", section "__llvm_prf_vns", align 1 +; LOWER: @llvm.used = appending global [5 x ptr] [ptr @__profvt__ZTV7Derived, ptr @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E", ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata" + +define i32 @_Z4funci(i32 %a) { +entry: + %call = call ptr @_Z10createTypev() + %add.ptr = getelementptr inbounds i8, ptr %call, i64 8 + %vtable = load ptr, ptr %add.ptr +; GEN: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64 +; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash:[0-9]+]], i64 [[P1]], i32 2, i32 0) +; LOWER: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64 +; LOWER: call void @__llvm_profile_instrument_target(i64 [[P1]], ptr @__profd__Z4funci, i32 2) + %vfunc1 = load ptr, ptr %vtable + %call1 = call i32 %vfunc1(ptr %add.ptr, i32 %a) + %vtable2 = load ptr, ptr %call +; GEN: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64 +; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash]], i64 [[P2]], i32 2, i32 1) +; LOWER: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64 +; LOWER: call void @__llvm_profile_instrument_target(i64 [[P2]], ptr @__profd__Z4funci, i32 3) + %vfunc2 = load ptr, ptr %vtable2 + %call4 = call i32 %vfunc2(ptr %call, i32 %a) + %add = add nsw i32 %call1, %call4 + ret i32 %add +} + +declare ptr @_Z10createTypev() +declare i32 @_ZN12_GLOBAL__N_15Base25func2Ei(ptr %this, i32 %a) +declare i32 @_ZN5Base15func1Ei(ptr, i32) + +!0 = !{i64 16, !"_ZTS5Base1"} +!3 = !{i64 16, !"_ZTS7Derived"} +!6 = !{i64 40, !7} +!7 = distinct !{} +!8 = !{i64 16, !9} +!9 = distinct !{} +!10 = !{i64 40, !9} +!11 = !{i64 16, !7} diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext new file mode 100644 index 000000000000..372f9f97b164 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext @@ -0,0 +1,74 @@ +# IR level Instrumentation Flag +:ir +_Z10createTypei +# Func Hash: +146835647075900052 +# Num Counters: +2 +# Counter Values: +750 +250 + +_ZN8Derived15func1Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +250 + +_ZN8Derived15func2Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +250 + +main +# Func Hash: +1124236338992350536 +# Num Counters: +2 +# Counter Values: +1000 +1 +# Num Value Kinds: +2 +# ValueKind = IPVK_IndirectCallTarget: +0 +# NumValueSites: +2 +2 +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750 +_ZN8Derived15func1Eii:250 +2 +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750 +_ZN8Derived15func2Eii:250 +# ValueKind = IPVK_VTableTarget: +2 +# NumValueSites: +2 +2 +vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +_ZTV8Derived1:250 +2 +vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +_ZTV8Derived1:250 + +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +750 + +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +750 + diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof.test b/llvm/test/tools/llvm-profdata/vtable-value-prof.test new file mode 100644 index 000000000000..378c2e11b236 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/vtable-value-prof.test @@ -0,0 +1,83 @@ +; RUN: rm -rf %t && mkdir %t && cd %t + +; Generate indexed profiles from text profiles +RUN: llvm-profdata merge %S/Inputs/vtable-value-prof.proftext -o indexed.profdata + +; Show indexed profiles +RUN: llvm-profdata show --function=main --ic-targets --show-vtables indexed.profdata | FileCheck %s --check-prefix=INDEXED + +; Show text profiles +RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %S/Inputs/vtable-value-prof.proftext | FileCheck %s --check-prefix=ICTEXT + +; Convert indexed profiles to its textual output and show it. +RUN: llvm-profdata merge --text -o text-from-indexed.proftext indexed.profdata +RUN: llvm-profdata show --function=main --ic-targets --show-vtables text-from-indexed.proftext | FileCheck %s --check-prefix=INDEXED +RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text text-from-indexed.proftext | FileCheck %s --check-prefix=ICTEXT + +INDEXED: Counters: +INDEXED-NEXT: main: +INDEXED-NEXT: Hash: 0x0f9a16fe6d398548 +INDEXED-NEXT: Counters: 2 +INDEXED-NEXT: Indirect Call Site Count: 2 +INDEXED-NEXT: Number of instrumented vtables: 2 +INDEXED-NEXT: Indirect Target Results: +INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) +INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) +INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +INDEXED-NEXT: VTable Results: +INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) +INDEXED-NEXT: Instrumentation level: IR entry_first = 0 +INDEXED-NEXT: Functions shown: 1 +INDEXED-NEXT: Total functions: 6 +INDEXED-NEXT: Maximum function count: 1000 +INDEXED-NEXT: Maximum internal block count: 250 +INDEXED-NEXT: Statistics for indirect call sites profile: +INDEXED-NEXT: Total number of sites: 2 +INDEXED-NEXT: Total number of sites with values: 2 +INDEXED-NEXT: Total number of profiled values: 4 +INDEXED-NEXT: Value sites histogram: +INDEXED-NEXT: NumTargets, SiteCount +INDEXED-NEXT: 2, 2 +INDEXED-NEXT: Statistics for vtable profile: +INDEXED-NEXT: Total number of sites: 2 +INDEXED-NEXT: Total number of sites with values: 2 +INDEXED-NEXT: Total number of profiled values: 4 +INDEXED-NEXT: Value sites histogram: +INDEXED-NEXT: NumTargets, SiteCount +INDEXED-NEXT: 2, 2 + +ICTEXT: :ir +ICTEXT: main +ICTEXT: # Func Hash: +ICTEXT: 1124236338992350536 +ICTEXT: # Num Counters: +ICTEXT: 2 +ICTEXT: # Counter Values: +ICTEXT: 1000 +ICTEXT: 1 +ICTEXT: # Num Value Kinds: +ICTEXT: 2 +ICTEXT: # ValueKind = IPVK_IndirectCallTarget: +ICTEXT: 0 +ICTEXT: # NumValueSites: +ICTEXT: 2 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750 +ICTEXT: _ZN8Derived15func1Eii:250 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750 +ICTEXT: _ZN8Derived15func2Eii:250 +ICTEXT: # ValueKind = IPVK_VTableTarget: +ICTEXT: 2 +ICTEXT: # NumValueSites: +ICTEXT: 2 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +ICTEXT: _ZTV8Derived1:250 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +ICTEXT: _ZTV8Derived1:250 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 911f9ab06ca4..646a8df18b9d 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -339,6 +339,9 @@ cl::opt ShowIndirectCallTargets( "ic-targets", cl::init(false), cl::desc("Show indirect call site target values for shown functions"), cl::sub(ShowSubcommand)); +cl::opt ShowVTables("show-vtables", cl::init(false), + cl::desc("Show vtable names for shown functions"), + cl::sub(ShowSubcommand)); cl::opt ShowMemOPSizes( "memop-sizes", cl::init(false), cl::desc("Show the profiled sizes of the memory intrinsic calls " @@ -708,6 +711,13 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, }); } + const InstrProfSymtab &symtab = Reader->getSymtab(); + const auto &VTableNames = symtab.getVTableNames(); + + for (const auto &kv : VTableNames) { + WC->Writer.addVTableName(kv.getKey()); + } + if (Reader->hasTemporalProfile()) { auto &Traces = Reader->getTemporalProfTraces(Input.Weight); if (!Traces.empty()) @@ -2934,6 +2944,10 @@ static int showInstrProfile( OS << " Indirect Call Site Count: " << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; + if (ShowVTables) + OS << " Number of instrumented vtables: " + << Func.getNumValueSites(IPVK_VTableTarget) << "\n"; + uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize); if (ShowMemOPSizes && NumMemOPCalls > 0) OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls @@ -2955,6 +2969,13 @@ static int showInstrProfile( &(Reader->getSymtab())); } + if (ShowVTables) { + OS << " VTable Results:\n"; + traverseAllValueSites(Func, IPVK_VTableTarget, + VPStats[IPVK_VTableTarget], OS, + &(Reader->getSymtab())); + } + if (ShowMemOPSizes && NumMemOPCalls > 0) { OS << " Memory Intrinsic Size Results:\n"; traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS, @@ -3003,6 +3024,11 @@ static int showInstrProfile( VPStats[IPVK_IndirectCallTarget]); } + if (ShownFunctions && ShowVTables) { + OS << "Statistics for vtable profile:\n"; + showValueSitesStats(OS, IPVK_VTableTarget, VPStats[IPVK_VTableTarget]); + } + if (ShownFunctions && ShowMemOPSizes) { OS << "Statistics for memory intrinsic calls sizes profile:\n"; showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]); diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 76f88c058424..867be90c9c67 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -633,56 +633,153 @@ TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) { Succeeded()); } +// callee1 to callee6 are from vtable1 to vtable6 respectively. static const char callee1[] = "callee1"; static const char callee2[] = "callee2"; static const char callee3[] = "callee3"; static const char callee4[] = "callee4"; static const char callee5[] = "callee5"; static const char callee6[] = "callee6"; +// callee7 and callee8 are not from any vtables. +static const char callee7[] = "callee7"; +static const char callee8[] = "callee8"; +// 'callee' is primarily used to create multiple-element vtables. +static const char callee[] = "callee"; +static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)}; +static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)}; +static const uint64_t vtable3[] = { + uint64_t(callee), + uint64_t(callee3), +}; +static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)}; +static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)}; +static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)}; + +// Returns the address of callee with a numbered suffix in vtable. +static uint64_t getCalleeAddress(const uint64_t *vtableAddr) { + uint64_t CalleeAddr; + // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3, + // and the 1st element in the rest of vtables. + if (vtableAddr == vtable1 || vtableAddr == vtable3) + CalleeAddr = uint64_t(vtableAddr) + 8; + else + CalleeAddr = uint64_t(vtableAddr); + return CalleeAddr; +} -TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) { +TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) { NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); - // 4 value sites. - Record1.reserveSites(IPVK_IndirectCallTarget, 4); - InstrProfValueData VD0[] = { - {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; - Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); - // No value profile data at the second site. - Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; - Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); - InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; - Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + // 4 indirect call value sites. + { + Record1.reserveSites(IPVK_IndirectCallTarget, 4); + InstrProfValueData VD0[] = { + {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; + Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); + // No value profile data at the second site. + Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); + InstrProfValueData VD3[] = {{(uint64_t)callee7, 1}, {(uint64_t)callee8, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); + } + + // 2 vtable value sites. + { + InstrProfValueData VD0[] = { + {getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}, + }; + InstrProfValueData VD2[] = { + {getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + }; + Record1.addValueData(IPVK_VTableTarget, 0, VD0, 3, nullptr); + Record1.addValueData(IPVK_VTableTarget, 2, VD2, 2, nullptr); + } Writer.addRecord(std::move(Record1), Err); Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee8", 0x1235, {3, 4}}, Err); + + // Set writer value prof data endianness. + Writer.setValueProfDataEndianness(getEndianness()); + auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); Expected R = Reader->getInstrProfRecord("caller", 0x1234); - EXPECT_THAT_ERROR(R.takeError(), Succeeded()); + ASSERT_THAT_ERROR(R.takeError(), Succeeded()); + + // Test the number of instrumented indirect call sites and the number of + // profiled values at each site. ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); - ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); - ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); - ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + EXPECT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + + // Test the number of instrumented vtable sites and the number of profiled + // values at each site. + ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U); + EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 3U); + EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 2U); + + // First indirect site. + { + uint64_t TotalC; + auto VD = R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); + + EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); + EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); + EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); + EXPECT_EQ(TotalC, 6U * getProfWeight()); + + EXPECT_STREQ((const char *)VD[0].Value, "callee3"); + EXPECT_STREQ((const char *)VD[1].Value, "callee2"); + EXPECT_STREQ((const char *)VD[2].Value, "callee1"); + } - uint64_t TotalC; - std::unique_ptr VD = - R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); + // First vtable site. + { + uint64_t TotalC; + auto VD = R->getValueForSite(IPVK_VTableTarget, 0, &TotalC); + + EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); + EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); + EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); + EXPECT_EQ(TotalC, 6U * getProfWeight()); + + EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1)); + } + + // Second vtable site. + { + uint64_t TotalC; + auto VD = R->getValueForSite(IPVK_VTableTarget, 1, &TotalC); - ASSERT_EQ(3U, VD[0].Count); - ASSERT_EQ(2U, VD[1].Count); - ASSERT_EQ(1U, VD[2].Count); - ASSERT_EQ(6U, TotalC); + EXPECT_EQ(VD[0].Count, 2U * getProfWeight()); + EXPECT_EQ(VD[1].Count, 1U * getProfWeight()); + EXPECT_EQ(TotalC, 3U * getProfWeight()); - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); + EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1)); + } } +INSTANTIATE_TEST_SUITE_P( + WeightAndEndiannessTest, InstrProfReaderWriterTest, + ::testing::Combine( + ::testing::Bool(), /* Sparse */ + ::testing::Values(1U, 10U), /* ProfWeight */ + ::testing::Values(llvm::endianness::big, + llvm::endianness::little) /* Endianness */ + )); TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { NamedInstrProfRecord Record("caller", 0x1234, {1, 2}); @@ -780,123 +877,53 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { ASSERT_EQ(1U, ValueData[3].Count); } -TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) { - NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); - - // 4 value sites. - Record1.reserveSites(IPVK_IndirectCallTarget, 4); - InstrProfValueData VD0[] = { - {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; - Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); - // No value profile data at the second site. - Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; - Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); - InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; - Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); - - Writer.addRecord(std::move(Record1), 10, Err); - Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); - Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); - Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); - auto Profile = Writer.writeBuffer(); - readProfile(std::move(Profile)); - - Expected R = Reader->getInstrProfRecord("caller", 0x1234); - EXPECT_THAT_ERROR(R.takeError(), Succeeded()); - ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); - ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); - ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); - ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); - - uint64_t TotalC; - std::unique_ptr VD = - R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); - ASSERT_EQ(30U, VD[0].Count); - ASSERT_EQ(20U, VD[1].Count); - ASSERT_EQ(10U, VD[2].Count); - ASSERT_EQ(60U, TotalC); - - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); -} - -TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) { - NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); - - // 4 value sites. - Record1.reserveSites(IPVK_IndirectCallTarget, 4); - InstrProfValueData VD0[] = { - {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; - Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); - // No value profile data at the second site. - Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; - Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); - InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; - Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); - - Writer.addRecord(std::move(Record1), Err); - Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); - Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); - Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); - - // Set big endian output. - Writer.setValueProfDataEndianness(support::big); - - auto Profile = Writer.writeBuffer(); - readProfile(std::move(Profile)); - - // Set big endian input. - Reader->setValueProfDataEndianness(support::big); - - Expected R = Reader->getInstrProfRecord("caller", 0x1234); - EXPECT_THAT_ERROR(R.takeError(), Succeeded()); - ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); - ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); - ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); - ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); - - std::unique_ptr VD = - R->getValueForSite(IPVK_IndirectCallTarget, 0); - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); - - // Restore little endian default: - Writer.setValueProfDataEndianness(support::little); -} - -TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { +TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { static const char caller[] = "caller"; NamedInstrProfRecord Record11(caller, 0x1234, {1, 2}); NamedInstrProfRecord Record12(caller, 0x1234, {1, 2}); - // 5 value sites. - Record11.reserveSites(IPVK_IndirectCallTarget, 5); - InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}, - {uint64_t(callee4), 4}}; - Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr); + // 5 value sites for indirect calls. + { + Record11.reserveSites(IPVK_IndirectCallTarget, 5); + InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}, + {uint64_t(callee4), 4}}; + Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr); - // No value profile data at the second site. - Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + // No value profile data at the second site. + Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = { - {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; - Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); + InstrProfValueData VD2[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); - InstrProfValueData VD3[] = {{uint64_t(callee1), 1}}; - Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + InstrProfValueData VD3[] = {{uint64_t(callee7), 1}, {uint64_t(callee8), 2}}; + Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); - InstrProfValueData VD4[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}}; - Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); + InstrProfValueData VD4[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); + } + // 3 value sites for vtables. + { + Record11.reserveSites(IPVK_VTableTarget, 3); + InstrProfValueData VD0[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}, + {getCalleeAddress(vtable4), 4}}; + Record11.addValueData(IPVK_VTableTarget, 0, VD0, 4, nullptr); + + InstrProfValueData VD2[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}}; + Record11.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr); + + InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}}; + Record11.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr); + } // A different record for the same caller. Record12.reserveSites(IPVK_IndirectCallTarget, 5); @@ -912,11 +939,28 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr); - InstrProfValueData VD42[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}}; + InstrProfValueData VD42[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr); + // 3 value sites for vtables. + { + Record12.reserveSites(IPVK_VTableTarget, 3); + InstrProfValueData VD0[] = {{getCalleeAddress(vtable2), 5}, + {getCalleeAddress(vtable3), 3}}; + Record12.addValueData(IPVK_VTableTarget, 0, VD0, 2, nullptr); + + InstrProfValueData VD2[] = {{getCalleeAddress(vtable2), 1}, + {getCalleeAddress(vtable3), 3}, + {getCalleeAddress(vtable4), 4}}; + Record12.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr); + + InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}}; + Record12.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr); + } + Writer.addRecord(std::move(Record11), Err); // Merge profile data. Writer.addRecord(std::move(Record12), Err); @@ -926,53 +970,95 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { Writer.addRecord({callee3, 0x1235, {3, 4}}, Err); Writer.addRecord({callee3, 0x1235, {3, 4}}, Err); Writer.addRecord({callee4, 0x1235, {3, 5}}, Err); + Writer.addRecord({callee7, 0x1235, {3, 5}}, Err); + Writer.addRecord({callee8, 0x1235, {3, 5}}, Err); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); + // Test the number of instrumented value sites and the number of profiled + // values for each site. Expected R = Reader->getInstrProfRecord("caller", 0x1234); EXPECT_THAT_ERROR(R.takeError(), Succeeded()); + // For indirect calls. ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget)); ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); + // For vtables. + ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U); + ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 4U); + ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); + ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); + + // Test the merged values for indirect calls. + { + auto VD = R->getValueForSite(IPVK_IndirectCallTarget, 0); + EXPECT_STREQ((const char *)VD[0].Value, "callee2"); + EXPECT_EQ(VD[0].Count, 7U); + EXPECT_STREQ((const char *)VD[1].Value, "callee3"); + EXPECT_EQ(VD[1].Count, 6U); + EXPECT_STREQ((const char *)VD[2].Value, "callee4"); + EXPECT_EQ(VD[2].Count, 4U); + EXPECT_STREQ((const char *)VD[3].Value, "callee1"); + EXPECT_EQ(VD[3].Count, 1U); + + auto VD_2(R->getValueForSite(IPVK_IndirectCallTarget, 2)); + EXPECT_STREQ((const char *)VD_2[0].Value, "callee3"); + EXPECT_EQ(VD_2[0].Count, 6U); + EXPECT_STREQ((const char *)VD_2[1].Value, "callee4"); + EXPECT_EQ(VD_2[1].Count, 4U); + EXPECT_STREQ((const char *)VD_2[2].Value, "callee2"); + EXPECT_EQ(VD_2[2].Count, 3U); + EXPECT_STREQ((const char *)VD_2[3].Value, "callee1"); + EXPECT_EQ(VD_2[3].Count, 1U); + + auto VD_3(R->getValueForSite(IPVK_IndirectCallTarget, 3)); + EXPECT_STREQ((const char *)VD_3[0].Value, "callee8"); + EXPECT_EQ(VD_3[0].Count, 2U); + EXPECT_STREQ((const char *)VD_3[1].Value, "callee7"); + EXPECT_EQ(VD_3[1].Count, 1U); + + auto VD_4(R->getValueForSite(IPVK_IndirectCallTarget, 4)); + EXPECT_STREQ((const char *)VD_4[0].Value, "callee3"); + EXPECT_EQ(VD_4[0].Count, 6U); + EXPECT_STREQ((const char *)VD_4[1].Value, "callee2"); + EXPECT_EQ(VD_4[1].Count, 4U); + EXPECT_STREQ((const char *)VD_4[2].Value, "callee1"); + EXPECT_EQ(VD_4[2].Count, 2U); + } - std::unique_ptr VD = - R->getValueForSite(IPVK_IndirectCallTarget, 0); - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(7U, VD[0].Count); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD[1].Count); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4")); - ASSERT_EQ(4U, VD[2].Count); - ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD[3].Count); - - std::unique_ptr VD_2( - R->getValueForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD_2[0].Count); - ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4")); - ASSERT_EQ(4U, VD_2[1].Count); - ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2")); - ASSERT_EQ(3U, VD_2[2].Count); - ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD_2[3].Count); - - std::unique_ptr VD_3( - R->getValueForSite(IPVK_IndirectCallTarget, 3)); - ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD_3[0].Count); - - std::unique_ptr VD_4( - R->getValueForSite(IPVK_IndirectCallTarget, 4)); - ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD_4[0].Count); - ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(4U, VD_4[1].Count); - ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1")); - ASSERT_EQ(2U, VD_4[2].Count); + // Test the merged values for vtables + { + auto VD0 = R->getValueForSite(IPVK_VTableTarget, 0); + EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD0[0].Count, 7U); + EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD0[1].Count, 6U); + EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD0[2].Count, 4U); + EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD0[3].Count, 1U); + + auto VD1 = R->getValueForSite(IPVK_VTableTarget, 1); + EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD1[0].Count, 6U); + EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD1[1].Count, 4U); + EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD1[2].Count, 3U); + EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD1[3].Count, 1U); + + auto VD2 = R->getValueForSite(IPVK_VTableTarget, 2); + EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD2[0].Count, 6U); + EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD2[1].Count, 4U); + EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD2[2].Count, 2U); + } } TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) { @@ -1083,27 +1169,66 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) { } } +INSTANTIATE_TEST_SUITE_P( + EdgeCaseTest, ValueProfileMergeEdgeCaseTest, + ::testing::Combine(::testing::Bool(), /* Sparse */ + ::testing::Values(IPVK_IndirectCallTarget, + IPVK_MemOPSize, + IPVK_VTableTarget) /* ValueKind */ + )); + static void addValueProfData(InstrProfRecord &Record) { - Record.reserveSites(IPVK_IndirectCallTarget, 5); - InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, - {uint64_t(callee2), 1000}, - {uint64_t(callee3), 500}, - {uint64_t(callee4), 300}, - {uint64_t(callee5), 100}}; - Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr); - InstrProfValueData VD1[] = {{uint64_t(callee5), 800}, - {uint64_t(callee3), 1000}, - {uint64_t(callee2), 2500}, - {uint64_t(callee1), 1300}}; - Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr); - InstrProfValueData VD2[] = {{uint64_t(callee6), 800}, - {uint64_t(callee3), 1000}, - {uint64_t(callee4), 5500}}; - Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); - InstrProfValueData VD3[] = {{uint64_t(callee2), 1800}, - {uint64_t(callee3), 2000}}; - Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); - Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); + // Add test data for indirect calls. + { + Record.reserveSites(IPVK_IndirectCallTarget, 6); + InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, + {uint64_t(callee2), 1000}, + {uint64_t(callee3), 500}, + {uint64_t(callee4), 300}, + {uint64_t(callee5), 100}}; + Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr); + InstrProfValueData VD1[] = {{uint64_t(callee5), 800}, + {uint64_t(callee3), 1000}, + {uint64_t(callee2), 2500}, + {uint64_t(callee1), 1300}}; + Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr); + InstrProfValueData VD2[] = {{uint64_t(callee6), 800}, + {uint64_t(callee3), 1000}, + {uint64_t(callee4), 5500}}; + Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); + InstrProfValueData VD3[] = {{uint64_t(callee2), 1800}, + {uint64_t(callee3), 2000}}; + Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); + Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); + InstrProfValueData VD5[] = {{uint64_t(callee7), 1234}, + {uint64_t(callee8), 5678}}; + Record.addValueData(IPVK_IndirectCallTarget, 5, VD5, 2, nullptr); + } + + // Add test data for vtables + { + Record.reserveSites(IPVK_VTableTarget, 4); + InstrProfValueData VD0[] = { + {getCalleeAddress(vtable1), 400}, {getCalleeAddress(vtable2), 1000}, + {getCalleeAddress(vtable3), 500}, {getCalleeAddress(vtable4), 300}, + {getCalleeAddress(vtable5), 100}, + }; + InstrProfValueData VD1[] = {{getCalleeAddress(vtable5), 800}, + {getCalleeAddress(vtable3), 1000}, + {getCalleeAddress(vtable2), 2500}, + {getCalleeAddress(vtable1), 1300}}; + InstrProfValueData VD2[] = { + {getCalleeAddress(vtable6), 800}, + {getCalleeAddress(vtable3), 1000}, + {getCalleeAddress(vtable4), 5500}, + }; + InstrProfValueData VD3[] = {{getCalleeAddress(vtable2), 1800}, + {getCalleeAddress(vtable3), 2000}}; + Record.addValueData(IPVK_VTableTarget, 0, VD0, 5, nullptr); + Record.addValueData(IPVK_VTableTarget, 1, VD1, 4, nullptr); + Record.addValueData(IPVK_VTableTarget, 2, VD2, 3, nullptr); + Record.addValueData(IPVK_VTableTarget, 3, VD3, 2, nullptr); + } } TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) { @@ -1116,59 +1241,107 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) { VPData->deserializeTo(Record, nullptr); // Now read data from Record and sanity check the data - ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget)); ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); + ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 5)); auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { return VD1.Count > VD2.Count; }; + std::unique_ptr VD_0( Record.getValueForSite(IPVK_IndirectCallTarget, 0)); llvm::sort(&VD_0[0], &VD_0[5], Cmp); - ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(1000U, VD_0[0].Count); - ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(500U, VD_0[1].Count); - ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1")); - ASSERT_EQ(400U, VD_0[2].Count); - ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4")); - ASSERT_EQ(300U, VD_0[3].Count); - ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5")); - ASSERT_EQ(100U, VD_0[4].Count); + EXPECT_STREQ((const char *)VD_0[0].Value, "callee2"); + EXPECT_EQ(1000U, VD_0[0].Count); + EXPECT_STREQ((const char *)VD_0[1].Value, "callee3"); + EXPECT_EQ(500U, VD_0[1].Count); + EXPECT_STREQ((const char *)VD_0[2].Value, "callee1"); + EXPECT_EQ(400U, VD_0[2].Count); + EXPECT_STREQ((const char *)VD_0[3].Value, "callee4"); + EXPECT_EQ(300U, VD_0[3].Count); + EXPECT_STREQ((const char *)VD_0[4].Value, "callee5"); + EXPECT_EQ(100U, VD_0[4].Count); std::unique_ptr VD_1( Record.getValueForSite(IPVK_IndirectCallTarget, 1)); llvm::sort(&VD_1[0], &VD_1[4], Cmp); - ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(2500U, VD_1[0].Count); - ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1")); - ASSERT_EQ(1300U, VD_1[1].Count); - ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3")); - ASSERT_EQ(1000U, VD_1[2].Count); - ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5")); - ASSERT_EQ(800U, VD_1[3].Count); + EXPECT_STREQ((const char *)VD_1[0].Value, "callee2"); + EXPECT_EQ(VD_1[0].Count, 2500U); + EXPECT_STREQ((const char *)VD_1[1].Value, "callee1"); + EXPECT_EQ(VD_1[1].Count, 1300U); + EXPECT_STREQ((const char *)VD_1[2].Value, "callee3"); + EXPECT_EQ(VD_1[2].Count, 1000U); + EXPECT_STREQ((const char *)VD_1[3].Value, "callee5"); + EXPECT_EQ(VD_1[3].Count, 800U); std::unique_ptr VD_2( Record.getValueForSite(IPVK_IndirectCallTarget, 2)); llvm::sort(&VD_2[0], &VD_2[3], Cmp); - ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4")); - ASSERT_EQ(5500U, VD_2[0].Count); - ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(1000U, VD_2[1].Count); - ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6")); - ASSERT_EQ(800U, VD_2[2].Count); + EXPECT_STREQ((const char *)VD_2[0].Value, "callee4"); + EXPECT_EQ(VD_2[0].Count, 5500U); + EXPECT_STREQ((const char *)VD_2[1].Value, "callee3"); + EXPECT_EQ(VD_2[1].Count, 1000U); + EXPECT_STREQ((const char *)VD_2[2].Value, "callee6"); + EXPECT_EQ(VD_2[2].Count, 800U); std::unique_ptr VD_3( Record.getValueForSite(IPVK_IndirectCallTarget, 3)); llvm::sort(&VD_3[0], &VD_3[2], Cmp); - ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(2000U, VD_3[0].Count); - ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(1800U, VD_3[1].Count); + EXPECT_STREQ((const char *)VD_3[0].Value, "callee3"); + EXPECT_EQ(VD_3[0].Count, 2000U); + EXPECT_STREQ((const char *)VD_3[1].Value, "callee2"); + EXPECT_EQ(VD_3[1].Count, 1800U); + + ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); + + auto VD0(Record.getValueForSite(IPVK_VTableTarget, 0)); + llvm::sort(&VD0[0], &VD0[5], Cmp); + EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD0[0].Count, 1000U); + EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD0[1].Count, 500U); + EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD0[2].Count, 400U); + EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD0[3].Count, 300U); + EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5)); + EXPECT_EQ(VD0[4].Count, 100U); + + auto VD1(Record.getValueForSite(IPVK_VTableTarget, 1)); + llvm::sort(&VD1[0], &VD1[4], Cmp); + EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD1[0].Count, 2500U); + EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD1[1].Count, 1300U); + EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD1[2].Count, 1000U); + EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5)); + EXPECT_EQ(VD1[3].Count, 800U); + + auto VD2(Record.getValueForSite(IPVK_VTableTarget, 2)); + llvm::sort(&VD2[0], &VD2[3], Cmp); + EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD2[0].Count, 5500U); + EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD2[1].Count, 1000U); + EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6)); + EXPECT_EQ(VD2[2].Count, 800U); + + auto VD3(Record.getValueForSite(IPVK_VTableTarget, 3)); + llvm::sort(&VD3[0], &VD3[2], Cmp); + EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD3[0].Count, 2000U); + EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD3[1].Count, 1800U); } TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) { @@ -1186,27 +1359,121 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) { Symtab.mapAddress(uint64_t(callee4), 0x4000ULL); // Missing mapping for callee5 + auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t { + return uint64_t(vtable); + }; + auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t { + return uint64_t(vtable) + 16; + }; + auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t { + return uint64_t(vtable) + 8; + }; + // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not + // mapped. + Symtab.mapVTableAddress(getVTableStartAddr(vtable1), + getVTableEndAddr(vtable1), MD5Hash("vtable1")); + Symtab.mapVTableAddress(getVTableStartAddr(vtable2), + getVTableEndAddr(vtable2), MD5Hash("vtable2")); + Symtab.mapVTableAddress(getVTableStartAddr(vtable3), + getVTableEndAddr(vtable3), MD5Hash("vtable3")); + Symtab.mapVTableAddress(getVTableStartAddr(vtable4), + getVTableEndAddr(vtable4), MD5Hash("vtable4")); + VPData->deserializeTo(Record, &Symtab); // Now read data from Record and sanity check the data - ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); - ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0), 5U); + + // Look up the value correpsonding to the middle of a vtable in symtab and + // test that it's the hash of the name. + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)), + MD5Hash("vtable1")); + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)), + MD5Hash("vtable2")); + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)), + MD5Hash("vtable3")); + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)), + MD5Hash("vtable4")); auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { return VD1.Count > VD2.Count; }; - std::unique_ptr VD_0( - Record.getValueForSite(IPVK_IndirectCallTarget, 0)); + auto VD_0(Record.getValueForSite(IPVK_IndirectCallTarget, 0)); llvm::sort(&VD_0[0], &VD_0[5], Cmp); ASSERT_EQ(VD_0[0].Value, 0x2000ULL); - ASSERT_EQ(1000U, VD_0[0].Count); + ASSERT_EQ(VD_0[0].Count, 1000U); ASSERT_EQ(VD_0[1].Value, 0x3000ULL); - ASSERT_EQ(500U, VD_0[1].Count); + ASSERT_EQ(VD_0[1].Count, 500U); ASSERT_EQ(VD_0[2].Value, 0x1000ULL); - ASSERT_EQ(400U, VD_0[2].Count); + ASSERT_EQ(VD_0[2].Count, 400U); // callee5 does not have a mapped value -- default to 0. ASSERT_EQ(VD_0[4].Value, 0ULL); + + // Sanity check the vtable value data + ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); + + { + // The first vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 0)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); + llvm::sort(&VD[0], &VD[5], Cmp); + EXPECT_EQ(VD[0].Count, 1000U); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable2")); + EXPECT_EQ(VD[1].Count, 500U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable3")); + EXPECT_EQ(VD[2].Value, MD5Hash("vtable1")); + EXPECT_EQ(VD[2].Count, 400U); + EXPECT_EQ(VD[3].Value, MD5Hash("vtable4")); + EXPECT_EQ(VD[3].Count, 300U); + + // vtable5 isn't mapped -- default to 0. + EXPECT_EQ(VD[4].Value, 0U); + EXPECT_EQ(VD[4].Count, 100U); + } + + { + // The second vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 1)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); + llvm::sort(&VD[0], &VD[4], Cmp); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable2")); + EXPECT_EQ(VD[0].Count, 2500U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable1")); + EXPECT_EQ(VD[1].Count, 1300U); + + EXPECT_EQ(VD[2].Value, MD5Hash("vtable3")); + EXPECT_EQ(VD[2].Count, 1000U); + // vtable5 isn't mapped -- default to 0. + EXPECT_EQ(VD[3].Value, 0U); + EXPECT_EQ(VD[3].Count, 800U); + } + + { + // The third vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 2)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); + llvm::sort(&VD[0], &VD[3], Cmp); + EXPECT_EQ(VD[0].Count, 5500U); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable4")); + EXPECT_EQ(VD[1].Count, 1000U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable3")); + // vtable6 isn't mapped -- default to 0. + EXPECT_EQ(VD[2].Value, 0U); + EXPECT_EQ(VD[2].Count, 800U); + } + + { + // The fourth vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 3)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); + llvm::sort(&VD[0], &VD[2], Cmp); + EXPECT_EQ(VD[0].Count, 2000U); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable3")); + EXPECT_EQ(VD[1].Count, 1800U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable2")); + } } TEST_P(MaybeSparseInstrProfTest, get_max_function_count) { @@ -1331,15 +1598,14 @@ TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) { std::string IRPGOName = getIRPGOFuncName(*F); auto IRPGOFuncName = - ProfSymtab.getFuncName(IndexedInstrProf::ComputeHash(IRPGOName)); - EXPECT_EQ(StringRef(IRPGOName), IRPGOFuncName); - EXPECT_EQ(StringRef(Funcs[I]), - getParsedIRPGOFuncName(IRPGOFuncName).second); + ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(IRPGOName)); + EXPECT_EQ(IRPGOName, IRPGOFuncName); + EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second); // Ensure we can still read this old record name. std::string PGOName = getPGOFuncName(*F); auto PGOFuncName = - ProfSymtab.getFuncName(IndexedInstrProf::ComputeHash(PGOName)); - EXPECT_EQ(StringRef(PGOName), PGOFuncName); + ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(PGOName)); + EXPECT_EQ(PGOName, PGOFuncName); EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str())); } } -- Gitee From 35a24cb02a8381232b888a473293166fefd0f47c Mon Sep 17 00:00:00 2001 From: Szymon Sobieszek Date: Thu, 7 Aug 2025 16:17:24 -0400 Subject: [PATCH 19/28] [DTP] Another batch of fixes related to porting dtp commits. --- llvm/include/llvm/ProfileData/InstrProf.h | 4 + .../llvm/ProfileData/InstrProfWriter.h | 10 +- .../Instrumentation/InstrProfiling.h | 14 +- llvm/lib/ProfileData/InstrProf.cpp | 10 +- llvm/lib/ProfileData/InstrProfWriter.cpp | 6 +- .../Instrumentation/InstrProfiling.cpp | 209 ++---------------- llvm/unittests/ProfileData/InstrProfTest.cpp | 36 ++- 7 files changed, 86 insertions(+), 203 deletions(-) diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 33937ad6469d..787a23b6af75 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -201,6 +201,10 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO = false); /// \c getIRPGOFuncName() std::pair getParsedIRPGOFuncName(StringRef IRPGOFuncName); +/// \return the filename and the function name parsed from the output of +/// \c getIRPGOFuncName() +std::pair getParsedIRPGOName(StringRef IRPGOName); + /// Return the name of the global variable used to store a function /// name in PGO instrumentation. \c FuncName is the name of the function /// returned by the \c getPGOFuncName call. diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index c38bc621469e..d58c465cfd88 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -72,10 +72,18 @@ private: // Use raw pointer here for the incomplete type object. InstrProfRecordWriterTrait *InfoObj; + // Temporary support for writing the previous version of the format, to enable + // some forward compatibility. Currently this suppresses the writing of the + // new vtable names section and header fields. + // TODO: Consider enabling this with future version changes as well, to ease + // deployment of newer versions of llvm-profdata. + bool WritePrevVersion = false; + public: InstrProfWriter(bool Sparse = false, uint64_t TemporalProfTraceReservoirSize = 0, - uint64_t MaxTemporalProfTraceLength = 0); + uint64_t MaxTemporalProfTraceLength = 0, + bool WritePrevVersion = false); ~InstrProfWriter(); StringMap &getProfileData() { return FunctionData; } diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index d8f3e75087ac..8399f894acd7 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -57,12 +57,18 @@ private: } }; DenseMap ProfileDataMap; + // Key is virtual table variable, value is 'VTableProfData' in the form of + // GlobalVariable. + DenseMap VTableDataMap; /// If runtime relocation is enabled, this maps functions to the load /// instruction that produces the profile relocation bias. DenseMap FunctionToProfileBiasMap; std::vector CompilerUsedVars; std::vector UsedVars; std::vector ReferencedNames; + // The list of virtual table variables of which the VTableProfData is + // collected. + std::vector ReferencedVTables; GlobalVariable *NamesVar; size_t NamesSize; @@ -149,7 +155,7 @@ private: GlobalValue::LinkageTypes Linkage); /// Set Comdat property of GV, if required. - void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName); + void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName); /// Setup the sections into which counters and bitmaps are allocated. GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, @@ -159,9 +165,15 @@ private: void createDataVariable(InstrProfCntrInstBase *Inc, InstrProfMCDCBitmapParameters *Update); + /// Get the counters for virtual table values, creating them if necessary. + void getOrCreateVTableProfData(GlobalVariable *GV); + /// Emit the section with compressed function names. void emitNameData(); + /// Emit the section with compressed vtable names. + void emitVTableNames(); + /// Emit value nodes section for value profiling. void emitVNodes(); diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 498d8708beaf..16083f399a13 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -422,7 +422,15 @@ std::string getPGOName(const GlobalVariable &V, bool InLTO) { // See getIRPGOObjectName() for a discription of the format. std::pair getParsedIRPGOFuncName(StringRef IRPGOName) { - auto [FileName, MangledName] = IRPGOFuncName.split(GlobalIdentifierDelimiter); + auto [FileName, MangledName] = IRPGOName.split(';'); + if (MangledName.empty()) + return std::make_pair(StringRef(), IRPGOName); + return std::make_pair(FileName, MangledName); +} + +// See getIRPGOObjectName() for a discription of the format. +std::pair getParsedIRPGOName(StringRef IRPGOName) { + auto [FileName, MangledName] = IRPGOName.split(';'); if (MangledName.empty()) return std::make_pair(StringRef(), IRPGOName); return std::make_pair(FileName, MangledName); diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 5534bc220356..b853f0a8dc66 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -181,10 +181,12 @@ public: InstrProfWriter::InstrProfWriter(bool Sparse, uint64_t TemporalProfTraceReservoirSize, - uint64_t MaxTemporalProfTraceLength) + uint64_t MaxTemporalProfTraceLength, + bool WritePrevVersion) : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength), TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize), - InfoObj(new InstrProfRecordWriterTrait()) {} + InfoObj(new InstrProfRecordWriterTrait()), + WritePrevVersion(WritePrevVersion) {} InstrProfWriter::~InstrProfWriter() { delete InfoObj; } diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 6493ae974cce..33742f5241ac 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -166,188 +166,6 @@ cl::opt SkipRetExitBlock( using LoadStorePair = std::pair; -static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { - auto *MD = dyn_cast_or_null(M.getModuleFlag(Flag)); - if (!MD) - return 0; - - // If the flag is a ConstantAsMetadata, it should be an integer representable - // in 64-bits. - return cast(MD->getValue())->getZExtValue(); -} - -static bool enablesValueProfiling(const Module &M) { - return isIRPGOFlagSet(&M) || - getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0; -} - -// Conservatively returns true if value profiling is enabled. -static bool profDataReferencedByCode(const Module &M) { - return enablesValueProfiling(M); -} - -class InstrLowerer final { -public: - InstrLowerer(Module &M, const InstrProfOptions &Options, - std::function GetTLI, - bool IsCS) - : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS), - GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {} - - bool lower(); - -private: - Module &M; - const InstrProfOptions Options; - const Triple TT; - // Is this lowering for the context-sensitive instrumentation. - const bool IsCS; - - std::function GetTLI; - - const bool DataReferencedByCode; - - struct PerFunctionProfileData { - uint32_t NumValueSites[IPVK_Last + 1] = {}; - GlobalVariable *RegionCounters = nullptr; - GlobalVariable *DataVar = nullptr; - GlobalVariable *RegionBitmaps = nullptr; - uint32_t NumBitmapBytes = 0; - - PerFunctionProfileData() = default; - }; - DenseMap ProfileDataMap; - // Key is virtual table variable, value is 'VTableProfData' in the form of - // GlobalVariable. - DenseMap VTableDataMap; - /// If runtime relocation is enabled, this maps functions to the load - /// instruction that produces the profile relocation bias. - DenseMap FunctionToProfileBiasMap; - std::vector CompilerUsedVars; - std::vector UsedVars; - std::vector ReferencedNames; - // The list of virtual table variables of which the VTableProfData is - // collected. - std::vector ReferencedVTables; - GlobalVariable *NamesVar = nullptr; - size_t NamesSize = 0; - - // vector of counter load/store pairs to be register promoted. - std::vector PromotionCandidates; - - int64_t TotalCountersPromoted = 0; - - /// Lower instrumentation intrinsics in the function. Returns true if there - /// any lowering. - bool lowerIntrinsics(Function *F); - - /// Register-promote counter loads and stores in loops. - void promoteCounterLoadStores(Function *F); - - /// Returns true if relocating counters at runtime is enabled. - bool isRuntimeCounterRelocationEnabled() const; - - /// Returns true if profile counter update register promotion is enabled. - bool isCounterPromotionEnabled() const; - - /// Count the number of instrumented value sites for the function. - void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins); - - /// Replace instrprof.value.profile with a call to runtime library. - void lowerValueProfileInst(InstrProfValueProfileInst *Ins); - - /// Replace instrprof.cover with a store instruction to the coverage byte. - void lowerCover(InstrProfCoverInst *Inc); - - /// Replace instrprof.timestamp with a call to - /// INSTR_PROF_PROFILE_SET_TIMESTAMP. - void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction); - - /// Replace instrprof.increment with an increment of the appropriate value. - void lowerIncrement(InstrProfIncrementInst *Inc); - - /// Force emitting of name vars for unused functions. - void lowerCoverageData(GlobalVariable *CoverageNamesVar); - - /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction - /// using the index represented by the a temp value into a bitmap. - void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); - - /// Replace instrprof.mcdc.temp.update with a shift and or instruction using - /// the corresponding condition ID. - void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); - - /// Compute the address of the counter value that this profiling instruction - /// acts on. - Value *getCounterAddress(InstrProfCntrInstBase *I); - - /// Get the region counters for an increment, creating them if necessary. - /// - /// If the counter array doesn't yet exist, the profile data variables - /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); - - /// Create the region counters. - GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, - StringRef Name, - GlobalValue::LinkageTypes Linkage); - - /// Compute the address of the test vector bitmap that this profiling - /// instruction acts on. - Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); - - /// Get the region bitmaps for an increment, creating them if necessary. - /// - /// If the bitmap array doesn't yet exist, the profile data variables - /// referring to them will also be created. - GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); - - /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with - /// an MC/DC Decision region. The number of bytes required is indicated by - /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called - /// as part of setupProfileSection() and is conceptually very similar to - /// what is done for profile data counters in createRegionCounters(). - GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, - StringRef Name, - GlobalValue::LinkageTypes Linkage); - - /// Set Comdat property of GV, if required. - void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName); - - /// Setup the sections into which counters and bitmaps are allocated. - GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, - InstrProfSectKind IPSK); - - /// Create INSTR_PROF_DATA variable for counters and bitmaps. - void createDataVariable(InstrProfCntrInstBase *Inc); - - /// Get the counters for virtual table values, creating them if necessary. - void getOrCreateVTableProfData(GlobalVariable *GV); - - /// Emit the section with compressed function names. - void emitNameData(); - - /// Emit the section with compressed vtable names. - void emitVTableNames(); - - /// Emit value nodes section for value profiling. - void emitVNodes(); - - /// Emit runtime registration functions for each profile data variable. - void emitRegistration(); - - /// Emit the necessary plumbing to pull in the runtime initialization. - /// Returns true if a change was made. - bool emitRuntimeHook(); - - /// Add uses of our data variables and runtime hook. - void emitUses(); - - /// Create a static initializer for our data, on platforms that need it, - /// and for any profile output file that was specified. - void emitInitialization(); -}; - /// /// A helper class to promote one counter RMW operation in the loop /// into register update. @@ -1254,14 +1072,15 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { return true; } -void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, - StringRef CounterGroupName) { +void InstrProfiling::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, + StringRef VarName) { // Place lowered global variables in a comdat group if the associated function // or global variable is a COMDAT. This will make sure that only one copy of // global variable (e.g. function counters) of the COMDAT function will be // emitted after linking. - bool NeedComdat = needsComdatForCounter(*GO, M); + bool NeedComdat = needsComdatForCounter(*GO, *M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); + const bool DataReferencedByCode = profDataReferencedByCode(*M); if (!UseComdat) return; @@ -1307,7 +1126,7 @@ static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) { return ConstantExpr::getBitCast(GV, Int8PtrTy); } -void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { +void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) { assert(!DebugInfoCorrelate && "Value profiling is not supported with lightweight instrumentation"); if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) @@ -1334,7 +1153,7 @@ void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { Visibility = GlobalValue::DefaultVisibility; } - LLVMContext &Ctx = M.getContext(); + LLVMContext &Ctx = M->getContext(); Type *DataTypes[] = { #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType, #include "llvm/ProfileData/InstrProfData.inc" @@ -1349,7 +1168,7 @@ void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { // Record the length of the vtable. This is needed since vtable pointers // loaded from C++ objects might be from the middle of a vtable definition. uint32_t VTableSizeVal = - M.getDataLayout().getTypeAllocSize(GV->getValueType()); + M->getDataLayout().getTypeAllocSize(GV->getValueType()); Constant *DataVals[] = { #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init, @@ -1358,7 +1177,7 @@ void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { }; auto *Data = - new GlobalVariable(M, DataTy, /*constant=*/false, Linkage, + new GlobalVariable(*M, DataTy, /*constant=*/false, Linkage, ConstantStruct::get(DataTy, DataVals), getInstrProfVTableVarPrefix() + PGOVTableName); @@ -1377,8 +1196,8 @@ void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { UsedVars.push_back(Data); } -GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, - InstrProfSectKind IPSK) { +GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK) { GlobalVariable *NamePtr = Inc->getName(); // Match the linkage and visibility of the name global. @@ -1792,7 +1611,7 @@ void InstrProfiling::emitNameData() { NamePtr->eraseFromParent(); } -void InstrLowerer::emitVTableNames() { +void InstrProfiling::emitVTableNames() { if (!EnableVTableValueProfiling || ReferencedVTables.empty()) return; @@ -1803,11 +1622,11 @@ void InstrLowerer::emitVTableNames() { report_fatal_error(Twine(toString(std::move(E))), false); } - auto &Ctx = M.getContext(); + auto &Ctx = M->getContext(); auto *VTableNamesVal = ConstantDataArray::getString( Ctx, StringRef(CompressedVTableNames), false /* AddNull */); GlobalVariable *VTableNamesVar = - new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */, + new GlobalVariable(*M, VTableNamesVal->getType(), true /* constant */, GlobalValue::PrivateLinkage, VTableNamesVal, getInstrProfVTableNamesVarName()); VTableNamesVar->setSection( @@ -1817,7 +1636,7 @@ void InstrLowerer::emitVTableNames() { UsedVars.push_back(VTableNamesVar); } -void InstrLowerer::emitRegistration() { +void InstrProfiling::emitRegistration() { if (!needsRuntimeRegistrationOfSectionRange(TT)) return; diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 867be90c9c67..39ee894f5bcd 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -68,6 +68,27 @@ struct SparseInstrProfTest : public InstrProfTest { void SetUp() override { Writer.setOutputSparse(true); } }; +struct InstrProfReaderWriterTest + : public InstrProfTest, + public ::testing::WithParamInterface< + std::tuple> { + void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); } + void TearDown() override { + // Reset writer value profile data endianness after each test case. Note + // it's not necessary to reset reader value profile endianness for each test + // case. Each test case creates a new reader; at reader initialization time, + // it uses the endianness from hash table object (which is little by + // default). + Writer.setValueProfDataEndianness(llvm::support::endianness::little); + } + + uint64_t getProfWeight() const { return std::get<1>(GetParam()); } + + llvm::support::endianness getEndianness() const { + return std::get<2>(GetParam()); + } +}; + struct MaybeSparseInstrProfTest : public InstrProfTest, public ::testing::WithParamInterface { void SetUp() override { Writer.setOutputSparse(GetParam()); } @@ -777,8 +798,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( ::testing::Bool(), /* Sparse */ ::testing::Values(1U, 10U), /* ProfWeight */ - ::testing::Values(llvm::endianness::big, - llvm::endianness::little) /* Endianness */ + ::testing::Values(llvm::support::endianness::big, + llvm::support::endianness::little) /* Endianness */ )); TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { @@ -1061,7 +1082,16 @@ TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { } } -TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) { +struct ValueProfileMergeEdgeCaseTest + : public InstrProfTest, + public ::testing::WithParamInterface> { + void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); } + + uint32_t getValueProfileKind() const { return std::get<1>(GetParam()); } +}; + +TEST_P(ValueProfileMergeEdgeCaseTest, get_icall_data_merge1_saturation) { + const uint32_t ValueKind = getValueProfileKind(); static const char bar[] = "bar"; const uint64_t MaxValCount = std::numeric_limits::max(); -- Gitee From a57af318cdece91c33a1b5e4569c6e53a1aad0f6 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 1 Apr 2024 15:14:49 -0700 Subject: [PATCH 20/28] [ThinLTO][TypeProf] Implement vtable def import (#79381) Add annotated vtable GUID as referenced variables in per function summary, and update bitcode writer to create value-ids for these referenced vtables. - This is the part3 of type profiling work, and described in the "Virtual Table Definition Import" [1] section of the RFC. [1] https://github.com/llvm/llvm-project/pull/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 --- llvm/include/llvm/ProfileData/InstrProf.h | 12 ++- .../IndirectCallPromotionAnalysis.cpp | 4 + llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 20 +++++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 13 +++- llvm/lib/ProfileData/InstrProf.cpp | 70 ++++++++++++------ .../thinlto-func-summary-vtableref-pgo.ll | 74 +++++++++++++++++++ 6 files changed, 169 insertions(+), 24 deletions(-) create mode 100644 llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 787a23b6af75..d4486572da67 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -293,7 +293,7 @@ void annotateValueSite(Module &M, Instruction &Inst, /// Extract the value profile data from \p Inst which is annotated with /// value profile meta data. Return false if there is no value data annotated, -/// otherwise return true. +/// otherwise return true. bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, @@ -301,6 +301,16 @@ bool getValueProfDataFromInst(const Instruction &Inst, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue = false); +/// Extract the value profile data from \p Inst and returns them if \p Inst is +/// annotated with value profile data. Returns nullptr otherwise. It's similar +/// to `getValueProfDataFromInst` above except that an array is allocated only +/// after a preliminary checking that the value profiles of kind `ValueKind` +/// exist. +std::unique_ptr +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue = false); + inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } inline StringRef getPGONameMetadataName() { return "PGOName"; } diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp index ebfa1c8fc08e..ab53717eb889 100644 --- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -45,6 +45,10 @@ static cl::opt cl::desc("Max number of promotions for a single indirect " "call callsite")); +cl::opt MaxNumVTableAnnotations( + "icp-max-num-vtables", cl::init(6), cl::Hidden, + cl::desc("Max number of vtables annotated for a vtable load instruction.")); + ICallPromotionAnalysis::ICallPromotionAnalysis() { ValueDataArray = std::make_unique(MaxNumPromotions); } diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 2076ed48ea34..dab3ce2e3a5a 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -82,6 +82,8 @@ static cl::opt ModuleSummaryDotFile( extern cl::opt ScalePartialSampleProfileWorkingSetSize; +extern cl::opt MaxNumVTableAnnotations; + // Walk through the operands of a given User via worklist iteration and populate // the set of GlobalValue references encountered. Invoked either on an // Instruction or a GlobalVariable (which walks its initializer). @@ -124,6 +126,24 @@ static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, Worklist.push_back(Operand); } } + + const Instruction *I = dyn_cast(CurUser); + if (I) { + uint32_t ActualNumValueData = 0; + uint64_t TotalCount = 0; + // MaxNumVTableAnnotations is the maximum number of vtables annotated on + // the instruction. + auto ValueDataArray = + getValueProfDataFromInst(*I, IPVK_VTableTarget, MaxNumVTableAnnotations, + ActualNumValueData, TotalCount); + + if (ValueDataArray.get()) { + for (uint32_t j = 0; j < ActualNumValueData; j++) { + RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */ + ValueDataArray[j].Value)); + } + } + } return HasBlockAddress; } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index d5bcd327a9b7..140a1624c6d5 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -198,7 +198,7 @@ public: for (const auto &GUIDSummaryLists : *Index) // Examine all summaries for this GUID. for (auto &Summary : GUIDSummaryLists.second.SummaryList) - if (auto FS = dyn_cast(Summary.get())) + if (auto FS = dyn_cast(Summary.get())) { // For each call in the function summary, see if the call // is to a GUID (which means it is for an indirect call, // otherwise we would have a Value for it). If so, synthesize @@ -206,6 +206,15 @@ public: for (auto &CallEdge : FS->calls()) if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue()) assignValueId(CallEdge.first.getGUID()); + + // For each referenced variables in the function summary, see if the + // variable is represented by a GUID (as opposed to a symbol to + // declarations or definitions in the module). If so, synthesize a + // value id. + for (auto &RefEdge : FS->refs()) + if (!RefEdge.haveGVs() || !RefEdge.getValue()) + assignValueId(RefEdge.getGUID()); + } } protected: @@ -3996,7 +4005,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( NameVals.push_back(SpecialRefCnts.second); // worefcnt for (auto &RI : FS->refs()) - NameVals.push_back(VE.getValueID(RI.getValue())); + NameVals.push_back(getValueId(RI)); bool HasProfileData = F.hasProfileData() || ForceSummaryEdgesCold != FunctionSummary::FSHT_None; diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 16083f399a13..b4cc5dfbe4f4 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1373,46 +1373,44 @@ void annotateValueSite(Module &M, Instruction &Inst, Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); } -bool getValueProfDataFromInst(const Instruction &Inst, - InstrProfValueKind ValueKind, - uint32_t MaxNumValueData, - InstrProfValueData ValueData[], - uint32_t &ActualNumValueData, uint64_t &TotalC, - bool GetNoICPValue) { +MDNode *mayHaveValueProfileOfKind(const Instruction &Inst, + InstrProfValueKind ValueKind) { MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); if (!MD) - return false; + return nullptr; - unsigned NOps = MD->getNumOperands(); + if (MD->getNumOperands() < 5) + return nullptr; - if (NOps < 5) - return false; - - // Operand 0 is a string tag "VP": MDString *Tag = cast(MD->getOperand(0)); - if (!Tag) - return false; - - if (!Tag->getString().equals("VP")) - return false; + if (!Tag || !Tag->getString().equals("VP")) + return nullptr; // Now check kind: ConstantInt *KindInt = mdconst::dyn_extract(MD->getOperand(1)); if (!KindInt) - return false; + return nullptr; if (KindInt->getZExtValue() != ValueKind) - return false; + return nullptr; + + return MD; +} +static bool getValueProfDataFromInstImpl(const MDNode *const MD, + const uint32_t MaxNumDataWant, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue) { + const unsigned NOps = MD->getNumOperands(); // Get total count ConstantInt *TotalCInt = mdconst::dyn_extract(MD->getOperand(2)); if (!TotalCInt) return false; TotalC = TotalCInt->getZExtValue(); - ActualNumValueData = 0; for (unsigned I = 3; I < NOps; I += 2) { - if (ActualNumValueData >= MaxNumValueData) + if (ActualNumValueData >= MaxNumDataWant) break; ConstantInt *Value = mdconst::dyn_extract(MD->getOperand(I)); ConstantInt *Count = @@ -1429,6 +1427,36 @@ bool getValueProfDataFromInst(const Instruction &Inst, return true; } +std::unique_ptr +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue) { + MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind); + if (!MD) + return nullptr; + auto ValueDataArray = std::make_unique(MaxNumValueData); + if (!getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueDataArray.get(), + ActualNumValueData, TotalC, GetNoICPValue)) + return nullptr; + return ValueDataArray; +} + +// FIXME: Migrate existing callers to the function above that returns an +// array. +bool getValueProfDataFromInst(const Instruction &Inst, + InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, uint64_t &TotalC, + bool GetNoICPValue) { + MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind); + if (!MD) + return false; + return getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueData, + ActualNumValueData, TotalC, + GetNoICPValue); +} + MDNode *getPGOFuncNameMetadata(const Function &F) { return F.getMetadata(getPGOFuncNameMetadataName()); } diff --git a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll new file mode 100644 index 000000000000..ba3ce9a75ee8 --- /dev/null +++ b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll @@ -0,0 +1,74 @@ +; Promote at most one function and annotate at most one vtable. +; As a result, only one value (of each relevant kind) shows up in the function +; summary. + +; RUN: opt -module-summary -icp-max-num-vtables=1 -icp-max-prom=1 %s -o %t.o + +; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s + +; RUN: llvm-dis -o - %t.o | FileCheck %s --check-prefix=DIS +; Round trip it through llvm-as +; RUN: llvm-dis -o - %t.o | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=DIS + +; CHECK: +; CHECK-NEXT: +; The `VALUE_GUID` below represents the "_ZTV4Base" referenced by the instruction +; that loads vtable pointers. +; CHECK-NEXT: +; The `VALUE_GUID` below represents the "_ZN4Base4funcEv" referenced by the +; indirect call instruction. +; CHECK-NEXT: +; NOTE vtables and functions from Derived class is dropped because +; `-icp-max-num-vtables` and `-icp-max-prom` are both set to one. +; has the format [valueid, flags, instcount, funcflags, +; numrefs, rorefcnt, worefcnt, +; m x valueid, +; n x (valueid, hotness+tailcall)] +; CHECK-NEXT: +; CHECK-NEXT: + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function has one BB and an entry count of 150, so the BB is hot according to +; ProfileSummary and reflected so in the bitcode (see llvm-dis output). +define i32 @_Z4testP4Base(ptr %0) !prof !15 { + %2 = load ptr, ptr %0, !prof !16 + %3 = load ptr, ptr %2 + %4 = tail call i32 %3(ptr %0), !prof !17 + ret i32 %4 +} + +!llvm.module.flags = !{!1} + +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 200} +!6 = !{!"MaxInternalCount", i64 200} +!7 = !{!"MaxFunctionCount", i64 200} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 990000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} + +!15 = !{!"function_entry_count", i32 150} +; 1960855528937986108 is the MD5 hash of _ZTV4Base, and +; 13870436605473471591 is the MD5 hash of _ZTV7Derived +!16 = !{!"VP", i32 2, i64 150, i64 1960855528937986108, i64 100, i64 13870436605473471591, i64 50} +; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv, and +; 6174874150489409711 is the MD5 hash of _ZN7Derived4funcEv +!17 = !{!"VP", i32 0, i64 150, i64 5459407273543877811, i64 100, i64 6174874150489409711, i64 50} + +; ModuleSummaryIndex stores map in std::map; so +; global value summares are printed out in the order that gv's guid increases. +; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0)) +; DIS: ^1 = gv: (guid: 1960855528937986108) +; DIS: ^2 = gv: (guid: 5459407273543877811) +; DIS: ^3 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^2, hotness: hot)), refs: (readonly ^1)))) ; guid = 15857150948103218965 +; DIS: ^4 = blockcount: 0 -- Gitee From 7c838a3cf0a5a010fe0b2b8b4631b05e171383c5 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Wed, 26 Jun 2024 11:38:20 -0700 Subject: [PATCH 21/28] [TypeProf][InstrFDO]Omit vtable symbols in indexed profiles by default (#96520) - The indexed iFDO profiles contains compressed vtable names for `llvm-profdata show --show-vtables` debugging usage. An optimized build doesn't need it and doesn't decompress the blob now [1], since optimized binary has the source code and IR to find vtable symbols. - The motivation is to avoid increasing profile size when it's not necessary. - This doesn't change the indexed profile format and thereby doesn't need a version change. [1] https://github.com/llvm/llvm-project/blob/eac925fb81f26342811ad1765e8f9919628e2254/llvm/include/llvm/ProfileData/InstrProfReader.h#L696-L699 --- .../profile/Linux/instrprof-vtable-value-prof.cpp | 10 +++++----- .../tools/llvm-profdata/vtable-value-prof.test | 4 ++-- llvm/tools/llvm-profdata/llvm-profdata.cpp | 14 ++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp index 5c8426b40892..c4d062ce9507 100644 --- a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp +++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp @@ -7,19 +7,19 @@ // RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profraw | FileCheck %s --check-prefixes=COMMON,RAW // Generate indexed profile from raw profile and show the data. -// RUN: llvm-profdata merge %t-test.profraw -o %t-test.profdata +// RUN: llvm-profdata merge --keep-vtable-symbols %t-test.profraw -o %t-test.profdata // RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED // Generate text profile from raw and indexed profiles respectively and show the data. -// RUN: llvm-profdata merge --text %t-test.profraw -o %t-raw.proftext +// RUN: llvm-profdata merge --keep-vtable-symbols --text %t-test.profraw -o %t-raw.proftext // RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-raw.proftext | FileCheck %s --check-prefix=ICTEXT -// RUN: llvm-profdata merge --text %t-test.profdata -o %t-indexed.proftext +// RUN: llvm-profdata merge --keep-vtable-symbols --text %t-test.profdata -o %t-indexed.proftext // RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-indexed.proftext | FileCheck %s --check-prefix=ICTEXT // Generate indexed profile from text profiles and show the data -// RUN: llvm-profdata merge --binary %t-raw.proftext -o %t-text.profraw +// RUN: llvm-profdata merge --keep-vtable-symbols --binary %t-raw.proftext -o %t-text.profraw // RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profraw | FileCheck %s --check-prefixes=COMMON,INDEXED -// RUN: llvm-profdata merge --binary %t-indexed.proftext -o %t-text.profdata +// RUN: llvm-profdata merge --keep-vtable-symbols --binary %t-indexed.proftext -o %t-text.profdata // RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED // COMMON: Counters: diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof.test b/llvm/test/tools/llvm-profdata/vtable-value-prof.test index 378c2e11b236..8dc8f6f0d480 100644 --- a/llvm/test/tools/llvm-profdata/vtable-value-prof.test +++ b/llvm/test/tools/llvm-profdata/vtable-value-prof.test @@ -1,7 +1,7 @@ ; RUN: rm -rf %t && mkdir %t && cd %t ; Generate indexed profiles from text profiles -RUN: llvm-profdata merge %S/Inputs/vtable-value-prof.proftext -o indexed.profdata +RUN: llvm-profdata merge --keep-vtable-symbols %S/Inputs/vtable-value-prof.proftext -o indexed.profdata ; Show indexed profiles RUN: llvm-profdata show --function=main --ic-targets --show-vtables indexed.profdata | FileCheck %s --check-prefix=INDEXED @@ -10,7 +10,7 @@ RUN: llvm-profdata show --function=main --ic-targets --show-vtables indexed.prof RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %S/Inputs/vtable-value-prof.proftext | FileCheck %s --check-prefix=ICTEXT ; Convert indexed profiles to its textual output and show it. -RUN: llvm-profdata merge --text -o text-from-indexed.proftext indexed.profdata +RUN: llvm-profdata merge --keep-vtable-symbols --text -o text-from-indexed.proftext indexed.profdata RUN: llvm-profdata show --function=main --ic-targets --show-vtables text-from-indexed.proftext | FileCheck %s --check-prefix=INDEXED RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text text-from-indexed.proftext | FileCheck %s --check-prefix=ICTEXT diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 646a8df18b9d..a3efc93dfbcf 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -285,6 +285,11 @@ cl::opt DropProfileSymbolList( cl::desc("Drop the profile symbol list when merging AutoFDO profiles " "(only meaningful for -sample)")); +cl::opt KeepVTableSymbols( + "keep-vtable-symbols", cl::init(false), cl::Hidden, + cl::sub(MergeSubcommand), + cl::desc("If true, keep the vtable symbols in indexed profiles")); + // Options specific to overlap subcommand. cl::opt BaseFilename(cl::Positional, cl::Required, cl::desc(""), @@ -711,11 +716,12 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, }); } - const InstrProfSymtab &symtab = Reader->getSymtab(); - const auto &VTableNames = symtab.getVTableNames(); + if (KeepVTableSymbols) { + const InstrProfSymtab &symtab = Reader->getSymtab(); + const auto &VTableNames = symtab.getVTableNames(); - for (const auto &kv : VTableNames) { - WC->Writer.addVTableName(kv.getKey()); + for (const auto &kv : VTableNames) + WC->Writer.addVTableName(kv.getKey()); } if (Reader->hasTemporalProfile()) { -- Gitee From bd179ba79c77b2ebcc0ecef396baaf697c981cb6 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Sat, 29 Jun 2024 23:21:33 -0700 Subject: [PATCH 22/28] [TypeProf][InstrFDO]Implement more efficient comparison sequence for indirect-call-promotion with vtable profiles. (#81442) Clang's `-fwhole-program-vtables` is required for this optimization to take place. If `-fwhole-program-vtables` is not enabled, this change is no-op. * Function-comparison (before): ``` %vtable = load ptr, ptr %obj %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 %func = load ptr, ptr %vfn %cond = icmp eq ptr %func, @callee br i1 %cond, label bb1, label bb2: bb1: call @callee bb2: call %func ``` * VTable-comparison (after): ``` %vtable = load ptr, ptr %obj %cond = icmp eq ptr %vtable, @vtable-address-point br i1 %cond, label bb1, label bb2: bb1: call @callee bb2: %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 %func = load ptr, ptr %vfn call %func ``` Key changes: 1. Find out virtual calls and the vtables they come from. - The ICP relies on type intrinsic `llvm.type.test` to find out virtual calls and the compatible vtables, and relies on type metadata to find the address point for comparison. 2. ICP pass does cost-benefit analysis and compares vtable only when the number of vtables for a function candidate is within (option specified) threshold. 3. Sink the function addressing and vtable load instruction to indirect fallback. - The sink helper functions are simplified versions of `InstCombinerImpl::tryToSinkInstruction`. Currently debug intrinsics are not handled. Ideally `InstCombinerImpl::tryToSinkInstructionDbgValues` and `InstCombinerImpl::tryToSinkInstructionDbgVariableRecords` could be moved into Transforms/Utils/Local.cpp (or another util cpp file) to handle debug intrinsics when moving instructions across basic blocks. 4. Keep value profiles updated 1) Update vtable value profiles after inline 2) For either function-based comparison or vtable-based comparison, update both vtable and indirect call value profiles. --- .../Linux/instrprof-vtable-value-prof.cpp | 157 ++-- .../Analysis/IndirectCallPromotionAnalysis.h | 6 +- .../llvm/Analysis/IndirectCallVisitor.h | 53 +- llvm/include/llvm/ProfileData/InstrProf.h | 15 + .../IndirectCallPromotionAnalysis.cpp | 6 +- llvm/lib/ProfileData/InstrProf.cpp | 36 +- .../Instrumentation/IndirectCallPromotion.cpp | 736 +++++++++++++++++- .../Instrumentation/PGOInstrumentation.cpp | 30 +- llvm/lib/Transforms/Utils/InlineFunction.cpp | 31 +- .../Transforms/Inline/update_invoke_prof.ll | 100 +++ .../Transforms/Inline/update_value_profile.ll | 85 ++ .../Transforms/PGOProfile/icp_vtable_cmp.ll | 139 ++++ .../PGOProfile/icp_vtable_invoke.ll | 125 +++ .../PGOProfile/icp_vtable_tail_call.ll | 68 ++ 14 files changed, 1454 insertions(+), 133 deletions(-) create mode 100644 llvm/test/Transforms/Inline/update_invoke_prof.ll create mode 100644 llvm/test/Transforms/Inline/update_value_profile.ll create mode 100644 llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll create mode 100644 llvm/test/Transforms/PGOProfile/icp_vtable_invoke.ll create mode 100644 llvm/test/Transforms/PGOProfile/icp_vtable_tail_call.ll diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp index c4d062ce9507..6cf73c6fdbd7 100644 --- a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp +++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp @@ -1,58 +1,65 @@ // REQUIRES: lld-available -// RUN: %clangxx_pgogen -fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling %s -o %t-test -// RUN: env LLVM_PROFILE_FILE=%t-test.profraw %t-test +// Building the instrumented binary will fail because lld doesn't support +// big-endian ELF for PPC (aka ABI 1). +// ld.lld: error: /lib/../lib64/Scrt1.o: ABI version 1 is not supported +// UNSUPPORTED: ppc && host-byteorder-big-endian + +// RUN: rm -rf %t && mkdir %t && cd %t + +// RUN: %clangxx_pgogen -fuse-ld=lld -O2 -fprofile-generate=. -mllvm -enable-vtable-value-profiling %s -o test +// RUN: env LLVM_PROFILE_FILE=test.profraw ./test // Show vtable profiles from raw profile. -// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profraw | FileCheck %s --check-prefixes=COMMON,RAW +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables test.profraw | FileCheck %s --check-prefixes=COMMON,RAW // Generate indexed profile from raw profile and show the data. -// RUN: llvm-profdata merge --keep-vtable-symbols %t-test.profraw -o %t-test.profdata -// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED +// RUN: llvm-profdata merge --keep-vtable-symbols test.profraw -o test.profdata +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables test.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED // Generate text profile from raw and indexed profiles respectively and show the data. -// RUN: llvm-profdata merge --keep-vtable-symbols --text %t-test.profraw -o %t-raw.proftext -// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-raw.proftext | FileCheck %s --check-prefix=ICTEXT -// RUN: llvm-profdata merge --keep-vtable-symbols --text %t-test.profdata -o %t-indexed.proftext -// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-indexed.proftext | FileCheck %s --check-prefix=ICTEXT +// RUN: llvm-profdata merge --keep-vtable-symbols --text test.profraw -o raw.proftext +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text raw.proftext | FileCheck %s --check-prefix=ICTEXT +// RUN: llvm-profdata merge --keep-vtable-symbols --text test.profdata -o indexed.proftext +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text indexed.proftext | FileCheck %s --check-prefix=ICTEXT // Generate indexed profile from text profiles and show the data -// RUN: llvm-profdata merge --keep-vtable-symbols --binary %t-raw.proftext -o %t-text.profraw -// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profraw | FileCheck %s --check-prefixes=COMMON,INDEXED -// RUN: llvm-profdata merge --keep-vtable-symbols --binary %t-indexed.proftext -o %t-text.profdata -// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED +// RUN: llvm-profdata merge --keep-vtable-symbols --binary raw.proftext -o text.profraw +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables text.profraw | FileCheck %s --check-prefixes=COMMON,INDEXED +// RUN: llvm-profdata merge --keep-vtable-symbols --binary indexed.proftext -o text.profdata +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables text.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED // COMMON: Counters: // COMMON-NEXT: main: -// COMMON-NEXT: Hash: 0x0f9a16fe6d398548 -// COMMON-NEXT: Counters: 2 +// COMMON-NEXT: Hash: 0x068617320ec408a0 +// COMMON-NEXT: Counters: 4 // COMMON-NEXT: Indirect Call Site Count: 2 // COMMON-NEXT: Number of instrumented vtables: 2 // RAW: Indirect Target Results: -// RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) -// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) -// RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) -// RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +// RAW-NEXT: [ 0, _ZN8Derived14funcEii, 50 ] (25.00%) +// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived24funcEii, 150 ] (75.00%) +// RAW-NEXT: [ 1, _ZN8Derived1D0Ev, 250 ] (25.00%) +// RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived2D0Ev, 750 ] (75.00%) // RAW-NEXT: VTable Results: -// RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) -// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// RAW-NEXT: [ 0, _ZTV8Derived1, 50 ] (25.00%) +// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 150 ] (75.00%) // RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) // RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) // INDEXED: Indirect Target Results: -// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) -// INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) -// INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) -// INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived24funcEii, 150 ] (75.00%) +// INDEXED-NEXT: [ 0, _ZN8Derived14funcEii, 50 ] (25.00%) +// INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived2D0Ev, 750 ] (75.00%) +// INDEXED-NEXT: [ 1, _ZN8Derived1D0Ev, 250 ] (25.00%) // INDEXED-NEXT: VTable Results: -// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) -// INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 150 ] (75.00%) +// INDEXED-NEXT: [ 0, _ZTV8Derived1, 50 ] (25.00%) // INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) // INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) // COMMON: Instrumentation level: IR entry_first = 0 // COMMON-NEXT: Functions shown: 1 -// COMMON-NEXT: Total functions: 6 +// COMMON-NEXT: Total functions: 7 // COMMON-NEXT: Maximum function count: 1000 -// COMMON-NEXT: Maximum internal block count: 250 +// COMMON-NEXT: Maximum internal block count: 1000 // COMMON-NEXT: Statistics for indirect call sites profile: // COMMON-NEXT: Total number of sites: 2 // COMMON-NEXT: Total number of sites with values: 2 @@ -71,11 +78,13 @@ // ICTEXT: :ir // ICTEXT: main // ICTEXT: # Func Hash: -// ICTEXT: 1124236338992350536 +// ICTEXT: 470088714870327456 // ICTEXT: # Num Counters: -// ICTEXT: 2 +// ICTEXT: 4 // ICTEXT: # Counter Values: // ICTEXT: 1000 +// ICTEXT: 1000 +// ICTEXT: 200 // ICTEXT: 1 // ICTEXT: # Num Value Kinds: // ICTEXT: 2 @@ -84,41 +93,98 @@ // ICTEXT: # NumValueSites: // ICTEXT: 2 // ICTEXT: 2 -// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii:750 -// ICTEXT: _ZN8Derived15func1Eii:250 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived24funcEii:150 +// ICTEXT: _ZN8Derived14funcEii:50 // ICTEXT: 2 -// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii:750 -// ICTEXT: _ZN8Derived15func2Eii:250 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived2D0Ev:750 +// ICTEXT: _ZN8Derived1D0Ev:250 // ICTEXT: # ValueKind = IPVK_VTableTarget: // ICTEXT: 2 // ICTEXT: # NumValueSites: // ICTEXT: 2 // ICTEXT: 2 -// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 -// ICTEXT: _ZTV8Derived1:250 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:150 +// ICTEXT: _ZTV8Derived1:50 // ICTEXT: 2 // ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 // ICTEXT: _ZTV8Derived1:250 +// Test indirect call promotion transformation using vtable profiles. +// - Build with `-g` to enable debug information. +// - In real world settings, ICP pass is disabled in prelink pipeline. In +// the postlink pipeline, ICP is enabled after whole-program-devirtualization +// pass. Do the same thing in this test. +// - Enable `-fwhole-program-vtables` generate type metadata and intrinsics. +// - Enable `-fno-split-lto-unit` and `-Wl,-lto-whole-program-visibility` to +// preserve type intrinsics for ICP pass. +// RUN: %clangxx -m64 -fprofile-use=test.profdata -Wl,--lto-whole-program-visibility \ +// RUN: -mllvm -disable-icp=true -Wl,-mllvm,-disable-icp=false -fuse-ld=lld \ +// RUN: -g -flto=thin -fwhole-program-vtables -fno-split-lto-unit -O2 \ +// RUN: -mllvm -enable-vtable-value-profiling -Wl,-mllvm,-enable-vtable-value-profiling \ +// RUN: -mllvm -enable-vtable-profile-use \ +// RUN: -Wl,-mllvm,-enable-vtable-profile-use -Rpass=pgo-icall-prom \ +// RUN: -Wl,-mllvm,-print-after=pgo-icall-prom \ +// RUN: -Wl,-mllvm,-filter-print-funcs=main %s 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=REMARK,IR --implicit-check-not="!VP" + +// For the indirect call site `ptr->func` +// REMARK: instrprof-vtable-value-prof.cpp:205:19: Promote indirect call to _ZN12_GLOBAL__N_18Derived24funcEii with count 150 out of 200, sink 1 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E} +// REMARK: instrprof-vtable-value-prof.cpp:205:19: Promote indirect call to _ZN8Derived14funcEii with count 50 out of 50, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1} +// +// For the indirect call site `delete ptr` +// REMARK: instrprof-vtable-value-prof.cpp:207:5: Promote indirect call to _ZN12_GLOBAL__N_18Derived2D0Ev with count 750 out of 1000, sink 2 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E} +// REMARK: instrprof-vtable-value-prof.cpp:207:5: Promote indirect call to _ZN8Derived1D0Ev with count 250 out of 250, sink 2 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1} + +// The IR matchers for indirect callsite `ptr->func`. +// IR-LABEL: @main +// IR: [[OBJ:%.*]] = {{.*}}call {{.*}} @_Z10createTypei +// IR: [[VTABLE:%.*]] = load ptr, ptr [[OBJ]] +// IR: [[CMP1:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @_ZTVN12_GLOBAL__N_18Derived2E, i32 16) +// IR: br i1 [[CMP1]], label %[[BB1:.*]], label %[[BB2:[a-zA-Z0-9_.]+]], +// +// IR: [[BB1]]: +// IR: [[RESBB1:%.*]] = {{.*}}call {{.*}} @_ZN12_GLOBAL__N_18Derived24funcEii +// IR: br label %[[MERGE0:[a-zA-Z0-9_.]+]] +// +// IR: [[BB2]]: +// IR: [[CMP2:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @_ZTV8Derived1, i32 16) +// IR: br i1 [[CMP2]], label %[[BB3:.*]], label %[[BB4:[a-zA-Z0-9_.]+]], +// +// IR: [[BB3]]: +// IR: [[RESBB3:%.*]] = {{.*}}call {{.*}} @_ZN8Derived14funcEii +// IR: br label %[[MERGE1:[a-zA-Z0-9_.]+]], +// +// IR: [[BB4]]: +// IR: [[FUNCPTR:%.*]] = load ptr, ptr [[VTABLE]] +// IR: [[RESBB4:%.*]] = {{.*}}call {{.*}} [[FUNCPTR]] +// IR: br label %[[MERGE1]] +// +// IR: [[MERGE1]]: +// IR: [[RES1:%.*]] = phi i32 [ [[RESBB4]], %[[BB4]] ], [ [[RESBB3]], %[[BB3]] ] +// IR: br label %[[MERGE0]] +// +// IR: [[MERGE0]]: +// IR: [[RES2:%.*]] = phi i32 [ [[RES1]], %[[MERGE1]] ], [ [[RESBB1]], %[[BB1]] ] #include #include class Base { public: - virtual int func1(int a, int b) = 0; - virtual int func2(int a, int b) = 0; + virtual int func(int a, int b) = 0; + + virtual ~Base() {}; }; class Derived1 : public Base { public: - int func1(int a, int b) override { return a + b; } + int func(int a, int b) override { return a * b; } - int func2(int a, int b) override { return a * b; } + ~Derived1() {} }; namespace { class Derived2 : public Base { public: - int func1(int a, int b) override { return a - b; } + int func(int a, int b) override { return a * (a - b); } - int func2(int a, int b) override { return a * (a - b); } + ~Derived2() {} }; } // namespace __attribute__((noinline)) Base *createType(int a) { @@ -135,7 +201,10 @@ int main(int argc, char **argv) { int a = rand(); int b = rand(); Base *ptr = createType(i); - sum += ptr->func1(a, b) + ptr->func2(b, a); + if (i % 5 == 0) + sum += ptr->func(b, a); + + delete ptr; } printf("sum is %d\n", sum); return 0; diff --git a/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h b/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h index 8a05e913a910..9c2be12fce2f 100644 --- a/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h +++ b/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h @@ -57,10 +57,8 @@ public: /// /// The returned array space is owned by this class, and overwritten on /// subsequent calls. - ArrayRef - getPromotionCandidatesForInstruction(const Instruction *I, uint32_t &NumVals, - uint64_t &TotalCount, - uint32_t &NumCandidates); + MutableArrayRef getPromotionCandidatesForInstruction( + const Instruction *I, uint64_t &TotalCount, uint32_t &NumCandidates); }; } // end namespace llvm diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h index 50815f4e3e83..67e70f5fd6d2 100644 --- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h +++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h @@ -27,30 +27,20 @@ struct PGOIndirectCallVisitor : public InstVisitor { std::vector ProfiledAddresses; PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {} - void visitCallBase(CallBase &Call) { - if (!Call.isIndirectCall()) - return; - - if (Type == InstructionType::kIndirectCall) { - IndirectCalls.push_back(&Call); - return; - } - - assert(Type == InstructionType::kVTableVal && "Control flow guaranteed"); - - LoadInst *LI = dyn_cast(Call.getCalledOperand()); - // The code pattern to look for - // - // %vtable = load ptr, ptr %b - // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 - // %2 = load ptr, ptr %vfn - // %call = tail call i32 %2(ptr %b) - // - // %vtable is the vtable address value to profile, and - // %2 is the indirect call target address to profile. + // Given an indirect call instruction, try to find the the following pattern + // + // %vtable = load ptr, ptr %obj + // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 + // %2 = load ptr, ptr %vfn + // $call = tail call i32 %2 + // + // A heuristic is used to find the address feeding instructions. + static Instruction *tryGetVTableInstruction(CallBase *CB) { + assert(CB != nullptr && "Caller guaranteed"); + LoadInst *LI = dyn_cast(CB->getCalledOperand()); if (LI != nullptr) { - Value *Ptr = LI->getPointerOperand(); - Value *VTablePtr = Ptr->stripInBoundsConstantOffsets(); + Value *FuncPtr = LI->getPointerOperand(); // GEP (or bitcast) + Value *VTablePtr = FuncPtr->stripInBoundsConstantOffsets(); // This is a heuristic to find address feeding instructions. // FIXME: Add support in the frontend so LLVM type intrinsics are // emitted without LTO. This way, added intrinsics could filter @@ -63,7 +53,22 @@ struct PGOIndirectCallVisitor : public InstVisitor { // address is negligible if exists at all. Comparing loaded address // with symbol address guarantees correctness. if (VTablePtr != nullptr && isa(VTablePtr)) - ProfiledAddresses.push_back(cast(VTablePtr)); + return cast(VTablePtr); + } + return nullptr; + } + + void visitCallBase(CallBase &Call) { + if (Call.isIndirectCall()) { + IndirectCalls.push_back(&Call); + + if (Type != InstructionType::kVTableVal) + return; + + Instruction *VPtr = + PGOIndirectCallVisitor::tryGetVTableInstruction(&Call); + if (VPtr) + ProfiledAddresses.push_back(VPtr); } } diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index d4486572da67..e408b5d4b1a6 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -311,6 +311,15 @@ getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue = false); +// TODO: Unify metadata name 'PGOFuncName' and 'PGOName', by supporting read +// of this metadata for backward compatibility and generating 'PGOName' only. +/// Extract the value profile data from \p Inst and returns them if \p Inst is +/// annotated with value profile data. Returns an empty vector otherwise. +SmallVector +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint64_t &TotalC, + bool GetNoICPValue = false); + inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } inline StringRef getPGONameMetadataName() { return "PGOName"; } @@ -323,8 +332,14 @@ std::string getPGOName(const GlobalVariable &V, bool InLTO = false); /// Create the PGOFuncName meta data if PGOFuncName is different from /// function's raw name. This should only apply to internal linkage functions /// declared by users only. +/// TODO: Update all callers to 'createPGONameMetadata' and deprecate this +/// function. void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); +/// Create the PGOName metadata if a global object's PGO name is different from +/// its mangled name. This should apply to local-linkage global objects only. +void createPGONameMetadata(GlobalObject &GO, StringRef PGOName); + /// Check if we can use Comdat for profile variables. This will eliminate /// the duplicated profile variables for Comdat functions. bool needsComdatForCounter(const GlobalObject &GV, const Module &M); diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp index ab53717eb889..643c155ba6d7 100644 --- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -87,7 +87,7 @@ uint32_t ICallPromotionAnalysis::getProfitablePromotionCandidates( return I; } -ArrayRef +MutableArrayRef ICallPromotionAnalysis::getPromotionCandidatesForInstruction( const Instruction *I, uint32_t &NumVals, uint64_t &TotalCount, uint32_t &NumCandidates) { @@ -96,8 +96,8 @@ ICallPromotionAnalysis::getPromotionCandidatesForInstruction( ValueDataArray.get(), NumVals, TotalCount); if (!Res) { NumCandidates = 0; - return ArrayRef(); + return MutableArrayRef(); } NumCandidates = getProfitablePromotionCandidates(I, NumVals, TotalCount); - return ArrayRef(ValueDataArray.get(), NumVals); + return MutableArrayRef(ValueDataArray.get(), NumVals); } diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index b4cc5dfbe4f4..35b47020c0e0 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -230,6 +230,12 @@ cl::opt EnableVTableValueProfiling( "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison.")); +cl::opt EnableVTableProfileUse( + "enable-vtable-profile-use", cl::init(false), + cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " + "profiles will be used by ICP pass for more efficient indirect " + "call sequence. If false, type profiles won't be used.")); + std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo) { @@ -1461,16 +1467,28 @@ MDNode *getPGOFuncNameMetadata(const Function &F) { return F.getMetadata(getPGOFuncNameMetadataName()); } -void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { - // Only for internal linkage functions. - if (PGOFuncName == F.getName()) - return; - // Don't create duplicated meta-data. - if (getPGOFuncNameMetadata(F)) +static void createPGONameMetadata(GlobalObject &GO, StringRef MetadataName, + StringRef PGOName) { + // Only for internal linkage functions or global variables. The name is not + // the same as PGO name for these global objects. + if (GO.getName() == PGOName) return; - LLVMContext &C = F.getContext(); - MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); - F.setMetadata(getPGOFuncNameMetadataName(), N); + + // Don't create duplicated metadata. + if (GO.getMetadata(MetadataName)) + return; + + LLVMContext &C = GO.getContext(); + MDNode *N = MDNode::get(C, MDString::get(C, PGOName)); + GO.setMetadata(MetadataName, N); +} + +void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { + return createPGONameMetadata(F, getPGOFuncNameMetadataName(), PGOFuncName); +} + +void createPGONameMetadata(GlobalObject &GO, StringRef PGOName) { + return createPGONameMetadata(GO, getPGONameMetadataName(), PGOName); } bool needsComdatForCounter(const GlobalObject &GO, const Module &M) { diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 146fa3e06996..e3d9c6ff67bd 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -13,13 +13,16 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/IndirectCallPromotionAnalysis.h" #include "llvm/Analysis/IndirectCallVisitor.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/Analysis/TypeMetadataUtils.h" #include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" @@ -39,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +54,12 @@ using namespace llvm; STATISTIC(NumOfPGOICallPromotion, "Number of indirect call promotions."); STATISTIC(NumOfPGOICallsites, "Number of indirect call candidate sites."); +extern cl::opt MaxNumVTableAnnotations; + +namespace llvm { +extern cl::opt EnableVTableProfileUse; +} + // Command line option to disable indirect-call promotion with the default as // false. This is for debug purpose. static cl::opt DisableICP("disable-icp", cl::init(false), cl::Hidden, @@ -102,13 +112,196 @@ static cl::opt ICPDUMPAFTER("icp-dumpafter", cl::init(false), cl::Hidden, cl::desc("Dump IR after transformation happens")); +// Indirect call promotion pass will fall back to function-based comparison if +// vtable-count / function-count is smaller than this threshold. +static cl::opt ICPVTablePercentageThreshold( + "icp-vtable-percentage-threshold", cl::init(0.99), cl::Hidden, + cl::desc("The percentage threshold of vtable-count / function-count for " + "cost-benefit analysis.")); + +// Although comparing vtables can save a vtable load, we may need to compare +// vtable pointer with multiple vtable address points due to class inheritance. +// Comparing with multiple vtables inserts additional instructions on hot code +// path, and doing so for an earlier candidate delays the comparisons for later +// candidates. For the last candidate, only the fallback path is affected. +// We allow multiple vtable comparison for the last function candidate and use +// the option below to cap the number of vtables. +static cl::opt ICPMaxNumVTableLastCandidate( + "icp-max-num-vtable-last-candidate", cl::init(1), cl::Hidden, + cl::desc("The maximum number of vtable for the last candidate.")); + namespace { +// The key is a vtable global variable, and the value is a map. +// In the inner map, the key represents address point offsets and the value is a +// constant for this address point. +using VTableAddressPointOffsetValMap = + SmallDenseMap>; + +// A struct to collect type information for a virtual call site. +struct VirtualCallSiteInfo { + // The offset from the address point to virtual function in the vtable. + uint64_t FunctionOffset; + // The instruction that computes the address point of vtable. + Instruction *VPtr; + // The compatible type used in LLVM type intrinsics. + StringRef CompatibleTypeStr; +}; + +// The key is a virtual call, and value is its type information. +using VirtualCallSiteTypeInfoMap = + SmallDenseMap; + +// The key is vtable GUID, and value is its value profile count. +using VTableGUIDCountsMap = SmallDenseMap; + +// Return the address point offset of the given compatible type. +// +// Type metadata of a vtable specifies the types that can contain a pointer to +// this vtable, for example, `Base*` can be a pointer to an derived type +// but not vice versa. See also https://llvm.org/docs/TypeMetadata.html +static std::optional +getAddressPointOffset(const GlobalVariable &VTableVar, + StringRef CompatibleType) { + SmallVector Types; + VTableVar.getMetadata(LLVMContext::MD_type, Types); + + for (MDNode *Type : Types) + if (auto *TypeId = dyn_cast(Type->getOperand(1).get()); + TypeId && TypeId->getString() == CompatibleType) + return cast( + cast(Type->getOperand(0))->getValue()) + ->getZExtValue(); + + return std::nullopt; +} + +// Return a constant representing the vtable's address point specified by the +// offset. +static Constant *getVTableAddressPointOffset(GlobalVariable *VTable, + uint32_t AddressPointOffset) { + Module &M = *VTable->getParent(); + LLVMContext &Context = M.getContext(); + assert(AddressPointOffset < + M.getDataLayout().getTypeAllocSize(VTable->getValueType()) && + "Out-of-bound access"); + + return ConstantExpr::getInBoundsGetElementPtr( + Type::getInt8Ty(Context), VTable, + llvm::ConstantInt::get(Type::getInt32Ty(Context), AddressPointOffset)); +} + +// Return the basic block in which Use `U` is used via its `UserInst`. +static BasicBlock *getUserBasicBlock(Use &U, Instruction *UserInst) { + if (PHINode *PN = dyn_cast(UserInst)) + return PN->getIncomingBlock(U); + + return UserInst->getParent(); +} + +// `DestBB` is a suitable basic block to sink `Inst` into when `Inst` have users +// and all users are in `DestBB`. The caller guarantees that `Inst->getParent()` +// is the sole predecessor of `DestBB` and `DestBB` is dominated by +// `Inst->getParent()`. +static bool isDestBBSuitableForSink(Instruction *Inst, BasicBlock *DestBB) { + // 'BB' is used only by assert. + [[maybe_unused]] BasicBlock *BB = Inst->getParent(); + + assert(BB != DestBB && BB->getTerminator()->getNumSuccessors() == 2 && + DestBB->getUniquePredecessor() == BB && + "Guaranteed by ICP transformation"); + + BasicBlock *UserBB = nullptr; + for (Use &Use : Inst->uses()) { + User *User = Use.getUser(); + // Do checked cast since IR verifier guarantees that the user of an + // instruction must be an instruction. See `Verifier::visitInstruction`. + Instruction *UserInst = cast(User); + // We can sink debug or pseudo instructions together with Inst. + if (UserInst->isDebugOrPseudoInst()) + continue; + UserBB = getUserBasicBlock(Use, UserInst); + // Do not sink if Inst is used in a basic block that is not DestBB. + // TODO: Sink to the common dominator of all user blocks. + if (UserBB != DestBB) + return false; + } + return UserBB != nullptr; +} + +// For the virtual call dispatch sequence, try to sink vtable load instructions +// to the cold indirect call fallback. +// FIXME: Move the sink eligibility check below to a utility function in +// Transforms/Utils/ directory. +static bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) { + if (!isDestBBSuitableForSink(I, DestBlock)) + return false; + + // Do not move control-flow-involving, volatile loads, vaarg, alloca + // instructions, etc. + if (isa(I) || I->isEHPad() || I->mayThrow() || !I->willReturn() || + isa(I)) + return false; + + // Do not sink convergent call instructions. + if (const auto *C = dyn_cast(I)) + if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent()) + return false; + + // Do not move an instruction that may write to memory. + if (I->mayWriteToMemory()) + return false; + + // We can only sink load instructions if there is nothing between the load and + // the end of block that could change the value. + if (I->mayReadFromMemory()) { + // We already know that SrcBlock is the unique predecessor of DestBlock. + for (BasicBlock::iterator Scan = std::next(I->getIterator()), + E = I->getParent()->end(); + Scan != E; ++Scan) { + // Note analysis analysis can tell whether two pointers can point to the + // same object in memory or not thereby find further opportunities to + // sink. + if (Scan->mayWriteToMemory()) + return false; + } + } + + BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt(); + I->moveBefore(*DestBlock, InsertPos); + + // TODO: Sink debug intrinsic users of I to 'DestBlock'. + // 'InstCombinerImpl::tryToSinkInstructionDbgValues' and + // 'InstCombinerImpl::tryToSinkInstructionDbgVariableRecords' already have + // the core logic to do this. + return true; +} + +// Try to sink instructions after VPtr to the indirect call fallback. +// Return the number of sunk IR instructions. +static int tryToSinkInstructions(BasicBlock *OriginalBB, + BasicBlock *IndirectCallBB) { + int SinkCount = 0; + // Do not sink across a critical edge for simplicity. + if (IndirectCallBB->getUniquePredecessor() != OriginalBB) + return SinkCount; + // Sink all eligible instructions in OriginalBB in reverse order. + for (Instruction &I : + llvm::make_early_inc_range(llvm::drop_begin(llvm::reverse(*OriginalBB)))) + if (tryToSinkInstruction(&I, IndirectCallBB)) + SinkCount++; + + return SinkCount; +} + // Promote indirect calls to conditional direct calls, keeping track of // thresholds. class IndirectCallPromoter { private: Function &F; + Module &M; + + ProfileSummaryInfo *PSI = nullptr; // Symtab that maps indirect call profile values to function names and // defines. @@ -116,6 +309,11 @@ private: const bool SamplePGO; + // A map from a virtual call to its type information. + const VirtualCallSiteTypeInfoMap &VirtualCSInfo; + + VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal; + OptimizationRemarkEmitter &ORE; // A struct that records the direct target and it's call count. @@ -123,6 +321,16 @@ private: Function *const TargetFunction; const uint64_t Count; + // The following fields only exists for promotion candidates with vtable + // information. + // + // Due to class inheritance, one virtual call candidate can come from + // multiple vtables. `VTableGUIDAndCounts` tracks the vtable GUIDs and + // counts for 'TargetFunction'. `AddressPoints` stores the vtable address + // points for comparison. + VTableGUIDCountsMap VTableGUIDAndCounts; + SmallVector AddressPoints; + PromotionCandidate(Function *F, uint64_t C) : TargetFunction(F), Count(C) {} }; @@ -135,16 +343,61 @@ private: const CallBase &CB, const ArrayRef &ValueDataRef, uint64_t TotalCount, uint32_t NumCandidates); - // Promote a list of targets for one indirect-call callsite. Return - // the number of promotions. - uint32_t tryToPromote(CallBase &CB, - const std::vector &Candidates, - uint64_t &TotalCount); + // Promote a list of targets for one indirect-call callsite by comparing + // indirect callee with functions. Return true if there are IR + // transformations and false otherwise. + bool tryToPromoteWithFuncCmp(CallBase &CB, Instruction *VPtr, + ArrayRef Candidates, + uint64_t TotalCount, + ArrayRef ICallProfDataRef, + uint32_t NumCandidates, + VTableGUIDCountsMap &VTableGUIDCounts); + + // Promote a list of targets for one indirect call by comparing vtables with + // functions. Return true if there are IR transformations and false + // otherwise. + bool tryToPromoteWithVTableCmp( + CallBase &CB, Instruction *VPtr, + const std::vector &Candidates, + uint64_t TotalFuncCount, uint32_t NumCandidates, + MutableArrayRef ICallProfDataRef, + VTableGUIDCountsMap &VTableGUIDCounts); + + // Return true if it's profitable to compare vtables for the callsite. + bool isProfitableToCompareVTables( + const CallBase &CB, const std::vector &Candidates, + uint64_t TotalCount); + + // Given an indirect callsite and the list of function candidates, compute + // the following vtable information in output parameters and return vtable + // pointer if type profiles exist. + // - Populate `VTableGUIDCounts` with using !prof + // metadata attached on the vtable pointer. + // - For each function candidate, finds out the vtables from which it gets + // called and stores the in promotion candidate. + Instruction *computeVTableInfos(const CallBase *CB, + VTableGUIDCountsMap &VTableGUIDCounts, + std::vector &Candidates); + + Constant *getOrCreateVTableAddressPointVar(GlobalVariable *GV, + uint64_t AddressPointOffset); + + void updateFuncValueProfiles(CallBase &CB, ArrayRef VDs, + uint64_t Sum, uint32_t MaxMDCount); + + void updateVPtrValueProfiles(Instruction *VPtr, + VTableGUIDCountsMap &VTableGUIDCounts); public: - IndirectCallPromoter(Function &Func, InstrProfSymtab *Symtab, bool SamplePGO, - OptimizationRemarkEmitter &ORE) - : F(Func), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {} + IndirectCallPromoter( + Function &Func, Module &M, ProfileSummaryInfo *PSI, + InstrProfSymtab *Symtab, bool SamplePGO, + const VirtualCallSiteTypeInfoMap &VirtualCSInfo, + VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal, + OptimizationRemarkEmitter &ORE) + : F(Func), M(M), PSI(PSI), Symtab(Symtab), SamplePGO(SamplePGO), + VirtualCSInfo(VirtualCSInfo), + VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE) {} IndirectCallPromoter(const IndirectCallPromoter &) = delete; IndirectCallPromoter &operator=(const IndirectCallPromoter &) = delete; @@ -240,26 +493,129 @@ IndirectCallPromoter::getPromotionCandidatesForCallSite( return Ret; } +Constant *IndirectCallPromoter::getOrCreateVTableAddressPointVar( + GlobalVariable *GV, uint64_t AddressPointOffset) { + auto [Iter, Inserted] = + VTableAddressPointOffsetVal[GV].try_emplace(AddressPointOffset, nullptr); + if (Inserted) + Iter->second = getVTableAddressPointOffset(GV, AddressPointOffset); + return Iter->second; +} + +Instruction *IndirectCallPromoter::computeVTableInfos( + const CallBase *CB, VTableGUIDCountsMap &GUIDCountsMap, + std::vector &Candidates) { + if (!EnableVTableProfileUse) + return nullptr; + + // Take the following code sequence as an example, here is how the code works + // @vtable1 = {[n x ptr] [... ptr @func1]} + // @vtable2 = {[m x ptr] [... ptr @func2]} + // + // %vptr = load ptr, ptr %d, !prof !0 + // %0 = tail call i1 @llvm.type.test(ptr %vptr, metadata !"vtable1") + // tail call void @llvm.assume(i1 %0) + // %vfn = getelementptr inbounds ptr, ptr %vptr, i64 1 + // %1 = load ptr, ptr %vfn + // call void %1(ptr %d), !prof !1 + // + // !0 = !{!"VP", i32 2, i64 100, i64 123, i64 50, i64 456, i64 50} + // !1 = !{!"VP", i32 0, i64 100, i64 789, i64 50, i64 579, i64 50} + // + // Step 1. Find out the %vptr instruction for indirect call and use its !prof + // to populate `GUIDCountsMap`. + // Step 2. For each vtable-guid, look up its definition from symtab. LTO can + // make vtable definitions visible across modules. + // Step 3. Compute the byte offset of the virtual call, by adding vtable + // address point offset and function's offset relative to vtable address + // point. For each function candidate, this step tells us the vtable from + // which it comes from, and the vtable address point to compare %vptr with. + + // Only virtual calls have virtual call site info. + auto Iter = VirtualCSInfo.find(CB); + if (Iter == VirtualCSInfo.end()) + return nullptr; + + LLVM_DEBUG(dbgs() << "\nComputing vtable infos for callsite #" + << NumOfPGOICallsites << "\n"); + + const auto &VirtualCallInfo = Iter->second; + Instruction *VPtr = VirtualCallInfo.VPtr; + + SmallDenseMap CalleeIndexMap; + for (size_t I = 0; I < Candidates.size(); I++) + CalleeIndexMap[Candidates[I].TargetFunction] = I; + + uint32_t ActualNumValueData = 0; + uint64_t TotalVTableCount = 0; + auto VTableValueDataArray = getValueProfDataFromInst( + *VirtualCallInfo.VPtr, IPVK_VTableTarget, MaxNumVTableAnnotations, + ActualNumValueData, TotalVTableCount); + if (VTableValueDataArray.get() == nullptr) + return VPtr; + + // Compute the functions and counts from by each vtable. + for (size_t j = 0; j < ActualNumValueData; j++) { + uint64_t VTableVal = VTableValueDataArray[j].Value; + GUIDCountsMap[VTableVal] = VTableValueDataArray[j].Count; + GlobalVariable *VTableVar = Symtab->getGlobalVariable(VTableVal); + if (!VTableVar) { + LLVM_DEBUG(dbgs() << " Cannot find vtable definition for " << VTableVal + << "; maybe the vtable isn't imported\n"); + continue; + } + + std::optional MaybeAddressPointOffset = + getAddressPointOffset(*VTableVar, VirtualCallInfo.CompatibleTypeStr); + if (!MaybeAddressPointOffset) + continue; + + const uint64_t AddressPointOffset = *MaybeAddressPointOffset; + + Function *Callee = nullptr; + std::tie(Callee, std::ignore) = getFunctionAtVTableOffset( + VTableVar, AddressPointOffset + VirtualCallInfo.FunctionOffset, M); + if (!Callee) + continue; + auto CalleeIndexIter = CalleeIndexMap.find(Callee); + if (CalleeIndexIter == CalleeIndexMap.end()) + continue; + + auto &Candidate = Candidates[CalleeIndexIter->second]; + // There shouldn't be duplicate GUIDs in one !prof metadata (except + // duplicated zeros), so assign counters directly won't cause overwrite or + // counter loss. + Candidate.VTableGUIDAndCounts[VTableVal] = VTableValueDataArray[j].Count; + Candidate.AddressPoints.push_back( + getOrCreateVTableAddressPointVar(VTableVar, AddressPointOffset)); + } + + return VPtr; +} + +// Creates 'branch_weights' prof metadata using TrueWeight and FalseWeight. +// Scales uint64_t counters down to uint32_t if necessary to prevent overflow. +static MDNode *createBranchWeights(LLVMContext &Context, uint64_t TrueWeight, + uint64_t FalseWeight) { + MDBuilder MDB(Context); + uint64_t Scale = calculateCountScale(std::max(TrueWeight, FalseWeight)); + return MDB.createBranchWeights(scaleBranchCount(TrueWeight, Scale), + scaleBranchCount(FalseWeight, Scale)); +} + CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE) { - - uint64_t ElseCount = TotalCount - Count; - uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount); - uint64_t Scale = calculateCountScale(MaxCount); - MDBuilder MDB(CB.getContext()); - MDNode *BranchWeights = MDB.createBranchWeights( - scaleBranchCount(Count, Scale), scaleBranchCount(ElseCount, Scale)); - - CallBase &NewInst = - promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights); + CallBase &NewInst = promoteCallWithIfThenElse( + CB, DirectCallee, + createBranchWeights(CB.getContext(), Count, TotalCount - Count)); if (AttachProfToDirectCall) { MDBuilder MDB(NewInst.getContext()); NewInst.setMetadata( LLVMContext::MD_prof, - MDB.createBranchWeights({static_cast(Count)})); + MDB.createBranchWeights({static_cast(Count)})); } using namespace ore; @@ -275,21 +631,177 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee, } // Promote indirect-call to conditional direct-call for one callsite. -uint32_t IndirectCallPromoter::tryToPromote( - CallBase &CB, const std::vector &Candidates, - uint64_t &TotalCount) { +bool IndirectCallPromoter::tryToPromoteWithFuncCmp( + CallBase &CB, Instruction *VPtr, ArrayRef Candidates, + uint64_t TotalCount, ArrayRef ICallProfDataRef, + uint32_t NumCandidates, VTableGUIDCountsMap &VTableGUIDCounts) { uint32_t NumPromoted = 0; for (const auto &C : Candidates) { - uint64_t Count = C.Count; - pgo::promoteIndirectCall(CB, C.TargetFunction, Count, TotalCount, SamplePGO, - &ORE); - assert(TotalCount >= Count); - TotalCount -= Count; + uint64_t FuncCount = C.Count; + pgo::promoteIndirectCall(CB, C.TargetFunction, FuncCount, TotalCount, + SamplePGO, &ORE); + assert(TotalCount >= FuncCount); + TotalCount -= FuncCount; NumOfPGOICallPromotion++; NumPromoted++; + + if (!EnableVTableProfileUse || C.VTableGUIDAndCounts.empty()) + continue; + + // After a virtual call candidate gets promoted, update the vtable's counts + // proportionally. Each vtable-guid in `C.VTableGUIDAndCounts` represents + // a vtable from which the virtual call is loaded. Compute the sum and use + // 128-bit APInt to improve accuracy. + uint64_t SumVTableCount = 0; + for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts) + SumVTableCount += VTableCount; + + for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts) { + APInt APFuncCount((unsigned)128, FuncCount, false /*signed*/); + APFuncCount *= VTableCount; + VTableGUIDCounts[GUID] -= APFuncCount.udiv(SumVTableCount).getZExtValue(); + } + } + if (NumPromoted == 0) + return false; + + assert(NumPromoted <= ICallProfDataRef.size() && + "Number of promoted functions should not be greater than the number " + "of values in profile metadata"); + + // Update value profiles on the indirect call. + updateFuncValueProfiles(CB, ICallProfDataRef.slice(NumPromoted), TotalCount, + NumCandidates); + updateVPtrValueProfiles(VPtr, VTableGUIDCounts); + return true; +} + +void IndirectCallPromoter::updateFuncValueProfiles( + CallBase &CB, ArrayRef CallVDs, uint64_t TotalCount, + uint32_t MaxMDCount) { + // First clear the existing !prof. + CB.setMetadata(LLVMContext::MD_prof, nullptr); + // Annotate the remaining value profiles if counter is not zero. + if (TotalCount != 0) + annotateValueSite(M, CB, CallVDs, TotalCount, IPVK_IndirectCallTarget, + MaxMDCount); +} + +void IndirectCallPromoter::updateVPtrValueProfiles( + Instruction *VPtr, VTableGUIDCountsMap &VTableGUIDCounts) { + if (!EnableVTableProfileUse || VPtr == nullptr || + !VPtr->getMetadata(LLVMContext::MD_prof)) + return; + VPtr->setMetadata(LLVMContext::MD_prof, nullptr); + std::vector VTableValueProfiles; + uint64_t TotalVTableCount = 0; + for (auto [GUID, Count] : VTableGUIDCounts) { + if (Count == 0) + continue; + + VTableValueProfiles.push_back({GUID, Count}); + TotalVTableCount += Count; + } + llvm::sort(VTableValueProfiles, + [](const InstrProfValueData &LHS, const InstrProfValueData &RHS) { + return LHS.Count > RHS.Count; + }); + + annotateValueSite(M, *VPtr, VTableValueProfiles, TotalVTableCount, + IPVK_VTableTarget, VTableValueProfiles.size()); +} + +bool IndirectCallPromoter::tryToPromoteWithVTableCmp( + CallBase &CB, Instruction *VPtr, + const std::vector &Candidates, uint64_t TotalFuncCount, + uint32_t NumCandidates, + MutableArrayRef ICallProfDataRef, + VTableGUIDCountsMap &VTableGUIDCounts) { + SmallVector PromotedFuncCount; + + for (const auto &Candidate : Candidates) { + for (auto &[GUID, Count] : Candidate.VTableGUIDAndCounts) + VTableGUIDCounts[GUID] -= Count; + + // 'OriginalBB' is the basic block of indirect call. After each candidate + // is promoted, a new basic block is created for the indirect fallback basic + // block and indirect call `CB` is moved into this new BB. + BasicBlock *OriginalBB = CB.getParent(); + promoteCallWithVTableCmp( + CB, VPtr, Candidate.TargetFunction, Candidate.AddressPoints, + createBranchWeights(CB.getContext(), Candidate.Count, + TotalFuncCount - Candidate.Count)); + + int SinkCount = tryToSinkInstructions(OriginalBB, CB.getParent()); + + ORE.emit([&]() { + OptimizationRemark Remark(DEBUG_TYPE, "Promoted", &CB); + + const auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts; + Remark << "Promote indirect call to " + << ore::NV("DirectCallee", Candidate.TargetFunction) + << " with count " << ore::NV("Count", Candidate.Count) + << " out of " << ore::NV("TotalCount", TotalFuncCount) << ", sink " + << ore::NV("SinkCount", SinkCount) + << " instruction(s) and compare " + << ore::NV("VTable", VTableGUIDAndCounts.size()) + << " vtable(s): {"; + + // Sort GUIDs so remark message is deterministic. + std::set GUIDSet; + for (auto [GUID, Count] : VTableGUIDAndCounts) + GUIDSet.insert(GUID); + for (auto Iter = GUIDSet.begin(); Iter != GUIDSet.end(); Iter++) { + if (Iter != GUIDSet.begin()) + Remark << ", "; + Remark << ore::NV("VTable", Symtab->getGlobalVariable(*Iter)); + } + + Remark << "}"; + + return Remark; + }); + + PromotedFuncCount.push_back(Candidate.Count); + + assert(TotalFuncCount >= Candidate.Count && + "Within one prof metadata, total count is the sum of counts from " + "individual pairs"); + // Use std::min since 'TotalFuncCount' is the saturated sum of individual + // counts, see + // https://github.com/llvm/llvm-project/blob/abedb3b8356d5d56f1c575c4f7682fba2cb19787/llvm/lib/ProfileData/InstrProf.cpp#L1281-L1288 + TotalFuncCount -= std::min(TotalFuncCount, Candidate.Count); + NumOfPGOICallPromotion++; } - return NumPromoted; + + if (PromotedFuncCount.empty()) + return false; + + // Update value profiles for 'CB' and 'VPtr', assuming that each 'CB' has a + // a distinct 'VPtr'. + // FIXME: When Clang `-fstrict-vtable-pointers` is enabled, a vtable might be + // used to load multiple virtual functions. The vtable profiles needs to be + // updated properly in that case (e.g, for each indirect call annotate both + // type profiles and function profiles in one !prof). + for (size_t I = 0; I < PromotedFuncCount.size(); I++) + ICallProfDataRef[I].Count -= + std::max(PromotedFuncCount[I], ICallProfDataRef[I].Count); + // Sort value profiles by count in descending order. + llvm::stable_sort(ICallProfDataRef, [](const InstrProfValueData &LHS, + const InstrProfValueData &RHS) { + return LHS.Count > RHS.Count; + }); + // Drop the pair if count is zero. + ArrayRef VDs( + ICallProfDataRef.begin(), + llvm::upper_bound(ICallProfDataRef, 0U, + [](uint64_t Count, const InstrProfValueData &ProfData) { + return ProfData.Count <= Count; + })); + updateFuncValueProfiles(CB, VDs, TotalFuncCount, NumCandidates); + updateVPtrValueProfiles(VPtr, VTableGUIDCounts); + return true; } // Traverse all the indirect-call callsite and get the value profile @@ -305,25 +817,151 @@ bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) { if (!NumCandidates || (PSI && PSI->hasProfileSummary() && !PSI->isHotCount(TotalCount, /*isForICP=*/true))) continue; + auto PromotionCandidates = getPromotionCandidatesForCallSite( *CB, ICallProfDataRef, TotalCount, NumCandidates); - uint32_t NumPromoted = tryToPromote(*CB, PromotionCandidates, TotalCount); - if (NumPromoted == 0) - continue; - Changed = true; - // Adjust the MD.prof metadata. First delete the old one. - CB->setMetadata(LLVMContext::MD_prof, nullptr); - // If all promoted, we don't need the MD.prof metadata. - if (TotalCount == 0 || NumPromoted == NumVals) - continue; - // Otherwise we need update with the un-promoted records back. - annotateValueSite(*F.getParent(), *CB, ICallProfDataRef.slice(NumPromoted), - TotalCount, IPVK_IndirectCallTarget, NumCandidates); + VTableGUIDCountsMap VTableGUIDCounts; + Instruction *VPtr = + computeVTableInfos(CB, VTableGUIDCounts, PromotionCandidates); + + if (isProfitableToCompareVTables(*CB, PromotionCandidates, TotalCount)) + Changed |= tryToPromoteWithVTableCmp(*CB, VPtr, PromotionCandidates, + TotalCount, NumCandidates, + ICallProfDataRef, VTableGUIDCounts); + else + Changed |= tryToPromoteWithFuncCmp(*CB, VPtr, PromotionCandidates, + TotalCount, ICallProfDataRef, + NumCandidates, VTableGUIDCounts); } return Changed; } +// TODO: Return false if the function addressing and vtable load instructions +// cannot sink to indirect fallback. +bool IndirectCallPromoter::isProfitableToCompareVTables( + const CallBase &CB, const std::vector &Candidates, + uint64_t TotalCount) { + if (!EnableVTableProfileUse || Candidates.empty()) + return false; + LLVM_DEBUG(dbgs() << "\nEvaluating vtable profitability for callsite #" + << NumOfPGOICallsites << CB << "\n"); + uint64_t RemainingVTableCount = TotalCount; + const size_t CandidateSize = Candidates.size(); + for (size_t I = 0; I < CandidateSize; I++) { + auto &Candidate = Candidates[I]; + auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts; + + LLVM_DEBUG(dbgs() << " Candidate " << I << " FunctionCount: " + << Candidate.Count << ", VTableCounts:"); + // Add [[maybe_unused]] since are only used by LLVM_DEBUG. + for ([[maybe_unused]] auto &[GUID, Count] : VTableGUIDAndCounts) + LLVM_DEBUG(dbgs() << " {" << Symtab->getGlobalVariable(GUID)->getName() + << ", " << Count << "}"); + LLVM_DEBUG(dbgs() << "\n"); + + uint64_t CandidateVTableCount = 0; + for (auto &[GUID, Count] : VTableGUIDAndCounts) + CandidateVTableCount += Count; + + if (CandidateVTableCount < Candidate.Count * ICPVTablePercentageThreshold) { + LLVM_DEBUG( + dbgs() << " function count " << Candidate.Count + << " and its vtable sum count " << CandidateVTableCount + << " have discrepancies. Bail out vtable comparison.\n"); + return false; + } + + RemainingVTableCount -= Candidate.Count; + + // 'MaxNumVTable' limits the number of vtables to make vtable comparison + // profitable. Comparing multiple vtables for one function candidate will + // insert additional instructions on the hot path, and allowing more than + // one vtable for non last candidates may or may not elongate the dependency + // chain for the subsequent candidates. Set its value to 1 for non-last + // candidate and allow option to override it for the last candidate. + int MaxNumVTable = 1; + if (I == CandidateSize - 1) + MaxNumVTable = ICPMaxNumVTableLastCandidate; + + if ((int)Candidate.AddressPoints.size() > MaxNumVTable) { + LLVM_DEBUG(dbgs() << " allow at most " << MaxNumVTable << " and got " + << Candidate.AddressPoints.size() + << " vtables. Bail out for vtable comparison.\n"); + return false; + } + } + + // If the indirect fallback is not cold, don't compare vtables. + if (PSI && PSI->hasProfileSummary() && + !PSI->isColdCount(RemainingVTableCount)) { + LLVM_DEBUG(dbgs() << " Indirect fallback basic block is not cold. Bail " + "out for vtable comparison.\n"); + return false; + } + + return true; +} + +// For virtual calls in the module, collect per-callsite information which will +// be used to associate an ICP candidate with a vtable and a specific function +// in the vtable. With type intrinsics (llvm.type.test), we can find virtual +// calls in a compile-time efficient manner (by iterating its users) and more +// importantly use the compatible type later to figure out the function byte +// offset relative to the start of vtables. +static void +computeVirtualCallSiteTypeInfoMap(Module &M, ModuleAnalysisManager &MAM, + VirtualCallSiteTypeInfoMap &VirtualCSInfo) { + // Right now only llvm.type.test is used to find out virtual call sites. + // With ThinLTO and whole-program-devirtualization, llvm.type.test and + // llvm.public.type.test are emitted, and llvm.public.type.test is either + // refined to llvm.type.test or dropped before indirect-call-promotion pass. + // + // FIXME: For fullLTO with VFE, `llvm.type.checked.load intrinsic` is emitted. + // Find out virtual calls by looking at users of llvm.type.checked.load in + // that case. + Function *TypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::type_test)); + if (!TypeTestFunc || TypeTestFunc->use_empty()) + return; + + auto &FAM = MAM.getResult(M).getManager(); + auto LookupDomTree = [&FAM](Function &F) -> DominatorTree & { + return FAM.getResult(F); + }; + // Iterate all type.test calls to find all indirect calls. + for (Use &U : llvm::make_early_inc_range(TypeTestFunc->uses())) { + auto *CI = dyn_cast(U.getUser()); + if (!CI) + continue; + auto *TypeMDVal = cast(CI->getArgOperand(1)); + if (!TypeMDVal) + continue; + auto *CompatibleTypeId = dyn_cast(TypeMDVal->getMetadata()); + if (!CompatibleTypeId) + continue; + + // Find out all devirtualizable call sites given a llvm.type.test + // intrinsic call. + SmallVector DevirtCalls; + SmallVector Assumes; + auto &DT = LookupDomTree(*CI->getFunction()); + findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT); + + for (auto &DevirtCall : DevirtCalls) { + CallBase &CB = DevirtCall.CB; + // Given an indirect call, try find the instruction which loads a + // pointer to virtual table. + Instruction *VTablePtr = + PGOIndirectCallVisitor::tryGetVTableInstruction(&CB); + if (!VTablePtr) + continue; + VirtualCSInfo[&CB] = {DevirtCall.Offset, VTablePtr, + CompatibleTypeId->getString()}; + } + } +} + // A wrapper function that does the actual work. static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, bool SamplePGO, ModuleAnalysisManager &MAM) { @@ -336,6 +974,20 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, return false; } bool Changed = false; + VirtualCallSiteTypeInfoMap VirtualCSInfo; + + if (EnableVTableProfileUse) + computeVirtualCallSiteTypeInfoMap(M, MAM, VirtualCSInfo); + + // VTableAddressPointOffsetVal stores the vtable address points. The vtable + // address point of a given is static (doesn't + // change after being computed once). + // IndirectCallPromoter::getOrCreateVTableAddressPointVar creates the map + // entry the first time a pair is seen, as + // promoteIndirectCalls processes an IR module and calls IndirectCallPromoter + // repeatedly on each function. + VTableAddressPointOffsetValMap VTableAddressPointOffsetVal; + for (auto &F : M) { if (F.isDeclaration() || F.hasOptNone()) continue; @@ -344,7 +996,9 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, MAM.getResult(M).getManager(); auto &ORE = FAM.getResult(F); - IndirectCallPromoter CallPromoter(F, &Symtab, SamplePGO, ORE); + IndirectCallPromoter CallPromoter(F, M, PSI, &Symtab, SamplePGO, + VirtualCSInfo, + VTableAddressPointOffsetVal, ORE); bool FuncChanged = CallPromoter.processFunction(PSI); if (ICPDUMPAFTER && FuncChanged) { LLVM_DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs())); diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index c5a68f355941..822c4b3d8175 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -318,6 +318,8 @@ static cl::opt PGOFunctionCriticalEdgeThreshold( cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold.")); +extern cl::opt MaxNumVTableAnnotations; + namespace llvm { // Command line option to turn on CFG dot dump after profile annotation. // Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts @@ -330,6 +332,7 @@ extern cl::opt ViewBlockFreqFuncName; // Command line option to enable vtable value profiling. Defined in // ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= extern cl::opt EnableVTableValueProfiling; +extern cl::opt EnableVTableProfileUse; extern cl::opt ProfileCorrelate; } // namespace llvm @@ -1706,6 +1709,14 @@ void SelectInstVisitor::visitSelectInst(SelectInst &SI) { llvm_unreachable("Unknown visiting mode"); } +static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind) { + if (ValueProfKind == IPVK_MemOPSize) + return MaxNumMemOPAnnotations; + if (ValueProfKind == llvm::IPVK_VTableTarget) + return MaxNumVTableAnnotations; + return MaxNumAnnotations; +} + // Traverse all valuesites and annotate the instructions for all value kind. void PGOUseFunc::annotateValueSites() { if (DisableValueProfiling) @@ -1740,10 +1751,10 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) { LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kind << "): Index = " << ValueSiteIndex << " out of " << NumValueSites << "\n"); - annotateValueSite(*M, *I.AnnotatedInst, ProfileRecord, - static_cast(Kind), ValueSiteIndex, - Kind == IPVK_MemOPSize ? MaxNumMemOPAnnotations - : MaxNumAnnotations); + annotateValueSite( + *M, *I.AnnotatedInst, ProfileRecord, + static_cast(Kind), ValueSiteIndex, + getMaxNumAnnotations(static_cast(Kind))); ValueSiteIndex++; } } @@ -2026,6 +2037,16 @@ static bool annotateAllFunctions( return false; } + if (EnableVTableProfileUse) { + for (GlobalVariable &G : M.globals()) { + if (!G.hasName() || !G.hasMetadata(LLVMContext::MD_type)) + continue; + + // Create the PGOFuncName meta data. + createPGONameMetadata(G, getPGOName(G, false /* InLTO*/)); + } + } + // Add the profile summary (read from the header of the indexed summary) here // so that we can use it below when reading counters (which checks if the // function should be marked with a cold or inlinehint attribute). @@ -2207,7 +2228,6 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M, }; auto *PSI = &MAM.getResult(M); - if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, *FS, LookupTLI, LookupBPI, LookupBFI, PSI, IsCS)) return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index f7b93fc8fd06..2e0c17cda8ba 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -23,6 +23,7 @@ #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/CaptureTracking.h" +#include "llvm/Analysis/IndirectCallVisitor.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/MemoryProfileInfo.h" #include "llvm/Analysis/ObjCARCAnalysisUtils.h" @@ -54,6 +55,7 @@ #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ProfDataUtils.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" @@ -1807,13 +1809,29 @@ void llvm::updateProfileCallee( ? 0 : PriorEntryCount + EntryDelta; + auto updateVTableProfWeight = [](CallBase *CB, const uint64_t NewEntryCount, + const uint64_t PriorEntryCount) { + Instruction *VPtr = PGOIndirectCallVisitor::tryGetVTableInstruction(CB); + if (VPtr) + scaleProfData(*VPtr, NewEntryCount, PriorEntryCount); + }; + // During inlining ? if (VMap) { uint64_t CloneEntryCount = PriorEntryCount - NewEntryCount; for (auto Entry : *VMap) if (isa(Entry.first)) - if (auto *CI = dyn_cast_or_null(Entry.second)) + if (auto *CI = dyn_cast_or_null(Entry.second)) { CI->updateProfWeight(CloneEntryCount, PriorEntryCount); + updateVTableProfWeight(CI, CloneEntryCount, PriorEntryCount); + } + + if (isa(Entry.first)) + if (auto *II = dyn_cast_or_null(Entry.second)) { + II->updateProfWeight(CloneEntryCount, PriorEntryCount); + updateVTableProfWeight(II, CloneEntryCount, PriorEntryCount); + } + } } if (EntryDelta) { @@ -1822,9 +1840,16 @@ void llvm::updateProfileCallee( for (BasicBlock &BB : *Callee) // No need to update the callsite if it is pruned during inlining. if (!VMap || VMap->count(&BB)) - for (Instruction &I : BB) - if (CallInst *CI = dyn_cast(&I)) + for (Instruction &I : BB) { + if (CallInst *CI = dyn_cast(&I)) { CI->updateProfWeight(NewEntryCount, PriorEntryCount); + updateVTableProfWeight(CI, NewEntryCount, PriorEntryCount); + } + if (InvokeInst *II = dyn_cast(&I)) { + II->updateProfWeight(NewEntryCount, PriorEntryCount); + updateVTableProfWeight(II, NewEntryCount, PriorEntryCount); + } + } } } diff --git a/llvm/test/Transforms/Inline/update_invoke_prof.ll b/llvm/test/Transforms/Inline/update_invoke_prof.ll new file mode 100644 index 000000000000..12eb7dbf418c --- /dev/null +++ b/llvm/test/Transforms/Inline/update_invoke_prof.ll @@ -0,0 +1,100 @@ +; Tests that instructions with value profiles and count-type branch weights are +; updated in both caller and callee after inline, but invoke instructions with +; taken or not taken branch probabilities are not updated. + +; RUN: opt < %s -passes='require,cgscc(inline)' -S | FileCheck %s + +declare i32 @__gxx_personality_v0(...) + +define void @caller(ptr %func) personality ptr @__gxx_personality_v0 !prof !15 { + call void @callee(ptr %func), !prof !16 + + ret void +} + +declare void @callee1(ptr %func) + +declare void @callee2(ptr %func) + +define void @callee(ptr %obj) personality ptr @__gxx_personality_v0 !prof !17 { + %vtable = load ptr, ptr %obj, !prof !21 + %func = load ptr, ptr %vtable + invoke void %func() + to label %next unwind label %lpad, !prof !18 + +next: + invoke void @callee1(ptr %func) + to label %cont unwind label %lpad, !prof !19 + +cont: + invoke void @callee2(ptr %func) + to label %ret unwind label %lpad, !prof !20 + +lpad: + %exn = landingpad {ptr, i32} + cleanup + unreachable + +ret: + ret void +} + +!llvm.module.flags = !{!1} +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"SampleProfile"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 10} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 2000} +!8 = !{!"NumCounts", i64 2} +!9 = !{!"NumFunctions", i64 2} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} +!15 = !{!"function_entry_count", i64 1000} +!16 = !{!"branch_weights", i64 1000} +!17 = !{!"function_entry_count", i32 1500} +!18 = !{!"VP", i32 0, i64 1500, i64 123, i64 900, i64 456, i64 600} +!19 = !{!"branch_weights", i32 1500} +!20 = !{!"branch_weights", i32 1234, i32 5678} +!21 = !{!"VP", i32 2, i64 1500, i64 789, i64 900, i64 321, i64 600} + +; CHECK-LABEL: define void @caller( +; CHECK-SAME: ptr [[FUNC:%.*]]) personality ptr @__gxx_personality_v0 !prof [[PROF14:![0-9]+]] { +; CHECK-NEXT: [[VTABLE_I:%.*]] = load ptr, ptr [[FUNC]], align 8, !prof [[PROF15:![0-9]+]] +; CHECK-NEXT: [[FUNC_I:%.*]] = load ptr, ptr [[VTABLE_I]], align 8 +; CHECK-NEXT: invoke void [[FUNC_I]]() +; CHECK-NEXT: to label %[[NEXT_I:.*]] unwind label %[[LPAD_I:.*]], !prof [[PROF16:![0-9]+]] +; CHECK: [[NEXT_I]]: +; CHECK-NEXT: invoke void @callee1(ptr [[FUNC_I]]) +; CHECK-NEXT: to label %[[CONT_I:.*]] unwind label %[[LPAD_I]], !prof [[PROF17:![0-9]+]] +; CHECK: [[CONT_I]]: +; CHECK-NEXT: invoke void @callee2(ptr [[FUNC_I]]) +; CHECK-NEXT: to label %[[CALLEE_EXIT:.*]] unwind label %[[LPAD_I]], !prof [[PROF18:![0-9]+]] +; + +; CHECK-LABEL: define void @callee( +; CHECK-SAME: ptr [[OBJ:%.*]]) personality ptr @__gxx_personality_v0 !prof [[PROF19:![0-9]+]] { +; CHECK-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[OBJ]], align 8, !prof [[PROF20:![0-9]+]] +; CHECK-NEXT: [[FUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8 +; CHECK-NEXT: invoke void [[FUNC]]() +; CHECK-NEXT: to label %[[NEXT:.*]] unwind label %[[LPAD:.*]], !prof [[PROF21:![0-9]+]] +; CHECK: [[NEXT]]: +; CHECK-NEXT: invoke void @callee1(ptr [[FUNC]]) +; CHECK-NEXT: to label %[[CONT:.*]] unwind label %[[LPAD]], !prof [[PROF22:![0-9]+]] +; CHECK: [[CONT]]: +; CHECK-NEXT: invoke void @callee2(ptr [[FUNC]]) +; CHECK-NEXT: to label %[[RET:.*]] unwind label %[[LPAD]], !prof [[PROF18]] + +; CHECK: [[PROF14]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF15]] = !{!"VP", i32 2, i64 1000, i64 789, i64 600, i64 321, i64 400} +; CHECK: [[PROF16]] = !{!"VP", i32 0, i64 1000, i64 123, i64 600, i64 456, i64 400} +; CHECK: [[PROF17]] = !{!"branch_weights", i32 1000} +; CHECK: [[PROF18]] = !{!"branch_weights", i32 1234, i32 5678} +; CHECK: [[PROF19]] = !{!"function_entry_count", i64 500} +; CHECK: [[PROF20]] = !{!"VP", i32 2, i64 500, i64 789, i64 300, i64 321, i64 200} +; CHECK: [[PROF21]] = !{!"VP", i32 0, i64 500, i64 123, i64 300, i64 456, i64 200} +; CHECK: [[PROF22]] = !{!"branch_weights", i32 500} diff --git a/llvm/test/Transforms/Inline/update_value_profile.ll b/llvm/test/Transforms/Inline/update_value_profile.ll new file mode 100644 index 000000000000..96aa35fb572d --- /dev/null +++ b/llvm/test/Transforms/Inline/update_value_profile.ll @@ -0,0 +1,85 @@ +; RUN: opt < %s -passes='require,cgscc(inline)' -inline-threshold=100 -S | FileCheck %s +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; When 'callee' is inlined into caller1 and caller2, the indirect call and vtable +; value profiles of the inlined copy should be scaled based on callers' profiles. +; The indirect call and vtable value profiles in 'callee' should be updated. +define i32 @callee(ptr %0, i32 %1) !prof !19 { +; CHECK-LABEL: define i32 @callee( +; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) !prof [[PROF0:![0-9]+]] { +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP0]], align 8, !prof [[PROF1:![0-9]+]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8 +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 [[TMP5]](ptr [[TMP0]], i32 [[TMP1]]), !prof [[PROF2:![0-9]+]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %3 = load ptr, ptr %0, !prof !15 + %5 = getelementptr inbounds i8, ptr %3, i64 8 + %6 = load ptr, ptr %5 + %7 = tail call i32 %6(ptr %0, i32 %1), !prof !16 + ret i32 %7 +} + +define i32 @caller1(i32 %0) !prof !17 { +; CHECK-LABEL: define i32 @caller1( +; CHECK-SAME: i32 [[TMP0:%.*]]) !prof [[PROF3:![0-9]+]] { +; CHECK-NEXT: [[TMP2:%.*]] = tail call ptr @_Z10createTypei(i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !prof [[PROF4:![0-9]+]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8 +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 [[TMP5]](ptr [[TMP2]], i32 [[TMP0]]), !prof [[PROF5:![0-9]+]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %2 = tail call ptr @_Z10createTypei(i32 %0) + %3 = tail call i32 @callee(ptr %2, i32 %0) + ret i32 %3 +} + +define i32 @caller2(i32 %0) !prof !18 { +; CHECK-LABEL: define i32 @caller2( +; CHECK-SAME: i32 [[TMP0:%.*]]) !prof [[PROF6:![0-9]+]] { +; CHECK-NEXT: [[TMP2:%.*]] = tail call ptr @_Z10createTypei(i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !prof [[PROF7:![0-9]+]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8 +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 [[TMP5]](ptr [[TMP2]], i32 [[TMP0]]), !prof [[PROF8:![0-9]+]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %2 = tail call ptr @_Z10createTypei(i32 %0) + %3 = tail call i32 @callee(ptr %2, i32 %0) + ret i32 %3 +} + +declare ptr @_Z10createTypei(i32) + +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 10} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 1000} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} +!15 = !{!"VP", i32 2, i64 1600, i64 321, i64 1000, i64 789, i64 600} +!16 = !{!"VP", i32 0, i64 1600, i64 123, i64 1000, i64 456, i64 600} +!17 = !{!"function_entry_count", i64 1000} +!18 = !{!"function_entry_count", i64 600} +!19 = !{!"function_entry_count", i64 1700} +;. +; CHECK: [[PROF0]] = !{!"function_entry_count", i64 100} +; CHECK: [[PROF1]] = !{!"VP", i32 2, i64 94, i64 321, i64 58, i64 789, i64 35} +; CHECK: [[PROF2]] = !{!"VP", i32 0, i64 94, i64 123, i64 58, i64 456, i64 35} +; CHECK: [[PROF3]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF4]] = !{!"VP", i32 2, i64 941, i64 321, i64 588, i64 789, i64 352} +; CHECK: [[PROF5]] = !{!"VP", i32 0, i64 941, i64 123, i64 588, i64 456, i64 352} +; CHECK: [[PROF6]] = !{!"function_entry_count", i64 600} +; CHECK: [[PROF7]] = !{!"VP", i32 2, i64 564, i64 321, i64 352, i64 789, i64 211} +; CHECK: [[PROF8]] = !{!"VP", i32 0, i64 564, i64 123, i64 352, i64 456, i64 211} +;. diff --git a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll new file mode 100644 index 000000000000..c77be3b1ed24 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll @@ -0,0 +1,139 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 + +; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=2 -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,VTABLE-CMP +; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=1 -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,FUNC-CMP + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@Base1 = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr null, ptr @Base1_foo, ptr @Base1_bar] }, !type !0 +@Base2 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @Base2_foo] }, !type !2 +@Base3 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @Base3_foo] }, !type !6 + +@Derived1 = constant { [3 x ptr], [4 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @Base2_foo], [4 x ptr] [ptr null, ptr null, ptr @Base1_foo, ptr @Derived1_bar] }, !type !1, !type !2, !type !3 +@Derived2 = constant { [3 x ptr], [3 x ptr], [4 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @Base3_foo], [3 x ptr] [ptr null, ptr null, ptr @Base2_foo], [4 x ptr] [ptr null, ptr null, ptr @Base1_foo, ptr @Derived2_bar] }, !type !4, !type !5, !type !6, !type !7 +@Derived3 = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr null, ptr @Base1_foo, ptr @Base1_bar] }, !type !0, !type !8 + +; VTABLE-CMP: remark: :0:0: Promote indirect call to Derived1_bar with count 600 out of 1600, sink 2 instruction(s) and compare 1 vtable(s): {Derived1} +; VTABLE-CMP: remark: :0:0: Promote indirect call to Derived2_bar with count 500 out of 1000, sink 2 instruction(s) and compare 1 vtable(s): {Derived2} +; VTABLE-CMP: remark: :0:0: Promote indirect call to Base1_bar with count 400 out of 500, sink 2 instruction(s) and compare 2 vtable(s): {Derived3, Base1} + +define void @test(ptr %d) { +; VTABLE-CMP-LABEL: define void @test( +; VTABLE-CMP-SAME: ptr [[D:%.*]]) { +; VTABLE-CMP-NEXT: [[ENTRY:.*:]] +; VTABLE-CMP-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[D]], align 8, !prof [[PROF9:![0-9]+]] +; VTABLE-CMP-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"Base1") +; VTABLE-CMP-NEXT: tail call void @llvm.assume(i1 [[TMP0]]) +; VTABLE-CMP-NEXT: [[TMP1:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @Derived1, i32 40) +; VTABLE-CMP-NEXT: br i1 [[TMP1]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF10:![0-9]+]] +; VTABLE-CMP: [[IF_TRUE_DIRECT_TARG]]: +; VTABLE-CMP-NEXT: call void @Derived1_bar(ptr [[D]]) +; VTABLE-CMP-NEXT: br label %[[IF_END_ICP:.*]] +; VTABLE-CMP: [[IF_FALSE_ORIG_INDIRECT]]: +; VTABLE-CMP-NEXT: [[TMP2:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @Derived2, i32 64) +; VTABLE-CMP-NEXT: br i1 [[TMP2]], label %[[IF_TRUE_DIRECT_TARG1:.*]], label %[[IF_FALSE_ORIG_INDIRECT2:.*]], !prof [[PROF11:![0-9]+]] +; VTABLE-CMP: [[IF_TRUE_DIRECT_TARG1]]: +; VTABLE-CMP-NEXT: call void @Derived2_bar(ptr [[D]]) +; VTABLE-CMP-NEXT: br label %[[IF_END_ICP3:.*]] +; VTABLE-CMP: [[IF_FALSE_ORIG_INDIRECT2]]: +; VTABLE-CMP-NEXT: [[TMP3:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @Base1, i32 16) +; VTABLE-CMP-NEXT: [[TMP4:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @Derived3, i32 16) +; VTABLE-CMP-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] +; VTABLE-CMP-NEXT: br i1 [[TMP5]], label %[[IF_TRUE_DIRECT_TARG4:.*]], label %[[IF_FALSE_ORIG_INDIRECT5:.*]], !prof [[PROF12:![0-9]+]] +; VTABLE-CMP: [[IF_TRUE_DIRECT_TARG4]]: +; VTABLE-CMP-NEXT: call void @Base1_bar(ptr [[D]]) +; VTABLE-CMP-NEXT: br label %[[IF_END_ICP6:.*]] +; VTABLE-CMP: [[IF_FALSE_ORIG_INDIRECT5]]: +; VTABLE-CMP-NEXT: [[VFN:%.*]] = getelementptr inbounds ptr, ptr [[VTABLE]], i64 1 +; VTABLE-CMP-NEXT: [[TMP6:%.*]] = load ptr, ptr [[VFN]], align 8 +; VTABLE-CMP-NEXT: call void [[TMP6]](ptr [[D]]) +; VTABLE-CMP-NEXT: br label %[[IF_END_ICP6]] +; VTABLE-CMP: [[IF_END_ICP6]]: +; VTABLE-CMP-NEXT: br label %[[IF_END_ICP3]] +; VTABLE-CMP: [[IF_END_ICP3]]: +; VTABLE-CMP-NEXT: br label %[[IF_END_ICP]] +; VTABLE-CMP: [[IF_END_ICP]]: +; VTABLE-CMP-NEXT: ret void +; +; FUNC-CMP-LABEL: define void @test( +; FUNC-CMP-SAME: ptr [[D:%.*]]) { +; FUNC-CMP-NEXT: [[ENTRY:.*:]] +; FUNC-CMP-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[D]], align 8, !prof [[PROF9:![0-9]+]] +; FUNC-CMP-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"Base1") +; FUNC-CMP-NEXT: tail call void @llvm.assume(i1 [[TMP0]]) +; FUNC-CMP-NEXT: [[VFN:%.*]] = getelementptr inbounds ptr, ptr [[VTABLE]], i64 1 +; FUNC-CMP-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VFN]], align 8 +; FUNC-CMP-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP1]], @Derived1_bar +; FUNC-CMP-NEXT: br i1 [[TMP2]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF10:![0-9]+]] +; FUNC-CMP: [[IF_TRUE_DIRECT_TARG]]: +; FUNC-CMP-NEXT: call void @Derived1_bar(ptr [[D]]) +; FUNC-CMP-NEXT: br label %[[IF_END_ICP:.*]] +; FUNC-CMP: [[IF_FALSE_ORIG_INDIRECT]]: +; FUNC-CMP-NEXT: [[TMP3:%.*]] = icmp eq ptr [[TMP1]], @Derived2_bar +; FUNC-CMP-NEXT: br i1 [[TMP3]], label %[[IF_TRUE_DIRECT_TARG1:.*]], label %[[IF_FALSE_ORIG_INDIRECT2:.*]], !prof [[PROF11:![0-9]+]] +; FUNC-CMP: [[IF_TRUE_DIRECT_TARG1]]: +; FUNC-CMP-NEXT: call void @Derived2_bar(ptr [[D]]) +; FUNC-CMP-NEXT: br label %[[IF_END_ICP3:.*]] +; FUNC-CMP: [[IF_FALSE_ORIG_INDIRECT2]]: +; FUNC-CMP-NEXT: [[TMP4:%.*]] = icmp eq ptr [[TMP1]], @Base1_bar +; FUNC-CMP-NEXT: br i1 [[TMP4]], label %[[IF_TRUE_DIRECT_TARG4:.*]], label %[[IF_FALSE_ORIG_INDIRECT5:.*]], !prof [[PROF12:![0-9]+]] +; FUNC-CMP: [[IF_TRUE_DIRECT_TARG4]]: +; FUNC-CMP-NEXT: call void @Base1_bar(ptr [[D]]) +; FUNC-CMP-NEXT: br label %[[IF_END_ICP6:.*]] +; FUNC-CMP: [[IF_FALSE_ORIG_INDIRECT5]]: +; FUNC-CMP-NEXT: call void [[TMP1]](ptr [[D]]) +; FUNC-CMP-NEXT: br label %[[IF_END_ICP6]] +; FUNC-CMP: [[IF_END_ICP6]]: +; FUNC-CMP-NEXT: br label %[[IF_END_ICP3]] +; FUNC-CMP: [[IF_END_ICP3]]: +; FUNC-CMP-NEXT: br label %[[IF_END_ICP]] +; FUNC-CMP: [[IF_END_ICP]]: +; FUNC-CMP-NEXT: ret void +; +entry: + %vtable = load ptr, ptr %d, !prof !9 + %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"Base1") + tail call void @llvm.assume(i1 %0) + %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 + %1 = load ptr, ptr %vfn + call void %1(ptr %d), !prof !10 + ret void +} + +define void @Base1_bar(ptr %this) { + ret void +} + +define void @Derived1_bar(ptr %this) { + ret void +} + +define void @Derived2_bar(ptr %this) { + ret void +} + + +declare i1 @llvm.type.test(ptr, metadata) +declare void @llvm.assume(i1) +declare i32 @Base2_foo(ptr) +declare i32 @Base1_foo(ptr) +declare void @Base3_foo(ptr) + +!0 = !{i64 16, !"Base1"} +!1 = !{i64 40, !"Base1"} +!2 = !{i64 16, !"Base2"} +!3 = !{i64 16, !"Derived1"} +!4 = !{i64 64, !"Base1"} +!5 = !{i64 40, !"Base2"} +!6 = !{i64 16, !"Base3"} +!7 = !{i64 16, !"Derived2"} +!8 = !{i64 16, !"Derived3"} +!9 = !{!"VP", i32 2, i64 1600, i64 -4123858694673519054, i64 600, i64 -7211198353767973908, i64 500, i64 -3574436251470806727, i64 200, i64 6288809125658696740, i64 200, i64 12345678, i64 100} +!10 = !{!"VP", i32 0, i64 1600, i64 3827408714133779784, i64 600, i64 5837445539218476403, i64 500, i64 -9064955852395570538, i64 400, i64 56781234, i64 100} +;. +; VTABLE-COMMON: [[PROF9]] = !{!"VP", i32 2, i64 100, i64 12345678, i64 100} +; VTABLE-COMMON: [[PROF10]] = !{!"branch_weights", i32 600, i32 1000} +; VTABLE-COMMON: [[PROF11]] = !{!"branch_weights", i32 500, i32 500} +; VTABLE-COMMON: [[PROF12]] = !{!"branch_weights", i32 400, i32 100} + diff --git a/llvm/test/Transforms/PGOProfile/icp_vtable_invoke.ll b/llvm/test/Transforms/PGOProfile/icp_vtable_invoke.ll new file mode 100644 index 000000000000..6d3a6972f688 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/icp_vtable_invoke.ll @@ -0,0 +1,125 @@ +; RUN: opt < %s -passes='pgo-icall-prom' -enable-vtable-profile-use -S | FileCheck %s --check-prefix=VTABLE + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@_ZTV4Base = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN4Base10get_ticketEv] }, !type !0, !type !1 +@_ZTV7Derived = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN7Derived10get_ticketEv] }, !type !0, !type !1, !type !2, !type !3 + +@.str = private constant [15 x i8] c"out of tickets\00" + +define i32 @test(ptr %b) personality ptr @__gxx_personality_v0 { +; VTABLE-LABEL: define i32 @test( +; VTABLE-SAME: ptr [[B:%.*]]) personality ptr @__gxx_personality_v0 { +; VTABLE-NEXT: [[ENTRY:.*:]] +; VTABLE-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[B]], align 8 +; VTABLE-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"_ZTS4Base") +; VTABLE-NEXT: tail call void @llvm.assume(i1 [[TMP0]]) +; VTABLE-NEXT: [[TMP3:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @_ZTV7Derived, i32 16) +; VTABLE-NEXT: br i1 [[TMP3]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF4:![0-9]+]] +; VTABLE: [[IF_TRUE_DIRECT_TARG]]: +; VTABLE-NEXT: [[TMP2:%.*]] = invoke i32 @_ZN7Derived10get_ticketEv(ptr [[B]]) +; VTABLE-NEXT: to label %[[IF_END_ICP:.*]] unwind label %[[LPAD:.*]] +; VTABLE: [[IF_FALSE_ORIG_INDIRECT]]: +; VTABLE-NEXT: [[TMP4:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @_ZTV4Base, i32 16) +; VTABLE-NEXT: br i1 [[TMP4]], label %[[IF_TRUE_DIRECT_TARG1:.*]], label %[[IF_FALSE_ORIG_INDIRECT2:.*]], !prof [[PROF5:![0-9]+]] +; VTABLE: [[IF_TRUE_DIRECT_TARG1]]: +; VTABLE-NEXT: [[TMP5:%.*]] = invoke i32 @_ZN4Base10get_ticketEv(ptr [[B]]) +; VTABLE-NEXT: to label %[[IF_END_ICP3:.*]] unwind label %[[LPAD]] +; VTABLE: [[IF_FALSE_ORIG_INDIRECT2]]: +; VTABLE-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VTABLE]], align 8 +; VTABLE-NEXT: [[CALL:%.*]] = invoke i32 [[TMP1]](ptr [[B]]) +; VTABLE-NEXT: to label %[[IF_END_ICP3]] unwind label %[[LPAD]] +; VTABLE: [[IF_END_ICP3]]: +; VTABLE-NEXT: [[TMP6:%.*]] = phi i32 [ [[CALL]], %[[IF_FALSE_ORIG_INDIRECT2]] ], [ [[TMP5]], %[[IF_TRUE_DIRECT_TARG1]] ] +; VTABLE-NEXT: br label %[[IF_END_ICP]] +; VTABLE: [[IF_END_ICP]]: +; VTABLE-NEXT: [[TMP7:%.*]] = phi i32 [ [[TMP6]], %[[IF_END_ICP3]] ], [ [[TMP2]], %[[IF_TRUE_DIRECT_TARG]] ] +; VTABLE-NEXT: br label %[[NEXT:.*]] +; VTABLE: [[NEXT]]: +; VTABLE-NEXT: ret i32 [[TMP7]] +; VTABLE: [[LPAD]]: +; VTABLE-NEXT: [[EXN:%.*]] = landingpad { ptr, i32 } +; VTABLE-NEXT: cleanup +; VTABLE-NEXT: unreachable +; +entry: + %vtable = load ptr, ptr %b, !prof !4 + %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS4Base") + tail call void @llvm.assume(i1 %0) + %1 = load ptr, ptr %vtable + %call = invoke i32 %1(ptr %b) to label %next unwind label %lpad, !prof !5 + +next: + ret i32 %call + +lpad: + %exn = landingpad {ptr, i32} + cleanup + unreachable +} + +declare void @make_error(ptr, ptr, i32) +declare i32 @get_ticket_id() +declare ptr @__cxa_allocate_exception(i64) + +define i32 @_ZN4Base10get_ticketEv(ptr %this) personality ptr @__gxx_personality_v0 { +entry: + %call = tail call i32 @get_ticket_id() + %cmp.not = icmp eq i32 %call, -1 + br i1 %cmp.not, label %if.end, label %if.then + +if.then: + ret i32 %call + +if.end: + %exception = tail call ptr @__cxa_allocate_exception(i64 1) + invoke void @make_error(ptr %exception, ptr @.str, i32 1) + to label %invoke.cont unwind label %lpad + +invoke.cont: + unreachable + +lpad: + %0 = landingpad { ptr, i32 } + cleanup + resume { ptr, i32 } %0 +} + +define i32 @_ZN7Derived10get_ticketEv(ptr %this) personality ptr @__gxx_personality_v0 { +entry: + %call = tail call i32 @get_ticket_id() + %cmp.not = icmp eq i32 %call, -1 + br i1 %cmp.not, label %if.end, label %if.then + +if.then: + ret i32 %call + +if.end: + %exception = tail call ptr @__cxa_allocate_exception(i64 1) + invoke void @make_error(ptr %exception, ptr @.str, i32 2) + to label %invoke.cont unwind label %lpad + +invoke.cont: + unreachable + +lpad: + %0 = landingpad { ptr, i32 } + cleanup + resume { ptr, i32 } %0 +} + +declare i1 @llvm.type.test(ptr, metadata) +declare void @llvm.assume(i1) +declare i32 @__gxx_personality_v0(...) + +!0 = !{i64 16, !"_ZTS4Base"} +!1 = !{i64 16, !"_ZTSM4BaseFivE.virtual"} +!2 = !{i64 16, !"_ZTS7Derived"} +!3 = !{i64 16, !"_ZTSM7DerivedFivE.virtual"} +!4 = !{!"VP", i32 2, i64 1600, i64 13870436605473471591, i64 900, i64 1960855528937986108, i64 700} +!5 = !{!"VP", i32 0, i64 1600, i64 14811317294552474744, i64 900, i64 9261744921105590125, i64 700} + +; VTABLE: [[PROF4]] = !{!"branch_weights", i32 900, i32 700} +; VTABLE: [[PROF5]] = !{!"branch_weights", i32 700, i32 0} +;. diff --git a/llvm/test/Transforms/PGOProfile/icp_vtable_tail_call.ll b/llvm/test/Transforms/PGOProfile/icp_vtable_tail_call.ll new file mode 100644 index 000000000000..fb9ec0d0c85f --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/icp_vtable_tail_call.ll @@ -0,0 +1,68 @@ +; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -S 2>&1 | FileCheck %s --check-prefixes=VTABLE,REMARK + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; REMARK: remark: :0:0: Promote indirect call to _ZN7Derived5func1Eii with count 900 out of 1600, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV7Derived} +; REMARK: remark: :0:0: Promote indirect call to _ZN4Base5func1Eii with count 700 out of 700, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV4Base} + +@_ZTV7Derived = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN7Derived5func1Eii] }, !type !0, !type !1, !type !2, !type !3 +@_ZTV4Base = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN4Base5func1Eii] }, !type !0, !type !1 + +define i32 @test_tail_call(ptr %ptr, i32 %a, i32 %b) { +; VTABLE-LABEL: define i32 @test_tail_call( +; VTABLE-SAME: ptr [[PTR:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; VTABLE-NEXT: entry: +; VTABLE-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[PTR]], align 8 +; VTABLE-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"_ZTS4Base") +; VTABLE-NEXT: tail call void @llvm.assume(i1 [[TMP0]]) +; VTABLE-NEXT: [[TMP2:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @_ZTV7Derived, i32 16) +; VTABLE-NEXT: br i1 [[TMP2]], label [[IF_TRUE_DIRECT_TARG:%.*]], label [[TMP4:%.*]], !prof [[PROF4:![0-9]+]] +; VTABLE: if.true.direct_targ: +; VTABLE-NEXT: [[TMP3:%.*]] = musttail call i32 @_ZN7Derived5func1Eii(ptr [[PTR]], i32 [[A]], i32 [[B]]) +; VTABLE-NEXT: ret i32 [[TMP3]] +; VTABLE: 3: +; VTABLE-NEXT: [[TMP4:%.*]] = icmp eq ptr [[VTABLE]], getelementptr inbounds (i8, ptr @_ZTV4Base, i32 16) +; VTABLE-NEXT: br i1 [[TMP4]], label [[IF_TRUE_DIRECT_TARG1:%.*]], label [[TMP7:%.*]], !prof [[PROF5:![0-9]+]] +; VTABLE: if.true.direct_targ1: +; VTABLE-NEXT: [[TMP6:%.*]] = musttail call i32 @_ZN4Base5func1Eii(ptr [[PTR]], i32 [[A]], i32 [[B]]) +; VTABLE-NEXT: ret i32 [[TMP6]] +; VTABLE: 6: +; VTABLE-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VTABLE]], align 8 +; VTABLE-NEXT: [[CALL:%.*]] = musttail call i32 [[TMP1]](ptr [[PTR]], i32 [[A]], i32 [[B]]) +; VTABLE-NEXT: ret i32 [[CALL]] +; +entry: + %vtable = load ptr, ptr %ptr, !prof !4 + %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS4Base") + tail call void @llvm.assume(i1 %0) + %1 = load ptr, ptr %vtable + %call = musttail call i32 %1(ptr %ptr, i32 %a, i32 %b), !prof !5 + ret i32 %call +} + +declare i1 @llvm.type.test(ptr, metadata) +declare void @llvm.assume(i1) + +define i32 @_ZN7Derived5func1Eii(ptr %this, i32 %a, i32 %b) { +entry: + %sub = sub nsw i32 %a, %b + ret i32 %sub +} + +define i32 @_ZN4Base5func1Eii(ptr %this, i32 %a, i32 %b) { +entry: + %add = add nsw i32 %b, %a + ret i32 %add +} + + +!0 = !{i64 16, !"_ZTS4Base"} +!1 = !{i64 16, !"_ZTSM4BaseFiiiE.virtual"} +!2 = !{i64 16, !"_ZTS7Derived"} +!3 = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"} +!4 = !{!"VP", i32 2, i64 1600, i64 13870436605473471591, i64 900, i64 1960855528937986108, i64 700} +!5 = !{!"VP", i32 0, i64 1600, i64 7889036118036845314, i64 900, i64 10495086226207060333, i64 700} + +; VTABLE: [[PROF4]] = !{!"branch_weights", i32 900, i32 700} +; VTABLE: [[PROF5]] = !{!"branch_weights", i32 700, i32 0} -- Gitee From 7839d30e27f565664716a112fc9fa6db758d2c02 Mon Sep 17 00:00:00 2001 From: Szymon Sobieszek Date: Fri, 8 Aug 2025 15:46:46 -0400 Subject: [PATCH 23/28] [ICP] Fix for failing test cases related to porting dtp --- .../CodeGen/coverage-profile-raw-version.c | 9 - .../include/llvm/Analysis/TypeMetadataUtils.h | 9 + llvm/include/llvm/IR/Instructions.h | 3 + llvm/include/llvm/IR/MDBuilder.h | 6 +- llvm/include/llvm/IR/ProfDataUtils.h | 34 + llvm/include/llvm/ProfileData/InstrProf.h | 16 + .../Transforms/Utils/CallPromotionUtils.h | 33 +- .../IndirectCallPromotionAnalysis.cpp | 6 +- llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 6 +- llvm/lib/Analysis/TypeMetadataUtils.cpp | 20 + llvm/lib/IR/Instructions.cpp | 12 + llvm/lib/IR/MDBuilder.cpp | 14 +- llvm/lib/IR/ProfDataUtils.cpp | 153 +++- .../Instrumentation/IndirectCallPromotion.cpp | 4 +- .../Transforms/Utils/CallPromotionUtils.cpp | 73 +- llvm/lib/Transforms/Utils/InlineFunction.cpp | 2 +- .../Transforms/Inline/update_value_profile.ll | 8 +- .../Transforms/PGOProfile/criticaledge.ll | 2 +- .../PGOProfile/statics_counter_naming.ll | 4 +- .../Transforms/PGOProfile/vtable_profile.ll | 8 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 48 +- llvm/unittests/ProfileData/InstrProfTest.cpp | 854 +++++------------- .../Utils/CallPromotionUtilsTest.cpp | 88 ++ 23 files changed, 676 insertions(+), 736 deletions(-) delete mode 100644 clang/test/CodeGen/coverage-profile-raw-version.c diff --git a/clang/test/CodeGen/coverage-profile-raw-version.c b/clang/test/CodeGen/coverage-profile-raw-version.c deleted file mode 100644 index 749dce50298f..000000000000 --- a/clang/test/CodeGen/coverage-profile-raw-version.c +++ /dev/null @@ -1,9 +0,0 @@ -// RUN: %clang_cc1 -debug-info-kind=standalone -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -debug-info-kind=standalone -mllvm -debug-info-correlate -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s --check-prefix=DEBUG_INFO - -// CHECK: @__llvm_profile_raw_version = {{.*}}constant i64 8 -// DEBUG_INFO: @__llvm_profile_raw_version = {{.*}}constant i64 576460752303423496 - -int main() { - return 0; -} diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h index dab67aad1ab0..8894945c28d9 100644 --- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h +++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h @@ -15,6 +15,7 @@ #define LLVM_ANALYSIS_TYPEMETADATAUTILS_H #include +#include namespace llvm { @@ -24,6 +25,7 @@ class CallInst; class Constant; class Function; class DominatorTree; +class GlobalVariable; class Instruction; class Module; @@ -77,6 +79,13 @@ void findDevirtualizableCallsForTypeCheckedLoad( Constant *getPointerAtOffset(Constant *I, uint64_t Offset, Module &M, Constant *TopLevelGlobal = nullptr); +/// Given a vtable and a specified offset, returns the function and the trivial +/// pointer at the specified offset in pair iff the pointer at the specified +/// offset is a function or an alias to a function. Returns a pair of nullptr +/// otherwise. +std::pair +getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M); + /// Finds the same "relative pointer" pattern as described above, where the /// target is `F`, and replaces the entire pattern with a constant zero. void replaceRelativePointerUsersWithZero(Function *F); diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index 9d638af6eeef..b32bb6d14302 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -4013,6 +4013,9 @@ public: unsigned getNumSuccessors() const { return 2; } + /// Updates profile metadata by scaling it by \p S / \p T. + void updateProfWeight(uint64_t S, uint64_t T); + // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return (I->getOpcode() == Instruction::Invoke); diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index 39165453de16..d972f85ecb47 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -59,10 +59,12 @@ public: //===------------------------------------------------------------------===// /// Return metadata containing two branch weights. - MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight); + MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, + bool IsExpected = false); /// Return metadata containing a number of branch weights. - MDNode *createBranchWeights(ArrayRef Weights); + MDNode *createBranchWeights(ArrayRef Weights, + bool IsExpected = false); /// Return metadata specifying that a branch or switch is unpredictable. MDNode *createUnpredictable(); diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h index 8019d0a3969b..4e1bcff528ef 100644 --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -55,6 +55,19 @@ MDNode *getBranchWeightMDNode(const Instruction &I); /// Nullptr otherwise. MDNode *getValidBranchWeightMDNode(const Instruction &I); +/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect* +/// intrinsic +bool hasBranchWeightOrigin(const Instruction &I); + +/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect* +/// intrinsic +bool hasBranchWeightOrigin(const MDNode *ProfileData); + +/// Return the offset to the first branch weight data +unsigned getBranchWeightOffset(const MDNode *ProfileData); + +unsigned getNumBranchWeights(const MDNode &ProfileData); + /// Extract branch weights from MD_prof metadata /// /// \param ProfileData A pointer to an MDNode. @@ -64,6 +77,16 @@ MDNode *getValidBranchWeightMDNode(const Instruction &I); bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl &Weights); +/// Faster version of extractBranchWeights() that skips checks and must only +/// be called with "branch_weights" metadata nodes. Supports uint32_t. +void extractFromBranchWeightMD32(const MDNode *ProfileData, + SmallVectorImpl &Weights); + +/// Faster version of extractBranchWeights() that skips checks and must only +/// be called with "branch_weights" metadata nodes. Supports uint64_t. +void extractFromBranchWeightMD64(const MDNode *ProfileData, + SmallVectorImpl &Weights); + /// Extract branch weights attatched to an Instruction /// /// \param I The Instruction to extract weights from. @@ -99,5 +122,16 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights); /// metadata was found. bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights); +/// Create a new `branch_weights` metadata node and add or overwrite +/// a `prof` metadata reference to instruction `I`. +/// \param I the Instruction to set branch weights on. +/// \param Weights an array of weights to set on instruction I. +/// \param IsExpected were these weights added from an llvm.expect* intrinsic. +void setBranchWeights(Instruction &I, ArrayRef Weights, + bool IsExpected); + +/// Scaling the profile data attached to 'I' using the ratio of S/T. +void scaleProfData(Instruction &I, uint64_t S, uint64_t T); + } // namespace llvm #endif diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index e408b5d4b1a6..92c85e10355d 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -593,6 +593,7 @@ public: Error create(const FuncNameIterRange &FuncIterRange, const VTableNameIterRange &VTableIterRange); + // Map the MD5 of the symbol name to the name. Error addSymbolName(StringRef SymbolName) { if (SymbolName.empty()) return make_error(instrprof_error::malformed, @@ -677,6 +678,11 @@ public: /// Return function from the name's md5 hash. Return nullptr if not found. inline Function *getFunction(uint64_t FuncMD5Hash); + /// Return the function's original assembly name by stripping off + /// the prefix attached (to symbols with priviate linkage). For + /// global functions, it returns the same string as getFuncName. + inline StringRef getOrigFuncName(uint64_t FuncMD5Hash); + /// Return the global variable corresponding to md5 hash. Return nullptr if /// not found. inline GlobalVariable *getGlobalVariable(uint64_t MD5Hash); @@ -777,6 +783,16 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { return nullptr; } +// See also getPGOFuncName implementation. These two need to be +// matched. +StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { + StringRef PGOName = getFuncName(FuncMD5Hash); + size_t S = PGOName.find_first_of(':'); + if (S == StringRef::npos) + return PGOName; + return PGOName.drop_front(S + 1); +} + GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) { return MD5VTableMap.lookup(MD5Hash); } diff --git a/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h b/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h index fcb384ec3613..530cf46ca38b 100644 --- a/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h +++ b/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h @@ -15,9 +15,12 @@ #define LLVM_TRANSFORMS_UTILS_CALLPROMOTIONUTILS_H namespace llvm { +template class ArrayRef; +class Constant; class CallBase; class CastInst; class Function; +class Instruction; class MDNode; class Value; @@ -41,7 +44,9 @@ bool isLegalToPromote(const CallBase &CB, Function *Callee, CallBase &promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast = nullptr); -/// Promote the given indirect call site to conditionally call \p Callee. +/// Promote the given indirect call site to conditionally call \p Callee. The +/// promoted direct call instruction is predicated on `CB.getCalledOperand() == +/// Callee`. /// /// This function creates an if-then-else structure at the location of the call /// site. The original call site is moved into the "else" block. A clone of the @@ -51,6 +56,22 @@ CallBase &promoteCall(CallBase &CB, Function *Callee, CallBase &promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights = nullptr); +/// This is similar to `promoteCallWithIfThenElse` except that the condition to +/// promote a virtual call is that \p VPtr is the same as any of \p +/// AddressPoints. +/// +/// This function is expected to be used on virtual calls (a subset of indirect +/// calls). \p VPtr is the virtual table address stored in the objects, and +/// \p AddressPoints contains vtable address points. A vtable address point is +/// a location inside the vtable that's referenced by vpointer in C++ objects. +/// +/// TODO: sink the address-calculation instructions of indirect callee to the +/// indirect call fallback after transformation +CallBase &promoteCallWithVTableCmp(CallBase &CB, Instruction *Vptr, + Function *Callee, + ArrayRef AddressPoints, + MDNode *BranchWeights); + /// Try to promote (devirtualize) a virtual call on an Alloca. Return true on /// success. /// @@ -76,11 +97,11 @@ bool tryPromoteCall(CallBase &CB); /// Predicate and clone the given call site. /// -/// This function creates an if-then-else structure at the location of the call -/// site. The "if" condition compares the call site's called value to the given -/// callee. The original call site is moved into the "else" block, and a clone -/// of the call site is placed in the "then" block. The cloned instruction is -/// returned. +/// This function creates an if-then-else structure at the location of the +/// call site. The "if" condition compares the call site's called value to +/// the given callee. The original call site is moved into the "else" block, +/// and a clone of the call site is placed in the "then" block. The cloned +/// instruction is returned. CallBase &versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights); } // end namespace llvm diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp index 643c155ba6d7..c4f896311a2d 100644 --- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -89,9 +89,9 @@ uint32_t ICallPromotionAnalysis::getProfitablePromotionCandidates( MutableArrayRef ICallPromotionAnalysis::getPromotionCandidatesForInstruction( - const Instruction *I, uint32_t &NumVals, uint64_t &TotalCount, - uint32_t &NumCandidates) { - bool Res = + const Instruction *I, uint64_t &TotalCount, uint32_t &NumCandidates) { + uint32_t NumVals; + auto Res = getValueProfDataFromInst(*I, IPVK_IndirectCallTarget, MaxNumPromotions, ValueDataArray.get(), NumVals, TotalCount); if (!Res) { diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index dab3ce2e3a5a..cd8ca50c4038 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -444,11 +444,11 @@ static void computeFunctionSummary( } } - uint32_t NumVals, NumCandidates; + uint32_t NumCandidates; uint64_t TotalCount; auto CandidateProfileData = - ICallAnalysis.getPromotionCandidatesForInstruction( - &I, NumVals, TotalCount, NumCandidates); + ICallAnalysis.getPromotionCandidatesForInstruction(&I, TotalCount, + NumCandidates); for (const auto &Candidate : CandidateProfileData) CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)] .updateHotness(getHotness(Candidate.Count, PSI)); diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp index bbaee06ed8a5..bcd2d5d08afe 100644 --- a/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp @@ -201,6 +201,26 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M, return nullptr; } +std::pair +llvm::getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, + Module &M) { + Constant *Ptr = getPointerAtOffset(GV->getInitializer(), Offset, M, GV); + if (!Ptr) + return std::pair(nullptr, nullptr); + + auto C = Ptr->stripPointerCasts(); + // Make sure this is a function or alias to a function. + auto Fn = dyn_cast(C); + auto A = dyn_cast(C); + if (!Fn && A) + Fn = dyn_cast(A->getAliasee()); + + if (!Fn) + return std::pair(nullptr, nullptr); + + return std::pair(Fn, C); +} + void llvm::replaceRelativePointerUsersWithZero(Function *F) { for (auto *U : F->users()) { auto *PtrExpr = dyn_cast(U); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index e614285df07a..9023352d690f 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -1128,6 +1128,18 @@ LandingPadInst *InvokeInst::getLandingPadInst() const { return cast(getUnwindDest()->getFirstNonPHI()); } +void InvokeInst::updateProfWeight(uint64_t S, uint64_t T) { + if (T == 0) { + LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in " + "div by 0. Ignoring. Likely the function " + << getParent()->getParent()->getName() + << " has 0 entry count, and contains call instruction " + "with non-zero prof info."); + return; + } + scaleProfData(*this, S, T); +} + //===----------------------------------------------------------------------===// // CallBrInst Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index 2490b3012bdc..510dba5fc40e 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -35,19 +35,23 @@ MDNode *MDBuilder::createFPMath(float Accuracy) { } MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight, - uint32_t FalseWeight) { - return createBranchWeights({TrueWeight, FalseWeight}); + uint32_t FalseWeight, bool IsExpected) { + return createBranchWeights({TrueWeight, FalseWeight}, IsExpected); } -MDNode *MDBuilder::createBranchWeights(ArrayRef Weights) { +MDNode *MDBuilder::createBranchWeights(ArrayRef Weights, + bool IsExpected) { assert(Weights.size() >= 1 && "Need at least one branch weights!"); - SmallVector Vals(Weights.size() + 1); + unsigned int Offset = IsExpected ? 2 : 1; + SmallVector Vals(Weights.size() + Offset); Vals[0] = createString("branch_weights"); + if (IsExpected) + Vals[1] = createString("expected"); Type *Int32Ty = Type::getInt32Ty(Context); for (unsigned i = 0, e = Weights.size(); i != e; ++i) - Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); + Vals[i + Offset] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); return MDNode::get(Context, Vals); } diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index e534368b05e4..a79d780292e3 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -17,6 +17,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/CommandLine.h" @@ -45,25 +46,8 @@ constexpr unsigned WeightsIdx = 1; // the minimum number of operands for MD_prof nodes with branch weights constexpr unsigned MinBWOps = 3; -bool extractWeights(const MDNode *ProfileData, - SmallVectorImpl &Weights) { - // Assume preconditions are already met (i.e. this is valid metadata) - assert(ProfileData && "ProfileData was nullptr in extractWeights"); - unsigned NOps = ProfileData->getNumOperands(); - - assert(WeightsIdx < NOps && "Weights Index must be less than NOps."); - Weights.resize(NOps - WeightsIdx); - - for (unsigned Idx = WeightsIdx, E = NOps; Idx != E; ++Idx) { - ConstantInt *Weight = - mdconst::dyn_extract(ProfileData->getOperand(Idx)); - assert(Weight && "Malformed branch_weight in MD_prof node"); - assert(Weight->getValue().getActiveBits() <= 32 && - "Too many bits for uint32_t"); - Weights[Idx - WeightsIdx] = Weight->getZExtValue(); - } - return true; -} +// the minimum number of operands for MD_prof nodes with value profiles +constexpr unsigned MinVPOps = 5; // We may want to add support for other MD_prof types, so provide an abstraction // for checking the metadata type. @@ -84,6 +68,27 @@ bool isTargetMD(const MDNode *ProfData, const char *Name, unsigned MinOps) { return ProfDataName->getString().equals(Name); } +template >> +static void extractFromBranchWeightMD(const MDNode *ProfileData, + SmallVectorImpl &Weights) { + assert(isBranchWeightMD(ProfileData) && "wrong metadata"); + + unsigned NOps = ProfileData->getNumOperands(); + unsigned WeightsIdx = getBranchWeightOffset(ProfileData); + assert(WeightsIdx < NOps && "Weights Index must be less than NOps."); + Weights.resize(NOps - WeightsIdx); + + for (unsigned Idx = WeightsIdx, E = NOps; Idx != E; ++Idx) { + ConstantInt *Weight = + mdconst::dyn_extract(ProfileData->getOperand(Idx)); + assert(Weight && "Malformed branch_weight in MD_prof node"); + assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) && + "Too many bits for MD_prof branch_weight"); + Weights[Idx - WeightsIdx] = Weight->getZExtValue(); + } +} + } // namespace namespace llvm { @@ -96,15 +101,53 @@ bool isBranchWeightMD(const MDNode *ProfileData) { return isTargetMD(ProfileData, "branch_weights", MinBWOps); } +bool isValueProfileMD(const MDNode *ProfileData) { + return isTargetMD(ProfileData, "VP", MinVPOps); +} + bool hasBranchWeightMD(const Instruction &I) { auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); return isBranchWeightMD(ProfileData); } +bool hasCountTypeMD(const Instruction &I) { + auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); + // Value profiles record count-type information. + if (isValueProfileMD(ProfileData)) + return true; + // Conservatively assume non CallBase instruction only get take/not-taken + // branch probability, so not interpret them as count. + return isa(I) && !isBranchWeightMD(ProfileData); +} + bool hasValidBranchWeightMD(const Instruction &I) { return getValidBranchWeightMDNode(I); } +bool hasBranchWeightOrigin(const Instruction &I) { + auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); + return hasBranchWeightOrigin(ProfileData); +} + +bool hasBranchWeightOrigin(const MDNode *ProfileData) { + if (!isBranchWeightMD(ProfileData)) + return false; + auto *ProfDataName = dyn_cast(ProfileData->getOperand(1)); + // NOTE: if we ever have more types of branch weight provenance, + // we need to check the string value is "expected". For now, we + // supply a more generic API, and avoid the spurious comparisons. + assert(ProfDataName == nullptr || ProfDataName->getString() == "expected"); + return ProfDataName != nullptr; +} + +unsigned getBranchWeightOffset(const MDNode *ProfileData) { + return hasBranchWeightOrigin(ProfileData) ? 2 : 1; +} + +unsigned getNumBranchWeights(const MDNode &ProfileData) { + return ProfileData.getNumOperands() - getBranchWeightOffset(&ProfileData); +} + MDNode *getBranchWeightMDNode(const Instruction &I) { auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); if (!isBranchWeightMD(ProfileData)) @@ -119,11 +162,22 @@ MDNode *getValidBranchWeightMDNode(const Instruction &I) { return nullptr; } +void extractFromBranchWeightMD32(const MDNode *ProfileData, + SmallVectorImpl &Weights) { + extractFromBranchWeightMD(ProfileData, Weights); +} + +void extractFromBranchWeightMD64(const MDNode *ProfileData, + SmallVectorImpl &Weights) { + extractFromBranchWeightMD(ProfileData, Weights); +} + bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl &Weights) { if (!isBranchWeightMD(ProfileData)) return false; - return extractWeights(ProfileData, Weights); + extractFromBranchWeightMD(ProfileData, Weights); + return true; } bool extractBranchWeights(const Instruction &I, @@ -184,4 +238,63 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) { return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal); } +void setBranchWeights(Instruction &I, ArrayRef Weights, + bool IsExpected) { + MDBuilder MDB(I.getContext()); + MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected); + I.setMetadata(LLVMContext::MD_prof, BranchWeights); +} +void scaleProfData(Instruction &I, uint64_t S, uint64_t T) { + assert(T != 0 && "Caller should guarantee"); + auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); + if (ProfileData == nullptr) + return; + + auto *ProfDataName = dyn_cast(ProfileData->getOperand(0)); + if (!ProfDataName || (ProfDataName->getString() != "branch_weights" && + ProfDataName->getString() != "VP")) + return; + + if (!hasCountTypeMD(I)) + return; + + LLVMContext &C = I.getContext(); + + MDBuilder MDB(C); + SmallVector Vals; + Vals.push_back(ProfileData->getOperand(0)); + APInt APS(128, S), APT(128, T); + if (ProfDataName->getString() == "branch_weights" && + ProfileData->getNumOperands() > 0) { + // Using APInt::div may be expensive, but most cases should fit 64 bits. + APInt Val(128, + mdconst::dyn_extract( + ProfileData->getOperand(getBranchWeightOffset(ProfileData))) + ->getValue() + .getZExtValue()); + Val *= APS; + Vals.push_back(MDB.createConstant(ConstantInt::get( + Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX)))); + } else if (ProfDataName->getString() == "VP") + for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) { + // The first value is the key of the value profile, which will not change. + Vals.push_back(ProfileData->getOperand(i)); + uint64_t Count = + mdconst::dyn_extract(ProfileData->getOperand(i + 1)) + ->getValue() + .getZExtValue(); + // Don't scale the magic number. + if (Count == NOMORE_ICP_MAGICNUM) { + Vals.push_back(ProfileData->getOperand(i + 1)); + continue; + } + // Using APInt::div may be expensive, but most cases should fit 64 bits. + APInt Val(128, Count); + Val *= APS; + Vals.push_back(MDB.createConstant(ConstantInt::get( + Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue()))); + } + I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals)); +} + } // namespace llvm diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index e3d9c6ff67bd..5f40373fa5d2 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -810,10 +810,10 @@ bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) { bool Changed = false; ICallPromotionAnalysis ICallAnalysis; for (auto *CB : findIndirectCalls(F)) { - uint32_t NumVals, NumCandidates; + uint32_t NumCandidates; uint64_t TotalCount; auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction( - CB, NumVals, TotalCount, NumCandidates); + CB, TotalCount, NumCandidates); if (!NumCandidates || (PSI && PSI->hasProfileSummary() && !PSI->isHotCount(TotalCount, /*isForICP=*/true))) continue; diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp index b488e3bb0cbd..0e1e5923722b 100644 --- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp +++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp @@ -12,9 +12,11 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/CallPromotionUtils.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/TypeMetadataUtils.h" #include "llvm/IR/AttributeMask.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -188,10 +190,9 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) { /// Predicate and clone the given call site. /// /// This function creates an if-then-else structure at the location of the call -/// site. The "if" condition compares the call site's called value to the given -/// callee. The original call site is moved into the "else" block, and a clone -/// of the call site is placed in the "then" block. The cloned instruction is -/// returned. +/// site. The "if" condition is specified by `Cond`. +/// The original call site is moved into the "else" block, and a clone of the +/// call site is placed in the "then" block. The cloned instruction is returned. /// /// For example, the call instruction below: /// @@ -202,7 +203,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) { /// Is replace by the following: /// /// orig_bb: -/// %cond = icmp eq i32 ()* %ptr, @func +/// %cond = Cond /// br i1 %cond, %then_bb, %else_bb /// /// then_bb: @@ -232,7 +233,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) { /// Is replace by the following: /// /// orig_bb: -/// %cond = icmp eq i32 ()* %ptr, @func +/// %cond = Cond /// br i1 %cond, %then_bb, %else_bb /// /// then_bb: @@ -267,7 +268,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) { /// Is replaced by the following: /// /// cond_bb: -/// %cond = icmp eq i32 ()* %ptr, @func +/// %cond = Cond /// br i1 %cond, %then_bb, %orig_bb /// /// then_bb: @@ -280,19 +281,13 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) { /// ; The original call instruction stays in its original block. /// %t0 = musttail call i32 %ptr() /// ret %t0 -CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee, - MDNode *BranchWeights) { +static CallBase &versionCallSiteWithCond(CallBase &CB, Value *Cond, + MDNode *BranchWeights) { IRBuilder<> Builder(&CB); CallBase *OrigInst = &CB; BasicBlock *OrigBlock = OrigInst->getParent(); - // Create the compare. The called value and callee must have the same type to - // be compared. - if (CB.getCalledOperand()->getType() != Callee->getType()) - Callee = Builder.CreateBitCast(Callee, CB.getCalledOperand()->getType()); - auto *Cond = Builder.CreateICmpEQ(CB.getCalledOperand(), Callee); - if (OrigInst->isMustTailCall()) { // Create an if-then structure. The original instruction stays in its block, // and a clone of the original instruction is placed in the "then" block. @@ -380,6 +375,22 @@ CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee, return *NewInst; } +// Predicate and clone the given call site using condition `CB.callee == +// Callee`. See the comment `versionCallSiteWithcond` for the transformation. +CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee, + MDNode *BranchWeights) { + + IRBuilder<> Builder(&CB); + + // Create the compare. The called value and callee must have the same type to + // be compared. + if (CB.getCalledOperand()->getType() != Callee->getType()) + Callee = Builder.CreateBitCast(Callee, CB.getCalledOperand()->getType()); + auto *Cond = Builder.CreateICmpEQ(CB.getCalledOperand(), Callee); + + return versionCallSiteWithCond(CB, Cond, BranchWeights); +} + bool llvm::isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason) { assert(!CB.getCalledFunction() && "Only indirect call sites can be promoted"); @@ -559,6 +570,27 @@ CallBase &llvm::promoteCallWithIfThenElse(CallBase &CB, Function *Callee, return promoteCall(NewInst, Callee); } +CallBase &llvm::promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, + Function *Callee, + ArrayRef AddressPoints, + MDNode *BranchWeights) { + assert(!AddressPoints.empty() && "Caller should guarantee"); + IRBuilder<> Builder(&CB); + SmallVector ICmps; + for (auto &AddressPoint : AddressPoints) + ICmps.push_back(Builder.CreateICmpEQ(VPtr, AddressPoint)); + + // TODO: Perform tree height reduction if the number of ICmps is high. + Value *Cond = Builder.CreateOr(ICmps); + + // Version the indirect call site. If Cond is true, 'NewInst' will be + // executed, otherwise the original call site will be executed. + CallBase &NewInst = versionCallSiteWithCond(CB, Cond, BranchWeights); + + // Promote 'NewInst' so that it directly calls the desired function. + return promoteCall(NewInst, Callee); +} + bool llvm::tryPromoteCall(CallBase &CB) { assert(!CB.getCalledFunction()); Module *M = CB.getCaller()->getParent(); @@ -597,16 +629,13 @@ bool llvm::tryPromoteCall(CallBase &CB) { // Not in the form of a global constant variable with an initializer. return false; - Constant *VTableGVInitializer = GV->getInitializer(); APInt VTableGVOffset = VTableOffsetGVBase + VTableOffset; if (!(VTableGVOffset.getActiveBits() <= 64)) return false; // Out of range. - Constant *Ptr = getPointerAtOffset(VTableGVInitializer, - VTableGVOffset.getZExtValue(), - *M); - if (!Ptr) - return false; // No constant (function) pointer found. - Function *DirectCallee = dyn_cast(Ptr->stripPointerCasts()); + + Function *DirectCallee = nullptr; + std::tie(DirectCallee, std::ignore) = + getFunctionAtVTableOffset(GV, VTableGVOffset.getZExtValue(), *M); if (!DirectCallee) return false; // No function pointer found. diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 2e0c17cda8ba..a098ea5322d9 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1819,7 +1819,7 @@ void llvm::updateProfileCallee( // During inlining ? if (VMap) { uint64_t CloneEntryCount = PriorEntryCount - NewEntryCount; - for (auto Entry : *VMap) + for (auto Entry : *VMap) { if (isa(Entry.first)) if (auto *CI = dyn_cast_or_null(Entry.second)) { CI->updateProfWeight(CloneEntryCount, PriorEntryCount); diff --git a/llvm/test/Transforms/Inline/update_value_profile.ll b/llvm/test/Transforms/Inline/update_value_profile.ll index 96aa35fb572d..5b458781f08b 100644 --- a/llvm/test/Transforms/Inline/update_value_profile.ll +++ b/llvm/test/Transforms/Inline/update_value_profile.ll @@ -15,10 +15,10 @@ define i32 @callee(ptr %0, i32 %1) !prof !19 { ; CHECK-NEXT: ret i32 [[TMP6]] ; %3 = load ptr, ptr %0, !prof !15 - %5 = getelementptr inbounds i8, ptr %3, i64 8 - %6 = load ptr, ptr %5 - %7 = tail call i32 %6(ptr %0, i32 %1), !prof !16 - ret i32 %7 + %4 = getelementptr inbounds i8, ptr %3, i64 8 + %5 = load ptr, ptr %4 + %6 = tail call i32 %5(ptr %0, i32 %1), !prof !16 + ret i32 %6 } define i32 @caller1(i32 %0) !prof !17 { diff --git a/llvm/test/Transforms/PGOProfile/criticaledge.ll b/llvm/test/Transforms/PGOProfile/criticaledge.ll index c24925c68fa3..616d0fa3a4cf 100644 --- a/llvm/test/Transforms/PGOProfile/criticaledge.ll +++ b/llvm/test/Transforms/PGOProfile/criticaledge.ll @@ -11,7 +11,7 @@ target triple = "x86_64-unknown-linux-gnu" ; GEN: $__llvm_profile_raw_version = comdat any ; GEN: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge" -; GEN: @__profn__stdin__bar = private constant [11 x i8] c";bar" +; GEN: @__profn__stdin__bar = private constant [11 x i8] c":bar" define i32 @test_criticalEdge(i32 %i, i32 %j) { entry: diff --git a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll index 4bffc87a9390..fe8df33324ab 100644 --- a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll +++ b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll @@ -4,8 +4,8 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -; NOPATH: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll;func" -; HASPATH-NOT: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll;func" +; NOPATH: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func" +; HASPATH-NOT: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func" define internal i32 @func() { entry: diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll index a8440031e149..93448b1785cd 100644 --- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll +++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll @@ -56,11 +56,11 @@ target triple = "x86_64-unknown-linux-gnu" ; GEN: __profn__Z4funci = private constant [8 x i8] c"_Z4funci" ; LOWER: $__profvt__ZTV7Derived = comdat nodeduplicate -; LOWER: $"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = comdat nodeduplicate +; LOWER: $"__profvt_vtable_local.ll:_ZTVN12_GLOBAL__N_15Base2E" = comdat nodeduplicate ; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8 -; LOWER: @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = internal global { i64, ptr, i32 } { i64 1419990121885302679, ptr @_ZTVN12_GLOBAL__N_15Base2E, i32 24 }, section "__llvm_prf_vtab", comdat, align 8 -; LOWER: @__llvm_prf_vnm = private constant [64 x i8] c"7>x\DA\8B\8F\0A\093wI-\CA,KMa,+IL\CAI\8D\CF\C9ON\CC\D1\CB\C9\B1\8E\07J\FA\19\1A\C5\BB\FB\F8;9\FA\C4\C7\FB\C5\1B\9A:%\16\A7\1A\B9\02\00\19:\12o", section "__llvm_prf_vns", align 1 -; LOWER: @llvm.used = appending global [5 x ptr] [ptr @__profvt__ZTV7Derived, ptr @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E", ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata" +; LOWER: @"__profvt_vtable_local.ll:_ZTVN12_GLOBAL__N_15Base2E" = internal global { i64, ptr, i32 } { i64 2756736352897687539, ptr @_ZTVN12_GLOBAL__N_15Base2E, i32 24 }, section "__llvm_prf_vtab", comdat, align 8 +; LOWER: @__llvm_prf_vnm = private constant [64 x i8] c"7>x\DA\8B\8F\0A\093wI-\CA,KMa,+IL\CAI\8D\CF\C9ON\CC\D1\CB\C9\B1\8A\07J\FA\19\1A\C5\BB\FB\F8;9\FA\C4\C7\FB\C5\1B\9A:%\16\A7\1A\B9\02\00\19\1F\12n", section "__llvm_prf_vns", align 1 +; LOWER: @llvm.used = appending global [5 x ptr] [ptr @__profvt__ZTV7Derived, ptr @"__profvt_vtable_local.ll:_ZTVN12_GLOBAL__N_15Base2E", ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata" define i32 @_Z4funci(i32 %a) { entry: diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index a3efc93dfbcf..05451f941516 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -608,7 +608,8 @@ static void overlapInput(const std::string &BaseFilename, /// Load an input into a writer context. static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, const InstrProfCorrelator *Correlator, - const StringRef ProfiledBinary, WriterContext *WC) { + const StringRef ProfiledBinary, WriterContext *WC, + const bool KeepVTableSymbols) { std::unique_lock CtxGuard{WC->Lock}; // Copy the filename, because llvm::ThreadPool copied the input "const @@ -791,11 +792,12 @@ static void writeInstrProfile(StringRef OutputFilename, static void mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, - SymbolRemapper *Remapper, StringRef OutputFilename, - ProfileFormat OutputFormat, uint64_t TraceReservoirSize, - uint64_t MaxTraceLength, bool OutputSparse, - unsigned NumThreads, FailureMode FailMode, - const StringRef ProfiledBinary) { + StringRef BinaryFilename, SymbolRemapper *Remapper, + StringRef OutputFilename, ProfileFormat OutputFormat, + uint64_t TraceReservoirSize, uint64_t MaxTraceLength, + bool OutputSparse, unsigned NumThreads, FailureMode FailMode, + const StringRef ProfiledBinary, + const bool KeepVTableSymbols) { if (OutputFormat == PF_Compact_Binary) exitWithError("Compact Binary is deprecated"); if (OutputFormat != PF_Binary && OutputFormat != PF_Ext_Binary && @@ -843,7 +845,7 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, if (NumThreads == 1) { for (const auto &Input : Inputs) loadInput(Input, Remapper, Correlator.get(), ProfiledBinary, - Contexts[0].get()); + Contexts[0].get(), KeepVTableSymbols); } else { ThreadPool Pool(hardware_concurrency(NumThreads)); @@ -851,7 +853,7 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, unsigned Ctx = 0; for (const auto &Input : Inputs) { Pool.async(loadInput, Input, Remapper, Correlator.get(), ProfiledBinary, - Contexts[Ctx].get()); + Contexts[Ctx].get(), KeepVTableSymbols); Ctx = (Ctx + 1) % NumThreads; } Pool.wait(); @@ -1244,7 +1246,7 @@ static void supplementInstrProfile( const WeightedFileVector &Inputs, StringRef SampleFilename, StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse, unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, - unsigned InstrProfColdThreshold) { + unsigned InstrProfColdThreshold, const bool KeepVTableSymbols) { if (OutputFilename.compare("-") == 0) exitWithError("cannot write indexed profdata format to stdout"); if (Inputs.size() != 1) @@ -1270,7 +1272,8 @@ static void supplementInstrProfile( SmallSet WriterErrorCodes; auto WC = std::make_unique(OutputSparse, ErrorLock, WriterErrorCodes); - loadInput(Inputs[0], nullptr, nullptr, /*ProfiledBinary=*/"", WC.get()); + loadInput(Inputs[0], nullptr, nullptr, /*ProfiledBinary=*/"", WC.get(), + KeepVTableSymbols); if (WC->Errors.size() > 0) exitWithError(std::move(WC->Errors[0].first), InstrFilename); @@ -1672,6 +1675,10 @@ static int merge_main(int argc, const char *argv[]) { cl::opt DebugInfoFilename( "debug-info", cl::init(""), cl::desc("Use the provided debug info to correlate the raw profile.")); + cl::opt + BinaryFilename("binary-file", cl::init(""), + cl::desc("For merge, use the provided unstripped binary to " + "correlate the raw profile.")); cl::opt ProfiledBinary( "profiled-binary", cl::init(""), cl::desc("Path to binary from which the profile was collected.")); @@ -1679,6 +1686,9 @@ static int merge_main(int argc, const char *argv[]) { "drop-profile-symbol-list", cl::init(false), cl::Hidden, cl::desc("Drop the profile symbol list when merging AutoFDO profiles " "(only meaningful for -sample)")); + cl::opt KeepVTableSymbols( + "keep-vtable-symbols", cl::init(false), cl::Hidden, + cl::desc("If true, keep the vtable symbols in indexed profiles")); // WARNING: This reservoir size value is propagated to any input indexed // profiles for simplicity. Changing this value between invocations could // result in sample bias. @@ -1725,16 +1735,17 @@ static int merge_main(int argc, const char *argv[]) { supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename, OutputFormat, OutputSparse, SupplMinSizeThreshold, - ZeroCounterThreshold, InstrProfColdThreshold); + ZeroCounterThreshold, InstrProfColdThreshold, + KeepVTableSymbols); return 0; } if (ProfileKind == instr) - mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), - OutputFilename, OutputFormat, + mergeInstrProfile(WeightedInputs, DebugInfoFilename, BinaryFilename, + Remapper.get(), OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, TemporalProfMaxTraceLength, OutputSparse, NumThreads, - FailureMode, ProfiledBinary); + FailureMode, ProfiledBinary, KeepVTableSymbols); else mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, ProfileSymbolListFile, CompressAllSections, @@ -1766,7 +1777,8 @@ static void overlapInstrProfile(const std::string &BaseFilename, OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n"; exit(0); } - loadInput(WeightedInput, nullptr, nullptr, /*ProfiledBinary=*/"", &Context); + loadInput(WeightedInput, nullptr, nullptr, /*ProfiledBinary=*/"", &Context, + /*KeepVTableSymbols=*/false); overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, IsCS); Overlap.dump(OS); @@ -2814,7 +2826,7 @@ static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK, static int showInstrProfile( const std::string &Filename, bool ShowCounts, uint32_t TopN, - bool ShowIndirectCallTargets, bool ShowMemOPSizes, bool ShowDetailedSummary, + bool ShowIndirectCallTargets, bool ShowVTables, bool ShowMemOPSizes, bool ShowDetailedSummary, std::vector DetailedSummaryCutoffs, bool ShowAllFunctions, bool ShowCS, uint64_t ValueCutoff, bool OnlyListBelow, const std::string &ShowFunction, bool TextFormat, bool ShowBinaryIds, @@ -3351,6 +3363,8 @@ static int show_main(int argc, const char *argv[]) { cl::opt ShowIndirectCallTargets( "ic-targets", cl::init(false), cl::desc("Show indirect call site target values for shown functions")); + cl::opt ShowVTables("show-vtables", cl::init(false), + cl::desc("Show vtable names for shown functions")); cl::opt ShowMemOPSizes( "memop-sizes", cl::init(false), cl::desc("Show the profiled sizes of the memory intrinsic calls " @@ -3446,7 +3460,7 @@ static int show_main(int argc, const char *argv[]) { if (ProfileKind == instr) return showInstrProfile( - Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets, + Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets, ShowVTables, ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs, ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction, TextFormat, ShowBinaryIds, ShowCovered, ShowProfileVersion, diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 39ee894f5bcd..20bd87b3e8a5 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -17,11 +17,11 @@ #include "llvm/Support/Compression.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Testing/Support/Error.h" +#include "llvm/Testing/Support/SupportHelpers.h" #include "gtest/gtest.h" #include using namespace llvm; -using ::testing::EndsWith; using ::testing::IsSubsetOf; using ::testing::SizeIs; using ::testing::UnorderedElementsAre; @@ -68,27 +68,6 @@ struct SparseInstrProfTest : public InstrProfTest { void SetUp() override { Writer.setOutputSparse(true); } }; -struct InstrProfReaderWriterTest - : public InstrProfTest, - public ::testing::WithParamInterface< - std::tuple> { - void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); } - void TearDown() override { - // Reset writer value profile data endianness after each test case. Note - // it's not necessary to reset reader value profile endianness for each test - // case. Each test case creates a new reader; at reader initialization time, - // it uses the endianness from hash table object (which is little by - // default). - Writer.setValueProfDataEndianness(llvm::support::endianness::little); - } - - uint64_t getProfWeight() const { return std::get<1>(GetParam()); } - - llvm::support::endianness getEndianness() const { - return std::get<2>(GetParam()); - } -}; - struct MaybeSparseInstrProfTest : public InstrProfTest, public ::testing::WithParamInterface { void SetUp() override { Writer.setOutputSparse(GetParam()); } @@ -541,266 +520,56 @@ TEST_F(InstrProfTest, test_memprof_merge) { EXPECT_THAT(WantRecord, EqualsRecord(Record)); } -TEST_F(InstrProfTest, test_irpgo_function_name) { - LLVMContext Ctx; - auto M = std::make_unique("MyModule.cpp", Ctx); - // Use Mach-O mangling so that non-private symbols get a `_` prefix. - M->setDataLayout(DataLayout("m:o")); - auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); - - std::vector> Data; - Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "_ExternalFoo"); - Data.emplace_back("InternalFoo", Function::InternalLinkage, - "MyModule.cpp;_InternalFoo"); - Data.emplace_back("PrivateFoo", Function::PrivateLinkage, - "MyModule.cpp;l_PrivateFoo"); - Data.emplace_back("WeakODRFoo", Function::WeakODRLinkage, "_WeakODRFoo"); - // Test Objective-C symbols - Data.emplace_back("\01-[C dynamicFoo:]", Function::ExternalLinkage, - "-[C dynamicFoo:]"); - Data.emplace_back("-", Function::ExternalLinkage, - "_-"); - Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage, - "MyModule.cpp;-[C internalFoo:]"); - - for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) - Function::Create(FTy, Linkage, Name, M.get()); - - for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) { - auto *F = M->getFunction(Name); - auto IRPGOFuncName = getIRPGOFuncName(*F); - EXPECT_EQ(IRPGOFuncName, ExpectedIRPGOFuncName); - - auto [Filename, ParsedIRPGOFuncName] = - getParsedIRPGOFuncName(IRPGOFuncName); - StringRef ExpectedParsedIRPGOFuncName = IRPGOFuncName; - if (ExpectedParsedIRPGOFuncName.consume_front("MyModule.cpp;")) { - EXPECT_EQ(Filename, "MyModule.cpp"); - } else { - EXPECT_EQ(Filename, ""); - } - EXPECT_EQ(ParsedIRPGOFuncName, ExpectedParsedIRPGOFuncName); - } -} - -TEST_F(InstrProfTest, test_pgo_function_name) { - LLVMContext Ctx; - auto M = std::make_unique("MyModule.cpp", Ctx); - auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); - - std::vector> Data; - Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo"); - Data.emplace_back("InternalFoo", Function::InternalLinkage, - "MyModule.cpp:InternalFoo"); - Data.emplace_back("PrivateFoo", Function::PrivateLinkage, - "MyModule.cpp:PrivateFoo"); - Data.emplace_back("WeakODRFoo", Function::WeakODRLinkage, "WeakODRFoo"); - // Test Objective-C symbols - Data.emplace_back("\01-[C externalFoo:]", Function::ExternalLinkage, - "-[C externalFoo:]"); - Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage, - "MyModule.cpp:-[C internalFoo:]"); - - for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) - Function::Create(FTy, Linkage, Name, M.get()); - - for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) { - auto *F = M->getFunction(Name); - EXPECT_EQ(getPGOFuncName(*F), ExpectedPGOFuncName); - } -} - -TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) { - LLVMContext Ctx; - auto M = std::make_unique("MyModule.cpp", Ctx); - // Use Mach-O mangling so that non-private symbols get a `_` prefix. - M->setDataLayout(DataLayout("m:o")); - auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); - auto *InternalFooF = - Function::Create(FTy, Function::InternalLinkage, "InternalFoo", M.get()); - auto *ExternalFooF = - Function::Create(FTy, Function::ExternalLinkage, "ExternalFoo", M.get()); - - auto *InternalBarF = - Function::Create(FTy, Function::InternalLinkage, "InternalBar", M.get()); - auto *ExternalBarF = - Function::Create(FTy, Function::ExternalLinkage, "ExternalBar", M.get()); - - Writer.addRecord({getIRPGOFuncName(*InternalFooF), 0x1234, {1}}, Err); - Writer.addRecord({getIRPGOFuncName(*ExternalFooF), 0x5678, {1}}, Err); - // Write a record with a deprecated name - Writer.addRecord({getPGOFuncName(*InternalBarF), 0x1111, {2}}, Err); - Writer.addRecord({getPGOFuncName(*ExternalBarF), 0x2222, {2}}, Err); - - auto Profile = Writer.writeBuffer(); - readProfile(std::move(Profile)); - - EXPECT_THAT_EXPECTED( - Reader->getInstrProfRecord(getIRPGOFuncName(*InternalFooF), 0x1234, - getPGOFuncName(*InternalFooF)), - Succeeded()); - EXPECT_THAT_EXPECTED( - Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalFooF), 0x5678, - getPGOFuncName(*ExternalFooF)), - Succeeded()); - // Ensure we can still read this old record name - EXPECT_THAT_EXPECTED( - Reader->getInstrProfRecord(getIRPGOFuncName(*InternalBarF), 0x1111, - getPGOFuncName(*InternalBarF)), - Succeeded()); - EXPECT_THAT_EXPECTED( - Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalBarF), 0x2222, - getPGOFuncName(*ExternalBarF)), - Succeeded()); -} - -// callee1 to callee6 are from vtable1 to vtable6 respectively. static const char callee1[] = "callee1"; static const char callee2[] = "callee2"; static const char callee3[] = "callee3"; static const char callee4[] = "callee4"; static const char callee5[] = "callee5"; static const char callee6[] = "callee6"; -// callee7 and callee8 are not from any vtables. -static const char callee7[] = "callee7"; -static const char callee8[] = "callee8"; -// 'callee' is primarily used to create multiple-element vtables. -static const char callee[] = "callee"; -static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)}; -static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)}; -static const uint64_t vtable3[] = { - uint64_t(callee), - uint64_t(callee3), -}; -static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)}; -static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)}; -static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)}; - -// Returns the address of callee with a numbered suffix in vtable. -static uint64_t getCalleeAddress(const uint64_t *vtableAddr) { - uint64_t CalleeAddr; - // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3, - // and the 1st element in the rest of vtables. - if (vtableAddr == vtable1 || vtableAddr == vtable3) - CalleeAddr = uint64_t(vtableAddr) + 8; - else - CalleeAddr = uint64_t(vtableAddr); - return CalleeAddr; -} -TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) { NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); - // 4 indirect call value sites. - { - Record1.reserveSites(IPVK_IndirectCallTarget, 4); - InstrProfValueData VD0[] = { - {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; - Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); - // No value profile data at the second site. - Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; - Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); - InstrProfValueData VD3[] = {{(uint64_t)callee7, 1}, {(uint64_t)callee8, 2}}; - Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); - } - - // 2 vtable value sites. - { - InstrProfValueData VD0[] = { - {getCalleeAddress(vtable1), 1}, - {getCalleeAddress(vtable2), 2}, - {getCalleeAddress(vtable3), 3}, - }; - InstrProfValueData VD2[] = { - {getCalleeAddress(vtable1), 1}, - {getCalleeAddress(vtable2), 2}, - }; - Record1.addValueData(IPVK_VTableTarget, 0, VD0, 3, nullptr); - Record1.addValueData(IPVK_VTableTarget, 2, VD2, 2, nullptr); - } + // 4 value sites. + Record1.reserveSites(IPVK_IndirectCallTarget, 4); + InstrProfValueData VD0[] = { + {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; + Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); + // No value profile data at the second site. + Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); + InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); Writer.addRecord(std::move(Record1), Err); Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); - Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err); - Writer.addRecord({"callee8", 0x1235, {3, 4}}, Err); - - // Set writer value prof data endianness. - Writer.setValueProfDataEndianness(getEndianness()); - auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); Expected R = Reader->getInstrProfRecord("caller", 0x1234); - ASSERT_THAT_ERROR(R.takeError(), Succeeded()); - - // Test the number of instrumented indirect call sites and the number of - // profiled values at each site. + EXPECT_THAT_ERROR(R.takeError(), Succeeded()); ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); - EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); - EXPECT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); - EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); - - // Test the number of instrumented vtable sites and the number of profiled - // values at each site. - ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U); - EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 3U); - EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 2U); - - // First indirect site. - { - uint64_t TotalC; - auto VD = R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); - - EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); - EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); - EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); - EXPECT_EQ(TotalC, 6U * getProfWeight()); - - EXPECT_STREQ((const char *)VD[0].Value, "callee3"); - EXPECT_STREQ((const char *)VD[1].Value, "callee2"); - EXPECT_STREQ((const char *)VD[2].Value, "callee1"); - } - - // First vtable site. - { - uint64_t TotalC; - auto VD = R->getValueForSite(IPVK_VTableTarget, 0, &TotalC); - - EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); - EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); - EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); - EXPECT_EQ(TotalC, 6U * getProfWeight()); - - EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1)); - } + ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); - // Second vtable site. - { - uint64_t TotalC; - auto VD = R->getValueForSite(IPVK_VTableTarget, 1, &TotalC); + uint64_t TotalC; + std::unique_ptr VD = + R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); - EXPECT_EQ(VD[0].Count, 2U * getProfWeight()); - EXPECT_EQ(VD[1].Count, 1U * getProfWeight()); - EXPECT_EQ(TotalC, 3U * getProfWeight()); + ASSERT_EQ(3U, VD[0].Count); + ASSERT_EQ(2U, VD[1].Count); + ASSERT_EQ(1U, VD[2].Count); + ASSERT_EQ(6U, TotalC); - EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1)); - } + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); } -INSTANTIATE_TEST_SUITE_P( - WeightAndEndiannessTest, InstrProfReaderWriterTest, - ::testing::Combine( - ::testing::Bool(), /* Sparse */ - ::testing::Values(1U, 10U), /* ProfWeight */ - ::testing::Values(llvm::support::endianness::big, - llvm::support::endianness::little) /* Endianness */ - )); TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { NamedInstrProfRecord Record("caller", 0x1234, {1, 2}); @@ -898,53 +667,123 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { ASSERT_EQ(1U, ValueData[3].Count); } -TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) { + NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); + + // 4 value sites. + Record1.reserveSites(IPVK_IndirectCallTarget, 4); + InstrProfValueData VD0[] = { + {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; + Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); + // No value profile data at the second site. + Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); + InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + + Writer.addRecord(std::move(Record1), 10, Err); + Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + Expected R = Reader->getInstrProfRecord("caller", 0x1234); + EXPECT_THAT_ERROR(R.takeError(), Succeeded()); + ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + + uint64_t TotalC; + std::unique_ptr VD = + R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); + ASSERT_EQ(30U, VD[0].Count); + ASSERT_EQ(20U, VD[1].Count); + ASSERT_EQ(10U, VD[2].Count); + ASSERT_EQ(60U, TotalC); + + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); +} + +TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) { + NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); + + // 4 value sites. + Record1.reserveSites(IPVK_IndirectCallTarget, 4); + InstrProfValueData VD0[] = { + {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; + Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); + // No value profile data at the second site. + Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); + InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + + Writer.addRecord(std::move(Record1), Err); + Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); + + // Set big endian output. + Writer.setValueProfDataEndianness((support::big)); + + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + // Set big endian input. + Reader->setValueProfDataEndianness(support::big); + + Expected R = Reader->getInstrProfRecord("caller", 0x1234); + EXPECT_THAT_ERROR(R.takeError(), Succeeded()); + ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + + std::unique_ptr VD = + R->getValueForSite(IPVK_IndirectCallTarget, 0); + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); + + // Restore little endian default: + Writer.setValueProfDataEndianness(support::little); +} + +TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { static const char caller[] = "caller"; NamedInstrProfRecord Record11(caller, 0x1234, {1, 2}); NamedInstrProfRecord Record12(caller, 0x1234, {1, 2}); - // 5 value sites for indirect calls. - { - Record11.reserveSites(IPVK_IndirectCallTarget, 5); - InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}, - {uint64_t(callee4), 4}}; - Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr); + // 5 value sites + Record11.reserveSites(IPVK_IndirectCallTarget, 5); + InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}, + {uint64_t(callee4), 4}}; + Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr); - // No value profile data at the second site. - Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + // No value profile data at the second site. + Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = { - {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; - Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); + InstrProfValueData VD2[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); - InstrProfValueData VD3[] = {{uint64_t(callee7), 1}, {uint64_t(callee8), 2}}; - Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); + InstrProfValueData VD3[] = {{uint64_t(callee1), 1}}; + Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); - InstrProfValueData VD4[] = { - {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; - Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); - } - // 3 value sites for vtables. - { - Record11.reserveSites(IPVK_VTableTarget, 3); - InstrProfValueData VD0[] = {{getCalleeAddress(vtable1), 1}, - {getCalleeAddress(vtable2), 2}, - {getCalleeAddress(vtable3), 3}, - {getCalleeAddress(vtable4), 4}}; - Record11.addValueData(IPVK_VTableTarget, 0, VD0, 4, nullptr); - - InstrProfValueData VD2[] = {{getCalleeAddress(vtable1), 1}, - {getCalleeAddress(vtable2), 2}, - {getCalleeAddress(vtable3), 3}}; - Record11.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr); - - InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1}, - {getCalleeAddress(vtable2), 2}, - {getCalleeAddress(vtable3), 3}}; - Record11.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr); - } + InstrProfValueData VD4[] = {{uint64_t(callee1), 1}, + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); // A different record for the same caller. Record12.reserveSites(IPVK_IndirectCallTarget, 5); @@ -960,28 +799,11 @@ TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr); - InstrProfValueData VD42[] = { - {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; + InstrProfValueData VD42[] = {{uint64_t(callee1), 1}, + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}}; Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr); - // 3 value sites for vtables. - { - Record12.reserveSites(IPVK_VTableTarget, 3); - InstrProfValueData VD0[] = {{getCalleeAddress(vtable2), 5}, - {getCalleeAddress(vtable3), 3}}; - Record12.addValueData(IPVK_VTableTarget, 0, VD0, 2, nullptr); - - InstrProfValueData VD2[] = {{getCalleeAddress(vtable2), 1}, - {getCalleeAddress(vtable3), 3}, - {getCalleeAddress(vtable4), 4}}; - Record12.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr); - - InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1}, - {getCalleeAddress(vtable2), 2}, - {getCalleeAddress(vtable3), 3}}; - Record12.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr); - } - Writer.addRecord(std::move(Record11), Err); // Merge profile data. Writer.addRecord(std::move(Record12), Err); @@ -991,107 +813,56 @@ TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { Writer.addRecord({callee3, 0x1235, {3, 4}}, Err); Writer.addRecord({callee3, 0x1235, {3, 4}}, Err); Writer.addRecord({callee4, 0x1235, {3, 5}}, Err); - Writer.addRecord({callee7, 0x1235, {3, 5}}, Err); - Writer.addRecord({callee8, 0x1235, {3, 5}}, Err); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); - // Test the number of instrumented value sites and the number of profiled - // values for each site. Expected R = Reader->getInstrProfRecord("caller", 0x1234); EXPECT_THAT_ERROR(R.takeError(), Succeeded()); - // For indirect calls. ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget)); ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); - // For vtables. - ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U); - ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 4U); - ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); - ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); - - // Test the merged values for indirect calls. - { - auto VD = R->getValueForSite(IPVK_IndirectCallTarget, 0); - EXPECT_STREQ((const char *)VD[0].Value, "callee2"); - EXPECT_EQ(VD[0].Count, 7U); - EXPECT_STREQ((const char *)VD[1].Value, "callee3"); - EXPECT_EQ(VD[1].Count, 6U); - EXPECT_STREQ((const char *)VD[2].Value, "callee4"); - EXPECT_EQ(VD[2].Count, 4U); - EXPECT_STREQ((const char *)VD[3].Value, "callee1"); - EXPECT_EQ(VD[3].Count, 1U); - - auto VD_2(R->getValueForSite(IPVK_IndirectCallTarget, 2)); - EXPECT_STREQ((const char *)VD_2[0].Value, "callee3"); - EXPECT_EQ(VD_2[0].Count, 6U); - EXPECT_STREQ((const char *)VD_2[1].Value, "callee4"); - EXPECT_EQ(VD_2[1].Count, 4U); - EXPECT_STREQ((const char *)VD_2[2].Value, "callee2"); - EXPECT_EQ(VD_2[2].Count, 3U); - EXPECT_STREQ((const char *)VD_2[3].Value, "callee1"); - EXPECT_EQ(VD_2[3].Count, 1U); - - auto VD_3(R->getValueForSite(IPVK_IndirectCallTarget, 3)); - EXPECT_STREQ((const char *)VD_3[0].Value, "callee8"); - EXPECT_EQ(VD_3[0].Count, 2U); - EXPECT_STREQ((const char *)VD_3[1].Value, "callee7"); - EXPECT_EQ(VD_3[1].Count, 1U); - - auto VD_4(R->getValueForSite(IPVK_IndirectCallTarget, 4)); - EXPECT_STREQ((const char *)VD_4[0].Value, "callee3"); - EXPECT_EQ(VD_4[0].Count, 6U); - EXPECT_STREQ((const char *)VD_4[1].Value, "callee2"); - EXPECT_EQ(VD_4[1].Count, 4U); - EXPECT_STREQ((const char *)VD_4[2].Value, "callee1"); - EXPECT_EQ(VD_4[2].Count, 2U); - } - // Test the merged values for vtables - { - auto VD0 = R->getValueForSite(IPVK_VTableTarget, 0); - EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD0[0].Count, 7U); - EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD0[1].Count, 6U); - EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4)); - EXPECT_EQ(VD0[2].Count, 4U); - EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1)); - EXPECT_EQ(VD0[3].Count, 1U); - - auto VD1 = R->getValueForSite(IPVK_VTableTarget, 1); - EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD1[0].Count, 6U); - EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4)); - EXPECT_EQ(VD1[1].Count, 4U); - EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD1[2].Count, 3U); - EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1)); - EXPECT_EQ(VD1[3].Count, 1U); - - auto VD2 = R->getValueForSite(IPVK_VTableTarget, 2); - EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD2[0].Count, 6U); - EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD2[1].Count, 4U); - EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1)); - EXPECT_EQ(VD2[2].Count, 2U); - } + std::unique_ptr VD = + R->getValueForSite(IPVK_IndirectCallTarget, 0); + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(7U, VD[0].Count); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD[1].Count); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4")); + ASSERT_EQ(4U, VD[2].Count); + ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD[3].Count); + + std::unique_ptr VD_2( + R->getValueForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD_2[0].Count); + ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4")); + ASSERT_EQ(4U, VD_2[1].Count); + ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2")); + ASSERT_EQ(3U, VD_2[2].Count); + ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD_2[3].Count); + + std::unique_ptr VD_3( + R->getValueForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD_3[0].Count); + + std::unique_ptr VD_4( + R->getValueForSite(IPVK_IndirectCallTarget, 4)); + ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD_4[0].Count); + ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(4U, VD_4[1].Count); + ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1")); + ASSERT_EQ(2U, VD_4[2].Count); } -struct ValueProfileMergeEdgeCaseTest - : public InstrProfTest, - public ::testing::WithParamInterface> { - void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); } - - uint32_t getValueProfileKind() const { return std::get<1>(GetParam()); } -}; - -TEST_P(ValueProfileMergeEdgeCaseTest, get_icall_data_merge1_saturation) { - const uint32_t ValueKind = getValueProfileKind(); +TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) { static const char bar[] = "bar"; const uint64_t MaxValCount = std::numeric_limits::max(); @@ -1199,18 +970,8 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) { } } -INSTANTIATE_TEST_SUITE_P( - EdgeCaseTest, ValueProfileMergeEdgeCaseTest, - ::testing::Combine(::testing::Bool(), /* Sparse */ - ::testing::Values(IPVK_IndirectCallTarget, - IPVK_MemOPSize, - IPVK_VTableTarget) /* ValueKind */ - )); - static void addValueProfData(InstrProfRecord &Record) { - // Add test data for indirect calls. - { - Record.reserveSites(IPVK_IndirectCallTarget, 6); + Record.reserveSites(IPVK_IndirectCallTarget, 5); InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, {uint64_t(callee2), 1000}, {uint64_t(callee3), 500}, @@ -1230,35 +991,6 @@ static void addValueProfData(InstrProfRecord &Record) { {uint64_t(callee3), 2000}}; Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); - InstrProfValueData VD5[] = {{uint64_t(callee7), 1234}, - {uint64_t(callee8), 5678}}; - Record.addValueData(IPVK_IndirectCallTarget, 5, VD5, 2, nullptr); - } - - // Add test data for vtables - { - Record.reserveSites(IPVK_VTableTarget, 4); - InstrProfValueData VD0[] = { - {getCalleeAddress(vtable1), 400}, {getCalleeAddress(vtable2), 1000}, - {getCalleeAddress(vtable3), 500}, {getCalleeAddress(vtable4), 300}, - {getCalleeAddress(vtable5), 100}, - }; - InstrProfValueData VD1[] = {{getCalleeAddress(vtable5), 800}, - {getCalleeAddress(vtable3), 1000}, - {getCalleeAddress(vtable2), 2500}, - {getCalleeAddress(vtable1), 1300}}; - InstrProfValueData VD2[] = { - {getCalleeAddress(vtable6), 800}, - {getCalleeAddress(vtable3), 1000}, - {getCalleeAddress(vtable4), 5500}, - }; - InstrProfValueData VD3[] = {{getCalleeAddress(vtable2), 1800}, - {getCalleeAddress(vtable3), 2000}}; - Record.addValueData(IPVK_VTableTarget, 0, VD0, 5, nullptr); - Record.addValueData(IPVK_VTableTarget, 1, VD1, 4, nullptr); - Record.addValueData(IPVK_VTableTarget, 2, VD2, 3, nullptr); - Record.addValueData(IPVK_VTableTarget, 3, VD3, 2, nullptr); - } } TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) { @@ -1271,107 +1003,59 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) { VPData->deserializeTo(Record, nullptr); // Now read data from Record and sanity check the data - ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); - ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 5)); auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { return VD1.Count > VD2.Count; }; - std::unique_ptr VD_0( Record.getValueForSite(IPVK_IndirectCallTarget, 0)); llvm::sort(&VD_0[0], &VD_0[5], Cmp); - EXPECT_STREQ((const char *)VD_0[0].Value, "callee2"); - EXPECT_EQ(1000U, VD_0[0].Count); - EXPECT_STREQ((const char *)VD_0[1].Value, "callee3"); - EXPECT_EQ(500U, VD_0[1].Count); - EXPECT_STREQ((const char *)VD_0[2].Value, "callee1"); - EXPECT_EQ(400U, VD_0[2].Count); - EXPECT_STREQ((const char *)VD_0[3].Value, "callee4"); - EXPECT_EQ(300U, VD_0[3].Count); - EXPECT_STREQ((const char *)VD_0[4].Value, "callee5"); - EXPECT_EQ(100U, VD_0[4].Count); + ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(1000U, VD_0[0].Count); + ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(500U, VD_0[1].Count); + ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1")); + ASSERT_EQ(400U, VD_0[2].Count); + ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4")); + ASSERT_EQ(300U, VD_0[3].Count); + ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5")); + ASSERT_EQ(100U, VD_0[4].Count); std::unique_ptr VD_1( Record.getValueForSite(IPVK_IndirectCallTarget, 1)); llvm::sort(&VD_1[0], &VD_1[4], Cmp); - EXPECT_STREQ((const char *)VD_1[0].Value, "callee2"); - EXPECT_EQ(VD_1[0].Count, 2500U); - EXPECT_STREQ((const char *)VD_1[1].Value, "callee1"); - EXPECT_EQ(VD_1[1].Count, 1300U); - EXPECT_STREQ((const char *)VD_1[2].Value, "callee3"); - EXPECT_EQ(VD_1[2].Count, 1000U); - EXPECT_STREQ((const char *)VD_1[3].Value, "callee5"); - EXPECT_EQ(VD_1[3].Count, 800U); + ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(2500U, VD_1[0].Count); + ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1")); + ASSERT_EQ(1300U, VD_1[1].Count); + ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3")); + ASSERT_EQ(1000U, VD_1[2].Count); + ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5")); + ASSERT_EQ(800U, VD_1[3].Count); std::unique_ptr VD_2( Record.getValueForSite(IPVK_IndirectCallTarget, 2)); llvm::sort(&VD_2[0], &VD_2[3], Cmp); - EXPECT_STREQ((const char *)VD_2[0].Value, "callee4"); - EXPECT_EQ(VD_2[0].Count, 5500U); - EXPECT_STREQ((const char *)VD_2[1].Value, "callee3"); - EXPECT_EQ(VD_2[1].Count, 1000U); - EXPECT_STREQ((const char *)VD_2[2].Value, "callee6"); - EXPECT_EQ(VD_2[2].Count, 800U); + ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4")); + ASSERT_EQ(5500U, VD_2[0].Count); + ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(1000U, VD_2[1].Count); + ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6")); + ASSERT_EQ(800U, VD_2[2].Count); std::unique_ptr VD_3( Record.getValueForSite(IPVK_IndirectCallTarget, 3)); llvm::sort(&VD_3[0], &VD_3[2], Cmp); - EXPECT_STREQ((const char *)VD_3[0].Value, "callee3"); - EXPECT_EQ(VD_3[0].Count, 2000U); - EXPECT_STREQ((const char *)VD_3[1].Value, "callee2"); - EXPECT_EQ(VD_3[1].Count, 1800U); - - ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); - - auto VD0(Record.getValueForSite(IPVK_VTableTarget, 0)); - llvm::sort(&VD0[0], &VD0[5], Cmp); - EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD0[0].Count, 1000U); - EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD0[1].Count, 500U); - EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1)); - EXPECT_EQ(VD0[2].Count, 400U); - EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4)); - EXPECT_EQ(VD0[3].Count, 300U); - EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5)); - EXPECT_EQ(VD0[4].Count, 100U); - - auto VD1(Record.getValueForSite(IPVK_VTableTarget, 1)); - llvm::sort(&VD1[0], &VD1[4], Cmp); - EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD1[0].Count, 2500U); - EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1)); - EXPECT_EQ(VD1[1].Count, 1300U); - EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD1[2].Count, 1000U); - EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5)); - EXPECT_EQ(VD1[3].Count, 800U); - - auto VD2(Record.getValueForSite(IPVK_VTableTarget, 2)); - llvm::sort(&VD2[0], &VD2[3], Cmp); - EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4)); - EXPECT_EQ(VD2[0].Count, 5500U); - EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD2[1].Count, 1000U); - EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6)); - EXPECT_EQ(VD2[2].Count, 800U); - - auto VD3(Record.getValueForSite(IPVK_VTableTarget, 3)); - llvm::sort(&VD3[0], &VD3[2], Cmp); - EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3)); - EXPECT_EQ(VD3[0].Count, 2000U); - EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2)); - EXPECT_EQ(VD3[1].Count, 1800U); + ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(2000U, VD_3[0].Count); + ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(1800U, VD_3[1].Count); } TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) { @@ -1389,121 +1073,27 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) { Symtab.mapAddress(uint64_t(callee4), 0x4000ULL); // Missing mapping for callee5 - auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t { - return uint64_t(vtable); - }; - auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t { - return uint64_t(vtable) + 16; - }; - auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t { - return uint64_t(vtable) + 8; - }; - // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not - // mapped. - Symtab.mapVTableAddress(getVTableStartAddr(vtable1), - getVTableEndAddr(vtable1), MD5Hash("vtable1")); - Symtab.mapVTableAddress(getVTableStartAddr(vtable2), - getVTableEndAddr(vtable2), MD5Hash("vtable2")); - Symtab.mapVTableAddress(getVTableStartAddr(vtable3), - getVTableEndAddr(vtable3), MD5Hash("vtable3")); - Symtab.mapVTableAddress(getVTableStartAddr(vtable4), - getVTableEndAddr(vtable4), MD5Hash("vtable4")); - VPData->deserializeTo(Record, &Symtab); // Now read data from Record and sanity check the data - ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0), 5U); - - // Look up the value correpsonding to the middle of a vtable in symtab and - // test that it's the hash of the name. - EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)), - MD5Hash("vtable1")); - EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)), - MD5Hash("vtable2")); - EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)), - MD5Hash("vtable3")); - EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)), - MD5Hash("vtable4")); + ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { return VD1.Count > VD2.Count; }; - auto VD_0(Record.getValueForSite(IPVK_IndirectCallTarget, 0)); + std::unique_ptr VD_0( + Record.getValueForSite(IPVK_IndirectCallTarget, 0)); llvm::sort(&VD_0[0], &VD_0[5], Cmp); ASSERT_EQ(VD_0[0].Value, 0x2000ULL); - ASSERT_EQ(VD_0[0].Count, 1000U); + ASSERT_EQ(1000U, VD_0[0].Count); ASSERT_EQ(VD_0[1].Value, 0x3000ULL); - ASSERT_EQ(VD_0[1].Count, 500U); + ASSERT_EQ(500U, VD_0[1].Count); ASSERT_EQ(VD_0[2].Value, 0x1000ULL); - ASSERT_EQ(VD_0[2].Count, 400U); + ASSERT_EQ(400U, VD_0[2].Count); // callee5 does not have a mapped value -- default to 0. ASSERT_EQ(VD_0[4].Value, 0ULL); - - // Sanity check the vtable value data - ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); - - { - // The first vtable site. - auto VD(Record.getValueForSite(IPVK_VTableTarget, 0)); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); - llvm::sort(&VD[0], &VD[5], Cmp); - EXPECT_EQ(VD[0].Count, 1000U); - EXPECT_EQ(VD[0].Value, MD5Hash("vtable2")); - EXPECT_EQ(VD[1].Count, 500U); - EXPECT_EQ(VD[1].Value, MD5Hash("vtable3")); - EXPECT_EQ(VD[2].Value, MD5Hash("vtable1")); - EXPECT_EQ(VD[2].Count, 400U); - EXPECT_EQ(VD[3].Value, MD5Hash("vtable4")); - EXPECT_EQ(VD[3].Count, 300U); - - // vtable5 isn't mapped -- default to 0. - EXPECT_EQ(VD[4].Value, 0U); - EXPECT_EQ(VD[4].Count, 100U); - } - - { - // The second vtable site. - auto VD(Record.getValueForSite(IPVK_VTableTarget, 1)); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); - llvm::sort(&VD[0], &VD[4], Cmp); - EXPECT_EQ(VD[0].Value, MD5Hash("vtable2")); - EXPECT_EQ(VD[0].Count, 2500U); - EXPECT_EQ(VD[1].Value, MD5Hash("vtable1")); - EXPECT_EQ(VD[1].Count, 1300U); - - EXPECT_EQ(VD[2].Value, MD5Hash("vtable3")); - EXPECT_EQ(VD[2].Count, 1000U); - // vtable5 isn't mapped -- default to 0. - EXPECT_EQ(VD[3].Value, 0U); - EXPECT_EQ(VD[3].Count, 800U); - } - - { - // The third vtable site. - auto VD(Record.getValueForSite(IPVK_VTableTarget, 2)); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); - llvm::sort(&VD[0], &VD[3], Cmp); - EXPECT_EQ(VD[0].Count, 5500U); - EXPECT_EQ(VD[0].Value, MD5Hash("vtable4")); - EXPECT_EQ(VD[1].Count, 1000U); - EXPECT_EQ(VD[1].Value, MD5Hash("vtable3")); - // vtable6 isn't mapped -- default to 0. - EXPECT_EQ(VD[2].Value, 0U); - EXPECT_EQ(VD[2].Count, 800U); - } - - { - // The fourth vtable site. - auto VD(Record.getValueForSite(IPVK_VTableTarget, 3)); - ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); - llvm::sort(&VD[0], &VD[2], Cmp); - EXPECT_EQ(VD[0].Count, 2000U); - EXPECT_EQ(VD[0].Value, MD5Hash("vtable3")); - EXPECT_EQ(VD[1].Count, 1800U); - EXPECT_EQ(VD[1].Value, MD5Hash("vtable2")); - } } TEST_P(MaybeSparseInstrProfTest, get_max_function_count) { @@ -1625,18 +1215,12 @@ TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) { for (unsigned I = 0; I < std::size(Funcs); I++) { Function *F = M->getFunction(Funcs[I]); - - std::string IRPGOName = getIRPGOFuncName(*F); - auto IRPGOFuncName = - ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(IRPGOName)); - EXPECT_EQ(IRPGOName, IRPGOFuncName); - EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second); - // Ensure we can still read this old record name. + ASSERT_TRUE(F != nullptr); std::string PGOName = getPGOFuncName(*F); - auto PGOFuncName = - ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(PGOName)); - EXPECT_EQ(PGOName, PGOFuncName); - EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str())); + uint64_t Key = IndexedInstrProf::ComputeHash(PGOName); + ASSERT_EQ(StringRef(PGOName), + ProfSymtab.getFuncName(Key)); + ASSERT_EQ(StringRef(Funcs[I]), ProfSymtab.getOrigFuncName(Key)); } } diff --git a/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp index eff8e27d36d6..6f5e32435692 100644 --- a/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp +++ b/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp @@ -8,9 +8,12 @@ #include "llvm/Transforms/Utils/CallPromotionUtils.h" #include "llvm/AsmParser/Parser.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" +#include "llvm/IR/NoFolder.h" #include "llvm/Support/SourceMgr.h" #include "gtest/gtest.h" @@ -24,6 +27,21 @@ static std::unique_ptr parseIR(LLVMContext &C, const char *IR) { return Mod; } +// Returns a constant representing the vtable's address point specified by the +// offset. +static Constant *getVTableAddressPointOffset(GlobalVariable *VTable, + uint32_t AddressPointOffset) { + Module &M = *VTable->getParent(); + LLVMContext &Context = M.getContext(); + assert(AddressPointOffset < + M.getDataLayout().getTypeAllocSize(VTable->getValueType()) && + "Out-of-bound access"); + + return ConstantExpr::getInBoundsGetElementPtr( + Type::getInt8Ty(Context), VTable, + llvm::ConstantInt::get(Type::getInt32Ty(Context), AddressPointOffset)); +} + TEST(CallPromotionUtilsTest, TryPromoteCall) { LLVMContext C; std::unique_ptr M = parseIR(C, @@ -368,3 +386,73 @@ declare %struct2 @_ZN4Impl3RunEv(%class.Impl* %this) bool IsPromoted = tryPromoteCall(*CI); EXPECT_FALSE(IsPromoted); } + +TEST(CallPromotionUtilsTest, promoteCallWithVTableCmp) { + LLVMContext C; + std::unique_ptr M = parseIR(C, + R"IR( +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@_ZTV5Base1 = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev] }, !type !0 +@_ZTV8Derived1 = constant { [4 x ptr], [3 x ptr] } { [4 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev], [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base25func2Ev] }, !type !0, !type !1, !type !2 +@_ZTV8Derived2 = constant { [3 x ptr], [3 x ptr], [4 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base35func3Ev], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN5Base25func2Ev], [4 x ptr] [ptr inttoptr (i64 -16 to ptr), ptr null ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev] }, !type !3, !type !4, !type !5, !type !6 + +define i32 @testfunc(ptr %d) { +entry: + %vtable = load ptr, ptr %d, !prof !7 + %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 + %0 = load ptr, ptr %vfn + %call = tail call i32 %0(ptr %d), !prof !8 + ret i32 %call +} + +define i32 @_ZN5Base15func1Ev(ptr %this) { +entry: + ret i32 2 +} + +declare i32 @_ZN5Base25func2Ev(ptr) +declare i32 @_ZN5Base15func0Ev(ptr) +declare void @_ZN5Base35func3Ev(ptr) + +!0 = !{i64 16, !"_ZTS5Base1"} +!1 = !{i64 48, !"_ZTS5Base2"} +!2 = !{i64 16, !"_ZTS8Derived1"} +!3 = !{i64 64, !"_ZTS5Base1"} +!4 = !{i64 40, !"_ZTS5Base2"} +!5 = !{i64 16, !"_ZTS5Base3"} +!6 = !{i64 16, !"_ZTS8Derived2"} +!7 = !{!"VP", i32 2, i64 1600, i64 -9064381665493407289, i64 800, i64 5035968517245772950, i64 500, i64 3215870116411581797, i64 300} +!8 = !{!"VP", i32 0, i64 1600, i64 6804820478065511155, i64 1600})IR"); + +Function *F = M->getFunction("testfunc"); +CallInst *CI = dyn_cast(&*std::next(F->front().rbegin())); +ASSERT_TRUE(CI && CI->isIndirectCall()); + +// Create the constant and the branch weights +SmallVector VTableAddressPoints; + +for (auto &[VTableName, AddressPointOffset] : {std::pair{"_ZTV5Base1", 16}, + {"_ZTV8Derived1", 16}, + {"_ZTV8Derived2", 64}}) + VTableAddressPoints.push_back(getVTableAddressPointOffset( + M->getGlobalVariable(VTableName), AddressPointOffset)); + + MDBuilder MDB(C); + MDNode *BranchWeights = MDB.createBranchWeights(1600, 0); + + size_t OrigEntryBBSize = F->front().size(); + + LoadInst *VPtr = dyn_cast(&*F->front().begin()); + + Function *Callee = M->getFunction("_ZN5Base15func1Ev"); + // Tests that promoted direct call is returned. + CallBase &DirectCB = promoteCallWithVTableCmp( + *CI, VPtr, Callee, VTableAddressPoints, BranchWeights); + EXPECT_EQ(DirectCB.getCalledOperand(), Callee); + + // Promotion inserts 3 icmp instructions and 2 or instructions, and removes + // 1 call instruction from the entry block. + EXPECT_EQ(F->front().size(), OrigEntryBBSize + 4); +} -- Gitee From bbbe6b9da68de7ed87c319d9723d0a92f6f76a95 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 22 Jul 2024 11:57:36 -0700 Subject: [PATCH 24/28] [InstrPGO][TypeProf]Annotate vtable types when they are present in the profile (#99402) Before this change, when `file.profdata` have vtable profiles but `--enable-vtable-value-profiling` is not on for optimized build, warnings from this line [1] will show up. They are benign for performance but confusing. It's better to automatically annotate vtable profiles if `file.profdata` has them. This PR implements it in profile use pass. * If `-icp-max-num-vtables` is zero (default value is 6), vtable profiles won't be annotated. [1] https://github.com/llvm/llvm-project/blob/464d321ee8dde1eaf14b5537eaf030e6df513849/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp#L1762-L1768 --- .../Linux/instrprof-vtable-value-prof.cpp | 27 ++++++++++++++++--- .../Instrumentation/PGOInstrumentation.cpp | 21 +++++++++++++-- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp index 6cf73c6fdbd7..46a925ce0f9f 100644 --- a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp +++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp @@ -109,6 +109,25 @@ // ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 // ICTEXT: _ZTV8Derived1:250 +// When vtable value profiles exist, pgo-instr-use pass should annotate them +// even if `-enable-vtable-value-profiling` is not explicitly on. +// RUN: %clangxx -m64 -fprofile-use=test.profdata -fuse-ld=lld -O2 \ +// RUN: -mllvm -print-after=pgo-instr-use -mllvm -filter-print-funcs=main \ +// RUN: -mllvm -print-module-scope %s 2>&1 | FileCheck %s --check-prefix=ANNOTATE + +// ANNOTATE-NOT: Inconsistent number of value sites +// ANNOTATE: !{!"VP", i32 2 + +// When vtable value profiles exist, pgo-instr-use pass will not annotate them +// if `-icp-max-num-vtables` is set to zero. +// RUN: %clangxx -m64 -fprofile-use=test.profdata -fuse-ld=lld -O2 \ +// RUN: -mllvm -icp-max-num-vtables=0 -mllvm -print-after=pgo-instr-use \ +// RUN: -mllvm -filter-print-funcs=main -mllvm -print-module-scope %s 2>&1 | \ +// RUN: FileCheck %s --check-prefix=OMIT + +// OMIT: Inconsistent number of value sites +// OMIT-NOT: !{!"VP", i32 2 + // Test indirect call promotion transformation using vtable profiles. // - Build with `-g` to enable debug information. // - In real world settings, ICP pass is disabled in prelink pipeline. In @@ -128,12 +147,12 @@ // RUN: | FileCheck %s --check-prefixes=REMARK,IR --implicit-check-not="!VP" // For the indirect call site `ptr->func` -// REMARK: instrprof-vtable-value-prof.cpp:205:19: Promote indirect call to _ZN12_GLOBAL__N_18Derived24funcEii with count 150 out of 200, sink 1 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E} -// REMARK: instrprof-vtable-value-prof.cpp:205:19: Promote indirect call to _ZN8Derived14funcEii with count 50 out of 50, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1} +// REMARK: instrprof-vtable-value-prof.cpp:226:19: Promote indirect call to _ZN12_GLOBAL__N_18Derived24funcEii with count 150 out of 200, sink 1 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E} +// REMARK: instrprof-vtable-value-prof.cpp:226:19: Promote indirect call to _ZN8Derived14funcEii with count 50 out of 50, sink 1 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1} // // For the indirect call site `delete ptr` -// REMARK: instrprof-vtable-value-prof.cpp:207:5: Promote indirect call to _ZN12_GLOBAL__N_18Derived2D0Ev with count 750 out of 1000, sink 2 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E} -// REMARK: instrprof-vtable-value-prof.cpp:207:5: Promote indirect call to _ZN8Derived1D0Ev with count 250 out of 250, sink 2 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1} +// REMARK: instrprof-vtable-value-prof.cpp:228:5: Promote indirect call to _ZN12_GLOBAL__N_18Derived2D0Ev with count 750 out of 1000, sink 2 instruction(s) and compare 1 vtable(s): {_ZTVN12_GLOBAL__N_18Derived2E} +// REMARK: instrprof-vtable-value-prof.cpp:228:5: Promote indirect call to _ZN8Derived1D0Ev with count 250 out of 250, sink 2 instruction(s) and compare 1 vtable(s): {_ZTV8Derived1} // The IR matchers for indirect callsite `ptr->func`. // IR-LABEL: @main diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 822c4b3d8175..f821cf40353a 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1071,7 +1071,7 @@ public: : F(Func), M(Modu), BFI(BFIin), PSI(PSI), FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS, InstrumentFuncEntry, HasSingleByteCoverage), - FreqAttr(FFA_Normal), IsCS(IsCS) {} + FreqAttr(FFA_Normal), IsCS(IsCS), VPC(Func, TLI) {} void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum); @@ -1153,6 +1153,8 @@ private: // Is to use the context sensitive profile. bool IsCS; + ValueProfileCollector VPC; + // Find the Instrumented BB and set the value. Return false on error. bool setInstrumentedCounts(const std::vector &CountFromProfile); @@ -1733,8 +1735,23 @@ void PGOUseFunc::annotateValueSites() { void PGOUseFunc::annotateValueSites(uint32_t Kind) { assert(Kind <= IPVK_Last); unsigned ValueSiteIndex = 0; - auto &ValueSites = FuncInfo.ValueSites[Kind]; + unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind); + + // Since there isn't a reliable or fast way for profile reader to tell if a + // profile is generated with `-enable-vtable-value-profiling` on, we run the + // value profile collector over the function IR to find the instrumented sites + // iff function profile records shows the number of instrumented vtable sites + // is not zero. Function cfg already takes the number of instrumented + // indirect call sites into account so it doesn't hash the number of + // instrumented vtables; as a side effect it makes it easier to enable + // profiling and profile use in two steps if needed. + // TODO: Remove this if/when -enable-vtable-value-profiling is on by default. + if (NumValueSites > 0 && Kind == IPVK_VTableTarget && + NumValueSites != FuncInfo.ValueSites[IPVK_VTableTarget].size() && + MaxNumVTableAnnotations != 0) + FuncInfo.ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget); + auto &ValueSites = FuncInfo.ValueSites[Kind]; if (NumValueSites != ValueSites.size()) { auto &Ctx = M->getContext(); Ctx.diagnose(DiagnosticInfoPGOProfile( -- Gitee From 2b2bafb608cfd2c2e8ad6357a269929980eae2c0 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Tue, 27 Aug 2024 11:51:24 -0700 Subject: [PATCH 25/28] [TypeProf][ICP]Allow vtable-comparison as long as vtable count is comparable with function count for each candidate (#106260) The current cost-benefit analysis between vtable comparison and function comparison require the indirect fallback branch to be cold. This is too conservative. This change allows vtable-comparison as long as vtable count is comparable with function count for each function candidate and removes the cold indirect fallback requirement. Tested: 1. Testing this on benchmarks uplifts the measurable performance wins. Counting the (possibly-duplicated) remarks (because of linkonce_odr functions, cross-module import of functions) show the number of vtable remarks increases from ~30k-ish to 50k-ish. 2. https://gcc.godbolt.org/z/sbGK7Pacn shows vtable-comparison doesn't happen today (using the same IR input) --- .../Instrumentation/IndirectCallPromotion.cpp | 10 +--------- .../Transforms/PGOProfile/icp_vtable_cmp.ll | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 5f40373fa5d2..c97b1e52e375 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -115,7 +115,7 @@ static cl::opt // Indirect call promotion pass will fall back to function-based comparison if // vtable-count / function-count is smaller than this threshold. static cl::opt ICPVTablePercentageThreshold( - "icp-vtable-percentage-threshold", cl::init(0.99), cl::Hidden, + "icp-vtable-percentage-threshold", cl::init(0.995), cl::Hidden, cl::desc("The percentage threshold of vtable-count / function-count for " "cost-benefit analysis.")); @@ -892,14 +892,6 @@ bool IndirectCallPromoter::isProfitableToCompareVTables( } } - // If the indirect fallback is not cold, don't compare vtables. - if (PSI && PSI->hasProfileSummary() && - !PSI->isColdCount(RemainingVTableCount)) { - LLVM_DEBUG(dbgs() << " Indirect fallback basic block is not cold. Bail " - "out for vtable comparison.\n"); - return false; - } - return true; } diff --git a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll index c77be3b1ed24..b6afce3d7c6d 100644 --- a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll +++ b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll @@ -120,6 +120,7 @@ declare i32 @Base2_foo(ptr) declare i32 @Base1_foo(ptr) declare void @Base3_foo(ptr) +!llvm.module.flags = !{!11} !0 = !{i64 16, !"Base1"} !1 = !{i64 40, !"Base1"} !2 = !{i64 16, !"Base2"} @@ -131,6 +132,23 @@ declare void @Base3_foo(ptr) !8 = !{i64 16, !"Derived3"} !9 = !{!"VP", i32 2, i64 1600, i64 -4123858694673519054, i64 600, i64 -7211198353767973908, i64 500, i64 -3574436251470806727, i64 200, i64 6288809125658696740, i64 200, i64 12345678, i64 100} !10 = !{!"VP", i32 0, i64 1600, i64 3827408714133779784, i64 600, i64 5837445539218476403, i64 500, i64 -9064955852395570538, i64 400, i64 56781234, i64 100} + +!11 = !{i32 1, !"ProfileSummary", !12} +!12 = !{!13, !14, !15, !16, !17, !18, !19, !20} +!13 = !{!"ProfileFormat", !"InstrProf"} +!14 = !{!"TotalCount", i64 10000} +!15 = !{!"MaxCount", i64 10} +!16 = !{!"MaxInternalCount", i64 1} +!17 = !{!"MaxFunctionCount", i64 1000} +!18 = !{!"NumCounts", i64 3} +!19 = !{!"NumFunctions", i64 3} +!20 = !{!"DetailedSummary", !21} +!21 = !{!22, !23, !24} +!22 = !{i32 10000, i64 101, i32 1} +!23 = !{i32 990000, i64 101, i32 1} +!24 = !{i32 999999, i64 1, i32 2} + + ;. ; VTABLE-COMMON: [[PROF9]] = !{!"VP", i32 2, i64 100, i64 12345678, i64 100} ; VTABLE-COMMON: [[PROF10]] = !{!"branch_weights", i32 600, i32 1000} -- Gitee From f3c04cf5b4504d4ac666cd0efa3abea1437c2131 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Wed, 2 Oct 2024 10:23:54 -0700 Subject: [PATCH 26/28] [TypeProf][PGO]Support skipping vtable comparisons for a class and its derived ones (#110575) Performance critical core libraries could be highly-optimized for arch or micro-arch features. For instance, the absl crc library specializes different templated classes among different hardwares [1]. In a practical setting, it's likely that instrumented profiles are collected on one type of machine and used to optimize binaries that run on multiple types of hardwares. While this kind of specialization is rare in terms of lines of code, compiler can do a better job to skip vtable-based ICP. * The per-class `Extend` implementation is arch-specific as well. If an instrumented profile is collected on one arch and applied to another arch where `Extend` implementation is different, `Extend` might be regarded as unlikely function in the latter case. `ABSL_ATTRIBUTE_HOT` annotation alleviates the problem by putting all `Extend` implementation into the hot text section [2] This change introduces a comma-separated list to specify the mangled vtable names, and ICP pass will skip vtable-based comparison if a vtable variable definition is shown to be in its class hierarchy (per LLVM type metadata). [1] https://github.com/abseil/abseil-cpp/blob/c6b27359c3d27438b1313dddd7598914c1274a50/absl/crc/internal/crc_x86_arm_combined.cc#L621-L650 [2] https://github.com/abseil/abseil-cpp/blame/c6b27359c3d27438b1313dddd7598914c1274a50/absl/crc/internal/crc_x86_arm_combined.cc#L370C3-L370C21 --- .../Instrumentation/IndirectCallPromotion.cpp | 61 +++++++++++++++++-- .../Transforms/PGOProfile/icp_vtable_cmp.ll | 4 ++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index c97b1e52e375..7515d636061d 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -130,6 +130,15 @@ static cl::opt ICPMaxNumVTableLastCandidate( "icp-max-num-vtable-last-candidate", cl::init(1), cl::Hidden, cl::desc("The maximum number of vtable for the last candidate.")); +static cl::list ICPIgnoredBaseTypes( + "icp-ignored-base-types", cl::Hidden, + cl::desc( + "A list of mangled vtable type info names. Classes specified by the " + "type info names and their derived ones will not be vtable-ICP'ed. " + "Useful when the profiled types and actual types in the optimized " + "binary could be different due to profiling limitations. Type info " + "names are those string literals used in LLVM type metadata")); + namespace { // The key is a vtable global variable, and the value is a map. @@ -316,6 +325,8 @@ private: OptimizationRemarkEmitter &ORE; + const DenseSet &IgnoredBaseTypes; + // A struct that records the direct target and it's call count. struct PromotionCandidate { Function *const TargetFunction; @@ -368,6 +379,10 @@ private: const CallBase &CB, const std::vector &Candidates, uint64_t TotalCount); + // Return true if the vtable corresponding to VTableGUID should be skipped + // for vtable-based comparison. + bool shouldSkipVTable(uint64_t VTableGUID); + // Given an indirect callsite and the list of function candidates, compute // the following vtable information in output parameters and return vtable // pointer if type profiles exist. @@ -394,10 +409,12 @@ public: InstrProfSymtab *Symtab, bool SamplePGO, const VirtualCallSiteTypeInfoMap &VirtualCSInfo, VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal, + const DenseSet &IgnoredBaseTypes, OptimizationRemarkEmitter &ORE) : F(Func), M(M), PSI(PSI), Symtab(Symtab), SamplePGO(SamplePGO), VirtualCSInfo(VirtualCSInfo), - VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE) {} + VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE), + IgnoredBaseTypes(IgnoredBaseTypes) {} IndirectCallPromoter(const IndirectCallPromoter &) = delete; IndirectCallPromoter &operator=(const IndirectCallPromoter &) = delete; @@ -861,9 +878,14 @@ bool IndirectCallPromoter::isProfitableToCompareVTables( LLVM_DEBUG(dbgs() << "\n"); uint64_t CandidateVTableCount = 0; - for (auto &[GUID, Count] : VTableGUIDAndCounts) + + for (auto &[GUID, Count] : VTableGUIDAndCounts) { CandidateVTableCount += Count; + if (shouldSkipVTable(GUID)) + return false; + } + if (CandidateVTableCount < Candidate.Count * ICPVTablePercentageThreshold) { LLVM_DEBUG( dbgs() << " function count " << Candidate.Count @@ -895,6 +917,27 @@ bool IndirectCallPromoter::isProfitableToCompareVTables( return true; } +bool IndirectCallPromoter::shouldSkipVTable(uint64_t VTableGUID) { + if (IgnoredBaseTypes.empty()) + return false; + + auto *VTableVar = Symtab->getGlobalVariable(VTableGUID); + + assert(VTableVar && "VTableVar must exist for GUID in VTableGUIDAndCounts"); + + SmallVector Types; + VTableVar->getMetadata(LLVMContext::MD_type, Types); + + for (auto *Type : Types) + if (auto *TypeId = dyn_cast(Type->getOperand(1).get())) + if (IgnoredBaseTypes.contains(TypeId->getString())) { + LLVM_DEBUG(dbgs() << " vtable profiles should be ignored. Bail " + "out of vtable comparison."); + return true; + } + return false; +} + // For virtual calls in the module, collect per-callsite information which will // be used to associate an ICP candidate with a vtable and a specific function // in the vtable. With type intrinsics (llvm.type.test), we can find virtual @@ -968,9 +1011,15 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, bool Changed = false; VirtualCallSiteTypeInfoMap VirtualCSInfo; - if (EnableVTableProfileUse) + DenseSet IgnoredBaseTypes; + + if (EnableVTableProfileUse) { computeVirtualCallSiteTypeInfoMap(M, MAM, VirtualCSInfo); + for (StringRef Str : ICPIgnoredBaseTypes) + IgnoredBaseTypes.insert(Str); + } + // VTableAddressPointOffsetVal stores the vtable address points. The vtable // address point of a given is static (doesn't // change after being computed once). @@ -988,9 +1037,9 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, MAM.getResult(M).getManager(); auto &ORE = FAM.getResult(F); - IndirectCallPromoter CallPromoter(F, M, PSI, &Symtab, SamplePGO, - VirtualCSInfo, - VTableAddressPointOffsetVal, ORE); + IndirectCallPromoter CallPromoter( + F, M, PSI, &Symtab, SamplePGO, VirtualCSInfo, + VTableAddressPointOffsetVal, IgnoredBaseTypes, ORE); bool FuncChanged = CallPromoter.processFunction(PSI); if (ICPDUMPAFTER && FuncChanged) { LLVM_DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs())); diff --git a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll index b6afce3d7c6d..84bb7a5830af 100644 --- a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll +++ b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll @@ -1,7 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; Tests that ICP compares vtables by checking IR. ; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=2 -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,VTABLE-CMP +; Require exactly one vtable candidate for each function candidate. Tests that ICP compares function by checking IR. ; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=1 -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,FUNC-CMP +; On top of line 4, ignore 'Base1' and its derived types for vtable-based comparison. Tests that ICP compares functions. +; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=2 -icp-ignored-base-types='Base1' -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,FUNC-CMP target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -- Gitee From 00713f7ead8a9e3dd22414082d9ed655363750ad Mon Sep 17 00:00:00 2001 From: Szymon Sobieszek Date: Tue, 12 Aug 2025 09:27:01 -0400 Subject: [PATCH 27/28] [ICP] Enabling extra icp Enabling extra icp with dynamic type profiling by including the llvm.public.type.test intrinsic into the analysis, apart from the llvm.type.test intrinsic. Also, reverted a dev_19.1.7 that we do not support. --- .../Instrumentation/IndirectCallPromotion.cpp | 5 +++++ llvm/tools/llvm-profdata/llvm-profdata.cpp | 21 +++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 7515d636061d..76a9decc7be5 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -957,6 +957,11 @@ computeVirtualCallSiteTypeInfoMap(Module &M, ModuleAnalysisManager &MAM, // that case. Function *TypeTestFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_test)); + + if (!TypeTestFunc || TypeTestFunc->use_empty()) + TypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test)); + if (!TypeTestFunc || TypeTestFunc->use_empty()) return; diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 05451f941516..7ed4221616b9 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -792,10 +792,10 @@ static void writeInstrProfile(StringRef OutputFilename, static void mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, - StringRef BinaryFilename, SymbolRemapper *Remapper, - StringRef OutputFilename, ProfileFormat OutputFormat, - uint64_t TraceReservoirSize, uint64_t MaxTraceLength, - bool OutputSparse, unsigned NumThreads, FailureMode FailMode, + SymbolRemapper *Remapper, StringRef OutputFilename, + ProfileFormat OutputFormat, uint64_t TraceReservoirSize, + uint64_t MaxTraceLength, bool OutputSparse, + unsigned NumThreads, FailureMode FailMode, const StringRef ProfiledBinary, const bool KeepVTableSymbols) { if (OutputFormat == PF_Compact_Binary) @@ -806,16 +806,13 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, // TODO: Maybe we should support correlation with mixture of different // correlation modes(w/wo debug-info/object correlation). - if (!DebugInfoFilename.empty() && !BinaryFilename.empty()) + if (!DebugInfoFilename.empty()) exitWithError("Expected only one of -debug-info, -binary-file"); std::string CorrelateFilename; ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE; if (!DebugInfoFilename.empty()) { CorrelateFilename = DebugInfoFilename; CorrelateKind = ProfCorrelatorKind::DEBUG_INFO; - } else if (!BinaryFilename.empty()) { - CorrelateFilename = BinaryFilename; - CorrelateKind = ProfCorrelatorKind::BINARY; } std::unique_ptr Correlator; @@ -1675,10 +1672,6 @@ static int merge_main(int argc, const char *argv[]) { cl::opt DebugInfoFilename( "debug-info", cl::init(""), cl::desc("Use the provided debug info to correlate the raw profile.")); - cl::opt - BinaryFilename("binary-file", cl::init(""), - cl::desc("For merge, use the provided unstripped binary to " - "correlate the raw profile.")); cl::opt ProfiledBinary( "profiled-binary", cl::init(""), cl::desc("Path to binary from which the profile was collected.")); @@ -1741,8 +1734,8 @@ static int merge_main(int argc, const char *argv[]) { } if (ProfileKind == instr) - mergeInstrProfile(WeightedInputs, DebugInfoFilename, BinaryFilename, - Remapper.get(), OutputFilename, OutputFormat, + mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), + OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, TemporalProfMaxTraceLength, OutputSparse, NumThreads, FailureMode, ProfiledBinary, KeepVTableSymbols); -- Gitee From dea9fe89a180734e7159c49dfe0fad45f1a487f6 Mon Sep 17 00:00:00 2001 From: Szymon Sobieszek Date: Tue, 12 Aug 2025 16:55:05 -0400 Subject: [PATCH 28/28] [ICP] Test cases fix Fix for test cases related to porting the dynamic type profiling feature + fixes regarding code formatting. --- .../test/profile/instrprof-binary-correlate.c | 46 ------ .../llvm/Analysis/IndirectCallVisitor.h | 6 +- llvm/include/llvm/IR/MDBuilder.h | 4 +- llvm/include/llvm/IR/ProfDataUtils.h | 6 +- .../Coverage/CoverageMappingReader.h | 2 +- llvm/include/llvm/ProfileData/InstrProf.h | 16 +- .../llvm/ProfileData/InstrProfWriter.h | 2 +- .../Transforms/Utils/CallPromotionUtils.h | 10 +- llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 2 +- llvm/lib/Analysis/TypeMetadataUtils.cpp | 2 +- llvm/lib/IR/Instructions.cpp | 8 +- llvm/lib/IR/MDBuilder.cpp | 2 +- llvm/lib/IR/ProfDataUtils.cpp | 38 ++--- .../Coverage/CoverageMappingReader.cpp | 16 +- llvm/lib/ProfileData/InstrProf.cpp | 52 +++---- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 22 +-- llvm/lib/ProfileData/InstrProfReader.cpp | 4 +- llvm/lib/ProfileData/InstrProfWriter.cpp | 12 +- .../Instrumentation/IndirectCallPromotion.cpp | 6 +- .../Instrumentation/PGOInstrumentation.cpp | 1 + .../Transforms/Utils/CallPromotionUtils.cpp | 14 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 22 ++- llvm/unittests/ProfileData/InstrProfTest.cpp | 138 +++++++++--------- .../Utils/CallPromotionUtilsTest.cpp | 40 ++--- 24 files changed, 212 insertions(+), 259 deletions(-) delete mode 100644 compiler-rt/test/profile/instrprof-binary-correlate.c diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c deleted file mode 100644 index 8f421014cf5c..000000000000 --- a/compiler-rt/test/profile/instrprof-binary-correlate.c +++ /dev/null @@ -1,46 +0,0 @@ -// REQUIRES: linux || windows -// Default -// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp -// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal -// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw -// RUN: llvm-cov report --instr-profile=%t.normal.profdata %t.normal > %t.normal.report -// RUN: llvm-cov show --instr-profile=%t.normal.profdata %t.normal > %t.normal.show - -// With -profile-correlate=binary flag -// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp -// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe -// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw -// RUN: llvm-cov report --instr-profile=%t-1.profdata %t-1.exe > %t-1.report -// RUN: llvm-cov show --instr-profile=%t-1.profdata %t-1.exe > %t-1.show -// RUN: diff %t.normal.profdata %t-1.profdata -// RUN: diff %t.normal.report %t-1.report -// RUN: diff %t.normal.show %t-1.show - -// Strip above binary and run -// RUN: llvm-strip %t-1.exe -o %t-2.exe -// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe -// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw -// RUN: llvm-cov report --instr-profile=%t-2.profdata %t-1.exe > %t-2.report -// RUN: llvm-cov show --instr-profile=%t-2.profdata %t-1.exe > %t-2.show -// RUN: diff %t.normal.profdata %t-2.profdata -// RUN: diff %t.normal.report %t-2.report -// RUN: diff %t.normal.show %t-2.show - -// Online merging. -// RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal -// RUN: env LLVM_PROFILE_FILE=%t-4.profraw %run %t.normal -// RUN: llvm-profdata merge -o %t.normal.merged.profdata %t-3.profraw %t-4.profraw -// RUN: llvm-cov report --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.report -// RUN: llvm-cov show --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.show - -// RUN: rm -rf %t.profdir && mkdir %t.profdir -// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe -// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe -// RUN: llvm-profdata merge -o %t-4.profdata --binary-file=%t-1.exe %t.profdir -// RUN: llvm-cov report --instr-profile=%t-4.profdata %t-1.exe > %t-4.report -// RUN: llvm-cov show --instr-profile=%t-4.profdata %t-1.exe > %t-4.show -// RUN: diff %t.normal.merged.profdata %t-4.profdata -// RUN: diff %t.normal.merged.report %t-4.report -// RUN: diff %t.normal.merged.show %t-4.show - -// TODO: After adding support for binary ID, test binaries with different binary IDs. diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h index 67e70f5fd6d2..549c6d9b9466 100644 --- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h +++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h @@ -53,7 +53,7 @@ struct PGOIndirectCallVisitor : public InstVisitor { // address is negligible if exists at all. Comparing loaded address // with symbol address guarantees correctness. if (VTablePtr != nullptr && isa(VTablePtr)) - return cast(VTablePtr); + return cast(VTablePtr); } return nullptr; } @@ -66,9 +66,9 @@ struct PGOIndirectCallVisitor : public InstVisitor { return; Instruction *VPtr = - PGOIndirectCallVisitor::tryGetVTableInstruction(&Call); + PGOIndirectCallVisitor::tryGetVTableInstruction(&Call); if (VPtr) - ProfiledAddresses.push_back(VPtr); + ProfiledAddresses.push_back(VPtr); } } diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index d972f85ecb47..865c364c778d 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -60,11 +60,11 @@ public: /// Return metadata containing two branch weights. MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, - bool IsExpected = false); + bool IsExpected = false); /// Return metadata containing a number of branch weights. MDNode *createBranchWeights(ArrayRef Weights, - bool IsExpected = false); + bool IsExpected = false); /// Return metadata specifying that a branch or switch is unpredictable. MDNode *createUnpredictable(); diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h index 4e1bcff528ef..0bea517df832 100644 --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -80,12 +80,12 @@ bool extractBranchWeights(const MDNode *ProfileData, /// Faster version of extractBranchWeights() that skips checks and must only /// be called with "branch_weights" metadata nodes. Supports uint32_t. void extractFromBranchWeightMD32(const MDNode *ProfileData, - SmallVectorImpl &Weights); + SmallVectorImpl &Weights); /// Faster version of extractBranchWeights() that skips checks and must only /// be called with "branch_weights" metadata nodes. Supports uint64_t. void extractFromBranchWeightMD64(const MDNode *ProfileData, - SmallVectorImpl &Weights); + SmallVectorImpl &Weights); /// Extract branch weights attatched to an Instruction /// @@ -128,7 +128,7 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights); /// \param Weights an array of weights to set on instruction I. /// \param IsExpected were these weights added from an llvm.expect* intrinsic. void setBranchWeights(Instruction &I, ArrayRef Weights, - bool IsExpected); + bool IsExpected); /// Scaling the profile data attached to 'I' using the ratio of S/T. void scaleProfData(Instruction &I, uint64_t S, uint64_t T); diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 48e4b78c6238..9ad4f8897443 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -196,7 +196,7 @@ private: FuncRecordsStorage FuncRecords; BinaryCoverageReader(std::unique_ptr Symtab, - FuncRecordsStorage &&FuncRecords) + FuncRecordsStorage &&FuncRecords) : ProfileNames(std::move(Symtab)), FuncRecords(std::move(FuncRecords)) {} public: diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 92c85e10355d..d63f1de9697f 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -234,15 +234,15 @@ StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, /// Given a vector of strings (names of global objects like functions or, /// virtual tables) \c NameStrs, the method generates a combined string \c -/// Result that is ready to be serialized. The \c result string is comprised of +/// Result that is ready to be serialized. The \c Result string is comprised of /// three fields: The first field is the length of the uncompressed strings, and /// the the second field is the length of the zlib-compressed string. Both -/// fields are encoded in ULEB128. If \c doCompress is false, the +/// fields are encoded in ULEB128. If \c doCompress is false, the /// third field is the uncompressed strings; otherwise it is the /// compressed string. When the string compression is off, the /// second field will have value zero. Error collectGlobalObjectNameStrings(ArrayRef NameStrs, - bool doCompression, std::string &Result); + bool doCompression, std::string &Result); /// Given a vector of strings (function PGO names) \c NameStrs, the /// method generates a combined string \c Result that is ready to be @@ -513,9 +513,9 @@ private: static StringRef getExternalSymbol() { return "** External Symbol **"; } - //Returns the canonical name of the given PGOName. In a canonical name, all - //suffixes that begins with "." except ".uniq." are stripped. - //FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. + // Returns the canonial name of the given PGOName. In a canonical name, all + // suffixes that begins with "." except ".__uniq." are stripped. + // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. static StringRef getCanonicalName(StringRef PGOName); // Add the function into the symbol table, by creating the following @@ -749,8 +749,8 @@ StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) { StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) { finalizeSymtab(); auto Result = llvm::lower_bound(MD5NameMap, MD5Hash, - [](const std::pair &LHS, - uint64_t RHS) { return LHS.first < RHS; }); + [](const std::pair &LHS, + uint64_t RHS) { return LHS.first < RHS; }); if (Result != MD5NameMap.end() && Result->first == MD5Hash) return Result->second; return StringRef(); diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index d58c465cfd88..b76e57faa537 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -83,7 +83,7 @@ public: InstrProfWriter(bool Sparse = false, uint64_t TemporalProfTraceReservoirSize = 0, uint64_t MaxTemporalProfTraceLength = 0, - bool WritePrevVersion = false); + bool WritePrevVersion = false); ~InstrProfWriter(); StringMap &getProfileData() { return FunctionData; } diff --git a/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h b/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h index 530cf46ca38b..385831f45703 100644 --- a/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h +++ b/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h @@ -66,11 +66,11 @@ CallBase &promoteCallWithIfThenElse(CallBase &CB, Function *Callee, /// a location inside the vtable that's referenced by vpointer in C++ objects. /// /// TODO: sink the address-calculation instructions of indirect callee to the -/// indirect call fallback after transformation -CallBase &promoteCallWithVTableCmp(CallBase &CB, Instruction *Vptr, - Function *Callee, - ArrayRef AddressPoints, - MDNode *BranchWeights); +/// indirect call fallback after transformation. +CallBase &promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, + Function *Callee, + ArrayRef AddressPoints, + MDNode *BranchWeights); /// Try to promote (devirtualize) a virtual call on an Alloca. Return true on /// success. diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index cd8ca50c4038..b8b3215bd134 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -448,7 +448,7 @@ static void computeFunctionSummary( uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction(&I, TotalCount, - NumCandidates); + NumCandidates); for (const auto &Candidate : CandidateProfileData) CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)] .updateHotness(getHotness(Candidate.Count, PSI)); diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp index bcd2d5d08afe..b8dcc39e9223 100644 --- a/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp @@ -203,7 +203,7 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M, std::pair llvm::getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, - Module &M) { + Module &M) { Constant *Ptr = getPointerAtOffset(GV->getInitializer(), Offset, M, GV); if (!Ptr) return std::pair(nullptr, nullptr); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 9023352d690f..918495bdae46 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -1131,10 +1131,10 @@ LandingPadInst *InvokeInst::getLandingPadInst() const { void InvokeInst::updateProfWeight(uint64_t S, uint64_t T) { if (T == 0) { LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in " - "div by 0. Ignoring. Likely the function " - << getParent()->getParent()->getName() - << " has 0 entry count, and contains call instruction " - "with non-zero prof info."); + "div by 0. Ignoring. Likely the function " + << getParent()->getParent()->getName() + << " has 0 entry count, and contains call instruction " + "with non-zero prof info."); return; } scaleProfData(*this, S, T); diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index 510dba5fc40e..7581ebecdaaf 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -40,7 +40,7 @@ MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight, } MDNode *MDBuilder::createBranchWeights(ArrayRef Weights, - bool IsExpected) { + bool IsExpected) { assert(Weights.size() >= 1 && "Need at least one branch weights!"); unsigned int Offset = IsExpected ? 2 : 1; diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index a79d780292e3..5d44b0605823 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -69,9 +69,9 @@ bool isTargetMD(const MDNode *ProfData, const char *Name, unsigned MinOps) { } template >> + typename = typename std::enable_if>> static void extractFromBranchWeightMD(const MDNode *ProfileData, - SmallVectorImpl &Weights) { + SmallVectorImpl &Weights) { assert(isBranchWeightMD(ProfileData) && "wrong metadata"); unsigned NOps = ProfileData->getNumOperands(); @@ -84,7 +84,7 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData, mdconst::dyn_extract(ProfileData->getOperand(Idx)); assert(Weight && "Malformed branch_weight in MD_prof node"); assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) && - "Too many bits for MD_prof branch_weight"); + "Too many bits for MD_prof branch_weight"); Weights[Idx - WeightsIdx] = Weight->getZExtValue(); } } @@ -115,7 +115,7 @@ bool hasCountTypeMD(const Instruction &I) { // Value profiles record count-type information. if (isValueProfileMD(ProfileData)) return true; - // Conservatively assume non CallBase instruction only get take/not-taken + // Conservatively assume non CallBase instruction only get taken/not-taken // branch probability, so not interpret them as count. return isa(I) && !isBranchWeightMD(ProfileData); } @@ -163,12 +163,12 @@ MDNode *getValidBranchWeightMDNode(const Instruction &I) { } void extractFromBranchWeightMD32(const MDNode *ProfileData, - SmallVectorImpl &Weights) { + SmallVectorImpl &Weights) { extractFromBranchWeightMD(ProfileData, Weights); } void extractFromBranchWeightMD64(const MDNode *ProfileData, - SmallVectorImpl &Weights) { + SmallVectorImpl &Weights) { extractFromBranchWeightMD(ProfileData, Weights); } @@ -239,7 +239,7 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) { } void setBranchWeights(Instruction &I, ArrayRef Weights, - bool IsExpected) { + bool IsExpected) { MDBuilder MDB(I.getContext()); MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected); I.setMetadata(LLVMContext::MD_prof, BranchWeights); @@ -252,7 +252,7 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) { auto *ProfDataName = dyn_cast(ProfileData->getOperand(0)); if (!ProfDataName || (ProfDataName->getString() != "branch_weights" && - ProfDataName->getString() != "VP")) + ProfDataName->getString() != "VP")) return; if (!hasCountTypeMD(I)) @@ -268,31 +268,31 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) { ProfileData->getNumOperands() > 0) { // Using APInt::div may be expensive, but most cases should fit 64 bits. APInt Val(128, - mdconst::dyn_extract( - ProfileData->getOperand(getBranchWeightOffset(ProfileData))) - ->getValue() - .getZExtValue()); + mdconst::dyn_extract( + ProfileData->getOperand(getBranchWeightOffset(ProfileData))) + ->getValue() + .getZExtValue()); Val *= APS; Vals.push_back(MDB.createConstant(ConstantInt::get( - Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX)))); + Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX)))); } else if (ProfDataName->getString() == "VP") for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) { // The first value is the key of the value profile, which will not change. Vals.push_back(ProfileData->getOperand(i)); uint64_t Count = - mdconst::dyn_extract(ProfileData->getOperand(i + 1)) - ->getValue() - .getZExtValue(); + mdconst::dyn_extract(ProfileData->getOperand(i + 1)) + ->getValue() + .getZExtValue(); // Don't scale the magic number. if (Count == NOMORE_ICP_MAGICNUM) { - Vals.push_back(ProfileData->getOperand(i + 1)); - continue; + Vals.push_back(ProfileData->getOperand(i + 1)); + continue; } // Using APInt::div may be expensive, but most cases should fit 64 bits. APInt Val(128, Count); Val *= APS; Vals.push_back(MDB.createConstant(ConstantInt::get( - Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue()))); + Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue()))); } I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals)); } diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 90797826e2be..b84f20cc0d41 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -837,34 +837,34 @@ BinaryCoverageReader::createCoverageReaderFromBuffer( StringRef Coverage, FuncRecordsStorage &&FuncRecords, std::unique_ptr ProfileNamesPtr, uint8_t BytesInAddress, support::endianness Endian, StringRef CompilationDir) { - if (ProfileNamesPtr == nullptr) - return make_error(coveragemap_error::malformed, - "Caller must provide ProfileNames"); + if (ProfileNamesPtr == nullptr) + return make_error(coveragemap_error::malformed, + "Caller must provide ProfileNames"); std::unique_ptr Reader(new BinaryCoverageReader( - std::move(ProfileNamesPtr), std::move(FuncRecords))); + std::move(ProfileNamesPtr), std::move(FuncRecords))); InstrProfSymtab &ProfileNames = *Reader->ProfileNames; StringRef FuncRecordsRef = Reader->FuncRecords->getBuffer(); if (BytesInAddress == 4 && Endian == support::endianness::little) { if (Error E = readCoverageMappingData( ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, - CompilationDir, Reader->Filenames)) + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 4 && Endian == support::endianness::big) { if (Error E = readCoverageMappingData( ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, - CompilationDir, Reader->Filenames)) + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 8 && Endian == support::endianness::little) { if (Error E = readCoverageMappingData( ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, - CompilationDir, Reader->Filenames)) + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 8 && Endian == support::endianness::big) { if (Error E = readCoverageMappingData( ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, - CompilationDir, Reader->Filenames)) + CompilationDir, Reader->Filenames)) return std::move(E); } else return make_error(coveragemap_error::malformed); diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 35b47020c0e0..6520b62b70b2 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -307,8 +307,8 @@ static StringRef getStrippedSourceFileName(const GlobalObject &GO) { // lookup functions from profiles built by older compilers. static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, - GlobalValue::LinkageTypes Linkage, - StringRef FileName) { + GlobalValue::LinkageTypes Linkage, + StringRef FileName) { return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName); } @@ -334,7 +334,7 @@ static std::optional lookupPGONameFromMetadata(MDNode *MD) { // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta // data, its original linkage must be non-internal. static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, - MDNode *PGONameMetadata) { + MDNode *PGONameMetadata) { if (!InLTO) { auto FileName = getStrippedSourceFileName(GO); return getIRPGONameForGlobalObject(GO, GO.getLinkage(), FileName); @@ -385,7 +385,7 @@ static std::optional lookupPGOFuncName(const Function &F) { } // Returns the IRPGO function name and does special handling when called -// in LTO optimization. See the comments of `getIRPGOObjectName` for details. +// in LTO optimization. See the comments of `getIRPGOObjectName` for details. std::string getIRPGOFuncName(const Function &F, bool InLTO) { return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F)); } @@ -426,7 +426,7 @@ std::string getPGOName(const GlobalVariable &V, bool InLTO) { return getIRPGOObjectName(V, InLTO, V.getMetadata(getPGONameMetadataName())); } -// See getIRPGOObjectName() for a discription of the format. +// See getIRPGOFuncName() for a discription of the format. std::pair getParsedIRPGOFuncName(StringRef IRPGOName) { auto [FileName, MangledName] = IRPGOName.split(';'); if (MangledName.empty()) @@ -529,15 +529,15 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) { } Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable, - StringRef VTablePGOName) { + StringRef VTablePGOName) { auto NameToGUIDMap = [&](StringRef Name) -> Error { if (Error E = addSymbolName(Name)) return E; - bool Inserted = true; - std::tie(std::ignore, Inserted) = - MD5VTableMap.try_emplace(GlobalValue::getGUID(Name), &VTable); - return Error::success(); + bool Inserted = true; + std::tie(std::ignore, Inserted) = + MD5VTableMap.try_emplace(GlobalValue::getGUID(Name), &VTable); + return Error::success(); }; if (Error E = NameToGUIDMap(VTablePGOName)) return E; @@ -554,7 +554,7 @@ Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable, /// method decodes the string and calls `NameCallback` for each substring. static Error readAndDecodeStrings(StringRef NameStrings, - std::function NameCallback) { + std::function NameCallback) { const uint8_t *P = NameStrings.bytes_begin(); const uint8_t *EndP = NameStrings.bytes_end(); while (P < EndP) { @@ -568,19 +568,19 @@ readAndDecodeStrings(StringRef NameStrings, StringRef NameStrings; if (IsCompressed) { if (!llvm::compression::zlib::isAvailable()) - return make_error(instrprof_error::zlib_unavailable); - + return make_error(instrprof_error::zlib_unavailable); + if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize), - UncompressedNameStrings, - UncompressedSize)) { + UncompressedNameStrings, + UncompressedSize)) { consumeError(std::move(E)); - return make_error(instrprof_error::uncompress_failed); + return make_error(instrprof_error::uncompress_failed); } P += CompressedSize; NameStrings = toStringRef(UncompressedNameStrings); } else { NameStrings = - StringRef(reinterpret_cast(P), UncompressedSize); + StringRef(reinterpret_cast(P), UncompressedSize); P += UncompressedSize; } // Now parse the name strings. @@ -588,7 +588,7 @@ readAndDecodeStrings(StringRef NameStrings, NameStrings.split(Names, getInstrProfNameSeparator()); for (StringRef &Name : Names) if (Error E = NameCallback(Name)) - return E; + return E; while (P < EndP && *P == 0) P++; @@ -637,9 +637,9 @@ StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) { Pos += UniqSuffix.length(); else Pos = 0; - + // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from - // the beginning. + // the beginning. Pos = PGOName.find('.', Pos); if (Pos != StringRef::npos && Pos != 0) return PGOName.substr(0, Pos); @@ -692,7 +692,7 @@ void InstrProfSymtab::dumpNames(raw_ostream &OS) const { } Error collectGlobalObjectNameStrings(ArrayRef NameStrs, - bool DoCompression, std::string &Result) { + bool DoCompression, std::string &Result) { assert(!NameStrs.empty() && "No name data to emit"); uint8_t Header[20], *P = Header; @@ -701,7 +701,7 @@ Error collectGlobalObjectNameStrings(ArrayRef NameStrs, assert( StringRef(UncompressedNameStrings).count(getInstrProfNameSeparator()) == - (NameStrs.size() - 1) && + (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"); unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); @@ -723,11 +723,11 @@ Error collectGlobalObjectNameStrings(ArrayRef NameStrs, SmallVector CompressedNameStrings; compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings), - CompressedNameStrings, - compression::zlib::BestSizeCompression); + CompressedNameStrings, + compression::zlib::BestSizeCompression); return WriteStringToResult(CompressedNameStrings.size(), - toStringRef(CompressedNameStrings)); + toStringRef(CompressedNameStrings)); } Error collectPGOFuncNameStrings(ArrayRef NameStrs, @@ -786,7 +786,7 @@ Error collectPGOFuncNameStrings(ArrayRef NameVars, } Error collectVTableStrings(ArrayRef VTables, - std::string &Result, bool DoCompression) { + std::string &Result, bool DoCompression) { std::vector VTableNameStrs; for (auto *VTable : VTables) VTableNameStrs.push_back(getPGOName(*VTable)); diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index fbd345156d85..f5a7c75e564f 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -239,7 +239,7 @@ template <> struct yaml::SequenceElementTraits { template Error InstrProfCorrelatorImpl::dumpYaml(int MaxWarnings, - raw_ostream &OS) { + raw_ostream &OS) { InstrProfCorrelator::CorrelationData Data; correlateProfileDataImpl(MaxWarnings, &Data); if (Data.Probes.empty()) @@ -363,10 +363,10 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( } if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { - WithColor::warning() - << "Incomplete DIE for function " << FunctionName - << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr - << " NumCounters=" << NumCounters << "\n"; + WithColor::warning() + << "Incomplete DIE for function " << FunctionName + << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr + << " NumCounters=" << NumCounters << "\n"; LLVM_DEBUG(Die.dump(dbgs())); } return; @@ -374,18 +374,18 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( uint64_t CountersStart = this->Ctx->CountersSectionStart; uint64_t CountersEnd = this->Ctx->CountersSectionEnd; if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { - if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { + if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { WithColor::warning() << format("CounterPtr out of range for function %s: Actual=0x%x " "Expected=[0x%x, 0x%x)\n", *FunctionName, *CounterPtr, CountersStart, CountersEnd); - LLVM_DEBUG(Die.dump(dbgs())); - } + LLVM_DEBUG(Die.dump(dbgs())); + } return; } if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) { - WithColor::warning() << format("Could not find address of function %s\n", - *FunctionName); + WithColor::warning() << format("Could not find address of function %s\n", + *FunctionName); LLVM_DEBUG(Die.dump(dbgs())); } // In debug info correlation mode, the CounterPtr is an absolute address of @@ -421,7 +421,7 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( if (!UnlimitedWarnings && NumSuppressedWarnings > 0) WithColor::warning() << format("Suppressed %d additional warnings\n", - NumSuppressedWarnings); + NumSuppressedWarnings); } template diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 413c41414eee..5eb4b6e1ff15 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1480,8 +1480,8 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, } Error IndexedInstrProfReader::getFunctionBitmap(StringRef FuncName, - uint64_t FuncHash, - BitVector &Bitmap) { + uint64_t FuncHash, + BitVector &Bitmap) { Expected Record = getInstrProfRecord(FuncName, FuncHash); if (Error E = Record.takeError()) return error(std::move(E)); diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index b853f0a8dc66..b2ec59f17433 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -182,7 +182,7 @@ public: InstrProfWriter::InstrProfWriter(bool Sparse, uint64_t TemporalProfTraceReservoirSize, uint64_t MaxTemporalProfTraceLength, - bool WritePrevVersion) + bool WritePrevVersion) : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength), TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize), InfoObj(new InstrProfRecordWriterTrait()), @@ -618,12 +618,12 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { const uint64_t CompressedStringLen = CompressedVTableNames.length(); - // Record the length of compressed string. - OS.write(CompressedStringLen); + // Record the length of compressed string. + OS.write(CompressedStringLen); - // Write the chars in compressed strings. - for (auto &c : CompressedVTableNames) - OS.writeByte(static_cast(c)); + // Write the chars in compressed strings. + for (auto &c : CompressedVTableNames) + OS.writeByte(static_cast(c)); // Pad up to a multiple of 8. // InstrProfReader could read bytes according to 'CompressedStringLen'. diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 76a9decc7be5..eef66f9a3664 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -632,7 +632,7 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee, MDBuilder MDB(NewInst.getContext()); NewInst.setMetadata( LLVMContext::MD_prof, - MDB.createBranchWeights({static_cast(Count)})); + MDB.createBranchWeights({static_cast(Count)})); } using namespace ore; @@ -960,7 +960,7 @@ computeVirtualCallSiteTypeInfoMap(Module &M, ModuleAnalysisManager &MAM, if (!TypeTestFunc || TypeTestFunc->use_empty()) TypeTestFunc = - M.getFunction(Intrinsic::getName(Intrinsic::public_type_test)); + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test)); if (!TypeTestFunc || TypeTestFunc->use_empty()) return; @@ -1043,7 +1043,7 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, auto &ORE = FAM.getResult(F); IndirectCallPromoter CallPromoter( - F, M, PSI, &Symtab, SamplePGO, VirtualCSInfo, + F, M, PSI, &Symtab, SamplePGO, VirtualCSInfo, VTableAddressPointOffsetVal, IgnoredBaseTypes, ORE); bool FuncChanged = CallPromoter.processFunction(PSI); if (ICPDUMPAFTER && FuncChanged) { diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index f821cf40353a..d24b463a0adf 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -2245,6 +2245,7 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M, }; auto *PSI = &MAM.getResult(M); + if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, *FS, LookupTLI, LookupBPI, LookupBFI, PSI, IsCS)) return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp index 0e1e5923722b..8333b08b7076 100644 --- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp +++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp @@ -282,7 +282,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) { /// %t0 = musttail call i32 %ptr() /// ret %t0 static CallBase &versionCallSiteWithCond(CallBase &CB, Value *Cond, - MDNode *BranchWeights) { + MDNode *BranchWeights) { IRBuilder<> Builder(&CB); CallBase *OrigInst = &CB; @@ -376,9 +376,9 @@ static CallBase &versionCallSiteWithCond(CallBase &CB, Value *Cond, } // Predicate and clone the given call site using condition `CB.callee == -// Callee`. See the comment `versionCallSiteWithcond` for the transformation. +// Callee`. See the comment `versionCallSiteWithCond` for the transformation. CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee, - MDNode *BranchWeights) { + MDNode *BranchWeights) { IRBuilder<> Builder(&CB); @@ -571,9 +571,9 @@ CallBase &llvm::promoteCallWithIfThenElse(CallBase &CB, Function *Callee, } CallBase &llvm::promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, - Function *Callee, - ArrayRef AddressPoints, - MDNode *BranchWeights) { + Function *Callee, + ArrayRef AddressPoints, + MDNode *BranchWeights) { assert(!AddressPoints.empty() && "Caller should guarantee"); IRBuilder<> Builder(&CB); SmallVector ICmps; @@ -632,7 +632,7 @@ bool llvm::tryPromoteCall(CallBase &CB) { APInt VTableGVOffset = VTableOffsetGVBase + VTableOffset; if (!(VTableGVOffset.getActiveBits() <= 64)) return false; // Out of range. - + Function *DirectCallee = nullptr; std::tie(DirectCallee, std::ignore) = getFunctionAtVTableOffset(GV, VTableGVOffset.getZExtValue(), *M); diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 7ed4221616b9..3d87d478a114 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -609,7 +609,7 @@ static void overlapInput(const std::string &BaseFilename, static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, const InstrProfCorrelator *Correlator, const StringRef ProfiledBinary, WriterContext *WC, - const bool KeepVTableSymbols) { + const bool KeepVTableSymbols) { std::unique_lock CtxGuard{WC->Lock}; // Copy the filename, because llvm::ThreadPool copied the input "const @@ -793,11 +793,11 @@ static void writeInstrProfile(StringRef OutputFilename, static void mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, SymbolRemapper *Remapper, StringRef OutputFilename, - ProfileFormat OutputFormat, uint64_t TraceReservoirSize, - uint64_t MaxTraceLength, bool OutputSparse, - unsigned NumThreads, FailureMode FailMode, + ProfileFormat OutputFormat, uint64_t TraceReservoirSize, + uint64_t MaxTraceLength, bool OutputSparse, + unsigned NumThreads, FailureMode FailMode, const StringRef ProfiledBinary, - const bool KeepVTableSymbols) { + const bool KeepVTableSymbols) { if (OutputFormat == PF_Compact_Binary) exitWithError("Compact Binary is deprecated"); if (OutputFormat != PF_Binary && OutputFormat != PF_Ext_Binary && @@ -806,8 +806,6 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, // TODO: Maybe we should support correlation with mixture of different // correlation modes(w/wo debug-info/object correlation). - if (!DebugInfoFilename.empty()) - exitWithError("Expected only one of -debug-info, -binary-file"); std::string CorrelateFilename; ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE; if (!DebugInfoFilename.empty()) { @@ -1270,7 +1268,7 @@ static void supplementInstrProfile( auto WC = std::make_unique(OutputSparse, ErrorLock, WriterErrorCodes); loadInput(Inputs[0], nullptr, nullptr, /*ProfiledBinary=*/"", WC.get(), - KeepVTableSymbols); + KeepVTableSymbols); if (WC->Errors.size() > 0) exitWithError(std::move(WC->Errors[0].first), InstrFilename); @@ -1729,13 +1727,13 @@ static int merge_main(int argc, const char *argv[]) { supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename, OutputFormat, OutputSparse, SupplMinSizeThreshold, ZeroCounterThreshold, InstrProfColdThreshold, - KeepVTableSymbols); + KeepVTableSymbols); return 0; } if (ProfileKind == instr) mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), - OutputFilename, OutputFormat, + OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, TemporalProfMaxTraceLength, OutputSparse, NumThreads, FailureMode, ProfiledBinary, KeepVTableSymbols); @@ -1771,7 +1769,7 @@ static void overlapInstrProfile(const std::string &BaseFilename, exit(0); } loadInput(WeightedInput, nullptr, nullptr, /*ProfiledBinary=*/"", &Context, - /*KeepVTableSymbols=*/false); + /*KeepVTableSymbols=*/false); overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, IsCS); Overlap.dump(OS); @@ -3357,7 +3355,7 @@ static int show_main(int argc, const char *argv[]) { "ic-targets", cl::init(false), cl::desc("Show indirect call site target values for shown functions")); cl::opt ShowVTables("show-vtables", cl::init(false), - cl::desc("Show vtable names for shown functions")); + cl::desc("Show vtable names for shown functions")); cl::opt ShowMemOPSizes( "memop-sizes", cl::init(false), cl::desc("Show the profiled sizes of the memory intrinsic calls " diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 20bd87b3e8a5..30c4dac0bd2f 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -540,7 +540,7 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) { InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; - Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); Writer.addRecord(std::move(Record1), Err); Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); @@ -561,14 +561,14 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) { std::unique_ptr VD = R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); - ASSERT_EQ(3U, VD[0].Count); - ASSERT_EQ(2U, VD[1].Count); - ASSERT_EQ(1U, VD[2].Count); - ASSERT_EQ(6U, TotalC); + ASSERT_EQ(3U, VD[0].Count); + ASSERT_EQ(2U, VD[1].Count); + ASSERT_EQ(1U, VD[2].Count); + ASSERT_EQ(6U, TotalC); - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); } TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { @@ -731,7 +731,7 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) { Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); // Set big endian output. - Writer.setValueProfDataEndianness((support::big)); + Writer.setValueProfDataEndianness(support::big); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); @@ -762,7 +762,7 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { NamedInstrProfRecord Record11(caller, 0x1234, {1, 2}); NamedInstrProfRecord Record12(caller, 0x1234, {1, 2}); - // 5 value sites + // 5 value sites. Record11.reserveSites(IPVK_IndirectCallTarget, 5); InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, {uint64_t(callee2), 2}, @@ -781,8 +781,8 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); InstrProfValueData VD4[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}}; + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}}; Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); // A different record for the same caller. @@ -800,8 +800,8 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr); InstrProfValueData VD42[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}}; + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}}; Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr); Writer.addRecord(std::move(Record11), Err); @@ -827,39 +827,39 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { std::unique_ptr VD = R->getValueForSite(IPVK_IndirectCallTarget, 0); - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(7U, VD[0].Count); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD[1].Count); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4")); - ASSERT_EQ(4U, VD[2].Count); - ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD[3].Count); - - std::unique_ptr VD_2( - R->getValueForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD_2[0].Count); - ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4")); - ASSERT_EQ(4U, VD_2[1].Count); - ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2")); - ASSERT_EQ(3U, VD_2[2].Count); - ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD_2[3].Count); - - std::unique_ptr VD_3( - R->getValueForSite(IPVK_IndirectCallTarget, 3)); - ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD_3[0].Count); - - std::unique_ptr VD_4( - R->getValueForSite(IPVK_IndirectCallTarget, 4)); - ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD_4[0].Count); - ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(4U, VD_4[1].Count); - ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1")); - ASSERT_EQ(2U, VD_4[2].Count); + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(7U, VD[0].Count); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD[1].Count); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4")); + ASSERT_EQ(4U, VD[2].Count); + ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD[3].Count); + + std::unique_ptr VD_2( + R->getValueForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD_2[0].Count); + ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4")); + ASSERT_EQ(4U, VD_2[1].Count); + ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2")); + ASSERT_EQ(3U, VD_2[2].Count); + ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD_2[3].Count); + + std::unique_ptr VD_3( + R->getValueForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD_3[0].Count); + + std::unique_ptr VD_4( + R->getValueForSite(IPVK_IndirectCallTarget, 4)); + ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD_4[0].Count); + ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(4U, VD_4[1].Count); + ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1")); + ASSERT_EQ(2U, VD_4[2].Count); } TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) { @@ -971,26 +971,26 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) { } static void addValueProfData(InstrProfRecord &Record) { - Record.reserveSites(IPVK_IndirectCallTarget, 5); - InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, - {uint64_t(callee2), 1000}, - {uint64_t(callee3), 500}, - {uint64_t(callee4), 300}, - {uint64_t(callee5), 100}}; - Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr); - InstrProfValueData VD1[] = {{uint64_t(callee5), 800}, - {uint64_t(callee3), 1000}, - {uint64_t(callee2), 2500}, - {uint64_t(callee1), 1300}}; - Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr); - InstrProfValueData VD2[] = {{uint64_t(callee6), 800}, - {uint64_t(callee3), 1000}, - {uint64_t(callee4), 5500}}; - Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); - InstrProfValueData VD3[] = {{uint64_t(callee2), 1800}, - {uint64_t(callee3), 2000}}; - Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); - Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); + Record.reserveSites(IPVK_IndirectCallTarget, 5); + InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, + {uint64_t(callee2), 1000}, + {uint64_t(callee3), 500}, + {uint64_t(callee4), 300}, + {uint64_t(callee5), 100}}; + Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr); + InstrProfValueData VD1[] = {{uint64_t(callee5), 800}, + {uint64_t(callee3), 1000}, + {uint64_t(callee2), 2500}, + {uint64_t(callee1), 1300}}; + Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr); + InstrProfValueData VD2[] = {{uint64_t(callee6), 800}, + {uint64_t(callee3), 1000}, + {uint64_t(callee4), 5500}}; + Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); + InstrProfValueData VD3[] = {{uint64_t(callee2), 1800}, + {uint64_t(callee3), 2000}}; + Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); + Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); } TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) { @@ -1218,8 +1218,8 @@ TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) { ASSERT_TRUE(F != nullptr); std::string PGOName = getPGOFuncName(*F); uint64_t Key = IndexedInstrProf::ComputeHash(PGOName); - ASSERT_EQ(StringRef(PGOName), - ProfSymtab.getFuncName(Key)); + ASSERT_EQ(StringRef(PGOName), + ProfSymtab.getFuncName(Key)); ASSERT_EQ(StringRef(Funcs[I]), ProfSymtab.getOrigFuncName(Key)); } } diff --git a/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp index 6f5e32435692..9c4e95c9a635 100644 --- a/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp +++ b/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp @@ -30,12 +30,12 @@ static std::unique_ptr parseIR(LLVMContext &C, const char *IR) { // Returns a constant representing the vtable's address point specified by the // offset. static Constant *getVTableAddressPointOffset(GlobalVariable *VTable, - uint32_t AddressPointOffset) { + uint32_t AddressPointOffset) { Module &M = *VTable->getParent(); LLVMContext &Context = M.getContext(); assert(AddressPointOffset < - M.getDataLayout().getTypeAllocSize(VTable->getValueType()) && - "Out-of-bound access"); + M.getDataLayout().getTypeAllocSize(VTable->getValueType()) && + "Out-of-bound access"); return ConstantExpr::getInBoundsGetElementPtr( Type::getInt8Ty(Context), VTable, @@ -390,14 +390,14 @@ declare %struct2 @_ZN4Impl3RunEv(%class.Impl* %this) TEST(CallPromotionUtilsTest, promoteCallWithVTableCmp) { LLVMContext C; std::unique_ptr M = parseIR(C, - R"IR( + R"IR( target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" - + @_ZTV5Base1 = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev] }, !type !0 @_ZTV8Derived1 = constant { [4 x ptr], [3 x ptr] } { [4 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev], [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base25func2Ev] }, !type !0, !type !1, !type !2 -@_ZTV8Derived2 = constant { [3 x ptr], [3 x ptr], [4 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base35func3Ev], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN5Base25func2Ev], [4 x ptr] [ptr inttoptr (i64 -16 to ptr), ptr null ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev] }, !type !3, !type !4, !type !5, !type !6 - +@_ZTV8Derived2 = constant { [3 x ptr], [3 x ptr], [4 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base35func3Ev], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN5Base25func2Ev], [4 x ptr] [ptr inttoptr (i64 -16 to ptr), ptr null, ptr @_ZN5Base15func0Ev, ptr @_ZN5Base15func1Ev] }, !type !3, !type !4, !type !5, !type !6 + define i32 @testfunc(ptr %d) { entry: %vtable = load ptr, ptr %d, !prof !7 @@ -406,16 +406,16 @@ entry: %call = tail call i32 %0(ptr %d), !prof !8 ret i32 %call } - + define i32 @_ZN5Base15func1Ev(ptr %this) { entry: ret i32 2 } - + declare i32 @_ZN5Base25func2Ev(ptr) declare i32 @_ZN5Base15func0Ev(ptr) declare void @_ZN5Base35func3Ev(ptr) - + !0 = !{i64 16, !"_ZTS5Base1"} !1 = !{i64 48, !"_ZTS5Base2"} !2 = !{i64 16, !"_ZTS8Derived1"} @@ -426,18 +426,18 @@ declare void @_ZN5Base35func3Ev(ptr) !7 = !{!"VP", i32 2, i64 1600, i64 -9064381665493407289, i64 800, i64 5035968517245772950, i64 500, i64 3215870116411581797, i64 300} !8 = !{!"VP", i32 0, i64 1600, i64 6804820478065511155, i64 1600})IR"); -Function *F = M->getFunction("testfunc"); -CallInst *CI = dyn_cast(&*std::next(F->front().rbegin())); -ASSERT_TRUE(CI && CI->isIndirectCall()); + Function *F = M->getFunction("testfunc"); + CallInst *CI = dyn_cast(&*std::next(F->front().rbegin())); + ASSERT_TRUE(CI && CI->isIndirectCall()); -// Create the constant and the branch weights -SmallVector VTableAddressPoints; + // Create the constant and the branch weights + SmallVector VTableAddressPoints; -for (auto &[VTableName, AddressPointOffset] : {std::pair{"_ZTV5Base1", 16}, - {"_ZTV8Derived1", 16}, - {"_ZTV8Derived2", 64}}) - VTableAddressPoints.push_back(getVTableAddressPointOffset( - M->getGlobalVariable(VTableName), AddressPointOffset)); + for (auto &[VTableName, AddressPointOffset] : {std::pair{"_ZTV5Base1", 16}, + {"_ZTV8Derived1", 16}, + {"_ZTV8Derived2", 64}}) + VTableAddressPoints.push_back(getVTableAddressPointOffset( + M->getGlobalVariable(VTableName), AddressPointOffset)); MDBuilder MDB(C); MDNode *BranchWeights = MDB.createBranchWeights(1600, 0); -- Gitee