diff --git a/src/mapleall/maple_ipa/include/ipa_clone.h b/src/mapleall/maple_ipa/include/ipa_clone.h index eaa560d69bcb8ddc5206cc4ff5e3ed858abacec8..db9ce2b455f1b906f0170aea599ec0f563ff7619 100644 --- a/src/mapleall/maple_ipa/include/ipa_clone.h +++ b/src/mapleall/maple_ipa/include/ipa_clone.h @@ -35,6 +35,7 @@ class IpaClone : public AnalysisResult { static void IpaCloneLabels(MIRFunction &newFunc, const MIRFunction &oldFunc); static void IpaClonePregTable(MIRFunction &newFunc, const MIRFunction &oldFunc); MIRFunction *IpaCloneFunction(MIRFunction &originalFunction, const std::string &newBaseFuncName) const; + MIRFunction *IpaCloneFunctionWithFreq(MIRFunction &originalFunction, const std::string &newBaseFuncName, int64_t) const; void DoIpaClone(); void InitParams(); void CopyFuncInfo(MIRFunction &originalFunction, MIRFunction &newFunc) const; diff --git a/src/mapleall/maple_ipa/src/ipa_clone.cpp b/src/mapleall/maple_ipa/src/ipa_clone.cpp index 5b62ab47c6210259c687538a2120620aaaf94ec4..f83170b1cc45d6707e4b227395bf0ba4f8736cbf 100644 --- a/src/mapleall/maple_ipa/src/ipa_clone.cpp +++ b/src/mapleall/maple_ipa/src/ipa_clone.cpp @@ -123,6 +123,47 @@ MIRFunction *IpaClone::IpaCloneFunction(MIRFunction &originalFunction, const std return newFunc; } +MIRFunction *IpaClone::IpaCloneFunctionWithFreq(MIRFunction &originalFunction, + const std::string &fullName, int64_t callsitefreq) const { + MapleAllocator cgAlloc(originalFunction.GetDataMemPool()); + ArgVector argument(cgAlloc.Adapter()); + IpaCloneArgument(originalFunction, argument); + MIRType *retType = originalFunction.GetReturnType(); + MIRFunction *newFunc = + mirBuilder.CreateFunction(fullName, *retType, argument, false, originalFunction.GetBody() != nullptr); + CHECK_FATAL(newFunc != nullptr, "create cloned function failed"); + mirBuilder.GetMirModule().AddFunction(newFunc); + newFunc->SetFlag(originalFunction.GetFlag()); + newFunc->SetSrcPosition(originalFunction.GetSrcPosition()); + newFunc->SetFuncAttrs(originalFunction.GetFuncAttrs()); + newFunc->SetBaseClassFuncNames(GlobalTables::GetStrTable().GetOrCreateStrIdxFromName(fullName)); + GcovFuncInfo *origProfData = originalFunction.GetFuncProfData(); + auto *moduleMp = mirBuilder.GetMirModule().GetMemPool(); + GcovFuncInfo *newProfData = moduleMp->New(&mirBuilder.GetMirModule().GetMPAllocator(), + newFunc->GetPuidx(), 0, 0); // skip checksum information + newFunc->SetFuncProfData(newProfData); + newProfData->SetFuncFrequency(callsitefreq); + newProfData->SetFuncRealFrequency(callsitefreq); + // TODO:: original function need to update frequency by real entry value + // update real left frequency + origProfData->SetFuncRealFrequency(origProfData->GetFuncRealFrequency() - callsitefreq); + if (originalFunction.GetBody() != nullptr) { + CopyFuncInfo(originalFunction, *newFunc); + BlockNode *newbody = originalFunction.GetBody()->CloneTreeWithFreqs(newFunc->GetCodeMempoolAllocator(), + newProfData->GetStmtFreqs(), origProfData->GetStmtFreqs(), + callsitefreq /*numer*/, origProfData->GetFuncFrequency() /*denom*/, + (kKeepOrigFreq | kUpdateFreqbyScale)); + newFunc->SetBody(newbody); + IpaCloneSymbols(*newFunc, originalFunction); + IpaCloneLabels(*newFunc, originalFunction); + IpaClonePregTable(*newFunc, originalFunction); + } + newFunc->SetFuncDesc(originalFunction.GetFuncDesc()); + // All the cloned functions cannot be accessed from other transform unit. + newFunc->SetAttr(FUNCATTR_static); + return newFunc; +} + void IpaClone::IpaCloneArgument(MIRFunction &originalFunction, ArgVector &argument) const { for (size_t i = 0; i < originalFunction.GetFormalCount(); ++i) { auto &formalName = originalFunction.GetFormalName(i); @@ -271,7 +312,26 @@ void IpaClone::DecideCloneFunction(std::vector &result, uint32 paramInd } std::string newFuncName = curFunc->GetName() + ".clone." + std::to_string(index++); MInline::ConvertPStaticToFStatic(*curFunc); - MIRFunction *newFunc = IpaCloneFunction(*curFunc, newFuncName); + MIRFunction *newFunc = nullptr; + if (Options::profileUse && curFunc->GetFuncProfData()) { + int clonedSiteFreqs = 0; + for (auto &value: calleeValue) { + for (auto &callSite : calleeInfo[keyPair][value]) { + MIRFunction *callerFunc = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callSite.GetPuidx()); + uint32 stmtId = callSite.GetStmtId(); + CallNode *oldCallNode = static_cast(callerFunc->GetStmtNodeFromMeId(stmtId)); + if (oldCallNode == nullptr) { + continue; + } + int64_t callsiteFreq = callerFunc->GetFuncProfData()->GetStmtFreq(stmtId); + CHECK_FATAL(callsiteFreq >= 0, "sanity check"); + clonedSiteFreqs += callsiteFreq; + } + } + newFunc = IpaCloneFunctionWithFreq(*curFunc, newFuncName, clonedSiteFreqs); + } else { + newFunc = IpaCloneFunction(*curFunc, newFuncName); + } ReplaceIfCondtion(newFunc, result, evalValue); for (auto &value: calleeValue) { bool optCallerParam = false; @@ -402,7 +462,25 @@ void IpaClone::CloneNoImportantExpressFunction(MIRFunction *func, uint32 paramIn auto &calleeInfo = mirModule->GetCalleeParamAboutInt(); std::string newFuncName = func->GetName() + ".constprop." + std::to_string(paramIndex); MInline::ConvertPStaticToFStatic(*func); - MIRFunction *newFunc = IpaCloneFunction(*func, newFuncName); + MIRFunction *newFunc = nullptr; + if (Options::profileUse && func->GetFuncProfData()) { + int clonedSiteFreqs = 0; + int64_t value = calleeInfo[keyPair].begin()->first; + for (auto &callSite : calleeInfo[keyPair][value]) { + MIRFunction *callerFunc = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callSite.GetPuidx()); + uint32 stmtId = callSite.GetStmtId(); + CallNode *oldCallNode = static_cast(callerFunc->GetStmtNodeFromMeId(stmtId)); + if (oldCallNode == nullptr) { + continue; + } + int64_t callsiteFreq = callerFunc->GetFuncProfData()->GetStmtFreq(stmtId); + CHECK_FATAL(callsiteFreq >= 0, "sanity check"); + clonedSiteFreqs += callsiteFreq; + } + newFunc = IpaCloneFunctionWithFreq(*func, newFuncName, clonedSiteFreqs); + } else { + newFunc = IpaCloneFunction(*func, newFuncName); + } int64_t value = calleeInfo[keyPair].begin()->first; RemoveUnneedParameter(newFunc, paramIndex, value); for (auto &callSite : calleeInfo[keyPair][value]) { diff --git a/src/mapleall/maple_ir/include/mir_nodes.h b/src/mapleall/maple_ir/include/mir_nodes.h index f782bdc03b6cb34e7d1513a0f3269bf81a5e4c05..0ae8153cc3bdae5537f7896a0484e171df56553e 100755 --- a/src/mapleall/maple_ir/include/mir_nodes.h +++ b/src/mapleall/maple_ir/include/mir_nodes.h @@ -2472,10 +2472,10 @@ class IfStmtNode : public UnaryStmtNode { node->SetOpnd(Opnd()->CloneTree(allocator), 0); if (fromFreqs.count(GetStmtID()) > 0) { int64_t oldFreq = fromFreqs[GetStmtID()]; - int64_t newFreq = denom > 0 ? (oldFreq * numer / denom) : oldFreq; - toFreqs[node->GetStmtID()] = newFreq > 0 ? newFreq : 1; + int64_t newFreq = numer == 0 ? 0 : (denom > 0 ? (oldFreq * numer / denom) : oldFreq); + toFreqs[node->GetStmtID()] = (newFreq > 0 || numer == 0) ? newFreq : 1; if (updateOp & kUpdateOrigFreq) { - int64_t left = (oldFreq - newFreq) > 0 ? (oldFreq - newFreq) : 1; + int64_t left = ((oldFreq - newFreq) > 0 || oldFreq == 0) ? (oldFreq - newFreq) : 1; fromFreqs[GetStmtID()] = left; } } @@ -2557,8 +2557,8 @@ class WhileStmtNode : public UnaryStmtNode { node->SetStmtID(stmtIDNext++); if (fromFreqs.count(GetStmtID()) > 0) { int64_t oldFreq = fromFreqs[GetStmtID()]; - int64_t newFreq = denom > 0 ? (oldFreq * numer / denom) : oldFreq; - toFreqs[node->GetStmtID()] = newFreq > 0 ? newFreq : 1; + int64_t newFreq = numer == 0 ? 0 : (denom > 0 ? (oldFreq * numer / denom) : oldFreq); + toFreqs[node->GetStmtID()] = (newFreq > 0 || numer == 0) ? newFreq : 1; if (updateOp & kUpdateOrigFreq) { int64_t left = (oldFreq - newFreq) > 0 ? (oldFreq - newFreq) : 1; fromFreqs[GetStmtID()] = left; @@ -2629,8 +2629,8 @@ class DoloopNode : public StmtNode { if (fromFreqs.count(GetStmtID()) > 0) { int64_t oldFreq = fromFreqs[GetStmtID()]; int64_t newFreq = oldFreq; - if (updateOp & kUpdateInlinedFreq) { // used in inline - newFreq = denom > 0 ? (oldFreq * numer / denom) : oldFreq; + if (updateOp & kUpdateFreqbyScale) { // used in inline/clone + newFreq = numer == 0 ? 0 : (denom > 0 ? (oldFreq * numer / denom) : oldFreq); } else if (updateOp & kUpdateUnrolledFreq) { // used in unrolled part int64_t bodyFreq = fromFreqs[GetDoBody()->GetStmtID()]; newFreq = denom > 0 ? (bodyFreq * numer / denom + (oldFreq - bodyFreq)) : oldFreq; diff --git a/src/mapleall/maple_ir/src/mir_nodes.cpp b/src/mapleall/maple_ir/src/mir_nodes.cpp index 573346e739521dbc61e3dbac4009a4c58958c4ff..0dc8ac87a3f5b141b5a6d7cb09a437a20b2ad1ba 100755 --- a/src/mapleall/maple_ir/src/mir_nodes.cpp +++ b/src/mapleall/maple_ir/src/mir_nodes.cpp @@ -255,9 +255,9 @@ BlockNode *BlockNode::CloneTreeWithFreqs(MapleAllocator &allocator, if (updateOp & kUpdateUnrollRemainderFreq) { newFreq = denom > 0 ? (oldFreq * numer % denom) : oldFreq; } else { - newFreq = denom > 0 ? (oldFreq * numer / denom) : oldFreq; + newFreq = numer == 0 ? 0 : (denom > 0 ? (oldFreq * numer / denom) : oldFreq); } - toFreqs[nnode->GetStmtID()] = (newFreq > 0 || (oldFreq == 0)) ? newFreq : 1; + toFreqs[nnode->GetStmtID()] = (newFreq > 0 || (numer == 0)) ? newFreq : 1; if (updateOp & kUpdateOrigFreq) { // upateOp & 1 : update from int64_t left = ((oldFreq - newFreq) > 0 || (oldFreq == 0)) ? (oldFreq - newFreq) : 1; fromFreqs[GetStmtID()] = left; @@ -285,9 +285,9 @@ BlockNode *BlockNode::CloneTreeWithFreqs(MapleAllocator &allocator, if (updateOp & kUpdateUnrollRemainderFreq) { newFreq = denom > 0 ? (oldFreq * numer % denom) : oldFreq; } else { - newFreq = denom > 0 ? (oldFreq * numer / denom) : oldFreq; + newFreq = numer == 0 ? 0 : (denom > 0 ? (oldFreq * numer / denom) : oldFreq); } - toFreqs[newStmt->GetStmtID()] = (newFreq > 0 || oldFreq == 0) ? newFreq : 1; + toFreqs[newStmt->GetStmtID()] = (newFreq > 0 || oldFreq == 0 || numer == 0) ? newFreq : 1; if (updateOp & kUpdateOrigFreq) { int64_t left = ((oldFreq - newFreq) > 0 || oldFreq == 0) ? (oldFreq - newFreq) : 1; fromFreqs[stmt.GetStmtID()] = left; diff --git a/src/mapleall/maple_me/include/bb.h b/src/mapleall/maple_me/include/bb.h index 08ed432e35fa65ab2b8647cced99f182e141c160..b0ced0184ec90cbc7a592d3495391ce6d89e94cf 100644 --- a/src/mapleall/maple_me/include/bb.h +++ b/src/mapleall/maple_me/include/bb.h @@ -402,6 +402,12 @@ class BB { MapleVector &GetSuccFreq() { return succFreq; } + void SetSuccFreq(int idx, uint64 freq) { + ASSERT(idx >= 0 && idx <= succFreq.size(), "sanity check"); + succFreq[idx] = freq; + } + // update edge frequency + void UpdateEdgeFreqs(); const MapleVector &GetSucc() const { return succ; diff --git a/src/mapleall/maple_me/include/me_cfg.h b/src/mapleall/maple_me/include/me_cfg.h index 1f16871834f913bf55b75e9ca05a9a0e715e8d7c..c84e80f33ed0cb9d981b83864d56bae36f2e7c7b 100644 --- a/src/mapleall/maple_me/include/me_cfg.h +++ b/src/mapleall/maple_me/include/me_cfg.h @@ -295,7 +295,9 @@ class MeCFG : public AnalysisResult { void SwapBBId(BB &bb1, BB &bb2); void ConstructBBFreqFromStmtFreq(); void ConstructStmtFreq(); - + void ConstructEdgeFreqFromBBFreq(); + void UpdateEdgeFreqWithNewBBFreq(); + void VerifyBBFreq(); private: void AddCatchHandlerForTryBB(BB &bb, MapleVector &exitBlocks); std::string ConstructFileNameToDump(const std::string &prefix) const; @@ -334,5 +336,7 @@ MAPLE_FUNC_PHASE_DECLARE_BEGIN(MEMeCfg, MeFunction) } MeCFG *theCFG = nullptr; MAPLE_MODULE_PHASE_DECLARE_END +MAPLE_FUNC_PHASE_DECLARE_BEGIN(MECfgVerifyFrequency, MeFunction) +MAPLE_FUNC_PHASE_DECLARE_END } // namespace maple #endif // MAPLE_ME_INCLUDE_ME_CFG_H diff --git a/src/mapleall/maple_me/src/bb.cpp b/src/mapleall/maple_me/src/bb.cpp index 1be0bb1bbc5c75437c5f3782ee1b99814389d980..f4f5585dfbe6b6cbdcc91f60ea67170c508062a6 100644 --- a/src/mapleall/maple_me/src/bb.cpp +++ b/src/mapleall/maple_me/src/bb.cpp @@ -187,7 +187,7 @@ void BB::RemoveBBFromPred(const BB &bb, bool updatePhi) { void BB::RemoveBBFromSucc(const BB &bb) { int ret = bb.RemoveBBFromVector(succ); - if (ret != -1 && frequency != 0 && !succFreq.empty()) { + if (ret != -1 && !succFreq.empty() && (Options::profileUse || frequency != 0)) { succFreq.erase(succFreq.cbegin() + ret); } } @@ -476,6 +476,25 @@ void BB::DumpMePhiList(const IRMap *irMap) { } } +// bb frequency is changed in tranform phase +// update its succ frequency by scaled value +void BB::UpdateEdgeFreqs() { + int len = GetSucc().size(); + ASSERT(len == GetSuccFreq().size(), "sanity check"); + int64_t succFreqs = 0; + for (int i = 0; i < len; i++) { + succFreqs += GetSuccFreq()[i]; + } + // early return if frequency is consistent + if (len == 0 || succFreqs == GetFrequency()) return; + for (int i = 0; i < len; i++) { + int64_t sfreq = GetSuccFreq()[i]; + int64_t scalefreq = (succFreqs == 0 ? + (frequency / len) : (sfreq * frequency / succFreqs)); + SetSuccFreq(i, scalefreq); + } +} + void SCCOfBBs::Dump() { std::cout << "SCC " << id << " contains\n"; for (BB *bb : bbs) { diff --git a/src/mapleall/maple_me/src/dse.cpp b/src/mapleall/maple_me/src/dse.cpp index 28351d69e5e0e6d1f5d5e99f4a47c0f93cfa5c0b..e072dfa5104b321013cac5e1c4732eb14247275b 100644 --- a/src/mapleall/maple_me/src/dse.cpp +++ b/src/mapleall/maple_me/src/dse.cpp @@ -212,6 +212,13 @@ void DSE::OnRemoveBranchStmt(BB &bb, const StmtNode &stmt) { succ->RemovePred(bb); } bb.SetKind(kBBFallthru); + if (Options::profileUse && !bb.GetSuccFreq().empty()) { + int64_t succ0Freq = bb.GetSuccFreq()[0]; + bb.GetSuccFreq().resize(1); + bb.SetSuccFreq(0, bb.GetFrequency()); + ASSERT(bb.GetFrequency() >= succ0Freq, "sanity check"); + bb.GetSucc(0)->SetFrequency(bb.GetSucc(0)->GetFrequency() + (bb.GetFrequency() - succ0Freq)); + } cfgUpdated = true; return; } @@ -228,6 +235,14 @@ void DSE::OnRemoveBranchStmt(BB &bb, const StmtNode &stmt) { break; } } + // merge all frequency of condition goto bb to fallthru succ + if (Options::profileUse && !bb.GetSuccFreq().empty()) { + int64_t succ0Freq = bb.GetSuccFreq()[0]; + bb.GetSuccFreq().resize(1); + bb.SetSuccFreq(0, bb.GetFrequency()); + ASSERT(bb.GetFrequency() >= succ0Freq, "sanity check"); + bb.GetSucc(0)->SetFrequency(bb.GetSucc(0)->GetFrequency() + (bb.GetFrequency() - succ0Freq)); + } } } diff --git a/src/mapleall/maple_me/src/me_cfg.cpp b/src/mapleall/maple_me/src/me_cfg.cpp index ac3cf1b8dbc4dee82d68285f3462b72aae7835c5..c1d452d10a3054e0f9a4cf252631bb5b1651d55d 100644 --- a/src/mapleall/maple_me/src/me_cfg.cpp +++ b/src/mapleall/maple_me/src/me_cfg.cpp @@ -1522,6 +1522,13 @@ void MeCFG::CreateBasicBlocks() { } labelBBIdMap[labelIdx] = curBB; curBB->SetBBLabel(labelIdx); + // label node is not real node in bb, get frequency information to bb + if (Options::profileUse && func.GetMirFunc()->GetFuncProfData()) { + auto freq = func.GetMirFunc()->GetFuncProfData()->GetStmtFreq(stmt->GetStmtID()); + if (freq >= 0) { + curBB->SetFrequency(freq); + } + } break; } case OP_jscatch: { @@ -1806,11 +1813,45 @@ void MeCFG::SwapBBId(BB &bb1, BB &bb2) { bb2.SetBBId(tmp); } +// set bb succ frequency from bb freq +// no critical edge is expected +void MeCFG::ConstructEdgeFreqFromBBFreq() { + // set succfreqs + auto eIt = valid_end(); + for (auto bIt = valid_begin(); bIt != eIt; ++bIt) { + auto *bb = *bIt; + if (!bb) continue; + if (bb->GetSucc().size() == 1) { + bb->PushBackSuccFreq(bb->GetFrequency()); + } else if (bb->GetSucc().size() == 2) { + auto *fallthru = bb->GetSucc(0); + auto *targetBB = bb->GetSucc(1); + if (fallthru->GetPred().size() == 1) { + auto succ0Freq = fallthru->GetFrequency(); + bb->PushBackSuccFreq(succ0Freq); + ASSERT(bb->GetFrequency() >= succ0Freq, "sanity check"); + bb->PushBackSuccFreq(bb->GetFrequency() - succ0Freq); + } else if (targetBB->GetPred().size() == 1) { + auto succ1Freq = targetBB->GetFrequency(); + ASSERT(bb->GetFrequency() >= succ1Freq, "sanity check"); + bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); + bb->PushBackSuccFreq(succ1Freq); + } else { + CHECK_FATAL(false, "ConstructEdgeFreqFromBBFreq::NYI critical edge"); + } + } else if (bb->GetSucc().size() > 2) { + // switch case, no critical edge is supposted + for (int i = 0; i < bb->GetSucc().size(); i++) { + bb->PushBackSuccFreq(bb->GetSucc(i)->GetFrequency()); + } + } + } + return; +} + // set bb frequency from stmt record void MeCFG::ConstructBBFreqFromStmtFreq() { - GcovProfileData* gcovData = func.GetMIRModule().GetGcovProfile(); - if (!gcovData) return; - GcovFuncInfo* funcData = gcovData->GetFuncProfile(func.GetUniqueID()); + GcovFuncInfo* funcData = func.GetMirFunc()->GetFuncProfData(); if (!funcData) return; if (funcData->stmtFreqs.size() == 0) return; auto eIt = valid_end(); @@ -1840,41 +1881,7 @@ void MeCFG::ConstructBBFreqFromStmtFreq() { } bb->SetFrequency(freq); // set succfreqs - for (auto bIt = valid_begin(); bIt != eIt; ++bIt) { - auto *bb = *bIt; - if (!bb) continue; - int64_t succSum = 0; // verify only - if (bb->GetSucc().size() == 1) { - bb->PushBackSuccFreq(bb->GetFrequency()); - } else if (bb->GetSucc().size() == 2) { - auto *fallthru = bb->GetSucc(0); - auto *targetBB = bb->GetSucc(1); - if (targetBB->GetFrequency() >= bb->GetFrequency()) { - // fixup frequency of target bb - uint32_t fallthruFreq = fallthru->GetFrequency(); - for (int i = 0; i < fallthru->GetPred().size(); i++) { - auto *pred = fallthru->GetPred(i); - if (pred->GetSucc().size() == 1 && (pred != bb)) { - fallthruFreq -= pred->GetFrequency(); - } - } - bb->PushBackSuccFreq(fallthruFreq); - bb->PushBackSuccFreq(bb->GetFrequency() - fallthruFreq); - } else { - bb->PushBackSuccFreq(fallthru->GetFrequency()); - bb->PushBackSuccFreq(targetBB->GetFrequency()); - } - } else if (bb->GetSucc().size() > 2) { - // switch case - for (int i = 0; i < bb->GetSucc().size(); i++) { - bb->PushBackSuccFreq(bb->GetSucc(i)->GetFrequency()); - succSum += bb->GetSucc(i)->GetFrequency(); - } - if (succSum != bb->GetFrequency()) { - LogInfo::MapleLogger() << "ERROR:: bb " << bb->GetBBId() << "frequency inconsistent with sum of succs" << "\n"; - } - } - } + ConstructEdgeFreqFromBBFreq(); // clear stmtFreqs since cfg frequency is create funcData->stmtFreqs.clear(); } @@ -1889,6 +1896,7 @@ void MeCFG::ConstructStmtFreq() { auto *bb = *bIt; if (bIt == common_entry()) { funcData->entry_freq = bb->GetFrequency(); + funcData->real_entryfreq = funcData->entry_freq; } for (auto &stmt : bb->GetStmtNodes()) { Opcode op = stmt.GetOpCode(); @@ -1902,6 +1910,43 @@ void MeCFG::ConstructStmtFreq() { } } +// bb frequency may be changed in transform phase, +// update edgeFreq with new BB frequency by scale +void MeCFG::UpdateEdgeFreqWithNewBBFreq() { + for (size_t idx = 0; idx < bbVec.size(); ++idx) { + BB *currBB = bbVec[idx]; + if (currBB == nullptr || currBB->GetSucc().empty()) { + continue; + } + // make bb frequency and succs frequency consistent + currBB->UpdateEdgeFreqs(); + } +} + +void MeCFG::VerifyBBFreq() { + for (size_t i = 2; i < bbVec.size(); ++i) { // skip common entry and common exit + auto *bb = bbVec[i]; + if (bb == nullptr || bb->GetAttributes(kBBAttrIsEntry) || bb->GetAttributes(kBBAttrIsExit)) { + continue; + } + // wontexit bb may has wrong succ, skip it + if (bb->GetSuccFreq().size() != bb->GetSucc().size() && !bb->GetAttributes(kBBAttrWontExit)) { + CHECK_FATAL(false, "VerifyBBFreq: succFreq size != succ size"); + } + // bb freq == sum(out edge freq) + uint64 succSumFreq = 0; + for (auto succFreq : bb->GetSuccFreq()) { + succSumFreq += succFreq; + } + if (succSumFreq != bb->GetFrequency()) { + LogInfo::MapleLogger() << "[VerifyFreq failure] BB" << bb->GetBBId() << " freq: " << + bb->GetFrequency() << ", all succ edge freq sum: " << succSumFreq << std::endl; + LogInfo::MapleLogger() << func.GetName() << std::endl; + CHECK_FATAL(false, "VerifyFreq failure: bb freq != succ freq sum"); + } + } +} + bool MEMeCfg::PhaseRun(MeFunction &f) { if (!f.IsPme() && f.GetPreMeFunc() != nullptr) { GetAnalysisInfoHook()->ForceEraseAllAnalysisPhase(); @@ -1934,10 +1979,31 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { theCFG->UnifyRetBBs(); } // construct bb freq from stmt freq - if (Options::profileUse) { + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { theCFG->ConstructBBFreqFromStmtFreq(); } theCFG->Verify(); return false; } + +bool MECfgVerifyFrequency::PhaseRun(MeFunction &f) { + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + f.GetCfg()->VerifyBBFreq(); + } + // hack code here: no use profile data after verifycation pass since + // following tranform phases related of cfg change are not touched + // TODO::following code will be deleted + f.GetMirFunc()->SetFuncProfData(nullptr); + auto &bbVec = f.GetCfg()->GetAllBBs(); + for (size_t i = 0; i < bbVec.size(); ++i) { // skip common entry and common exit + auto *bb = bbVec[i]; + if (bb == nullptr) { + continue; + } + bb->SetFrequency(0); + bb->GetSuccFreq().clear(); + } + + return false; +} } // namespace maple diff --git a/src/mapleall/maple_me/src/me_critical_edge.cpp b/src/mapleall/maple_me/src/me_critical_edge.cpp index 0d2f2046fa95f8861cddea63f236067aec5c3630..71bd031762cb3d8d8197c8bea5f711cb03490b6b 100644 --- a/src/mapleall/maple_me/src/me_critical_edge.cpp +++ b/src/mapleall/maple_me/src/me_critical_edge.cpp @@ -158,6 +158,16 @@ void MeSplitCEdge::BreakCriticalEdge(MeFunction &func, BB &pred, BB &succ) const newBB->SetKind(kBBFallthru); // default kind newBB->SetAttributes(kBBAttrArtificial); + // update newBB frequency : copy predBB succFreq as newBB frequency + if (Options::profileUse && func.GetMirFunc()->GetFuncProfData() && + (!(func.IsPme() || func.IsLfo()))) { + int idx = pred.GetSuccIndex(*newBB); + ASSERT(idx >= 0 && idx < pred.GetSucc().size(), "sanity check"); + uint64_t freq = pred.GetEdgeFreq(idx); + newBB->SetFrequency(freq); + newBB->PushBackSuccFreq(freq); + } + if (needUpdateTryAttr && isInsertAfterPred && pred.GetAttributes(kBBAttrIsTry)) { UpdateNewBBInTry(func, *newBB, pred); } @@ -292,13 +302,15 @@ void MESplitCEdge::GetAnalysisDependence(maple::AnalysisDep &aDep) const { bool MESplitCEdge::PhaseRun(maple::MeFunction &f) { bool enableDebug = DEBUGFUNC_NEWPM(f); MeSplitCEdge mscedge = MeSplitCEdge(enableDebug); - bool split = mscedge.SplitCriticalEdgeForMeFunc(f); - if (split && Options::profileUse && (f.IsPme() || f.IsLfo()) && f.GetPreMeFunc()) { - // new inserted BB will break while/if label information and IR layout - f.GetPreMeFunc()->label2IfInfo.clear(); - f.GetPreMeFunc()->label2WhileInfo.clear(); - f.GetPreMeFunc()->pmeCreatedIfLabelSet.clear(); - f.GetPreMeFunc()->pmeCreatedWhileLabelSet.clear(); + mscedge.SplitCriticalEdgeForMeFunc(f); + if (Options::profileUse) { + if ((f.IsPme() || f.IsLfo()) && f.GetPreMeFunc()) { + // new inserted BB will break while/if label information and IR layout + f.GetPreMeFunc()->label2IfInfo.clear(); + f.GetPreMeFunc()->label2WhileInfo.clear(); + f.GetPreMeFunc()->pmeCreatedIfLabelSet.clear(); + f.GetPreMeFunc()->pmeCreatedWhileLabelSet.clear(); + } } return false; } diff --git a/src/mapleall/maple_me/src/me_dse.cpp b/src/mapleall/maple_me/src/me_dse.cpp index 0842ad6fd60e711e2f9cff0e769bec5ee625c6f0..e5c3823f923b30f66426d14076e38a1c48e1fcaf 100644 --- a/src/mapleall/maple_me/src/me_dse.cpp +++ b/src/mapleall/maple_me/src/me_dse.cpp @@ -70,6 +70,9 @@ bool MEDse::PhaseRun(maple::MeFunction &f) { f.Verify(); // cfg change , invalid results in MeFuncResultMgr if (dse.UpdatedCfg()) { + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + f.GetCfg()->UpdateEdgeFreqWithNewBBFreq(); + } GetAnalysisInfoHook()->ForceEraseAnalysisPhase(f.GetUniqueID(), &MEDominance::id); } } diff --git a/src/mapleall/maple_me/src/me_loop_inversion.cpp b/src/mapleall/maple_me/src/me_loop_inversion.cpp index 0a41245c7f58d6e973bddd6d8deb3cc14fa2977d..622c0fb14104491f6ddbbc561e8adef5c3226ca7 100644 --- a/src/mapleall/maple_me/src/me_loop_inversion.cpp +++ b/src/mapleall/maple_me/src/me_loop_inversion.cpp @@ -133,6 +133,16 @@ void MeLoopInversion::Convert(MeFunction &func, BB &bb, BB &pred, MapleMapNewBasicBlock(); latchBB->SetAttributes(kBBAttrArtificial); latchBB->SetAttributes(kBBAttrIsInLoop); // latchBB is inloop + // update newBB frequency : copy predBB succFreq as latch frequency + if (Options::profileUse && func.GetMirFunc()->GetFuncProfData()) { + int idx = pred.GetSuccIndex(bb); + ASSERT(idx >= 0 && idx < pred.GetSucc().size(), "sanity check"); + uint64_t freq = pred.GetEdgeFreq(idx); + latchBB->SetFrequency(freq); + // update bb frequency: remove pred frequency since pred is deleted + ASSERT(bb.GetFrequency() >= freq, "sanity check"); + bb.SetFrequency(bb.GetFrequency() - freq); + } // update pred bb pred.ReplaceSucc(&bb, latchBB); // replace pred.succ with latchBB and set pred of latchBB // update pred stmt if needed @@ -183,6 +193,45 @@ void MeLoopInversion::Convert(MeFunction &func, BB &bb, BB &pred, MapleMapAddSucc(*succ); } latchBB->SetKind(bb.GetKind()); + // update succFreq + if (Options::profileUse && func.GetMirFunc()->GetFuncProfData()) { + int succFreqSize = bb.GetSuccFreq().size(); + ASSERT(latchBB->GetSucc().size() == succFreqSize, "sanity check"); + // copy bb succFreq as latch frequency + for (int i = 0; i < succFreqSize; i++) { + latchBB->PushBackSuccFreq(bb.GetSuccFreq()[i]); + } + if (bb.GetFrequency() > 0) { + // swap frequency and fixed the frequency value + if (swapSuccOfLatch) { + ASSERT(latchBB->GetKind() == kBBCondGoto, "impossible"); + int64_t newftFreq = (latchBB->GetFrequency() == 0 || bb.GetSuccFreq()[1] == 0) ? 0 : 1; + int64_t newtgFreq= latchBB->GetFrequency() == 0 ? 0 : (latchBB->GetFrequency() - newftFreq); + latchBB->SetSuccFreq(0, newftFreq); + latchBB->SetSuccFreq(1, newtgFreq); // loop is changed to do-while format + int64_t bbsucc0Freq = bb.GetSucc(0)->GetFrequency() - newtgFreq; + int64_t bbsucc1Freq = bb.GetFrequency() - bbsucc0Freq; + bb.SetSuccFreq(0, bbsucc0Freq); + bb.SetSuccFreq(1, bbsucc1Freq); + } else if (latchBB->GetKind() == kBBCondGoto) { + int64_t newftFreq = (latchBB->GetFrequency() == 0 || bb.GetSuccFreq()[0] == 0) ? 0 : 1; + int64_t newtgFreq = latchBB->GetFrequency() == 0 ? 0 : (latchBB->GetFrequency() - newftFreq); + latchBB->SetSuccFreq(0, newftFreq); // freq to exit BB + latchBB->SetSuccFreq(1, newtgFreq); // loop is changed to do-while format + // update bb succ frequency + int64_t bbsucc0Freq = bb.GetSucc(0)->GetFrequency() - newftFreq; + int64_t bbsucc1Freq = bb.GetFrequency() - bbsucc0Freq; + bb.SetSuccFreq(0, bbsucc0Freq); + bb.SetSuccFreq(1, bbsucc1Freq); + } else if (latchBB->GetKind() == kBBFallthru || latchBB->GetKind() == kBBGoto) { + int64_t newsuccFreq = (latchBB->GetFrequency() == 0) ? 0 : latchBB->GetSuccFreq()[0] - 1; + latchBB->SetSuccFreq(0, newsuccFreq); // loop is changed to do-while format + bb.SetSuccFreq(0, bb.GetFrequency()); + } else { + ASSERT(0, "NYI:: unexpected bb type"); + } + } + } // swap latchBB's succ if needed if (swapSuccOfLatch) { // modify condBr stmt diff --git a/src/mapleall/maple_me/src/me_phase_manager.cpp b/src/mapleall/maple_me/src/me_phase_manager.cpp index 027294329bbb8a27a705e3d6484d4833ece94f6f..09a1b64e5a5f69c382302fb8e0d23f15462ab9b8 100644 --- a/src/mapleall/maple_me/src/me_phase_manager.cpp +++ b/src/mapleall/maple_me/src/me_phase_manager.cpp @@ -212,6 +212,7 @@ void MeFuncPM::GetAnalysisDependence(maple::AnalysisDep &aDep) const { MAPLE_TRANSFORM_PHASE_REGISTER_CANSKIP(MeFuncPM, meFuncPM) MAPLE_ANALYSIS_PHASE_REGISTER_CANSKIP(MEMeCfg, mecfgbuild) +MAPLE_ANALYSIS_PHASE_REGISTER_CANSKIP(MECfgVerifyFrequency, cfgverifyfreq) MAPLE_ANALYSIS_PHASE_REGISTER_CANSKIP(MEVerify, meverify) MAPLE_ANALYSIS_PHASE_REGISTER_CANSKIP(MEAliasClass, aliasclass) MAPLE_ANALYSIS_PHASE_REGISTER_CANSKIP(MESSATab, ssatab) diff --git a/src/mapleall/maple_me/src/me_profile_use.cpp b/src/mapleall/maple_me/src/me_profile_use.cpp index 7c6eb2fda9578cdd727c868d9afb12686bc6dce5..05bf473328af0b6288527461754ae3bec6b1a41f 100644 --- a/src/mapleall/maple_me/src/me_profile_use.cpp +++ b/src/mapleall/maple_me/src/me_profile_use.cpp @@ -362,6 +362,7 @@ bool MEProfUse::PhaseRun(maple::MeFunction &f) { bool result = true; if (Options::profileUse) { result = profUse.GcovRun(); + f.GetCfg()->VerifyBBFreq(); } else { profUse.Run(); } diff --git a/src/mapleall/maple_me/src/optimizeCFG.cpp b/src/mapleall/maple_me/src/optimizeCFG.cpp index a2c38be67dab28b48ad2abb70e12d184adb6e36b..9891fc3feb2ebffaf773149bba5c5d05a65eb77f 100644 --- a/src/mapleall/maple_me/src/optimizeCFG.cpp +++ b/src/mapleall/maple_me/src/optimizeCFG.cpp @@ -1,4 +1,5 @@ /* + } * Copyright (c) [2021] Huawei Technologies Co.,Ltd.All rights reserved. * * OpenArkCompiler is licensed under Mulan PSL v2. @@ -881,6 +882,7 @@ void OptimizeBB::DeleteBB(BB *bb) { bb->GetSucc().clear(); bb->GetPred().clear(); cfg->DeleteBasicBlock(*bb); + bb->GetSuccFreq().clear(); } // eliminate dead BB : @@ -950,6 +952,10 @@ bool OptimizeBB::BranchBB2UncondBB(BB &bb) { ASSERT(bb.GetSucc().back() == destBB, "[FUNC: %s]Goto BB%d has different destination", funcName, LOG_BBID(&bb)); bb.RemoveSucc(*bb.GetSucc().back()); } + if (Options::profileUse && !bb.GetSuccFreq().empty()) { + ASSERT(bb.GetSuccFreq().size() == bb.GetSucc().size(), "sanity check"); + bb.SetSuccFreq(0, bb.GetFrequency()); + } // bb must be one of fallthru pred of destBB, if there is another one, // we should add gotoStmt to avoid duplicate fallthru pred if (GetFallthruPredNum(*destBB) > 1) { @@ -994,6 +1000,7 @@ bool OptimizeBB::OptimizeCondBB2UnCond() { // case 1 BB *ftBB = currBB->GetSucc(0); BB *gtBB = currBB->GetSucc(1); + uint64_t removedSuccFreq = 0; if (isMeIR) { MeStmt *brStmt = currBB->GetLastMe(); MeExpr *condExpr = brStmt->GetOpnd(0); @@ -1003,12 +1010,28 @@ bool OptimizeBB::OptimizeCondBB2UnCond() { if (isCondTrue ^ isBrtrue) { // goto fallthru BB currBB->RemoveLastMeStmt(); currBB->SetKind(kBBFallthru); + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + removedSuccFreq = currBB->GetSuccFreq()[1]; + } currBB->RemoveSucc(*gtBB, true); + // update frequency + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + currBB->SetSuccFreq(0, currBB->GetFrequency()); + ftBB->SetFrequency(ftBB->GetFrequency() + removedSuccFreq); + } } else { MeStmt *gotoStmt = irmap->CreateGotoMeStmt(f.GetOrCreateBBLabel(*gtBB), currBB, &brStmt->GetSrcPosition()); currBB->ReplaceMeStmt(brStmt, gotoStmt); currBB->SetKind(kBBGoto); + // update frequency + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + removedSuccFreq = currBB->GetSuccFreq()[0]; + } currBB->RemoveSucc(*ftBB, true); + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + currBB->SetSuccFreq(0, currBB->GetFrequency()); + gtBB->SetFrequency(gtBB->GetFrequency() + removedSuccFreq); + } } SetBBRunAgain(); return true; @@ -1023,12 +1046,26 @@ bool OptimizeBB::OptimizeCondBB2UnCond() { if (isCondTrue ^ isBrTrue) { currBB->RemoveLastStmt(); currBB->SetKind(kBBFallthru); + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + removedSuccFreq = currBB->GetSuccFreq()[1]; + } currBB->RemoveSucc(*gtBB); + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + currBB->SetSuccFreq(0, currBB->GetFrequency()); + ftBB->SetFrequency(ftBB->GetFrequency() + removedSuccFreq); + } } else { StmtNode *gotoStmt = irBuilder->CreateStmtGoto(OP_goto, f.GetOrCreateBBLabel(*gtBB)); currBB->ReplaceStmt(&brStmt, gotoStmt); currBB->SetKind(kBBGoto); + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + removedSuccFreq = currBB->GetSuccFreq()[0]; + } currBB->RemoveSucc(*ftBB); + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + currBB->SetSuccFreq(0, currBB->GetFrequency()); + gtBB->SetFrequency(gtBB->GetFrequency() + removedSuccFreq); + } } SetBBRunAgain(); return true; @@ -1163,9 +1200,16 @@ BB *OptimizeBB::MergeSuccIntoPred(BB *pred, BB *succ) { } } } + // update succFreqs before update succs + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData() && + !succ->GetSuccFreq().empty()) { + // copy succFreqs of succ to pred + for (int i = 0; i < succ->GetSuccFreq().size(); i++) { + pred->PushBackSuccFreq(succ->GetSuccFreq()[i]); + } + } succ->MoveAllSuccToPred(pred, cfg->GetCommonExitBB()); pred->RemoveSucc(*succ, false); // philist will not be associated with pred BB, no need to update phi here. - if (!isSuccEmpty) { // only when we move stmt from succ to pred should we set attr and kind here pred->SetAttributes(succ->GetAttributes()); @@ -1725,6 +1769,19 @@ bool OptimizeBB::SkipRedundantCond(BB &pred, BB &succ) { } newTarget->AddPred(*newBB); UpdatePhiForMovingPred(predPredIdx, newBB, &succ, newTarget); + // update newBB frequency : copy predBB succFreq as newBB frequency + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + int idx = pred.GetSuccIndex(*newBB); + ASSERT(idx >= 0 && idx < pred.GetSucc().size(), "sanity check"); + uint64_t freq = pred.GetEdgeFreq(idx); + newBB->SetFrequency(freq); + newBB->PushBackSuccFreq(freq); + // update frequency of succ because one of its pred is removed + // frequency of + uint32_t freqOfSucc = succ.GetFrequency(); + ASSERT(freqOfSucc >= freq, "sanity check"); + succ.SetFrequency(freqOfSucc - freq); + } return true; } return false; @@ -2354,6 +2411,10 @@ bool OptimizeFuntionCFG::OptimizeFuncCFGIteratively() { if (currBB == nullptr) { continue; } + // bb frequency may be updated, make bb frequency and succs frequency consistent + if (Options::profileUse && !currBB->GetSuccFreq().empty()) { + currBB->UpdateEdgeFreqs(); + } changed |= OptimizeCFGForBB(currBB); } everChanged |= changed; diff --git a/src/mapleall/maple_me/src/pme_emit.cpp b/src/mapleall/maple_me/src/pme_emit.cpp index e880637ccdca2dce8b5a883b72b681ef763474e8..227cb6a761f6297a9ab6c99aa066f9063f7763a8 100755 --- a/src/mapleall/maple_me/src/pme_emit.cpp +++ b/src/mapleall/maple_me/src/pme_emit.cpp @@ -740,6 +740,7 @@ void PreMeEmitter::EmitBB(BB *bb, BlockNode *curblk) { CHECK_FATAL(curblk != nullptr, "null ptr check"); bool setFirstFreq = (GetFuncProfData() != nullptr); bool setLastFreq = false; + bool bbIsEmpty = bb->GetMeStmts().empty(); // emit head. label LabelIdx labidx = bb->GetBBLabel(); if (labidx != 0 && !preMeFunc->WhileLabelCreatedByPreMe(labidx) && !preMeFunc->IfLabelCreatedByPreMe(labidx)) { @@ -760,7 +761,7 @@ void PreMeEmitter::EmitBB(BB *bb, BlockNode *curblk) { curblk->AddStatement(stmt); // add for first stmt in bb in curblk if (GetFuncProfData() != nullptr) { - if (setFirstFreq || (stmt->GetOpCode() == OP_call) || (stmt->GetOpCode() == OP_callassigned)) { + if (setFirstFreq || (stmt->GetOpCode() == OP_call) || IsCallAssigned(stmt->GetOpCode())) { GetFuncProfData()->SetStmtFreq(stmt->GetStmtID(), bb->GetFrequency()); setFirstFreq = false; } else { @@ -780,8 +781,12 @@ void PreMeEmitter::EmitBB(BB *bb, BlockNode *curblk) { if (GetFuncProfData()) { if (setLastFreq) { GetFuncProfData()->SetStmtFreq(curblk->GetLast()->GetStmtID(), bb->GetFrequency()); - } else { - LogInfo::MapleLogger() << " bb " << bb->GetBBId() << "no stmt used to add frequency\n"; + } else if (bbIsEmpty) { + LogInfo::MapleLogger() << " bb " << bb->GetBBId() << "no stmt used to add frequency, add commentnode\n"; + CommentNode *commentNode = codeMP->New(*(mirFunc->GetModule())); + commentNode->SetComment("freqStmt"+std::to_string(commentNode->GetStmtID())); + GetFuncProfData()->SetStmtFreq(commentNode->GetStmtID(), bb->GetFrequency()); + curblk->AddStatement(commentNode); } } } @@ -910,8 +915,9 @@ uint32 PreMeEmitter::Raise2PreMeIf(uint32 curj, BlockNode *curblk) { while (mestmt->GetOp() != OP_brfalse && mestmt->GetOp() != OP_brtrue) { StmtNode *stmt = EmitPreMeStmt(mestmt, curblk); curblk->AddStatement(stmt); - if (setFirstFreq) { - // add frequency of first stmt of curbb + if (GetFuncProfData() && + (setFirstFreq || (stmt->GetOpCode() == OP_call) || IsCallAssigned(stmt->GetOpCode()))) { + // add frequency of first/call stmt of curbb GetFuncProfData()->SetStmtFreq(stmt->GetStmtID(), curbb->GetFrequency()); setFirstFreq = false; } diff --git a/src/mapleall/maple_util/include/gcov_profile.h b/src/mapleall/maple_util/include/gcov_profile.h index 3c30e62428fc08cd0166a6b66fe63c1444ae7a81..b4d93878114214783b57a4f12bfaa8781c1f2e75 100644 --- a/src/mapleall/maple_util/include/gcov_profile.h +++ b/src/mapleall/maple_util/include/gcov_profile.h @@ -15,7 +15,7 @@ typedef unsigned location_t; enum UpdateFreqOp { kKeepOrigFreq = 0, kUpdateOrigFreq = 0x1, - kUpdateInlinedFreq = 0x2, + kUpdateFreqbyScale = 0x2, kUpdateUnrolledFreq = 0x4, kUpdateUnrollRemainderFreq = 0x8, }; @@ -29,6 +29,9 @@ public: int64_t GetFuncFrequency() const { return entry_freq; } void SetFuncFrequency(int64_t freq) { entry_freq = freq; } + int64_t GetFuncRealFrequency() const { return real_entryfreq; } + void SetFuncRealFrequency(int64_t freq) { real_entryfreq = freq; } + std::unordered_map& GetStmtFreqs() { return stmtFreqs; } @@ -68,6 +71,7 @@ public: MapleVector counts; int64_t entry_freq; // record entry bb frequence std::unordered_map stmtFreqs; // stmt_id is key, counter value + int64_t real_entryfreq; // function prof data may be modified after clone/inline }; class GcovProfileData { @@ -80,7 +84,10 @@ public: } return nullptr; } - + void AddFuncProfile(unsigned puidx, GcovFuncInfo *funcData) { + ASSERT(funcsCounter.count(puidx) == 0, "sanity check"); + funcsCounter[puidx] = funcData; + } MapleUnorderedMap funcsCounter; // use puidx as key }; diff --git a/src/mapleall/mpl2mpl/src/constantfold.cpp b/src/mapleall/mpl2mpl/src/constantfold.cpp index 75d9fbe2e4a15cdea0306c1a9b41a3f31e5fde5b..860ed2846a2f8c8c181406f265ba399cff95825a 100644 --- a/src/mapleall/mpl2mpl/src/constantfold.cpp +++ b/src/mapleall/mpl2mpl/src/constantfold.cpp @@ -2292,6 +2292,8 @@ StmtNode *ConstantFold::SimplifyIassignWithAddrofBaseNode(IassignNode &node, con dassignNode->SetStIdx(base.GetStIdx()); dassignNode->SetRHS(node.GetRHS()); dassignNode->SetFieldID(base.GetFieldID() + node.GetFieldID()); + // reuse stmtid to maintain stmtFreqs if profileUse is on + dassignNode->SetStmtID(node.GetStmtID()); return dassignNode; } return &node; @@ -2363,6 +2365,10 @@ StmtNode *ConstantFold::SimplifyIassign(IassignNode *node) { StmtNode *ConstantFold::SimplifyCondGoto(CondGotoNode *node) { CHECK_NULL_FATAL(node); + // optimize condgoto need to update frequency, skip here + if (Options::profileUse && mirModule->CurFunction()->GetFuncProfData()) { + return node; + } BaseNode *returnValue = nullptr; returnValue = Fold(node->Opnd(0)); returnValue = (returnValue == nullptr) ? node : returnValue; @@ -2383,6 +2389,9 @@ StmtNode *ConstantFold::SimplifyCondGoto(CondGotoNode *node) { uint32 freq = mirModule->CurFunction()->GetFreqFromLastStmt(node->GetStmtID()); GotoNode *gotoNode = mirModule->CurFuncCodeMemPool()->New(OP_goto); gotoNode->SetOffset(node->GetOffset()); + if (Options::profileUse && mirModule->CurFunction()->GetFuncProfData()) { + gotoNode->SetStmtID(node->GetStmtID()); // reuse condnode stmtid + } mirModule->CurFunction()->SetLastFreqMap(gotoNode->GetStmtID(), freq); return gotoNode; } else { @@ -2618,6 +2627,8 @@ StmtNode *ConstantFold::SimplifyIcall(IcallNode *node) { callNode->GetNopnd().push_back(node->GetNopndAt(i)); } callNode->SetNumOpnds(callNode->GetNopndSize()); + // reuse stmtID to skip update stmtFreqs when profileUse is on + callNode->SetStmtID(node->GetStmtID()); return callNode; } default: diff --git a/src/mapleall/mpl2mpl/src/inline.cpp b/src/mapleall/mpl2mpl/src/inline.cpp index 2349977e18e968b59eb35aabcdd3c9e99825cae4..e7ea3f5ceaeb377a8417ad056d49896be762ca5d 100644 --- a/src/mapleall/mpl2mpl/src/inline.cpp +++ b/src/mapleall/mpl2mpl/src/inline.cpp @@ -729,7 +729,7 @@ bool MInline::PerformInline(MIRFunction &caller, BlockNode &enclosingBlk, CallNo // multiple definition for these static symbols ConvertPStaticToFStatic(callee); // Step 1: Clone CALLEE's body. - auto getBody = [caller, callee, this] (BlockNode* funcBody, CallNode &callStmt) { + auto getBody = [caller, callee, this] (BlockNode* funcBody, CallNode &callStmt, bool recursiveFirstClone) { if (callee.IsFromMpltInline()) { return funcBody->CloneTree(module.GetCurFuncCodeMPAllocator()); } @@ -739,12 +739,22 @@ bool MInline::PerformInline(MIRFunction &caller, BlockNode &enclosingBlk, CallNo ASSERT(callerProfData && calleeProfData, "nullptr check"); int64_t callsiteFreq = callerProfData->GetStmtFreq(callStmt.GetStmtID()); int64_t calleeEntryFreq = calleeProfData->GetFuncFrequency(); - auto *blockNode = funcBody->CloneTreeWithFreqs(module.GetCurFuncCodeMPAllocator(), + uint32_t updateOp = (kKeepOrigFreq | kUpdateFreqbyScale); + BlockNode *blockNode; + if (recursiveFirstClone) { + blockNode = funcBody->CloneTreeWithFreqs(module.GetCurFuncCodeMPAllocator(), callerProfData->GetStmtFreqs(), calleeProfData->GetStmtFreqs(), - callsiteFreq, calleeEntryFreq, (kUpdateOrigFreq | kUpdateInlinedFreq)); - // update func Frequency - calleeProfData->SetFuncFrequency(calleeEntryFreq - callsiteFreq); + 1, 1, updateOp); + } else { + blockNode = funcBody->CloneTreeWithFreqs(module.GetCurFuncCodeMPAllocator(), + callerProfData->GetStmtFreqs(), + calleeProfData->GetStmtFreqs(), + callsiteFreq, calleeEntryFreq, updateOp); + // update callee left entry Frequency + int calleeFreq = calleeProfData->GetFuncRealFrequency(); + calleeProfData->SetFuncRealFrequency(calleeFreq - callsiteFreq); + } return blockNode; } else { return funcBody->CloneTreeWithSrcPosition(module); @@ -764,7 +774,7 @@ bool MInline::PerformInline(MIRFunction &caller, BlockNode &enclosingBlk, CallNo } if (currFuncBody == nullptr) { // For Inline recursive, we save the original function body before first inline. - currFuncBody = getBody(callee.GetBody(), callStmt); + currFuncBody = getBody(callee.GetBody(), callStmt, true); // update inlining levels if (recursiveFuncToInlineLevel.find(&callee) != recursiveFuncToInlineLevel.end()) { recursiveFuncToInlineLevel[&callee]++; @@ -772,9 +782,9 @@ bool MInline::PerformInline(MIRFunction &caller, BlockNode &enclosingBlk, CallNo recursiveFuncToInlineLevel[&callee] = 1; } } - newBody = getBody(currFuncBody, callStmt); + newBody = getBody(currFuncBody, callStmt, false); } else { - newBody = getBody(callee.GetBody(), callStmt); + newBody = getBody(callee.GetBody(), callStmt, false); } // Step 2: Rename symbols, labels, pregs