From c49480d81fa697da97f92edfc1ac5ac0c0263d86 Mon Sep 17 00:00:00 2001 From: Fred Chow Date: Mon, 19 Sep 2022 22:18:36 -0700 Subject: [PATCH 1/4] Enabled raising back to while and if stmts under --profileUse and fixed -O3 issues that resulted from this change Avoided creating critical edge in lowering IF stmts. Auto-vectorization is no longer disabled under --profileUse. --- src/mapleall/maple_me/src/lfo_loop_vec.cpp | 4 + src/mapleall/maple_me/src/me_cfg.cpp | 43 ++++--- .../maple_me/src/me_critical_edge.cpp | 9 -- .../maple_me/src/me_loop_unrolling.cpp | 6 + .../maple_me/src/me_value_range_prop.cpp | 3 +- src/mapleall/maple_me/src/optimizeCFG.cpp | 4 - src/mapleall/maple_me/src/pme_emit.cpp | 8 ++ src/mapleall/maple_me/src/pme_mir_lower.cpp | 112 +++++------------- src/mapleall/maple_me/src/seqvec.cpp | 6 + 9 files changed, 79 insertions(+), 116 deletions(-) diff --git a/src/mapleall/maple_me/src/lfo_loop_vec.cpp b/src/mapleall/maple_me/src/lfo_loop_vec.cpp index b44c1fe97b..189a49ec6e 100644 --- a/src/mapleall/maple_me/src/lfo_loop_vec.cpp +++ b/src/mapleall/maple_me/src/lfo_loop_vec.cpp @@ -1528,6 +1528,10 @@ void LoopVectorization::VectorizeReductionStmt(StmtNode *stmt, LoopTransPlan *tp StmtNode *copyStmt = stmt->CloneTree(*codeMPAlloc); IntrinsicopNode *intrnvecSum = GenSumVecStmt(regReadlhsvec, regReadlhsvec->GetPrimType()); copyStmt->GetRHS()->SetOpnd(intrnvecSum, 1); + FuncProfInfo *profData = mirFunc->GetFuncProfData(); + if (profData) { + profData->CopyStmtFreq(copyStmt->GetStmtID(), stmt->GetStmtID()); + } tp->vecInfo->afterLoopStmts.push_back(copyStmt); doloopbody->RemoveStmt(stmt); } diff --git a/src/mapleall/maple_me/src/me_cfg.cpp b/src/mapleall/maple_me/src/me_cfg.cpp index 52957520ee..21a5525c58 100644 --- a/src/mapleall/maple_me/src/me_cfg.cpp +++ b/src/mapleall/maple_me/src/me_cfg.cpp @@ -1309,14 +1309,6 @@ void MeCFG::CreateBasicBlocks() { } case OP_dassign: { DassignNode *dass = static_cast(stmt); - // delete identity assignments inserted by LFO - if (dass->GetRHS()->GetOpCode() == OP_dread) { - DreadNode *dread = static_cast(dass->GetRHS()); - if (dass->GetStIdx() == dread->GetStIdx() && dass->GetFieldID() == dread->GetFieldID()) { - func.CurFunction()->GetBody()->RemoveStmt(stmt); - break; - } - } if (curBB->IsEmpty()) { curBB->SetFirst(stmt); } @@ -1539,8 +1531,8 @@ void MeCFG::CreateBasicBlocks() { SetBBTryNodeMap(*newBB, *tryStmt); } curBB = newBB; - } else if (func.GetPreMeFunc() && (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != - func.GetPreMeFunc()->label2WhileInfo.end())) { + } else if (func.GetPreMeFunc() && + (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != func.GetPreMeFunc()->label2WhileInfo.end())) { curBB->SetKind(kBBFallthru); BB *newBB = NewBasicBlock(); if (tryStmt != nullptr) { @@ -1625,6 +1617,10 @@ void MeCFG::CreateBasicBlocks() { lastBB->SetFirst(func.GetMIRModule().GetMIRBuilder()->CreateStmtReturn(nullptr)); lastBB->SetLast(lastBB->GetStmtNodes().begin().d()); lastBB->SetKindReturn(); + if (updateFreq) { + FuncProfInfo *funcData = func.GetMirFunc()->GetFuncProfData(); + funcData->stmtFreqs[lastBB->GetLast().GetStmtID()] = 0; + } } else if (lastBB->GetKind() == kBBUnknown) { lastBB->SetKindReturn(); lastBB->SetAttributes(kBBAttrIsExit); @@ -1878,15 +1874,25 @@ void MeCFG::ConstructEdgeFreqFromBBFreq() { if (fallthru->GetPred().size() == 1) { auto succ0Freq = fallthru->GetFrequency(); bb->PushBackSuccFreq(succ0Freq); - ASSERT(bb->GetFrequency() >= succ0Freq, "sanity check"); + ASSERT(bb->GetFrequency() + 1 >= succ0Freq, "sanity check"); bb->PushBackSuccFreq(bb->GetFrequency() - succ0Freq); } else if (targetBB->GetPred().size() == 1) { auto succ1Freq = targetBB->GetFrequency(); ASSERT(bb->GetFrequency() >= succ1Freq, "sanity check"); - bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); - bb->PushBackSuccFreq(succ1Freq); + if (bb->GetAttributes(kBBAttrWontExit) && bb->GetSuccFreq().size() == 1) { + // special case: WontExitAnalysis() has pushed 0 to bb->succFreq + bb->GetSuccFreq()[0] = bb->GetFrequency(); + bb->PushBackSuccFreq(0); + } else { + bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); + bb->PushBackSuccFreq(succ1Freq); + } + } else if (fallthru->GetFrequency() > targetBB->GetFrequency()) { + bb->PushBackSuccFreq(bb->GetFrequency()); + bb->PushBackSuccFreq(0); } else { - CHECK_FATAL(false, "ConstructEdgeFreqFromBBFreq::NYI critical edge"); + bb->PushBackSuccFreq(0); + bb->PushBackSuccFreq(bb->GetFrequency()); } } else if (bb->GetSucc().size() > 2) { // switch case, no critical edge is supposted @@ -1937,9 +1943,6 @@ void MeCFG::ConstructBBFreqFromStmtFreq() { ConstructEdgeFreqFromBBFreq(); // clear stmtFreqs since cfg frequency is create funcData->stmtFreqs.clear(); - - // set updateFrequency with true - updateFreq = true; } void MeCFG::ConstructStmtFreq() { @@ -2086,6 +2089,10 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { MemPool *meCfgMp = GetPhaseMemPool(); theCFG = meCfgMp->New(meCfgMp, f); f.SetTheCfg(theCFG); + + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + theCFG->SetUpdateCFGFreq(true); + } theCFG->CreateBasicBlocks(); if (theCFG->NumBBs() == 0) { /* there's no basicblock generated */ @@ -2104,7 +2111,7 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { } theCFG->Verify(); // construct bb freq from stmt freq - if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + if (theCFG->UpdateCFGFreq()) { theCFG->ConstructBBFreqFromStmtFreq(); if (theCFG->DumpIRProfileFile()) { std::string fileName = "after-mecfgbuild"; diff --git a/src/mapleall/maple_me/src/me_critical_edge.cpp b/src/mapleall/maple_me/src/me_critical_edge.cpp index fd97bb2452..dc52c67bc2 100644 --- a/src/mapleall/maple_me/src/me_critical_edge.cpp +++ b/src/mapleall/maple_me/src/me_critical_edge.cpp @@ -304,15 +304,6 @@ bool MESplitCEdge::PhaseRun(maple::MeFunction &f) { bool enableDebug = DEBUGFUNC_NEWPM(f); MeSplitCEdge mscedge = MeSplitCEdge(enableDebug); mscedge.SplitCriticalEdgeForMeFunc(f); - if (Options::profileUse) { - if ((f.IsPme() || f.IsLfo()) && f.GetPreMeFunc()) { - // new inserted BB break cached while/if label information and IR layout - f.GetPreMeFunc()->label2IfInfo.clear(); - f.GetPreMeFunc()->label2WhileInfo.clear(); - f.GetPreMeFunc()->pmeCreatedIfLabelSet.clear(); - f.GetPreMeFunc()->pmeCreatedWhileLabelSet.clear(); - } - } if (f.GetCfg()->UpdateCFGFreq() && (f.GetCfg()->DumpIRProfileFile())) { f.GetCfg()->DumpToFile("after-splitcriticaledge", false, true); } diff --git a/src/mapleall/maple_me/src/me_loop_unrolling.cpp b/src/mapleall/maple_me/src/me_loop_unrolling.cpp index 5e39fd56c1..4b85f7c4f6 100644 --- a/src/mapleall/maple_me/src/me_loop_unrolling.cpp +++ b/src/mapleall/maple_me/src/me_loop_unrolling.cpp @@ -487,9 +487,15 @@ bool LoopUnrolling::SplitCondGotoBB() { for (auto *bb : exitBB->GetSucc()) { bb->ReplacePred(exitBB, newCondGotoBB); + if (profValid) { + newCondGotoBB->GetSuccFreq().push_back(0); + } } exitBB->AddSucc(*newCondGotoBB); + if (profValid) { + exitBB->GetSuccFreq().push_back(0); + } exitBB->RemoveMeStmt(lastStmt); newCondGotoBB->InsertMeStmtLastBr(lastStmt); return true; diff --git a/src/mapleall/maple_me/src/me_value_range_prop.cpp b/src/mapleall/maple_me/src/me_value_range_prop.cpp index 39fd3e9567..3fe4cf6b9e 100644 --- a/src/mapleall/maple_me/src/me_value_range_prop.cpp +++ b/src/mapleall/maple_me/src/me_value_range_prop.cpp @@ -3980,13 +3980,12 @@ bool ValueRangePropagation::RemoveTheEdgeOfPredBB( uint64 edgeFreq = 0; if (func.GetCfg()->UpdateCFGFreq()) { edgeFreq = pred.GetSuccFreq()[index]; - ASSERT(bb.GetFrequency() >= edgeFreq, "sanity check"); } pred.RemoveSucc(bb); pred.AddSucc(*newBB, index); newBB->AddSucc(trueBranch); if (func.GetCfg()->UpdateCFGFreq()) { - if (bb.GetFrequency() > edgeFreq) { + if (bb.GetFrequency() >= edgeFreq) { bb.SetFrequency(static_cast(bb.GetFrequency() - edgeFreq)); bb.UpdateEdgeFreqs(); } diff --git a/src/mapleall/maple_me/src/optimizeCFG.cpp b/src/mapleall/maple_me/src/optimizeCFG.cpp index 4e47f5e40f..005d09a602 100644 --- a/src/mapleall/maple_me/src/optimizeCFG.cpp +++ b/src/mapleall/maple_me/src/optimizeCFG.cpp @@ -1157,10 +1157,6 @@ bool OptimizeBB::RemoveSuccFromNoReturnBB() { cfg->GetCommonExitBB()->AddExit(*currBB); currBB->SetAttributes(kBBAttrIsExit); ResetBBRunAgain(); - // add edge Frequency to exitbb - if (cfg->UpdateCFGFreq()) { - currBB->PushBackSuccFreq(0); - } return true; } return false; diff --git a/src/mapleall/maple_me/src/pme_emit.cpp b/src/mapleall/maple_me/src/pme_emit.cpp index efd76f295e..f3e348931b 100644 --- a/src/mapleall/maple_me/src/pme_emit.cpp +++ b/src/mapleall/maple_me/src/pme_emit.cpp @@ -365,6 +365,10 @@ StmtNode* PreMeEmitter::EmitPreMeStmt(MeStmt *meStmt, BaseNode *parent) { switch (meStmt->GetOp()) { case OP_dassign: { DassignMeStmt *dsmestmt = static_cast(meStmt); + if (dsmestmt->GetRHS()->GetMeOp() == kMeOpVar && + static_cast(dsmestmt->GetRHS())->GetOst() == dsmestmt->GetLHS()->GetOst()) { + return nullptr; // identity assignment introduced by LFO + } DassignNode *dass = codeMP->New(); MIRSymbol *sym = dsmestmt->GetLHS()->GetOst()->GetMIRSymbol(); dass->SetStIdx(sym->GetStIdx()); @@ -968,6 +972,10 @@ uint32 PreMeEmitter::Raise2PreMeIf(uint32 curJ, BlockNode *curBlk) { MeStmt *mestmt = curbb->GetFirstMe(); while (mestmt->GetOp() != OP_brfalse && mestmt->GetOp() != OP_brtrue) { StmtNode *stmt = EmitPreMeStmt(mestmt, curBlk); + if (stmt == nullptr) { + mestmt = mestmt->GetNext(); + continue; + } UpdateStmtInfo(*mestmt, *stmt, *curBlk, curbb->GetFrequency()); curBlk->AddStatement(stmt); if (GetFuncProfData() && diff --git a/src/mapleall/maple_me/src/pme_mir_lower.cpp b/src/mapleall/maple_me/src/pme_mir_lower.cpp index e2a45d62d4..048ff7464a 100644 --- a/src/mapleall/maple_me/src/pme_mir_lower.cpp +++ b/src/mapleall/maple_me/src/pme_mir_lower.cpp @@ -121,80 +121,6 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { if (GetFuncProfData()) { GetFuncProfData()->CopyStmtFreq(evalstmt->GetStmtID(), ifstmt.GetStmtID()); } - } else if (elseempty) { - // brfalse - // - // label - CondGotoNode *brfalsestmt = mirModule.CurFuncCodeMemPool()->New(OP_brfalse); - brfalsestmt->SetOpnd(ifstmt.Opnd(), 0); - brfalsestmt->SetSrcPos(ifstmt.GetSrcPos()); - LabelIdx endlabelidx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('e'); - mirFunc->GetLabelTab()->AddToStringLabelMap(endlabelidx); - if (canRaiseBack) { - preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); - } - PreMeIfInfo *ifInfo = preMeFunc->pmemp->New(); - brfalsestmt->SetOffset(endlabelidx); - blk->AddStatement(brfalsestmt); - // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(brfalsestmt->GetStmtID(), ifstmt.GetStmtID()); - } - blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); - - LabelNode *labstmt = mirModule.CurFuncCodeMemPool()->New(); - labstmt->SetLabelIdx(endlabelidx); - ifInfo->endLabel = endlabelidx; - if (canRaiseBack) { - preMeFunc->label2IfInfo.insert(std::make_pair(endlabelidx, ifInfo)); - } - SrcPosition pos = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetThenPart()->GetSrcPos()); - labstmt->SetSrcPos(pos); - blk->AddStatement(labstmt); - // set stmtfreqs - if (GetFuncProfData()) { - ASSERT(GetFuncProfData()->GetStmtFreq(ifstmt.GetThenPart()->GetStmtID()) >= 0, "sanity check"); - int64_t freq = GetFuncProfData()->GetStmtFreq(ifstmt.GetStmtID()) - - GetFuncProfData()->GetStmtFreq(ifstmt.GetThenPart()->GetStmtID()); - GetFuncProfData()->SetStmtFreq(labstmt->GetStmtID(), freq); - } - } else if (thenempty) { - // brtrue - // - // label - CondGotoNode *brtruestmt = mirModule.CurFuncCodeMemPool()->New(OP_brtrue); - brtruestmt->SetOpnd(ifstmt.Opnd(), 0); - brtruestmt->SetSrcPos(ifstmt.GetSrcPos()); - LabelIdx endlabelidx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('e'); - if (canRaiseBack) { - preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); - } - PreMeIfInfo *ifInfo = preMeFunc->pmemp->New(); - mirFunc->GetLabelTab()->AddToStringLabelMap(endlabelidx); - brtruestmt->SetOffset(endlabelidx); - blk->AddStatement(brtruestmt); - - // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(brtruestmt->GetStmtID(), ifstmt.GetStmtID()); - } - blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); - LabelNode *labstmt = mirModule.CurFuncCodeMemPool()->New(); - labstmt->SetLabelIdx(endlabelidx); - ifInfo->endLabel = endlabelidx; - if (canRaiseBack) { - preMeFunc->label2IfInfo.insert(std::make_pair(endlabelidx, ifInfo)); - } - SrcPosition pos = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); - labstmt->SetSrcPos(pos); - blk->AddStatement(labstmt); - // set stmtfreqs - if (GetFuncProfData()) { - ASSERT(GetFuncProfData()->GetStmtFreq(ifstmt.GetElsePart()->GetStmtID()) > 0, "sanity check"); - int64_t freq = GetFuncProfData()->GetStmtFreq(ifstmt.GetStmtID()) - - GetFuncProfData()->GetStmtFreq(ifstmt.GetElsePart()->GetStmtID()); - GetFuncProfData()->SetStmtFreq(labstmt->GetStmtID(), freq); - } } else { // brfalse // @@ -222,8 +148,11 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { preMeFunc->label2IfInfo.insert(std::make_pair(elselabelidx, ifInfo)); } - blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); - bool fallthru_from_then = !OpCodeNoFallThrough(ifstmt.GetThenPart()->GetLast()->GetOpCode()); + bool fallthru_from_then = true; + if (!thenempty) { + blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); + fallthru_from_then = !OpCodeNoFallThrough(ifstmt.GetThenPart()->GetLast()->GetOpCode()); + } LabelIdx endlabelidx = 0; if (fallthru_from_then) { @@ -237,7 +166,11 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { blk->AddStatement(gotostmt); // set stmtfreqs if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(gotostmt->GetStmtID(), ifstmt.GetThenPart()->GetStmtID()); + if (!thenempty) { + GetFuncProfData()->CopyStmtFreq(gotostmt->GetStmtID(), ifstmt.GetThenPart()->GetStmtID()); + } else { + GetFuncProfData()->SetStmtFreq(gotostmt->GetStmtID(), 0); + } } } @@ -247,17 +180,21 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { labstmt->SetSrcPos(pos); blk->AddStatement(labstmt); - blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); + if (!elseempty) { + blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); + } if (fallthru_from_then) { labstmt = mirModule.CurFuncCodeMemPool()->New(); labstmt->SetLabelIdx(endlabelidx); - SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); - labstmt->SetSrcPos(position); + if (!elseempty) { + SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); + labstmt->SetSrcPos(position); + } blk->AddStatement(labstmt); // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(labstmt->GetStmtID(), ifstmt.GetElsePart()->GetStmtID()); + if (!elseempty && GetFuncProfData()) { + GetFuncProfData()->CopyStmtFreq(labstmt->GetStmtID(), ifstmt.GetStmtID()); } } if (endlabelidx == 0) { // create end label @@ -266,7 +203,7 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); } LabelNode *endlabelnode = mirbuilder->CreateStmtLabel(endlabelidx); - SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); + SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetSrcPos()); endlabelnode->SetSrcPos(position); blk->AddStatement(endlabelnode); // set stmtfreqs @@ -276,5 +213,14 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { } ifInfo->endLabel = endlabelidx; } + // generate extra label to avoid critical edge + LabelIdx extraLabelIdx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('x'); + preMeFunc->SetIfLabelCreatedByPreMe(extraLabelIdx); + LabelNode *extraLabelNode = mirbuilder->CreateStmtLabel(extraLabelIdx); + blk->AddStatement(extraLabelNode); + // set stmtfreqs + if (GetFuncProfData()) { + GetFuncProfData()->CopyStmtFreq(extraLabelNode->GetStmtID(), ifstmt.GetStmtID()); + } return blk; } diff --git a/src/mapleall/maple_me/src/seqvec.cpp b/src/mapleall/maple_me/src/seqvec.cpp index da70e8a3e4..1feb9978d1 100644 --- a/src/mapleall/maple_me/src/seqvec.cpp +++ b/src/mapleall/maple_me/src/seqvec.cpp @@ -571,6 +571,12 @@ void SeqVectorize::MergeIassigns(MapleVector &cands) { RegreadNode *regreadNode = codeMP->New(vecType->GetPrimType(), dupScalarStmt->GetRegIdx()); blockParent->InsertBefore(iassign, dupScalarStmt); iassign->SetRHS(regreadNode); + if (Options::profileUse && mirFunc->GetFuncProfData()) { + std::unordered_map &stmtFreqs = mirFunc->GetFuncProfData()->stmtFreqs; + if (stmtFreqs.find(iassign->GetStmtID()) != stmtFreqs.end()) { + mirFunc->GetFuncProfData()->CopyStmtFreq(dupScalarStmt->GetStmtID(), iassign->GetStmtID()); + } + } } else if (iassign->GetRHS()->GetOpCode() == OP_iread) { // rhs is iread IreadNode *ireadnode = static_cast(iassign->GetRHS()); -- Gitee From 59181af22fbb006ca005504218cb33e6437121d5 Mon Sep 17 00:00:00 2001 From: Fred Chow Date: Mon, 19 Sep 2022 22:18:36 -0700 Subject: [PATCH 2/4] Enabled raising back to while and if stmts under --profileUse and fixed -O3 issues that resulted from this change Avoided creating critical edge in lowering IF stmts. Auto-vectorization is no longer disabled under --profileUse. --- src/mapleall/maple_me/src/lfo_loop_vec.cpp | 4 + src/mapleall/maple_me/src/me_cfg.cpp | 43 ++++--- .../maple_me/src/me_critical_edge.cpp | 9 -- .../maple_me/src/me_loop_unrolling.cpp | 6 + .../maple_me/src/me_value_range_prop.cpp | 3 +- src/mapleall/maple_me/src/optimizeCFG.cpp | 4 - src/mapleall/maple_me/src/pme_emit.cpp | 8 ++ src/mapleall/maple_me/src/pme_mir_lower.cpp | 112 +++++------------- src/mapleall/maple_me/src/seqvec.cpp | 5 +- 9 files changed, 77 insertions(+), 117 deletions(-) diff --git a/src/mapleall/maple_me/src/lfo_loop_vec.cpp b/src/mapleall/maple_me/src/lfo_loop_vec.cpp index b44c1fe97b..189a49ec6e 100644 --- a/src/mapleall/maple_me/src/lfo_loop_vec.cpp +++ b/src/mapleall/maple_me/src/lfo_loop_vec.cpp @@ -1528,6 +1528,10 @@ void LoopVectorization::VectorizeReductionStmt(StmtNode *stmt, LoopTransPlan *tp StmtNode *copyStmt = stmt->CloneTree(*codeMPAlloc); IntrinsicopNode *intrnvecSum = GenSumVecStmt(regReadlhsvec, regReadlhsvec->GetPrimType()); copyStmt->GetRHS()->SetOpnd(intrnvecSum, 1); + FuncProfInfo *profData = mirFunc->GetFuncProfData(); + if (profData) { + profData->CopyStmtFreq(copyStmt->GetStmtID(), stmt->GetStmtID()); + } tp->vecInfo->afterLoopStmts.push_back(copyStmt); doloopbody->RemoveStmt(stmt); } diff --git a/src/mapleall/maple_me/src/me_cfg.cpp b/src/mapleall/maple_me/src/me_cfg.cpp index 202d51f7fe..a0d57c6efe 100644 --- a/src/mapleall/maple_me/src/me_cfg.cpp +++ b/src/mapleall/maple_me/src/me_cfg.cpp @@ -1309,14 +1309,6 @@ void MeCFG::CreateBasicBlocks() { } case OP_dassign: { DassignNode *dass = static_cast(stmt); - // delete identity assignments inserted by LFO - if (dass->GetRHS()->GetOpCode() == OP_dread) { - DreadNode *dread = static_cast(dass->GetRHS()); - if (dass->GetStIdx() == dread->GetStIdx() && dass->GetFieldID() == dread->GetFieldID()) { - func.CurFunction()->GetBody()->RemoveStmt(stmt); - break; - } - } if (curBB->IsEmpty()) { curBB->SetFirst(stmt); } @@ -1539,8 +1531,8 @@ void MeCFG::CreateBasicBlocks() { SetBBTryNodeMap(*newBB, *tryStmt); } curBB = newBB; - } else if (func.GetPreMeFunc() && (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != - func.GetPreMeFunc()->label2WhileInfo.end())) { + } else if (func.GetPreMeFunc() && + (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != func.GetPreMeFunc()->label2WhileInfo.end())) { curBB->SetKind(kBBFallthru); BB *newBB = NewBasicBlock(); if (tryStmt != nullptr) { @@ -1625,6 +1617,10 @@ void MeCFG::CreateBasicBlocks() { lastBB->SetFirst(func.GetMIRModule().GetMIRBuilder()->CreateStmtReturn(nullptr)); lastBB->SetLast(lastBB->GetStmtNodes().begin().d()); lastBB->SetKindReturn(); + if (updateFreq) { + FuncProfInfo *funcData = func.GetMirFunc()->GetFuncProfData(); + funcData->stmtFreqs[lastBB->GetLast().GetStmtID()] = 0; + } } else if (lastBB->GetKind() == kBBUnknown) { lastBB->SetKindReturn(); lastBB->SetAttributes(kBBAttrIsExit); @@ -1878,15 +1874,25 @@ void MeCFG::ConstructEdgeFreqFromBBFreq() { if (fallthru->GetPred().size() == 1) { auto succ0Freq = fallthru->GetFrequency(); bb->PushBackSuccFreq(succ0Freq); - ASSERT(bb->GetFrequency() >= succ0Freq, "sanity check"); + ASSERT(bb->GetFrequency() + 1 >= succ0Freq, "sanity check"); bb->PushBackSuccFreq(bb->GetFrequency() - succ0Freq); } else if (targetBB->GetPred().size() == 1) { auto succ1Freq = targetBB->GetFrequency(); ASSERT(bb->GetFrequency() >= succ1Freq, "sanity check"); - bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); - bb->PushBackSuccFreq(succ1Freq); + if (bb->GetAttributes(kBBAttrWontExit) && bb->GetSuccFreq().size() == 1) { + // special case: WontExitAnalysis() has pushed 0 to bb->succFreq + bb->GetSuccFreq()[0] = bb->GetFrequency(); + bb->PushBackSuccFreq(0); + } else { + bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); + bb->PushBackSuccFreq(succ1Freq); + } + } else if (fallthru->GetFrequency() > targetBB->GetFrequency()) { + bb->PushBackSuccFreq(bb->GetFrequency()); + bb->PushBackSuccFreq(0); } else { - CHECK_FATAL(false, "ConstructEdgeFreqFromBBFreq::NYI critical edge"); + bb->PushBackSuccFreq(0); + bb->PushBackSuccFreq(bb->GetFrequency()); } } else if (bb->GetSucc().size() > 2) { // switch case, no critical edge is supposted @@ -1937,9 +1943,6 @@ void MeCFG::ConstructBBFreqFromStmtFreq() { ConstructEdgeFreqFromBBFreq(); // clear stmtFreqs since cfg frequency is create funcData->stmtFreqs.clear(); - - // set updateFrequency with true - updateFreq = true; } void MeCFG::ConstructStmtFreq() { @@ -2086,6 +2089,10 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { MemPool *meCfgMp = GetPhaseMemPool(); theCFG = meCfgMp->New(meCfgMp, f); f.SetTheCfg(theCFG); + + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + theCFG->SetUpdateCFGFreq(true); + } theCFG->CreateBasicBlocks(); if (theCFG->NumBBs() == 0) { /* there's no basicblock generated */ @@ -2104,7 +2111,7 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { } theCFG->Verify(); // construct bb freq from stmt freq - if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + if (theCFG->UpdateCFGFreq()) { theCFG->ConstructBBFreqFromStmtFreq(); if (theCFG->DumpIRProfileFile()) { std::string fileName = "after-mecfgbuild"; diff --git a/src/mapleall/maple_me/src/me_critical_edge.cpp b/src/mapleall/maple_me/src/me_critical_edge.cpp index fd97bb2452..dc52c67bc2 100644 --- a/src/mapleall/maple_me/src/me_critical_edge.cpp +++ b/src/mapleall/maple_me/src/me_critical_edge.cpp @@ -304,15 +304,6 @@ bool MESplitCEdge::PhaseRun(maple::MeFunction &f) { bool enableDebug = DEBUGFUNC_NEWPM(f); MeSplitCEdge mscedge = MeSplitCEdge(enableDebug); mscedge.SplitCriticalEdgeForMeFunc(f); - if (Options::profileUse) { - if ((f.IsPme() || f.IsLfo()) && f.GetPreMeFunc()) { - // new inserted BB break cached while/if label information and IR layout - f.GetPreMeFunc()->label2IfInfo.clear(); - f.GetPreMeFunc()->label2WhileInfo.clear(); - f.GetPreMeFunc()->pmeCreatedIfLabelSet.clear(); - f.GetPreMeFunc()->pmeCreatedWhileLabelSet.clear(); - } - } if (f.GetCfg()->UpdateCFGFreq() && (f.GetCfg()->DumpIRProfileFile())) { f.GetCfg()->DumpToFile("after-splitcriticaledge", false, true); } diff --git a/src/mapleall/maple_me/src/me_loop_unrolling.cpp b/src/mapleall/maple_me/src/me_loop_unrolling.cpp index 5e39fd56c1..4b85f7c4f6 100644 --- a/src/mapleall/maple_me/src/me_loop_unrolling.cpp +++ b/src/mapleall/maple_me/src/me_loop_unrolling.cpp @@ -487,9 +487,15 @@ bool LoopUnrolling::SplitCondGotoBB() { for (auto *bb : exitBB->GetSucc()) { bb->ReplacePred(exitBB, newCondGotoBB); + if (profValid) { + newCondGotoBB->GetSuccFreq().push_back(0); + } } exitBB->AddSucc(*newCondGotoBB); + if (profValid) { + exitBB->GetSuccFreq().push_back(0); + } exitBB->RemoveMeStmt(lastStmt); newCondGotoBB->InsertMeStmtLastBr(lastStmt); return true; diff --git a/src/mapleall/maple_me/src/me_value_range_prop.cpp b/src/mapleall/maple_me/src/me_value_range_prop.cpp index 39fd3e9567..3fe4cf6b9e 100644 --- a/src/mapleall/maple_me/src/me_value_range_prop.cpp +++ b/src/mapleall/maple_me/src/me_value_range_prop.cpp @@ -3980,13 +3980,12 @@ bool ValueRangePropagation::RemoveTheEdgeOfPredBB( uint64 edgeFreq = 0; if (func.GetCfg()->UpdateCFGFreq()) { edgeFreq = pred.GetSuccFreq()[index]; - ASSERT(bb.GetFrequency() >= edgeFreq, "sanity check"); } pred.RemoveSucc(bb); pred.AddSucc(*newBB, index); newBB->AddSucc(trueBranch); if (func.GetCfg()->UpdateCFGFreq()) { - if (bb.GetFrequency() > edgeFreq) { + if (bb.GetFrequency() >= edgeFreq) { bb.SetFrequency(static_cast(bb.GetFrequency() - edgeFreq)); bb.UpdateEdgeFreqs(); } diff --git a/src/mapleall/maple_me/src/optimizeCFG.cpp b/src/mapleall/maple_me/src/optimizeCFG.cpp index 0526e05e0c..a01ee97b68 100644 --- a/src/mapleall/maple_me/src/optimizeCFG.cpp +++ b/src/mapleall/maple_me/src/optimizeCFG.cpp @@ -1157,10 +1157,6 @@ bool OptimizeBB::RemoveSuccFromNoReturnBB() { cfg->GetCommonExitBB()->AddExit(*currBB); currBB->SetAttributes(kBBAttrIsExit); ResetBBRunAgain(); - // add edge Frequency to exitbb - if (cfg->UpdateCFGFreq()) { - currBB->PushBackSuccFreq(0); - } return true; } return false; diff --git a/src/mapleall/maple_me/src/pme_emit.cpp b/src/mapleall/maple_me/src/pme_emit.cpp index efd76f295e..f3e348931b 100644 --- a/src/mapleall/maple_me/src/pme_emit.cpp +++ b/src/mapleall/maple_me/src/pme_emit.cpp @@ -365,6 +365,10 @@ StmtNode* PreMeEmitter::EmitPreMeStmt(MeStmt *meStmt, BaseNode *parent) { switch (meStmt->GetOp()) { case OP_dassign: { DassignMeStmt *dsmestmt = static_cast(meStmt); + if (dsmestmt->GetRHS()->GetMeOp() == kMeOpVar && + static_cast(dsmestmt->GetRHS())->GetOst() == dsmestmt->GetLHS()->GetOst()) { + return nullptr; // identity assignment introduced by LFO + } DassignNode *dass = codeMP->New(); MIRSymbol *sym = dsmestmt->GetLHS()->GetOst()->GetMIRSymbol(); dass->SetStIdx(sym->GetStIdx()); @@ -968,6 +972,10 @@ uint32 PreMeEmitter::Raise2PreMeIf(uint32 curJ, BlockNode *curBlk) { MeStmt *mestmt = curbb->GetFirstMe(); while (mestmt->GetOp() != OP_brfalse && mestmt->GetOp() != OP_brtrue) { StmtNode *stmt = EmitPreMeStmt(mestmt, curBlk); + if (stmt == nullptr) { + mestmt = mestmt->GetNext(); + continue; + } UpdateStmtInfo(*mestmt, *stmt, *curBlk, curbb->GetFrequency()); curBlk->AddStatement(stmt); if (GetFuncProfData() && diff --git a/src/mapleall/maple_me/src/pme_mir_lower.cpp b/src/mapleall/maple_me/src/pme_mir_lower.cpp index e2a45d62d4..048ff7464a 100644 --- a/src/mapleall/maple_me/src/pme_mir_lower.cpp +++ b/src/mapleall/maple_me/src/pme_mir_lower.cpp @@ -121,80 +121,6 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { if (GetFuncProfData()) { GetFuncProfData()->CopyStmtFreq(evalstmt->GetStmtID(), ifstmt.GetStmtID()); } - } else if (elseempty) { - // brfalse - // - // label - CondGotoNode *brfalsestmt = mirModule.CurFuncCodeMemPool()->New(OP_brfalse); - brfalsestmt->SetOpnd(ifstmt.Opnd(), 0); - brfalsestmt->SetSrcPos(ifstmt.GetSrcPos()); - LabelIdx endlabelidx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('e'); - mirFunc->GetLabelTab()->AddToStringLabelMap(endlabelidx); - if (canRaiseBack) { - preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); - } - PreMeIfInfo *ifInfo = preMeFunc->pmemp->New(); - brfalsestmt->SetOffset(endlabelidx); - blk->AddStatement(brfalsestmt); - // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(brfalsestmt->GetStmtID(), ifstmt.GetStmtID()); - } - blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); - - LabelNode *labstmt = mirModule.CurFuncCodeMemPool()->New(); - labstmt->SetLabelIdx(endlabelidx); - ifInfo->endLabel = endlabelidx; - if (canRaiseBack) { - preMeFunc->label2IfInfo.insert(std::make_pair(endlabelidx, ifInfo)); - } - SrcPosition pos = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetThenPart()->GetSrcPos()); - labstmt->SetSrcPos(pos); - blk->AddStatement(labstmt); - // set stmtfreqs - if (GetFuncProfData()) { - ASSERT(GetFuncProfData()->GetStmtFreq(ifstmt.GetThenPart()->GetStmtID()) >= 0, "sanity check"); - int64_t freq = GetFuncProfData()->GetStmtFreq(ifstmt.GetStmtID()) - - GetFuncProfData()->GetStmtFreq(ifstmt.GetThenPart()->GetStmtID()); - GetFuncProfData()->SetStmtFreq(labstmt->GetStmtID(), freq); - } - } else if (thenempty) { - // brtrue - // - // label - CondGotoNode *brtruestmt = mirModule.CurFuncCodeMemPool()->New(OP_brtrue); - brtruestmt->SetOpnd(ifstmt.Opnd(), 0); - brtruestmt->SetSrcPos(ifstmt.GetSrcPos()); - LabelIdx endlabelidx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('e'); - if (canRaiseBack) { - preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); - } - PreMeIfInfo *ifInfo = preMeFunc->pmemp->New(); - mirFunc->GetLabelTab()->AddToStringLabelMap(endlabelidx); - brtruestmt->SetOffset(endlabelidx); - blk->AddStatement(brtruestmt); - - // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(brtruestmt->GetStmtID(), ifstmt.GetStmtID()); - } - blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); - LabelNode *labstmt = mirModule.CurFuncCodeMemPool()->New(); - labstmt->SetLabelIdx(endlabelidx); - ifInfo->endLabel = endlabelidx; - if (canRaiseBack) { - preMeFunc->label2IfInfo.insert(std::make_pair(endlabelidx, ifInfo)); - } - SrcPosition pos = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); - labstmt->SetSrcPos(pos); - blk->AddStatement(labstmt); - // set stmtfreqs - if (GetFuncProfData()) { - ASSERT(GetFuncProfData()->GetStmtFreq(ifstmt.GetElsePart()->GetStmtID()) > 0, "sanity check"); - int64_t freq = GetFuncProfData()->GetStmtFreq(ifstmt.GetStmtID()) - - GetFuncProfData()->GetStmtFreq(ifstmt.GetElsePart()->GetStmtID()); - GetFuncProfData()->SetStmtFreq(labstmt->GetStmtID(), freq); - } } else { // brfalse // @@ -222,8 +148,11 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { preMeFunc->label2IfInfo.insert(std::make_pair(elselabelidx, ifInfo)); } - blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); - bool fallthru_from_then = !OpCodeNoFallThrough(ifstmt.GetThenPart()->GetLast()->GetOpCode()); + bool fallthru_from_then = true; + if (!thenempty) { + blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); + fallthru_from_then = !OpCodeNoFallThrough(ifstmt.GetThenPart()->GetLast()->GetOpCode()); + } LabelIdx endlabelidx = 0; if (fallthru_from_then) { @@ -237,7 +166,11 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { blk->AddStatement(gotostmt); // set stmtfreqs if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(gotostmt->GetStmtID(), ifstmt.GetThenPart()->GetStmtID()); + if (!thenempty) { + GetFuncProfData()->CopyStmtFreq(gotostmt->GetStmtID(), ifstmt.GetThenPart()->GetStmtID()); + } else { + GetFuncProfData()->SetStmtFreq(gotostmt->GetStmtID(), 0); + } } } @@ -247,17 +180,21 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { labstmt->SetSrcPos(pos); blk->AddStatement(labstmt); - blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); + if (!elseempty) { + blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); + } if (fallthru_from_then) { labstmt = mirModule.CurFuncCodeMemPool()->New(); labstmt->SetLabelIdx(endlabelidx); - SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); - labstmt->SetSrcPos(position); + if (!elseempty) { + SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); + labstmt->SetSrcPos(position); + } blk->AddStatement(labstmt); // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(labstmt->GetStmtID(), ifstmt.GetElsePart()->GetStmtID()); + if (!elseempty && GetFuncProfData()) { + GetFuncProfData()->CopyStmtFreq(labstmt->GetStmtID(), ifstmt.GetStmtID()); } } if (endlabelidx == 0) { // create end label @@ -266,7 +203,7 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); } LabelNode *endlabelnode = mirbuilder->CreateStmtLabel(endlabelidx); - SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); + SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetSrcPos()); endlabelnode->SetSrcPos(position); blk->AddStatement(endlabelnode); // set stmtfreqs @@ -276,5 +213,14 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { } ifInfo->endLabel = endlabelidx; } + // generate extra label to avoid critical edge + LabelIdx extraLabelIdx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('x'); + preMeFunc->SetIfLabelCreatedByPreMe(extraLabelIdx); + LabelNode *extraLabelNode = mirbuilder->CreateStmtLabel(extraLabelIdx); + blk->AddStatement(extraLabelNode); + // set stmtfreqs + if (GetFuncProfData()) { + GetFuncProfData()->CopyStmtFreq(extraLabelNode->GetStmtID(), ifstmt.GetStmtID()); + } return blk; } diff --git a/src/mapleall/maple_me/src/seqvec.cpp b/src/mapleall/maple_me/src/seqvec.cpp index d9788f6c0a..1feb9978d1 100644 --- a/src/mapleall/maple_me/src/seqvec.cpp +++ b/src/mapleall/maple_me/src/seqvec.cpp @@ -572,7 +572,10 @@ void SeqVectorize::MergeIassigns(MapleVector &cands) { blockParent->InsertBefore(iassign, dupScalarStmt); iassign->SetRHS(regreadNode); if (Options::profileUse && mirFunc->GetFuncProfData()) { - mirFunc->GetFuncProfData()->CopyStmtFreq(dupScalarStmt->GetStmtID(), iassign->GetStmtID()); + std::unordered_map &stmtFreqs = mirFunc->GetFuncProfData()->stmtFreqs; + if (stmtFreqs.find(iassign->GetStmtID()) != stmtFreqs.end()) { + mirFunc->GetFuncProfData()->CopyStmtFreq(dupScalarStmt->GetStmtID(), iassign->GetStmtID()); + } } } else if (iassign->GetRHS()->GetOpCode() == OP_iread) { // rhs is iread -- Gitee From 23ebfcd46e3ee47cc05cf5f6a987320d5a9c0184 Mon Sep 17 00:00:00 2001 From: Fred Chow Date: Mon, 19 Sep 2022 22:18:36 -0700 Subject: [PATCH 3/4] Enabled raising back to while and if stmts under --profileUse and fixed -O3 issues that resulted from this change Avoided creating critical edge in lowering IF stmts. Auto-vectorization is no longer disabled under --profileUse. --- src/mapleall/maple_me/src/lfo_loop_vec.cpp | 4 + src/mapleall/maple_me/src/me_cfg.cpp | 43 ++++--- .../maple_me/src/me_critical_edge.cpp | 9 -- .../maple_me/src/me_loop_unrolling.cpp | 6 + .../maple_me/src/me_value_range_prop.cpp | 3 +- src/mapleall/maple_me/src/optimizeCFG.cpp | 4 - src/mapleall/maple_me/src/pme_emit.cpp | 8 ++ src/mapleall/maple_me/src/pme_mir_lower.cpp | 112 +++++------------- src/mapleall/maple_me/src/seqvec.cpp | 5 +- 9 files changed, 77 insertions(+), 117 deletions(-) diff --git a/src/mapleall/maple_me/src/lfo_loop_vec.cpp b/src/mapleall/maple_me/src/lfo_loop_vec.cpp index b44c1fe97b..189a49ec6e 100644 --- a/src/mapleall/maple_me/src/lfo_loop_vec.cpp +++ b/src/mapleall/maple_me/src/lfo_loop_vec.cpp @@ -1528,6 +1528,10 @@ void LoopVectorization::VectorizeReductionStmt(StmtNode *stmt, LoopTransPlan *tp StmtNode *copyStmt = stmt->CloneTree(*codeMPAlloc); IntrinsicopNode *intrnvecSum = GenSumVecStmt(regReadlhsvec, regReadlhsvec->GetPrimType()); copyStmt->GetRHS()->SetOpnd(intrnvecSum, 1); + FuncProfInfo *profData = mirFunc->GetFuncProfData(); + if (profData) { + profData->CopyStmtFreq(copyStmt->GetStmtID(), stmt->GetStmtID()); + } tp->vecInfo->afterLoopStmts.push_back(copyStmt); doloopbody->RemoveStmt(stmt); } diff --git a/src/mapleall/maple_me/src/me_cfg.cpp b/src/mapleall/maple_me/src/me_cfg.cpp index 202d51f7fe..a0d57c6efe 100644 --- a/src/mapleall/maple_me/src/me_cfg.cpp +++ b/src/mapleall/maple_me/src/me_cfg.cpp @@ -1309,14 +1309,6 @@ void MeCFG::CreateBasicBlocks() { } case OP_dassign: { DassignNode *dass = static_cast(stmt); - // delete identity assignments inserted by LFO - if (dass->GetRHS()->GetOpCode() == OP_dread) { - DreadNode *dread = static_cast(dass->GetRHS()); - if (dass->GetStIdx() == dread->GetStIdx() && dass->GetFieldID() == dread->GetFieldID()) { - func.CurFunction()->GetBody()->RemoveStmt(stmt); - break; - } - } if (curBB->IsEmpty()) { curBB->SetFirst(stmt); } @@ -1539,8 +1531,8 @@ void MeCFG::CreateBasicBlocks() { SetBBTryNodeMap(*newBB, *tryStmt); } curBB = newBB; - } else if (func.GetPreMeFunc() && (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != - func.GetPreMeFunc()->label2WhileInfo.end())) { + } else if (func.GetPreMeFunc() && + (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != func.GetPreMeFunc()->label2WhileInfo.end())) { curBB->SetKind(kBBFallthru); BB *newBB = NewBasicBlock(); if (tryStmt != nullptr) { @@ -1625,6 +1617,10 @@ void MeCFG::CreateBasicBlocks() { lastBB->SetFirst(func.GetMIRModule().GetMIRBuilder()->CreateStmtReturn(nullptr)); lastBB->SetLast(lastBB->GetStmtNodes().begin().d()); lastBB->SetKindReturn(); + if (updateFreq) { + FuncProfInfo *funcData = func.GetMirFunc()->GetFuncProfData(); + funcData->stmtFreqs[lastBB->GetLast().GetStmtID()] = 0; + } } else if (lastBB->GetKind() == kBBUnknown) { lastBB->SetKindReturn(); lastBB->SetAttributes(kBBAttrIsExit); @@ -1878,15 +1874,25 @@ void MeCFG::ConstructEdgeFreqFromBBFreq() { if (fallthru->GetPred().size() == 1) { auto succ0Freq = fallthru->GetFrequency(); bb->PushBackSuccFreq(succ0Freq); - ASSERT(bb->GetFrequency() >= succ0Freq, "sanity check"); + ASSERT(bb->GetFrequency() + 1 >= succ0Freq, "sanity check"); bb->PushBackSuccFreq(bb->GetFrequency() - succ0Freq); } else if (targetBB->GetPred().size() == 1) { auto succ1Freq = targetBB->GetFrequency(); ASSERT(bb->GetFrequency() >= succ1Freq, "sanity check"); - bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); - bb->PushBackSuccFreq(succ1Freq); + if (bb->GetAttributes(kBBAttrWontExit) && bb->GetSuccFreq().size() == 1) { + // special case: WontExitAnalysis() has pushed 0 to bb->succFreq + bb->GetSuccFreq()[0] = bb->GetFrequency(); + bb->PushBackSuccFreq(0); + } else { + bb->PushBackSuccFreq(bb->GetFrequency() - succ1Freq); + bb->PushBackSuccFreq(succ1Freq); + } + } else if (fallthru->GetFrequency() > targetBB->GetFrequency()) { + bb->PushBackSuccFreq(bb->GetFrequency()); + bb->PushBackSuccFreq(0); } else { - CHECK_FATAL(false, "ConstructEdgeFreqFromBBFreq::NYI critical edge"); + bb->PushBackSuccFreq(0); + bb->PushBackSuccFreq(bb->GetFrequency()); } } else if (bb->GetSucc().size() > 2) { // switch case, no critical edge is supposted @@ -1937,9 +1943,6 @@ void MeCFG::ConstructBBFreqFromStmtFreq() { ConstructEdgeFreqFromBBFreq(); // clear stmtFreqs since cfg frequency is create funcData->stmtFreqs.clear(); - - // set updateFrequency with true - updateFreq = true; } void MeCFG::ConstructStmtFreq() { @@ -2086,6 +2089,10 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { MemPool *meCfgMp = GetPhaseMemPool(); theCFG = meCfgMp->New(meCfgMp, f); f.SetTheCfg(theCFG); + + if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + theCFG->SetUpdateCFGFreq(true); + } theCFG->CreateBasicBlocks(); if (theCFG->NumBBs() == 0) { /* there's no basicblock generated */ @@ -2104,7 +2111,7 @@ bool MEMeCfg::PhaseRun(MeFunction &f) { } theCFG->Verify(); // construct bb freq from stmt freq - if (Options::profileUse && f.GetMirFunc()->GetFuncProfData()) { + if (theCFG->UpdateCFGFreq()) { theCFG->ConstructBBFreqFromStmtFreq(); if (theCFG->DumpIRProfileFile()) { std::string fileName = "after-mecfgbuild"; diff --git a/src/mapleall/maple_me/src/me_critical_edge.cpp b/src/mapleall/maple_me/src/me_critical_edge.cpp index fd97bb2452..dc52c67bc2 100644 --- a/src/mapleall/maple_me/src/me_critical_edge.cpp +++ b/src/mapleall/maple_me/src/me_critical_edge.cpp @@ -304,15 +304,6 @@ bool MESplitCEdge::PhaseRun(maple::MeFunction &f) { bool enableDebug = DEBUGFUNC_NEWPM(f); MeSplitCEdge mscedge = MeSplitCEdge(enableDebug); mscedge.SplitCriticalEdgeForMeFunc(f); - if (Options::profileUse) { - if ((f.IsPme() || f.IsLfo()) && f.GetPreMeFunc()) { - // new inserted BB break cached while/if label information and IR layout - f.GetPreMeFunc()->label2IfInfo.clear(); - f.GetPreMeFunc()->label2WhileInfo.clear(); - f.GetPreMeFunc()->pmeCreatedIfLabelSet.clear(); - f.GetPreMeFunc()->pmeCreatedWhileLabelSet.clear(); - } - } if (f.GetCfg()->UpdateCFGFreq() && (f.GetCfg()->DumpIRProfileFile())) { f.GetCfg()->DumpToFile("after-splitcriticaledge", false, true); } diff --git a/src/mapleall/maple_me/src/me_loop_unrolling.cpp b/src/mapleall/maple_me/src/me_loop_unrolling.cpp index 5e39fd56c1..4b85f7c4f6 100644 --- a/src/mapleall/maple_me/src/me_loop_unrolling.cpp +++ b/src/mapleall/maple_me/src/me_loop_unrolling.cpp @@ -487,9 +487,15 @@ bool LoopUnrolling::SplitCondGotoBB() { for (auto *bb : exitBB->GetSucc()) { bb->ReplacePred(exitBB, newCondGotoBB); + if (profValid) { + newCondGotoBB->GetSuccFreq().push_back(0); + } } exitBB->AddSucc(*newCondGotoBB); + if (profValid) { + exitBB->GetSuccFreq().push_back(0); + } exitBB->RemoveMeStmt(lastStmt); newCondGotoBB->InsertMeStmtLastBr(lastStmt); return true; diff --git a/src/mapleall/maple_me/src/me_value_range_prop.cpp b/src/mapleall/maple_me/src/me_value_range_prop.cpp index 94b0311cfd..bcfa6a8a5f 100644 --- a/src/mapleall/maple_me/src/me_value_range_prop.cpp +++ b/src/mapleall/maple_me/src/me_value_range_prop.cpp @@ -3981,13 +3981,12 @@ bool ValueRangePropagation::RemoveTheEdgeOfPredBB( uint64 edgeFreq = 0; if (func.GetCfg()->UpdateCFGFreq()) { edgeFreq = pred.GetSuccFreq()[index]; - ASSERT(bb.GetFrequency() >= edgeFreq, "sanity check"); } pred.RemoveSucc(bb); pred.AddSucc(*newBB, index); newBB->AddSucc(trueBranch); if (func.GetCfg()->UpdateCFGFreq()) { - if (bb.GetFrequency() > edgeFreq) { + if (bb.GetFrequency() >= edgeFreq) { bb.SetFrequency(static_cast(bb.GetFrequency() - edgeFreq)); bb.UpdateEdgeFreqs(); } diff --git a/src/mapleall/maple_me/src/optimizeCFG.cpp b/src/mapleall/maple_me/src/optimizeCFG.cpp index 0526e05e0c..a01ee97b68 100644 --- a/src/mapleall/maple_me/src/optimizeCFG.cpp +++ b/src/mapleall/maple_me/src/optimizeCFG.cpp @@ -1157,10 +1157,6 @@ bool OptimizeBB::RemoveSuccFromNoReturnBB() { cfg->GetCommonExitBB()->AddExit(*currBB); currBB->SetAttributes(kBBAttrIsExit); ResetBBRunAgain(); - // add edge Frequency to exitbb - if (cfg->UpdateCFGFreq()) { - currBB->PushBackSuccFreq(0); - } return true; } return false; diff --git a/src/mapleall/maple_me/src/pme_emit.cpp b/src/mapleall/maple_me/src/pme_emit.cpp index efd76f295e..f3e348931b 100644 --- a/src/mapleall/maple_me/src/pme_emit.cpp +++ b/src/mapleall/maple_me/src/pme_emit.cpp @@ -365,6 +365,10 @@ StmtNode* PreMeEmitter::EmitPreMeStmt(MeStmt *meStmt, BaseNode *parent) { switch (meStmt->GetOp()) { case OP_dassign: { DassignMeStmt *dsmestmt = static_cast(meStmt); + if (dsmestmt->GetRHS()->GetMeOp() == kMeOpVar && + static_cast(dsmestmt->GetRHS())->GetOst() == dsmestmt->GetLHS()->GetOst()) { + return nullptr; // identity assignment introduced by LFO + } DassignNode *dass = codeMP->New(); MIRSymbol *sym = dsmestmt->GetLHS()->GetOst()->GetMIRSymbol(); dass->SetStIdx(sym->GetStIdx()); @@ -968,6 +972,10 @@ uint32 PreMeEmitter::Raise2PreMeIf(uint32 curJ, BlockNode *curBlk) { MeStmt *mestmt = curbb->GetFirstMe(); while (mestmt->GetOp() != OP_brfalse && mestmt->GetOp() != OP_brtrue) { StmtNode *stmt = EmitPreMeStmt(mestmt, curBlk); + if (stmt == nullptr) { + mestmt = mestmt->GetNext(); + continue; + } UpdateStmtInfo(*mestmt, *stmt, *curBlk, curbb->GetFrequency()); curBlk->AddStatement(stmt); if (GetFuncProfData() && diff --git a/src/mapleall/maple_me/src/pme_mir_lower.cpp b/src/mapleall/maple_me/src/pme_mir_lower.cpp index e2a45d62d4..048ff7464a 100644 --- a/src/mapleall/maple_me/src/pme_mir_lower.cpp +++ b/src/mapleall/maple_me/src/pme_mir_lower.cpp @@ -121,80 +121,6 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { if (GetFuncProfData()) { GetFuncProfData()->CopyStmtFreq(evalstmt->GetStmtID(), ifstmt.GetStmtID()); } - } else if (elseempty) { - // brfalse - // - // label - CondGotoNode *brfalsestmt = mirModule.CurFuncCodeMemPool()->New(OP_brfalse); - brfalsestmt->SetOpnd(ifstmt.Opnd(), 0); - brfalsestmt->SetSrcPos(ifstmt.GetSrcPos()); - LabelIdx endlabelidx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('e'); - mirFunc->GetLabelTab()->AddToStringLabelMap(endlabelidx); - if (canRaiseBack) { - preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); - } - PreMeIfInfo *ifInfo = preMeFunc->pmemp->New(); - brfalsestmt->SetOffset(endlabelidx); - blk->AddStatement(brfalsestmt); - // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(brfalsestmt->GetStmtID(), ifstmt.GetStmtID()); - } - blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); - - LabelNode *labstmt = mirModule.CurFuncCodeMemPool()->New(); - labstmt->SetLabelIdx(endlabelidx); - ifInfo->endLabel = endlabelidx; - if (canRaiseBack) { - preMeFunc->label2IfInfo.insert(std::make_pair(endlabelidx, ifInfo)); - } - SrcPosition pos = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetThenPart()->GetSrcPos()); - labstmt->SetSrcPos(pos); - blk->AddStatement(labstmt); - // set stmtfreqs - if (GetFuncProfData()) { - ASSERT(GetFuncProfData()->GetStmtFreq(ifstmt.GetThenPart()->GetStmtID()) >= 0, "sanity check"); - int64_t freq = GetFuncProfData()->GetStmtFreq(ifstmt.GetStmtID()) - - GetFuncProfData()->GetStmtFreq(ifstmt.GetThenPart()->GetStmtID()); - GetFuncProfData()->SetStmtFreq(labstmt->GetStmtID(), freq); - } - } else if (thenempty) { - // brtrue - // - // label - CondGotoNode *brtruestmt = mirModule.CurFuncCodeMemPool()->New(OP_brtrue); - brtruestmt->SetOpnd(ifstmt.Opnd(), 0); - brtruestmt->SetSrcPos(ifstmt.GetSrcPos()); - LabelIdx endlabelidx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('e'); - if (canRaiseBack) { - preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); - } - PreMeIfInfo *ifInfo = preMeFunc->pmemp->New(); - mirFunc->GetLabelTab()->AddToStringLabelMap(endlabelidx); - brtruestmt->SetOffset(endlabelidx); - blk->AddStatement(brtruestmt); - - // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(brtruestmt->GetStmtID(), ifstmt.GetStmtID()); - } - blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); - LabelNode *labstmt = mirModule.CurFuncCodeMemPool()->New(); - labstmt->SetLabelIdx(endlabelidx); - ifInfo->endLabel = endlabelidx; - if (canRaiseBack) { - preMeFunc->label2IfInfo.insert(std::make_pair(endlabelidx, ifInfo)); - } - SrcPosition pos = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); - labstmt->SetSrcPos(pos); - blk->AddStatement(labstmt); - // set stmtfreqs - if (GetFuncProfData()) { - ASSERT(GetFuncProfData()->GetStmtFreq(ifstmt.GetElsePart()->GetStmtID()) > 0, "sanity check"); - int64_t freq = GetFuncProfData()->GetStmtFreq(ifstmt.GetStmtID()) - - GetFuncProfData()->GetStmtFreq(ifstmt.GetElsePart()->GetStmtID()); - GetFuncProfData()->SetStmtFreq(labstmt->GetStmtID(), freq); - } } else { // brfalse // @@ -222,8 +148,11 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { preMeFunc->label2IfInfo.insert(std::make_pair(elselabelidx, ifInfo)); } - blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); - bool fallthru_from_then = !OpCodeNoFallThrough(ifstmt.GetThenPart()->GetLast()->GetOpCode()); + bool fallthru_from_then = true; + if (!thenempty) { + blk->AppendStatementsFromBlock(*ifstmt.GetThenPart()); + fallthru_from_then = !OpCodeNoFallThrough(ifstmt.GetThenPart()->GetLast()->GetOpCode()); + } LabelIdx endlabelidx = 0; if (fallthru_from_then) { @@ -237,7 +166,11 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { blk->AddStatement(gotostmt); // set stmtfreqs if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(gotostmt->GetStmtID(), ifstmt.GetThenPart()->GetStmtID()); + if (!thenempty) { + GetFuncProfData()->CopyStmtFreq(gotostmt->GetStmtID(), ifstmt.GetThenPart()->GetStmtID()); + } else { + GetFuncProfData()->SetStmtFreq(gotostmt->GetStmtID(), 0); + } } } @@ -247,17 +180,21 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { labstmt->SetSrcPos(pos); blk->AddStatement(labstmt); - blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); + if (!elseempty) { + blk->AppendStatementsFromBlock(*ifstmt.GetElsePart()); + } if (fallthru_from_then) { labstmt = mirModule.CurFuncCodeMemPool()->New(); labstmt->SetLabelIdx(endlabelidx); - SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); - labstmt->SetSrcPos(position); + if (!elseempty) { + SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); + labstmt->SetSrcPos(position); + } blk->AddStatement(labstmt); // set stmtfreqs - if (GetFuncProfData()) { - GetFuncProfData()->CopyStmtFreq(labstmt->GetStmtID(), ifstmt.GetElsePart()->GetStmtID()); + if (!elseempty && GetFuncProfData()) { + GetFuncProfData()->CopyStmtFreq(labstmt->GetStmtID(), ifstmt.GetStmtID()); } } if (endlabelidx == 0) { // create end label @@ -266,7 +203,7 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { preMeFunc->SetIfLabelCreatedByPreMe(endlabelidx); } LabelNode *endlabelnode = mirbuilder->CreateStmtLabel(endlabelidx); - SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetElsePart()->GetSrcPos()); + SrcPosition position = func->GetMirFunc()->GetScope()->GetScopeEndPos(ifstmt.GetSrcPos()); endlabelnode->SetSrcPos(position); blk->AddStatement(endlabelnode); // set stmtfreqs @@ -276,5 +213,14 @@ BlockNode *PreMeMIRLower::LowerIfStmt(IfStmtNode &ifstmt, bool recursive) { } ifInfo->endLabel = endlabelidx; } + // generate extra label to avoid critical edge + LabelIdx extraLabelIdx = mirFunc->GetLabelTab()->CreateLabelWithPrefix('x'); + preMeFunc->SetIfLabelCreatedByPreMe(extraLabelIdx); + LabelNode *extraLabelNode = mirbuilder->CreateStmtLabel(extraLabelIdx); + blk->AddStatement(extraLabelNode); + // set stmtfreqs + if (GetFuncProfData()) { + GetFuncProfData()->CopyStmtFreq(extraLabelNode->GetStmtID(), ifstmt.GetStmtID()); + } return blk; } diff --git a/src/mapleall/maple_me/src/seqvec.cpp b/src/mapleall/maple_me/src/seqvec.cpp index d9788f6c0a..1feb9978d1 100644 --- a/src/mapleall/maple_me/src/seqvec.cpp +++ b/src/mapleall/maple_me/src/seqvec.cpp @@ -572,7 +572,10 @@ void SeqVectorize::MergeIassigns(MapleVector &cands) { blockParent->InsertBefore(iassign, dupScalarStmt); iassign->SetRHS(regreadNode); if (Options::profileUse && mirFunc->GetFuncProfData()) { - mirFunc->GetFuncProfData()->CopyStmtFreq(dupScalarStmt->GetStmtID(), iassign->GetStmtID()); + std::unordered_map &stmtFreqs = mirFunc->GetFuncProfData()->stmtFreqs; + if (stmtFreqs.find(iassign->GetStmtID()) != stmtFreqs.end()) { + mirFunc->GetFuncProfData()->CopyStmtFreq(dupScalarStmt->GetStmtID(), iassign->GetStmtID()); + } } } else if (iassign->GetRHS()->GetOpCode() == OP_iread) { // rhs is iread -- Gitee From 6d06679c033a81ff56f5f88f0e4fec05ad146a6d Mon Sep 17 00:00:00 2001 From: Fred Chow Date: Mon, 26 Sep 2022 08:11:38 -0700 Subject: [PATCH 4/4] fixed warnings --- src/mapleall/maple_me/src/me_cfg.cpp | 5 +++-- src/mapleall/maple_me/src/pme_emit.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mapleall/maple_me/src/me_cfg.cpp b/src/mapleall/maple_me/src/me_cfg.cpp index a0d57c6efe..4b3547b7a7 100644 --- a/src/mapleall/maple_me/src/me_cfg.cpp +++ b/src/mapleall/maple_me/src/me_cfg.cpp @@ -1531,8 +1531,9 @@ void MeCFG::CreateBasicBlocks() { SetBBTryNodeMap(*newBB, *tryStmt); } curBB = newBB; - } else if (func.GetPreMeFunc() && - (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != func.GetPreMeFunc()->label2WhileInfo.end())) { + } else if (func.GetPreMeFunc() && + (func.GetPreMeFunc()->label2WhileInfo.find(labelIdx) != + func.GetPreMeFunc()->label2WhileInfo.end())) { curBB->SetKind(kBBFallthru); BB *newBB = NewBasicBlock(); if (tryStmt != nullptr) { diff --git a/src/mapleall/maple_me/src/pme_emit.cpp b/src/mapleall/maple_me/src/pme_emit.cpp index f3e348931b..928446b366 100644 --- a/src/mapleall/maple_me/src/pme_emit.cpp +++ b/src/mapleall/maple_me/src/pme_emit.cpp @@ -365,7 +365,7 @@ StmtNode* PreMeEmitter::EmitPreMeStmt(MeStmt *meStmt, BaseNode *parent) { switch (meStmt->GetOp()) { case OP_dassign: { DassignMeStmt *dsmestmt = static_cast(meStmt); - if (dsmestmt->GetRHS()->GetMeOp() == kMeOpVar && + if (dsmestmt->GetRHS()->GetMeOp() == kMeOpVar && static_cast(dsmestmt->GetRHS())->GetOst() == dsmestmt->GetLHS()->GetOst()) { return nullptr; // identity assignment introduced by LFO } -- Gitee