diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h index e23bf815ce1c07ed1e5379ec35b76f05d5195192..572a252766286c22640b05fe7c48e49653fcd001 100644 --- a/llvm/include/llvm/ADT/DenseMap.h +++ b/llvm/include/llvm/ADT/DenseMap.h @@ -141,10 +141,15 @@ public: setNumTombstones(0); } + /// Return true if the specified key is in the map, false otherwise. + bool contains(const_arg_type_t Val) const { + const BucketT *TheBucket; + return LookupBucketFor(Val, TheBucket); + } + /// Return 1 if the specified key is in the map, 0 otherwise. size_type count(const_arg_type_t Val) const { - const BucketT *TheBucket; - return LookupBucketFor(Val, TheBucket) ? 1 : 0; + return contains(Val) ? 1 : 0; } iterator find(const_arg_type_t Val) { diff --git a/llvm/include/llvm/ADT/MapVector.h b/llvm/include/llvm/ADT/MapVector.h index 9d908f3af4ed0d447c53bb1abcd4c71655166b1c..a129b63f8be7a70e91fd7e71ffdcaa6e7b603f16 100644 --- a/llvm/include/llvm/ADT/MapVector.h +++ b/llvm/include/llvm/ADT/MapVector.h @@ -140,10 +140,9 @@ public: return std::make_pair(begin() + I, false); } - size_type count(const KeyT &Key) const { - typename MapType::const_iterator Pos = Map.find(Key); - return Pos == Map.end()? 0 : 1; - } + bool contains(const KeyT &Key) const { return Map.find(Key) != Map.end(); } + + size_type count(const KeyT &Key) const { return contains(Key) ? 1 : 0; } iterator find(const KeyT &Key) { typename MapType::const_iterator Pos = Map.find(Key); diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h index 443714c02389fc7d8d020591cd23a8bd1adebf93..17e81bcc2ab941bf5993bbba562f8602fefe8fe8 100644 --- a/llvm/include/llvm/ADT/StringMap.h +++ b/llvm/include/llvm/ADT/StringMap.h @@ -249,8 +249,11 @@ public: /// if the key is not in the map. ValueTy &operator[](StringRef Key) { return try_emplace(Key).first->second; } + /// contains - Return true if the element is in the map, false otherwise. + bool contains(StringRef Key) const { return find(Key) != end(); } + /// count - Return 1 if the element is in the map, 0 otherwise. - size_type count(StringRef Key) const { return find(Key) == end() ? 0 : 1; } + size_type count(StringRef Key) const { return contains(Key) ? 1 : 0; } template size_type count(const StringMapEntry &MapEntry) const { diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index f0dbb8fc2c2dbf3f6830e116fce6315870333dc6..af8db60b1e8e5ea4baa203d918c2bb078e5c1a41 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -811,12 +811,12 @@ public: /// it. If an explicit branch to the fallthrough block is not allowed, /// set JumpToFallThrough to be false. Non-null return is a conservative /// answer. - MachineBasicBlock *getFallThrough(bool JumpToFallThrough = false); + MachineBasicBlock *getFallThrough(bool JumpToFallThrough = true); /// Return the fallthrough block if the block can implicitly /// transfer control to it's successor, whether by a branch or /// a fallthrough. Non-null return is a conservative answer. - MachineBasicBlock *getLogicalFallThrough() { return getFallThrough(true); } + MachineBasicBlock *getLogicalFallThrough() { return getFallThrough(false); } /// Return true if the block can implicitly transfer control to the /// block after it by falling off the end of it. This should return diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index 6361373ba71b8d44c0d7ca97f9dbd224b857f3b3..684f206731291b24c3f2c612221e93d49067001a 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -232,6 +232,9 @@ public: /// target default. CodeModel::Model getCodeModel() const { return CMModel; } + /// Returns the maximum code size possible under the code model. + uint64_t getMaxCodeSize() const; + /// Set the code model. void setCodeModel(CodeModel::Model CM) { CMModel = CM; } diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp index a5e9f2e97143c81cbdaf62e21ec1539b0c71f3fc..9e8fa23efe44e05e9d021e2e673bd4366b84d020 100644 --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -268,7 +268,9 @@ void llvm::avoidZeroOffsetLandingPad(MachineFunction &MF) { MachineBasicBlock::iterator MI = MBB.begin(); while (!MI->isEHLabel()) ++MI; - MF.getSubtarget().getInstrInfo()->insertNoop(MBB, MI); + MCInst Nop = MF.getSubtarget().getInstrInfo()->getNop(); + BuildMI(MBB, MI, DebugLoc(), + MF.getSubtarget().getInstrInfo()->get(Nop.getOpcode())); } } } diff --git a/llvm/lib/CodeGen/BranchRelaxation.cpp b/llvm/lib/CodeGen/BranchRelaxation.cpp index 016c81dc5aa4bf275927917bdc1d9735b7790aeb..5b967b075f16bb4ba77b9c92aa605756105da046 100644 --- a/llvm/lib/CodeGen/BranchRelaxation.cpp +++ b/llvm/lib/CodeGen/BranchRelaxation.cpp @@ -26,6 +26,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" #include #include #include @@ -84,6 +85,7 @@ class BranchRelaxation : public MachineFunctionPass { MachineFunction *MF; const TargetRegisterInfo *TRI; const TargetInstrInfo *TII; + const TargetMachine *TM; bool relaxBranchInstructions(); void scanFunction(); @@ -219,6 +221,11 @@ BranchRelaxation::createNewBlockAfter(MachineBasicBlock &OrigMBB, MachineBasicBlock *NewBB = MF->CreateMachineBasicBlock(BB); MF->insert(++OrigMBB.getIterator(), NewBB); + // Place the new block in the same section as OrigBB + NewBB->setSectionID(OrigMBB.getSectionID()); + NewBB->setIsEndSection(OrigMBB.isEndSection()); + OrigMBB.setIsEndSection(false); + // Insert an entry into BlockInfo to align it properly with the block numbers. BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo()); @@ -228,8 +235,9 @@ BranchRelaxation::createNewBlockAfter(MachineBasicBlock &OrigMBB, /// Split the basic block containing MI into two blocks, which are joined by /// an unconditional branch. Update data structures and renumber blocks to /// account for this change and returns the newly created block. -MachineBasicBlock *BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI, - MachineBasicBlock *DestBB) { +MachineBasicBlock * +BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI, + MachineBasicBlock *DestBB) { MachineBasicBlock *OrigBB = MI.getParent(); // Create a new MBB for the code after the OrigBB. @@ -237,6 +245,11 @@ MachineBasicBlock *BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI, MF->CreateMachineBasicBlock(OrigBB->getBasicBlock()); MF->insert(++OrigBB->getIterator(), NewBB); + // Place the new block in the same section as OrigBB. + NewBB->setSectionID(OrigBB->getSectionID()); + NewBB->setIsEndSection(OrigBB->isEndSection()); + OrigBB->setIsEndSection(false); + // Splice the instructions starting with MI over to NewBB. NewBB->splice(NewBB->end(), OrigBB, MI.getIterator(), OrigBB->end()); @@ -287,7 +300,12 @@ bool BranchRelaxation::isBlockInRange( int64_t BrOffset = getInstrOffset(MI); int64_t DestOffset = BlockInfo[DestBB.getNumber()].Offset; - if (TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - BrOffset)) + const MachineBasicBlock *SrcBB = MI.getParent(); + + if (TII->isBranchOffsetInRange(MI.getOpcode(), + SrcBB->getSectionID() != DestBB.getSectionID() + ? TM->getMaxCodeSize() + : DestOffset - BrOffset)) return true; LLVM_DEBUG(dbgs() << "Out of range branch to destination " @@ -449,7 +467,10 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) { int64_t DestOffset = BlockInfo[DestBB->getNumber()].Offset; int64_t SrcOffset = getInstrOffset(MI); - assert(!TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - SrcOffset)); + assert(!TII->isBranchOffsetInRange( + MI.getOpcode(), MBB->getSectionID() != DestBB->getSectionID() + ? TM->getMaxCodeSize() + : DestOffset - SrcOffset)); BlockInfo[MBB->getNumber()].Size -= OldBrSize; @@ -479,9 +500,15 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) { // be erased. MachineBasicBlock *RestoreBB = createNewBlockAfter(MF->back(), DestBB->getBasicBlock()); + std::prev(RestoreBB->getIterator()) + ->setIsEndSection(RestoreBB->isEndSection()); + RestoreBB->setIsEndSection(false); TII->insertIndirectBranch(*BranchBB, *DestBB, *RestoreBB, DL, - DestOffset - SrcOffset, RS.get()); + BranchBB->getSectionID() != DestBB->getSectionID() + ? TM->getMaxCodeSize() + : DestOffset - SrcOffset, + RS.get()); BlockInfo[BranchBB->getNumber()].Size = computeBlockSize(*BranchBB); adjustBlockOffsets(*MBB); @@ -512,6 +539,11 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) { BlockInfo[RestoreBB->getNumber()].Size = computeBlockSize(*RestoreBB); // Update the offset starting from the previous block. adjustBlockOffsets(*PrevBB); + + // Fix up section information for RestoreBB and DestBB + RestoreBB->setSectionID(DestBB->getSectionID()); + RestoreBB->setIsBeginSection(DestBB->isBeginSection()); + DestBB->setIsBeginSection(false); } else { // Remove restore block if it's not required. MF->erase(RestoreBB); @@ -540,7 +572,7 @@ bool BranchRelaxation::relaxBranchInstructions() { // Unconditional branch destination might be unanalyzable, assume these // are OK. if (MachineBasicBlock *DestBB = TII->getBranchDestBlock(*Last)) { - if (!isBlockInRange(*Last, *DestBB)) { + if (!isBlockInRange(*Last, *DestBB) && !TII->isTailCall(*Last)) { fixupUnconditionalBranch(*Last); ++NumUnconditionalRelaxed; Changed = true; @@ -594,6 +626,7 @@ bool BranchRelaxation::runOnMachineFunction(MachineFunction &mf) { const TargetSubtargetInfo &ST = MF->getSubtarget(); TII = ST.getInstrInfo(); + TM = &MF->getTarget(); TRI = ST.getRegisterInfo(); if (TRI->trackLivenessAfterRegAlloc(*MF)) diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp index e79586550eb84df9ea47404a3ebb3cb3eeb74c8c..ed7da0b0d2d430b6aba17dd9828d02cc80199943 100644 --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -977,8 +977,8 @@ MachineBasicBlock *MachineBasicBlock::getFallThrough(bool JumpToFallThrough) { // If there is some explicit branch to the fallthrough block, it can obviously // reach, even though the branch should get folded to fall through implicitly. - if (!JumpToFallThrough && (MachineFunction::iterator(TBB) == Fallthrough || - MachineFunction::iterator(FBB) == Fallthrough)) + if (JumpToFallThrough && (MachineFunction::iterator(TBB) == Fallthrough || + MachineFunction::iterator(FBB) == Fallthrough)) return &*Fallthrough; // If it's an unconditional branch to some block not the fall through, it diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 72b7450d5a131581bd5434d19bb1b4df8b46ab8f..f0abf379619309598571be4ba8cca986aac91bc8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -28,6 +28,7 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -69,6 +70,10 @@ static cl::opt BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)")); +static cl::opt + BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), + cl::desc("Restrict range of B instructions (DEBUG)")); + AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI) : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP, AArch64::CATCHRET), @@ -184,7 +189,7 @@ static unsigned getBranchDisplacementBits(unsigned Opc) { default: llvm_unreachable("unexpected opcode!"); case AArch64::B: - return 64; + return BDisplacementBits; case AArch64::TBNZW: case AArch64::TBZW: case AArch64::TBNZX: @@ -229,6 +234,68 @@ AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const { } } +void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, + MachineBasicBlock &NewDestBB, + MachineBasicBlock &RestoreBB, + const DebugLoc &DL, + int64_t BrOffset, + RegScavenger *RS) const { + assert(RS && "RegScavenger required for long branching"); + assert(MBB.empty() && + "new block should be inserted for expanding unconditional branch"); + assert(MBB.pred_size() == 1); + assert(RestoreBB.empty() && + "restore block should be inserted for restoring clobbered registers"); + + auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) { + // Offsets outside of the signed 33-bit range are not supported for ADRP + + // ADD. + if (!isInt<33>(BrOffset)) + report_fatal_error( + "Branch offsets outside of the signed 33-bit range not supported"); + + BuildMI(MBB, MBB.end(), DL, get(AArch64::ADRP), Reg) + .addSym(DestBB.getSymbol(), AArch64II::MO_PAGE); + BuildMI(MBB, MBB.end(), DL, get(AArch64::ADDXri), Reg) + .addReg(Reg) + .addSym(DestBB.getSymbol(), AArch64II::MO_PAGEOFF | AArch64II::MO_NC) + .addImm(0); + BuildMI(MBB, MBB.end(), DL, get(AArch64::BR)).addReg(Reg); + }; + + RS->enterBasicBlockEnd(MBB); + Register Reg = RS->FindUnusedReg(&AArch64::GPR64RegClass); + + // If there's a free register, manually insert the indirect branch using it. + if (Reg != AArch64::NoRegister) { + buildIndirectBranch(Reg, NewDestBB); + RS->setRegUsed(Reg); + return; + } + + // Otherwise, spill and use X16. This briefly moves the stack pointer, making + // it incompatible with red zones. + AArch64FunctionInfo *AFI = MBB.getParent()->getInfo(); + if (!AFI || AFI->hasRedZone().value_or(true)) + report_fatal_error( + "Unable to insert indirect branch inside function that has red zone"); + + Reg = AArch64::X16; + BuildMI(MBB, MBB.end(), DL, get(AArch64::STRXpre)) + .addReg(AArch64::SP, RegState::Define) + .addReg(Reg) + .addReg(AArch64::SP) + .addImm(-16); + + buildIndirectBranch(Reg, RestoreBB); + + BuildMI(RestoreBB, RestoreBB.end(), DL, get(AArch64::LDRXpost)) + .addReg(AArch64::SP, RegState::Define) + .addReg(Reg, RegState::Define) + .addReg(AArch64::SP) + .addImm(16); +} + // Branch analysis. bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 861c96bced3ba9813419d1014fa3589363895d08..a2de91564d74d007978bdbdd8c0363ca29c0ee45 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -209,6 +209,11 @@ public: MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override; + void insertIndirectBranch(MachineBasicBlock &MBB, + MachineBasicBlock &NewDestBB, + MachineBasicBlock &RestoreBB, const DebugLoc &DL, + int64_t BrOffset, RegScavenger *RS) const override; + bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl &Cond, diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp index 8d1ad617889c9810a80d2d7d2c17fda396a94b1d..98e71b25b6226e22f916665f5105d7cabfc6a519 100644 --- a/llvm/lib/Target/TargetMachine.cpp +++ b/llvm/lib/Target/TargetMachine.cpp @@ -67,6 +67,19 @@ void TargetMachine::resetTargetOptions(const Function &F) const { /// and dynamic-no-pic. Reloc::Model TargetMachine::getRelocationModel() const { return RM; } +uint64_t TargetMachine::getMaxCodeSize() const { + switch (getCodeModel()) { + case CodeModel::Tiny: + return llvm::maxUIntN(10); + case CodeModel::Small: + case CodeModel::Kernel: + case CodeModel::Medium: + return llvm::maxUIntN(31); + case CodeModel::Large: + return llvm::maxUIntN(64); + } +} + /// Get the IR-specified TLS model for Var. static TLSModel::Model getSelectedTLSModel(const GlobalValue *GV) { switch (GV->getThreadLocalMode()) { diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 3f328431b140a4956dc3f08931cc92f1157b7ac1..e804122adae368c417aa12858141e71b5663f69a 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -8364,12 +8364,6 @@ void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { MI.setDesc(get(table[Domain - 1])); } -void X86InstrInfo::insertNoop(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI) const { - DebugLoc DL; - BuildMI(MBB, MI, DL, get(X86::NOOP)); -} - /// Return the noop instruction to use for a noop. MCInst X86InstrInfo::getNop() const { MCInst Nop; diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h index 73262461dff5c30b737c0d3200d6e5471a401354..02cb5f761747ea9bb7bb3af35bef9872fcb277d6 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.h +++ b/llvm/lib/Target/X86/X86InstrInfo.h @@ -455,9 +455,6 @@ public: int64_t Offset2, unsigned NumLoads) const override; - void insertNoop(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI) const override; - MCInst getNop() const override; bool diff --git a/llvm/test/CodeGen/AArch64/branch-relax-b.ll b/llvm/test/CodeGen/AArch64/branch-relax-b.ll new file mode 100644 index 0000000000000000000000000000000000000000..cbf0152d811d56409ff462b7524b8c5919892ef2 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/branch-relax-b.ll @@ -0,0 +1,139 @@ +; RUN: llc < %s -mtriple=aarch64-none-linux-gnu --verify-machineinstrs -aarch64-b-offset-bits=9 -aarch64-tbz-offset-bits=6 -aarch64-cbz-offset-bits=6 -aarch64-bcc-offset-bits=6 | FileCheck %s + +define void @relax_b_nospill(i1 zeroext %0) { +; CHECK-LABEL: relax_b_nospill: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: tbnz w0, +; CHECK-SAME: LBB0_1 +; CHECK-NEXT: // %bb.3: // %entry +; CHECK-NEXT: adrp [[SCAVENGED_REGISTER:x[0-9]+]], .LBB0_2 +; CHECK-NEXT: add [[SCAVENGED_REGISTER]], [[SCAVENGED_REGISTER]], :lo12:.LBB0_2 +; CHECK-NEXT: br [[SCAVENGED_REGISTER]] +; CHECK-NEXT: .LBB0_1: // %iftrue +; CHECK-NEXT: //APP +; CHECK-NEXT: .zero 2048 +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB0_2: // %iffalse +; CHECK-NEXT: //APP +; CHECK-NEXT: .zero 8 +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: ret +entry: + br i1 %0, label %iftrue, label %iffalse + +iftrue: + call void asm sideeffect ".space 2048", ""() + ret void + +iffalse: + call void asm sideeffect ".space 8", ""() + ret void +} + +define void @relax_b_spill() { +; CHECK-LABEL: relax_b_spill: // @relax_b_spill +; CHECK: // %bb.0: // %entry +; CHECK-COUNT-5: // 16-byte Folded Spill +; CHECK-NOT: // 16-byte Folded Spill +; CHECK: //APP +; CHECK-COUNT-29: mov {{x[0-9]+}}, +; CHECK-NOT: mov {{x[0-9]+}}, +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: b.eq .LBB1_1 +; CHECK-NEXT: // %bb.4: // %entry +; CHECK-NEXT: str [[SPILL_REGISTER:x[0-9]+]], [sp, +; CHECK-SAME: -16]! +; CHECK-NEXT: adrp [[SPILL_REGISTER:x[0-9]+]], .LBB1_5 +; CHECK-NEXT: add [[SPILL_REGISTER:x[0-9]+]], [[SPILL_REGISTER:x[0-9]+]], :lo12:.LBB1_5 +; CHECK-NEXT: br [[SPILL_REGISTER:x[0-9]+]] +; CHECK-NEXT: .LBB1_1: // %iftrue +; CHECK-NEXT: //APP +; CHECK-NEXT: .zero 2048 +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: b .LBB1_3 +; CHECK-NEXT: .LBB1_5: // %iffalse +; CHECK-NEXT: ldr [[SPILL_REGISTER:x[0-9]+]], [sp], +; CHECK-SAME: 16 +; CHECK-NEXT: // %bb.2: // %iffalse +; CHECK-NEXT: //APP +; CHECK-COUNT-29: // reg use {{x[0-9]+}} +; CHECK-NOT: // reg use {{x[0-9]+}} +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: .LBB1_3: // %common.ret +; CHECK-COUNT-5: // 16-byte Folded Reload +; CHECK-NOT: // 16-byte Folded Reload +; CHECK-NEXT: ret +entry: + %x0 = call i64 asm sideeffect "mov x0, 1", "={x0}"() + %x1 = call i64 asm sideeffect "mov x1, 1", "={x1}"() + %x2 = call i64 asm sideeffect "mov x2, 1", "={x2}"() + %x3 = call i64 asm sideeffect "mov x3, 1", "={x3}"() + %x4 = call i64 asm sideeffect "mov x4, 1", "={x4}"() + %x5 = call i64 asm sideeffect "mov x5, 1", "={x5}"() + %x6 = call i64 asm sideeffect "mov x6, 1", "={x6}"() + %x7 = call i64 asm sideeffect "mov x7, 1", "={x7}"() + %x8 = call i64 asm sideeffect "mov x8, 1", "={x8}"() + %x9 = call i64 asm sideeffect "mov x9, 1", "={x9}"() + %x10 = call i64 asm sideeffect "mov x10, 1", "={x10}"() + %x11 = call i64 asm sideeffect "mov x11, 1", "={x11}"() + %x12 = call i64 asm sideeffect "mov x12, 1", "={x12}"() + %x13 = call i64 asm sideeffect "mov x13, 1", "={x13}"() + %x14 = call i64 asm sideeffect "mov x14, 1", "={x14}"() + %x15 = call i64 asm sideeffect "mov x15, 1", "={x15}"() + %x16 = call i64 asm sideeffect "mov x16, 1", "={x16}"() + %x17 = call i64 asm sideeffect "mov x17, 1", "={x17}"() + %x18 = call i64 asm sideeffect "mov x18, 1", "={x18}"() + %x19 = call i64 asm sideeffect "mov x19, 1", "={x19}"() + %x20 = call i64 asm sideeffect "mov x20, 1", "={x20}"() + %x21 = call i64 asm sideeffect "mov x21, 1", "={x21}"() + %x22 = call i64 asm sideeffect "mov x22, 1", "={x22}"() + %x23 = call i64 asm sideeffect "mov x23, 1", "={x23}"() + %x24 = call i64 asm sideeffect "mov x24, 1", "={x24}"() + %x25 = call i64 asm sideeffect "mov x25, 1", "={x25}"() + %x26 = call i64 asm sideeffect "mov x26, 1", "={x26}"() + %x27 = call i64 asm sideeffect "mov x27, 1", "={x27}"() + %x28 = call i64 asm sideeffect "mov x28, 1", "={x28}"() + + %cmp = icmp eq i64 %x16, %x15 + br i1 %cmp, label %iftrue, label %iffalse + +iftrue: + call void asm sideeffect ".space 2048", ""() + ret void + +iffalse: + call void asm sideeffect "# reg use $0", "{x0}"(i64 %x0) + call void asm sideeffect "# reg use $0", "{x1}"(i64 %x1) + call void asm sideeffect "# reg use $0", "{x2}"(i64 %x2) + call void asm sideeffect "# reg use $0", "{x3}"(i64 %x3) + call void asm sideeffect "# reg use $0", "{x4}"(i64 %x4) + call void asm sideeffect "# reg use $0", "{x5}"(i64 %x5) + call void asm sideeffect "# reg use $0", "{x6}"(i64 %x6) + call void asm sideeffect "# reg use $0", "{x7}"(i64 %x7) + call void asm sideeffect "# reg use $0", "{x8}"(i64 %x8) + call void asm sideeffect "# reg use $0", "{x9}"(i64 %x9) + call void asm sideeffect "# reg use $0", "{x10}"(i64 %x10) + call void asm sideeffect "# reg use $0", "{x11}"(i64 %x11) + call void asm sideeffect "# reg use $0", "{x12}"(i64 %x12) + call void asm sideeffect "# reg use $0", "{x13}"(i64 %x13) + call void asm sideeffect "# reg use $0", "{x14}"(i64 %x14) + call void asm sideeffect "# reg use $0", "{x15}"(i64 %x15) + call void asm sideeffect "# reg use $0", "{x16}"(i64 %x16) + call void asm sideeffect "# reg use $0", "{x17}"(i64 %x17) + call void asm sideeffect "# reg use $0", "{x18}"(i64 %x18) + call void asm sideeffect "# reg use $0", "{x19}"(i64 %x19) + call void asm sideeffect "# reg use $0", "{x20}"(i64 %x20) + call void asm sideeffect "# reg use $0", "{x21}"(i64 %x21) + call void asm sideeffect "# reg use $0", "{x22}"(i64 %x22) + call void asm sideeffect "# reg use $0", "{x23}"(i64 %x23) + call void asm sideeffect "# reg use $0", "{x24}"(i64 %x24) + call void asm sideeffect "# reg use $0", "{x25}"(i64 %x25) + call void asm sideeffect "# reg use $0", "{x26}"(i64 %x26) + call void asm sideeffect "# reg use $0", "{x27}"(i64 %x27) + call void asm sideeffect "# reg use $0", "{x28}"(i64 %x28) + ret void +} + +declare i32 @bar() +declare i32 @baz() \ No newline at end of file diff --git a/llvm/test/CodeGen/AArch64/branch-relax-cross-section.mir b/llvm/test/CodeGen/AArch64/branch-relax-cross-section.mir new file mode 100644 index 0000000000000000000000000000000000000000..1cf307cd16ecffd4e43af8a76105550905671377 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/branch-relax-cross-section.mir @@ -0,0 +1,71 @@ +# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass branch-relaxation -aarch64-b-offset-bits=64 %s -o - | FileCheck %s + +--- | + declare i32 @bar() + declare i32 @baz() + declare i32 @qux() + + define void @relax_tbz(i1 zeroext %0) { + br i1 %0, label %false_block, label %true_block + + false_block: ; preds = %1 + %2 = call i32 @baz() + br label %end + + end: ; preds = %true_block, %false_block + %3 = tail call i32 @qux() + ret void + + true_block: ; preds = %1 + %4 = call i32 @bar() + br label %end + } + +... +--- +name: relax_tbz +tracksRegLiveness: true +liveins: + - { reg: '$w0', virtual-reg: '' } +stack: + - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16, + stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true, + debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } +body: | + ; CHECK-LABEL: name: relax_tbz + ; COM: Check that cross-section conditional branches are + ; COM: relaxed. + ; CHECK: bb.0 (%ir-block.1, bbsections 1): + ; CHECK-NEXT: successors: %bb.3(0x40000000) + ; CHECK: TBNZW + ; CHECK-SAME: %bb.3 + ; CHECK: B %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3 (%ir-block.1, bbsections 1): + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: B %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.false_block (bbsections 2): + ; CHECK: TCRETURNdi @qux, 0, csr_aarch64_aapcs, implicit $sp + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.true_block (bbsections 3): + ; CHECK: TCRETURNdi @qux, 0, csr_aarch64_aapcs, implicit $sp + bb.0 (%ir-block.1, bbsections 1): + successors: %bb.1(0x40000000), %bb.2(0x40000000) + liveins: $w0, $lr + + early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) + TBZW killed renamable $w0, 0, %bb.2 + B %bb.1 + + bb.1.false_block (bbsections 2): + BL @baz, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $w0 + early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) + TCRETURNdi @qux, 0, csr_aarch64_aapcs, implicit $sp + + bb.2.true_block (bbsections 3): + BL @bar, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $w0 + early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) + TCRETURNdi @qux, 0, csr_aarch64_aapcs, implicit $sp +... diff --git a/llvm/test/CodeGen/X86/machine-function-splitter.ll b/llvm/test/CodeGen/X86/machine-function-splitter.ll index 900a2e4f7f998c2f43d7d86ad6fff8d6d717c9a7..ad3357129fa93d6268f5deb67946671b4e0f2e5e 100644 --- a/llvm/test/CodeGen/X86/machine-function-splitter.ll +++ b/llvm/test/CodeGen/X86/machine-function-splitter.ll @@ -386,7 +386,7 @@ define void @foo16(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ; MFS-O0: .section .text.split.foo16 ; MFS-O0-NEXT: foo16.cold %2 = call i32 @baz() - br i1 false, label %3, label %5, !prof !25 + br i1 false, label %3, label %5, !prof !24 3: ; preds = %1 %4 = call i32 @bar() @@ -445,7 +445,7 @@ define i32 @foo18(i32 %in) !prof !14 !section_prefix !15 { i32 1, label %hot2 i32 2, label %cold1 i32 3, label %cold2 - ], !prof !28 + ], !prof !25 common.ret: ; preds = %0 ret i32 0 @@ -473,10 +473,10 @@ define i32 @foo19(i32 %in) !prof !14 !section_prefix !15 { ; MFS-DEFAULTS-LABEL: foo19 ; MFS-DEFAULTS: .section .text.split.foo19 ; MFS-DEFAULTS-NEXT: foo19.cold: -; MFS-DEFAULTS-X86: .LJTI18_0 -; MFS-DEFAULTS-AARCH64-NOT: .LJTI18_0 +; MFS-DEFAULTS-X86: .LJTI16_0 +; MFS-DEFAULTS-AARCH64-NOT: .LJTI16_0 ; MFS-DEFAULTS: .section .rodata -; MFS-DEFAULTS: .LJTI18_0 +; MFS-DEFAULTS: .LJTI16_0 %cmp = icmp sgt i32 %in, 3 br i1 %cmp, label %hot, label %cold_switch, !prof !17 @@ -489,7 +489,7 @@ cold_switch: ; preds = %0 i32 1, label %hot2 i32 2, label %cold1 i32 3, label %cold2 - ], !prof !28 + ], !prof !25 common.ret: ; preds = %0 ret i32 0 @@ -574,4 +574,5 @@ attributes #0 = { "implicit-section-name"="nosplit" } !21 = !{!"branch_weights", i32 6000, i32 4000} !22 = !{!"branch_weights", i32 80, i32 9920} !23 = !{!"function_entry_count", i64 7} -!24 = !{!"branch_weights", i32 0, i32 4000, i32 4000, i32 0, i32 0} +!24 = !{!"branch_weights", i32 0, i32 700} +!25 = !{!"branch_weights", i32 0, i32 4000, i32 4000, i32 0, i32 0} diff --git a/llvm/unittests/ADT/DenseMapTest.cpp b/llvm/unittests/ADT/DenseMapTest.cpp index e23f1809a58ba96a8f4f55673b547442690d06a4..0d71731c4d1e13a34117bd8756b4210e60476f44 100644 --- a/llvm/unittests/ADT/DenseMapTest.cpp +++ b/llvm/unittests/ADT/DenseMapTest.cpp @@ -134,6 +134,7 @@ TYPED_TEST(DenseMapTest, EmptyIntMapTest) { // Lookup tests EXPECT_FALSE(this->Map.count(this->getKey())); + EXPECT_FALSE(this->Map.contains(this->getKey())); EXPECT_TRUE(this->Map.find(this->getKey()) == this->Map.end()); EXPECT_EQ(typename TypeParam::mapped_type(), this->Map.lookup(this->getKey())); @@ -165,6 +166,7 @@ TYPED_TEST(DenseMapTest, SingleEntryMapTest) { // Lookup tests EXPECT_TRUE(this->Map.count(this->getKey())); + EXPECT_TRUE(this->Map.contains(this->getKey())); EXPECT_TRUE(this->Map.find(this->getKey()) == this->Map.begin()); EXPECT_EQ(this->getValue(), this->Map.lookup(this->getKey())); EXPECT_EQ(this->getValue(), this->Map[this->getKey()]); diff --git a/llvm/unittests/ADT/MapVectorTest.cpp b/llvm/unittests/ADT/MapVectorTest.cpp index 552f9956bdc2c6148f2e338b02074b7c479994a9..1a371cbfba81e8ea4b57c4077ca94c86c3db8991 100644 --- a/llvm/unittests/ADT/MapVectorTest.cpp +++ b/llvm/unittests/ADT/MapVectorTest.cpp @@ -87,8 +87,10 @@ TEST(MapVectorTest, erase) { MV.insert(std::make_pair(5, 6)); ASSERT_EQ(MV.size(), 3u); + ASSERT_TRUE(MV.contains(1)); MV.erase(MV.find(1)); ASSERT_EQ(MV.size(), 2u); + ASSERT_FALSE(MV.contains(1)); ASSERT_EQ(MV.find(1), MV.end()); ASSERT_EQ(MV[3], 4); ASSERT_EQ(MV[5], 6); diff --git a/llvm/unittests/ADT/StringMapTest.cpp b/llvm/unittests/ADT/StringMapTest.cpp index 431b3973d6d6c1261fff5c435521b279a190ea7a..f9b138e9a472137139397d9cae76823711594211 100644 --- a/llvm/unittests/ADT/StringMapTest.cpp +++ b/llvm/unittests/ADT/StringMapTest.cpp @@ -41,6 +41,7 @@ protected: EXPECT_TRUE(testMap.begin() == testMap.end()); // Lookup tests + EXPECT_FALSE(testMap.contains(testKey)); EXPECT_EQ(0u, testMap.count(testKey)); EXPECT_EQ(0u, testMap.count(StringRef(testKeyFirst, testKeyLength))); EXPECT_EQ(0u, testMap.count(testKeyStr)); @@ -64,6 +65,7 @@ protected: EXPECT_TRUE(it == testMap.end()); // Lookup tests + EXPECT_TRUE(testMap.contains(testKey)); EXPECT_EQ(1u, testMap.count(testKey)); EXPECT_EQ(1u, testMap.count(StringRef(testKeyFirst, testKeyLength))); EXPECT_EQ(1u, testMap.count(testKeyStr));