diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 8856fe0ea5554f1bd17287b66a852da6b0b726fd..53b55e247ae36d887896dfdea901424ba32fb9ad 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -1904,6 +1904,402 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, } } +RegScavenger *RS; +/// StackObjSet - A set of stack object indexes +using StackObjSet = SmallSetVector; +// MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved +// stack frame indexes. +unsigned MinCSFrameIndex = std::numeric_limits::max(); +unsigned MaxCSFrameIndex = 0; +/// AdjustStackOffset - Helper function used to adjust the stack frame offset. +static inline void AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, + bool StackGrowsDown, int64_t &Offset, + Align &MaxAlign, unsigned Skew) { + // If the stack grows down, add the object size to find the lowest address. + if (StackGrowsDown) + Offset += MFI.getObjectSize(FrameIdx); + + Align Alignment = MFI.getObjectAlign(FrameIdx); + + // If the alignment of this object is greater than that of the stack, then + // increase the stack alignment to match. + MaxAlign = std::max(MaxAlign, Alignment); + + // Adjust to alignment boundary. + Offset = alignTo(Offset, Alignment, Skew); + + if (StackGrowsDown) { + LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset + << "]\n"); + MFI.setObjectOffset(FrameIdx, -Offset); // Set the computed offset + } else { + LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << Offset + << "]\n"); + MFI.setObjectOffset(FrameIdx, Offset); + Offset += MFI.getObjectSize(FrameIdx); + } +} + +/// AssignProtectedObjSet - Helper function to assign large stack objects (i.e., +/// those required to be close to the Stack Protector) to stack offsets. +static void AssignProtectedObjSet(const StackObjSet &UnassignedObjs, + SmallSet &ProtectedObjs, + MachineFrameInfo &MFI, bool StackGrowsDown, + int64_t &Offset, Align &MaxAlign, + unsigned Skew) { + + for (int i : UnassignedObjs) { + AdjustStackOffset(MFI, i, StackGrowsDown, Offset, MaxAlign, Skew); + ProtectedObjs.insert(i); + } +} + +/// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the +/// abstract stack objects. +static void calculateFrameObjectOffsets(MachineFunction &MF) { + const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); + + bool StackGrowsDown = + TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; + + // Loop over all of the stack objects, assigning sequential addresses... + MachineFrameInfo &MFI = MF.getFrameInfo(); + + // Start at the beginning of the local area. + // The Offset is the distance from the stack top in the direction + // of stack growth -- so it's always nonnegative. + // OHOS_LOCAL begin + auto CC = MF.getFunction().getCallingConv(); + int LocalAreaOffset = TFI.getOffsetOfLocalArea(CC); + // OHOS_LOCAL end + + if (StackGrowsDown) + LocalAreaOffset = -LocalAreaOffset; + assert(LocalAreaOffset >= 0 && + "Local area offset should be in direction of stack growth"); + int64_t Offset = LocalAreaOffset; + + // Skew to be applied to alignment. + unsigned Skew = TFI.getStackAlignmentSkew(MF); + +#ifdef EXPENSIVE_CHECKS + for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) + if (!MFI.isDeadObjectIndex(i) && + MFI.getStackID(i) == TargetStackID::Default) + assert(MFI.getObjectAlign(i) <= MFI.getMaxAlign() && + "MaxAlignment is invalid"); +#endif + + // If there are fixed sized objects that are preallocated in the local area, + // non-fixed objects can't be allocated right at the start of local area. + // Adjust 'Offset' to point to the end of last fixed sized preallocated + // object. + for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) { + // Only allocate objects on the default stack. + if (MFI.getStackID(i) != TargetStackID::Default) + continue; + + int64_t FixedOff; + if (StackGrowsDown) { + // The maximum distance from the stack pointer is at lower address of + // the object -- which is given by offset. For down growing stack + // the offset is negative, so we negate the offset to get the distance. + FixedOff = -MFI.getObjectOffset(i); + } else { + // The maximum distance from the start pointer is at the upper + // address of the object. + FixedOff = MFI.getObjectOffset(i) + MFI.getObjectSize(i); + } + if (FixedOff > Offset) + Offset = FixedOff; + } + + Align MaxAlign = MFI.getMaxAlign(); + // First assign frame offsets to stack objects that are used to spill + // callee saved registers. + if (MaxCSFrameIndex >= MinCSFrameIndex) { + for (unsigned i = 0; i <= MaxCSFrameIndex - MinCSFrameIndex; ++i) { + unsigned FrameIndex = + StackGrowsDown ? MinCSFrameIndex + i : MaxCSFrameIndex - i; + + // Only allocate objects on the default stack. + if (MFI.getStackID(FrameIndex) != TargetStackID::Default) + continue; + + // TODO: should this just be if (MFI.isDeadObjectIndex(FrameIndex)) + if (!StackGrowsDown && MFI.isDeadObjectIndex(FrameIndex)) + continue; + + AdjustStackOffset(MFI, FrameIndex, StackGrowsDown, Offset, MaxAlign, + Skew); + } + } + + assert(MaxAlign == MFI.getMaxAlign() && + "MFI.getMaxAlign should already account for all callee-saved " + "registers without a fixed stack slot"); + + // FixedCSEnd is the stack offset to the end of the fixed and callee-save + // stack area. + int64_t FixedCSEnd = Offset; + +#ifdef ARK_GC_SUPPORT + int CalleeSavedFrameSize = 0; + Triple::ArchType archType = TFI.GetArkSupportTarget(); + if (archType == Triple::aarch64 && TFI.hasFP(MF)) { + int fpPosition = TFI.GetFixedFpPosition(); + int slotSize = sizeof(uint64_t); + int fpToCallerSpDelta = 0; + // 0:not exist +:count from head -:count from tail + // for x86-64 + // +--------------------------+ + // | caller Frame | + // +--------------------------+--- + // | returnAddr | ^ + // +--------------------------+ 2 slot(fpToCallerSpDelta) + // | Fp | V fpPosition = 2 + // +--------------------------+--- + // | type | + // +--------------------------+ + // | ReServeSize | + // +--------------------------+ + // | R14 | + // +--------------------------+ + // | R13 | + // +--------------------------+ + // | R12 | + // +--------------------------+ + // | RBX | + // +--------------------------+ + // for ARM64 + // +--------------------------+ + // | caller Frame | + // +--------------------------+--- + // | callee save registers | ^ + // | (exclude Fp) | | + // | | callee save registers + // size(fpToCallerSpDelta) + // +--------------------------+ | + // | Fp | V fpPosition = -1 + // +--------------------------+--- FixedCSEnd + // | type | + // +--------------------------+ + // | ReServeSize | + // +--------------------------+ + if (fpPosition >= 0) { + fpToCallerSpDelta = fpPosition * slotSize; + } else { + fpToCallerSpDelta = FixedCSEnd + (fpPosition + 1) * slotSize; + } + Function &func = const_cast(MF.getFunction()); + Attribute attr = Attribute::get(func.getContext(), "fpToCallerSpDelta", + std::to_string(fpToCallerSpDelta).c_str()); + func.addAttributeAtIndex(AttributeList::FunctionIndex, attr); + + CalleeSavedFrameSize = TFI.GetFrameReserveSize(MF); + Offset += CalleeSavedFrameSize; + } + + if ((archType == Triple::x86_64) && TFI.hasFP(MF)) { + // Determine which of the registers in the callee save list should be saved. + int fpPosition = TFI.GetFixedFpPosition(); + int fpToCallerSpDelta = 0; + int slotSize = sizeof(uint64_t); + if (fpPosition >= 0) { + fpToCallerSpDelta = fpPosition * slotSize; + } else { + fpToCallerSpDelta = FixedCSEnd + (fpPosition + 1) * slotSize; + } + Function &func = const_cast(MF.getFunction()); + Attribute attr = Attribute::get(func.getContext(), "fpToCallerSpDelta", + std::to_string(fpToCallerSpDelta).c_str()); + func.addAttributeAtIndex(AttributeList::FunctionIndex, attr); + + CalleeSavedFrameSize = TFI.GetFrameReserveSize(MF); + std::vector &CSI = MFI.getCalleeSavedInfo(); + LLVM_DEBUG(dbgs() << " CSI size: " << CSI.size() + << " CalleeSavedFrameSize " << CalleeSavedFrameSize + << "\n"); + // if callee-saved is empty, the reserved-size can't be passed to the + // computation of local zone because the assignCalleeSavedSpillSlots() + // directly return. Otherwise, the reserved-size don't need to add to the + // computation of local zone because it has been considered while computing + // the offsets of callee-saved-zone that will be passed to the computation + // of local-zone + if (CSI.empty()) { + Offset += CalleeSavedFrameSize; + } + } +#endif + + // Make sure the special register scavenging spill slot is closest to the + // incoming stack pointer if a frame pointer is required and is closer + // to the incoming rather than the final stack pointer. + const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); + bool EarlyScavengingSlots = + TFI.allocateScavengingFrameIndexesNearIncomingSP(MF); + if (RS && EarlyScavengingSlots) { + SmallVector SFIs; + RS->getScavengingFrameIndices(SFIs); + for (int SFI : SFIs) + AdjustStackOffset(MFI, SFI, StackGrowsDown, Offset, MaxAlign, Skew); + } + + // FIXME: Once this is working, then enable flag will change to a target + // check for whether the frame is large enough to want to use virtual + // frame index registers. Functions which don't want/need this optimization + // will continue to use the existing code path. + if (MFI.getUseLocalStackAllocationBlock()) { + Align Alignment = MFI.getLocalFrameMaxAlign(); + + // Adjust to alignment boundary. + Offset = alignTo(Offset, Alignment, Skew); + + LLVM_DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n"); + + // Resolve offsets for objects in the local block. + for (unsigned i = 0, e = MFI.getLocalFrameObjectCount(); i != e; ++i) { + std::pair Entry = MFI.getLocalFrameObjectMap(i); + int64_t FIOffset = (StackGrowsDown ? -Offset : Offset) + Entry.second; + LLVM_DEBUG(dbgs() << "alloc FI(" << Entry.first << ") at SP[" << FIOffset + << "]\n"); + MFI.setObjectOffset(Entry.first, FIOffset); + } + // Allocate the local block + Offset += MFI.getLocalFrameSize(); + + MaxAlign = std::max(Alignment, MaxAlign); + } + + // Retrieve the Exception Handler registration node. + int EHRegNodeFrameIndex = std::numeric_limits::max(); + if (const WinEHFuncInfo *FuncInfo = MF.getWinEHFuncInfo()) + EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex; + + // Make sure that the stack protector comes before the local variables on the + // stack. + Function &F = MF.getFunction(); // OHOS_LOCAL + SmallSet ProtectedObjs; + if (MFI.hasStackProtectorIndex()) { + int StackProtectorFI = MFI.getStackProtectorIndex(); + StackObjSet LargeArrayObjs; + StackObjSet SmallArrayObjs; + StackObjSet AddrOfObjs; + + // If we need a stack protector, we need to make sure that + // LocalStackSlotPass didn't already allocate a slot for it. + // If we are told to use the LocalStackAllocationBlock, the stack protector + // is expected to be already pre-allocated. + if (MFI.getStackID(StackProtectorFI) != TargetStackID::Default) { + // If the stack protector isn't on the default stack then it's up to the + // target to set the stack offset. + assert(MFI.getObjectOffset(StackProtectorFI) != 0 && + "Offset of stack protector on non-default stack expected to be " + "already set."); + assert(!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()) && + "Stack protector on non-default stack expected to not be " + "pre-allocated by LocalStackSlotPass."); + } else if (!MFI.getUseLocalStackAllocationBlock()) { + AdjustStackOffset(MFI, StackProtectorFI, StackGrowsDown, Offset, MaxAlign, + Skew); + } else if (!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex())) { + llvm_unreachable( + "Stack protector not pre-allocated by LocalStackSlotPass."); + } + + // Assign large stack objects first. + for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { + if (MFI.isObjectPreAllocated(i) && MFI.getUseLocalStackAllocationBlock()) + continue; + if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) + continue; + if (RS && RS->isScavengingFrameIndex((int)i)) + continue; + if (MFI.isDeadObjectIndex(i)) + continue; + if (StackProtectorFI == (int)i || EHRegNodeFrameIndex == (int)i) + continue; + // Only allocate objects on the default stack. + if (MFI.getStackID(i) != TargetStackID::Default) + continue; + + switch (MFI.getObjectSSPLayout(i)) { + case MachineFrameInfo::SSPLK_None: + continue; + case MachineFrameInfo::SSPLK_SmallArray: + SmallArrayObjs.insert(i); + continue; + case MachineFrameInfo::SSPLK_AddrOf: + AddrOfObjs.insert(i); + continue; + case MachineFrameInfo::SSPLK_LargeArray: + LargeArrayObjs.insert(i); + continue; + } + llvm_unreachable("Unexpected SSPLayoutKind."); + } + + // We expect **all** the protected stack objects to be pre-allocated by + // LocalStackSlotPass. If it turns out that PEI still has to allocate some + // of them, we may end up messing up the expected order of the objects. + if (MFI.getUseLocalStackAllocationBlock() && + !(LargeArrayObjs.empty() && SmallArrayObjs.empty() && + AddrOfObjs.empty())) + llvm_unreachable("Found protected stack objects not pre-allocated by " + "LocalStackSlotPass."); + + // OHOS_LOCAL begin + AssignProtectedObjSet(LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown, + Offset, MaxAlign, Skew); + AssignProtectedObjSet(SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown, + Offset, MaxAlign, Skew); + AssignProtectedObjSet(AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown, + Offset, MaxAlign, Skew); + } else if (F.hasFnAttribute(Attribute::StackProtectRetReq) || + F.hasFnAttribute(Attribute::StackProtectRetStrong)) { + StackObjSet LargeArrayObjs; + StackObjSet SmallArrayObjs; + StackObjSet AddrOfObjs; + // Assign large stack objects first. + for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { + if (MFI.isObjectPreAllocated(i) && MFI.getUseLocalStackAllocationBlock()) + continue; + if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) + continue; + if (RS && RS->isScavengingFrameIndex((int)i)) + continue; + if (MFI.isDeadObjectIndex(i)) + continue; + if (EHRegNodeFrameIndex == (int)i) + continue; + if (MFI.getStackID(i) != + TargetStackID::Default) // Only allocate objects on the default stack. + continue; + switch (MFI.getObjectSSPLayout(i)) { + case MachineFrameInfo::SSPLK_None: + continue; + case MachineFrameInfo::SSPLK_SmallArray: + SmallArrayObjs.insert(i); + continue; + case MachineFrameInfo::SSPLK_AddrOf: + AddrOfObjs.insert(i); + continue; + case MachineFrameInfo::SSPLK_LargeArray: + LargeArrayObjs.insert(i); + continue; + } + llvm_unreachable("Unexpected SSPLayoutKind."); + } + // OHOS_LOCAL end + + AssignProtectedObjSet(LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown, + Offset, MaxAlign, Skew); + AssignProtectedObjSet(SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown, + Offset, MaxAlign, Skew); + AssignProtectedObjSet(AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown, + Offset, MaxAlign, Skew); + } +} static void InsertReturnAddressAuth(MachineFunction &MF, MachineBasicBlock &MBB) { const auto &MFI = *MF.getInfo(); @@ -1940,6 +2336,7 @@ static void InsertReturnAddressAuth(MachineFunction &MF, .addCFIIndex(CFIIndex) .setMIFlags(MachineInstr::FrameDestroy); } + calculateFrameObjectOffsets(MF); } static bool isFuncletReturnInstr(const MachineInstr &MI) { @@ -3307,7 +3704,7 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, Assign(I, -Offset); } } - + Align MaxAlign = MFI.getMaxAlign(); // Ensure that the Callee-save area is aligned to 16bytes. Offset = alignTo(Offset, Align(16U));