diff --git a/src/mapleall/bin/debug/dex2mpl.tar.gz b/src/mapleall/bin/debug/dex2mpl.tar.gz index bbd47238fa0bb8eead38f6e4b007153de63de347..51b04921fd6e48acd9cd08095abd7f6239f0fac2 100644 Binary files a/src/mapleall/bin/debug/dex2mpl.tar.gz and b/src/mapleall/bin/debug/dex2mpl.tar.gz differ diff --git a/src/mapleall/bin/dex2mpl b/src/mapleall/bin/dex2mpl index 5b18e255062f6a1a74f0129973a12194927b8efc..cb78732486e16c60b7b13880067538ad9dc45d78 100755 Binary files a/src/mapleall/bin/dex2mpl and b/src/mapleall/bin/dex2mpl differ diff --git a/src/mapleall/bin/dex2mpl_android b/src/mapleall/bin/dex2mpl_android index c813a0a408a9b335ee306567ea2645f3330158fc..2746a49fcd25715b9f83b6c2a3c24f865bbf9ef4 100755 Binary files a/src/mapleall/bin/dex2mpl_android and b/src/mapleall/bin/dex2mpl_android differ diff --git a/src/mapleall/maple_be/BUILD.gn b/src/mapleall/maple_be/BUILD.gn index 66dfb9d99bed45744367f6df0afd4ac51cb3b92c..cddb2d3011ebb701ba8fca0f0233f00a05c81f5e 100644 --- a/src/mapleall/maple_be/BUILD.gn +++ b/src/mapleall/maple_be/BUILD.gn @@ -77,6 +77,7 @@ src_libcgaarch64 = [ "src/cg/aarch64/aarch64_reaching.cpp", "src/cg/aarch64/aarch64_schedule.cpp", "src/cg/aarch64/aarch64_strldr.cpp", + "src/cg/aarch64/aarch64_lsra.cpp", ] src_libcg = [ diff --git a/src/mapleall/maple_be/include/cg/aarch64/aarch64_lsra.h b/src/mapleall/maple_be/include/cg/aarch64/aarch64_lsra.h new file mode 100644 index 0000000000000000000000000000000000000000..f07fee9268df5d660112db821db14664be6e6ebb --- /dev/null +++ b/src/mapleall/maple_be/include/cg/aarch64/aarch64_lsra.h @@ -0,0 +1,529 @@ +/* + * Copyright (c) [2020] Huawei Technologies Co.,Ltd.All rights reserved. + * + * OpenArkCompiler is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR + * FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ +#ifndef MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_LSRA_H +#define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_LSRA_H +#include "aarch64_reg_alloc.h" +#include "aarch64_operand.h" +#include "aarch64_insn.h" +#include "aarch64_abi.h" + +namespace maplebe { +class LSRALinearScanRegAllocator : public AArch64RegAllocator { + enum RegInCatch : uint8 { + /* + * RA do not want to allocate certain registers if a live interval is + * only inside of catch blocks. + */ + kRegCatchNotInit = 0, /* unitialized state */ + kRegNOtInCatch = 1, /* interval is part or all outside of catch */ + kRegAllInCatch = 2, /* inteval is completely inside catch */ + }; + + enum RegInCleanup : uint8 { + /* Similar to reg_in_catch_t */ + kRegCleanupNotInit = 0, /* unitialized state */ + kRegAllInFirstbb = 1, /* interval is all in the first bb */ + kRegAllOutCleanup = 2, /* interval is all outside of cleanup, must in normal bb, may in first bb. */ + kRegInCleanupAndFirstbb = 3, /* inteval is in cleanup and first bb. */ + kRegInCleanupAndNormalbb = 4, /* inteval is in cleanup and non-first bb. */ + kRegAllInCleanup = 5 /* inteval is inside cleanup, except for bb 1 */ + }; + + class LiveInterval { + public: + explicit LiveInterval(MapleAllocator &mallocator) + : ranges(mallocator.Adapter()), + holes(mallocator.Adapter()), + usePositions(mallocator.Adapter()) {} + + virtual ~LiveInterval() = default; + + void AddRange(uint32 from, uint32 to); + void AddUsePos(uint32 pos); + + const Insn *GetIsCall() const { + return isCall; + } + + void SetIsCall(Insn &newIsCall) { + isCall = &newIsCall; + } + + uint32 GetPhysUse() const { + return physUse; + } + + void SetPhysUse(uint32 newPhysUse) { + physUse = newPhysUse; + } + + uint32 GetLastUse() const { + return lastUse; + } + + void SetLastUse(uint32 newLastUse) { + lastUse = newLastUse; + } + + uint32 GetRegNO() const { + return regNO; + } + + void SetRegNO(uint32 newRegNO) { + regNO = newRegNO; + } + + uint32 GetAssignedReg() const { + return assignedReg; + } + + void SetAssignedReg(uint32 newAssignedReg) { + assignedReg = newAssignedReg; + } + + uint32 GetFirstDef() const { + return firstDef; + } + + void SetFirstDef(uint32 newFirstDef) { + firstDef = newFirstDef; + } + + uint32 GetStackSlot() const { + return stackSlot; + } + + void SetStackSlot(uint32 newStkSlot) { + stackSlot = newStkSlot; + } + + RegType GetRegType() const { + return regType; + } + + void SetRegType(RegType newRegType) { + regType = newRegType; + } + + uint32 GetFirstAcrossedCall() const { + return firstAcrossedCall; + } + + void SetFirstAcrossedCall(uint32 newFirstAcrossedCall) { + firstAcrossedCall = newFirstAcrossedCall; + } + + bool IsEndByCall() const { + return endByCall; + } + + bool IsUseBeforeDef() const { + return useBeforeDef; + } + + void SetUseBeforeDef(bool newUseBeforeDef) { + useBeforeDef = newUseBeforeDef; + } + + bool IsShouldSave() const { + return shouldSave; + } + + void SetShouldSave(bool newShouldSave) { + shouldSave = newShouldSave; + } + + bool IsMultiUseInBB() const { + return multiUseInBB; + } + + void SetMultiUseInBB(bool newMultiUseInBB) { + multiUseInBB = newMultiUseInBB; + } + + bool IsThrowVal() const { + return isThrowVal; + } + + bool IsCallerSpilled() const { + return isCallerSpilled; + } + + void SetIsCallerSpilled(bool newIsCallerSpilled) { + isCallerSpilled = newIsCallerSpilled; + } + + bool IsMustAllocate() const { + return mustAllocate; + } + + void SetMustAllocate(bool newMustAllocate) { + mustAllocate = newMustAllocate; + } + + uint32 GetRefCount() const{ + return refCount; + } + + void SetRefCount(uint32 newRefCount) { + refCount = newRefCount; + } + + float GetPriority() const { + return priority; + } + + void SetPriority(float newPriority) { + priority = newPriority; + } + + const MapleVector> &GetRanges() const { + return ranges; + } + + MapleVector> &GetRanges() { + return ranges; + } + + size_t GetRangesSize () const { + return ranges.size(); + } + + const MapleVector> &GetHoles() const { + return holes; + } + + void HolesPushBack(uint32 pair1, uint32 pair2) { + holes.push_back(std::pair(pair1, pair2)); + } + + void UsePositionsInsert(uint32 insertId) { + (void)usePositions.insert(insertId); + } + + const LiveInterval *GetLiParent() const { + return liveParent; + } + + void SetLiParent(LiveInterval *newLiParent) { + liveParent = newLiParent; + } + + void SetLiParentChild(LiveInterval *child) { + liveParent->SetLiChild(child); + } + + const LiveInterval *GetLiChild() const { + return liveChild; + } + + void SetLiChild(LiveInterval *newLiChild) { + liveChild = newLiChild; + } + + uint32 GetResultCount() const { + return resultCount; + } + + void SetResultCount(uint32 newResultCount) { + resultCount = newResultCount; + } + + void SetInCatchState() { + /* + * Once in REG_NOT_IN_CATCH, it is irreversible since once an interval + * is not in a catch, it is not completely in a catch. + */ + if (inCatchState == kRegNOtInCatch) { + return; + } + inCatchState = kRegAllInCatch; + } + + void SetNotInCatchState() { + inCatchState = kRegNOtInCatch; + } + + bool IsInCatch() const { + return (inCatchState == kRegAllInCatch); + } + + void SetInCleanupState() { + switch (inCleanUpState) { + case kRegCleanupNotInit: + inCleanUpState = kRegAllInCleanup; + break; + case kRegAllInFirstbb: + inCleanUpState = kRegInCleanupAndFirstbb; + break; + case kRegAllOutCleanup: + inCleanUpState = kRegInCleanupAndNormalbb; + break; + case kRegInCleanupAndFirstbb: + break; + case kRegInCleanupAndNormalbb: + break; + case kRegAllInCleanup: + break; + default: + ASSERT(false, "CG Internal error."); + break; + } + } + + void SetNotInCleanupState(bool isFirstBB) { + switch (inCleanUpState) { + case kRegCleanupNotInit: { + if (isFirstBB) { + inCleanUpState = kRegAllInFirstbb; + } else { + inCleanUpState = kRegAllOutCleanup; + } + break; + } + case kRegAllInFirstbb: { + if (!isFirstBB) { + inCleanUpState = kRegAllOutCleanup; + } + break; + } + case kRegAllOutCleanup: + break; + case kRegInCleanupAndFirstbb: { + if (!isFirstBB) { + inCleanUpState = kRegInCleanupAndNormalbb; + } + break; + } + case kRegInCleanupAndNormalbb: + break; + case kRegAllInCleanup: { + if (isFirstBB) { + inCleanUpState = kRegInCleanupAndFirstbb; + } else { + inCleanUpState = kRegInCleanupAndNormalbb; + } + break; + } + default: + ASSERT(false, "CG Internal error."); + break; + } + } + + bool IsAllInCleanupOrFirstBB() const { + return (inCleanUpState == kRegAllInCleanup) || (inCleanUpState == kRegInCleanupAndFirstbb); + } + + bool IsAllOutCleanup() const { + return (inCleanUpState == kRegAllInFirstbb) || (inCleanUpState == kRegAllOutCleanup); + } + + private: + Insn *isCall = nullptr; + uint32 firstDef = 0; + uint32 lastUse = 0; + uint32 physUse = 0; + uint32 regNO = 0; + /* physical register, using cg defined reg based on R0/V0. */ + uint32 assignedReg = 0; + uint32 stackSlot = -1; + RegType regType = kRegTyUndef; + uint32 firstAcrossedCall = 0; + bool endByCall = false; + bool useBeforeDef = false; + bool shouldSave = false; + bool multiUseInBB = false; /* vreg has more than 1 use in bb */ + bool isThrowVal = false; + bool isCallerSpilled = false; /* only for R0(R1?) which are used for explicit incoming value of throwval; */ + bool mustAllocate = false; /* The register cannot be spilled (clinit pair) */ + uint32 refCount = 0; + float priority = 0.0; + MapleVector> ranges; + MapleVector> holes; + MapleSet usePositions; + LiveInterval *liveParent = nullptr; /* Current li is in aother li's hole. */ + LiveInterval *liveChild = nullptr; /* Another li is in current li's hole. */ + uint32 resultCount = 0; /* number of times this vreg has been written */ + uint8 inCatchState = kRegCatchNotInit; /* part or all of live interval is outside of catch blocks */ + uint8 inCleanUpState = kRegCleanupNotInit; /* part or all of live interval is outside of cleanup blocks */ + }; + + struct ActiveCmp { + bool operator()(const LiveInterval *lhs, const LiveInterval *rhs) const { + CHECK_NULL_FATAL(lhs); + CHECK_NULL_FATAL(rhs); + /* elements considered equal if return false */ + if (lhs == rhs) { + return false; + } + if (lhs->GetFirstDef() == rhs->GetFirstDef() && lhs->GetLastUse() == rhs->GetLastUse() && + lhs->GetRegNO() == rhs->GetRegNO() && lhs->GetRegType() == rhs->GetRegType() && + lhs->GetAssignedReg() == rhs->GetAssignedReg()) { + return false; + } + if (lhs->GetPhysUse() != 0 && rhs->GetPhysUse() != 0) { + if (lhs->GetFirstDef() == rhs->GetFirstDef()) { + return lhs->GetPhysUse() < rhs->GetPhysUse(); + } else { + return lhs->GetFirstDef() < rhs->GetFirstDef(); + } + } + /* At this point, lhs != rhs */ + if (lhs->GetLastUse() == rhs->GetLastUse()) { + return lhs->GetFirstDef() <= rhs->GetFirstDef(); + } + return lhs->GetLastUse() < rhs->GetLastUse(); + } + }; + + public: + LSRALinearScanRegAllocator(CGFunc &cgFunc, MemPool &memPool) + : AArch64RegAllocator(cgFunc, memPool), + liveIntervalsArray(alloc.Adapter()), + lastIntParamLi(alloc.Adapter()), + lastFpParamLi(alloc.Adapter()), + initialQue(alloc.Adapter()), + intParamQueue(alloc.Adapter()), + fpParamQueue(alloc.Adapter()), + callList(alloc.Adapter()), + active(alloc.Adapter()), + intCallerRegSet(alloc.Adapter()), + intCalleeRegSet(alloc.Adapter()), + intParamRegSet(alloc.Adapter()), + intSpillRegSet(alloc.Adapter()), + fpCallerRegSet(alloc.Adapter()), + fpCalleeRegSet(alloc.Adapter()), + fpParamRegSet(alloc.Adapter()), + fpSpillRegSet(alloc.Adapter()), + calleeUseCnt(alloc.Adapter()) { + for (int32 i = 0; i < AArch64Abi::kNumIntParmRegs; ++i) { + intParamQueue.push_back(initialQue); + fpParamQueue.push_back(initialQue); + } + } + ~LSRALinearScanRegAllocator() override = default; + + bool AllocateRegisters() override; + bool CheckForReg(Operand &opnd, Insn &insn, LiveInterval &li, regno_t regNO, bool isDef) const; + void PrintRegSet(const MapleSet &set, const std::string &str) const; + void PrintLiveInterval(LiveInterval &li, const std::string &str) const; + void PrintLiveRanges() const; + void PrintParamQueue(const std::string &str); + void PrintCallQueue(const std::string &str) const; + void PrintActiveList(const std::string &str, uint32 len = 0) const; + void PrintActiveListSimple() const; + void PrintLiveIntervals() const; + void DebugCheckActiveList() const; + void InitFreeRegPool(); + void RecordCall(Insn &insn); + void RecordPhysRegs(const RegOperand ®Opnd, uint32 insnNum, bool isDef); + void UpdateLiveIntervalState(const BB &bb, LiveInterval &li); + void SetupLiveInterval(Operand &opnd, Insn &insn, bool isDef, uint32 &nUses); + void UpdateLiveIntervalByLiveIn(const BB &bb, uint32 insnNum); + void UpdateParamLiveIntervalByLiveIn(const BB &bb, uint32 insnNum); + void ComputeLiveIn(BB &bb, uint32 insnNum); + void ComputeLiveOut(BB &bb, uint32 insnNum); + void ComputeLiveIntervalForEachOperand(Insn &insn); + void ComputeLiveInterval(); + bool PropagateRenameReg(Insn &insn, uint32 replaceReg, Operand &renameOperand); + void PropagateX0(); + void FindLowestPrioInActive(LiveInterval *&li, RegType regType = kRegTyInt, bool startRa = false); + void LiveIntervalAnalysis(); + bool OpndNeedAllocation(Insn &insn, Operand &opnd, bool isDef, uint32 insnNum); + void InsertParamToActive(Operand &opnd); + void InsertToActive(Operand &opnd, uint32 insnNum); + void ReturnPregToSet(LiveInterval &li, uint32 preg); + void ReleasePregToSet(LiveInterval &li, uint32 preg); + void UpdateActiveAtRetirement(uint32 insnID); + void RetireFromActive(const Insn &insn); + void AssignPhysRegsForInsn(Insn &insn); + RegOperand *GetReplaceOpnd(Insn &insn, Operand &opnd, uint32 &spillIdx, bool isDef); + void SetAllocMode(); + void CheckSpillCallee(); + void LinearScanRegAllocator(); + void FinalizeRegisters(); + void SpillOperand(Insn &insn, Operand &opnd, bool isDef, uint32 spillIdx); + void SetOperandSpill(Operand &opnd); + RegOperand *HandleSpillForInsn(Insn &insn, Operand &opnd); + MemOperand *GetSpillMem(uint32 vregNO, bool isDest, Insn &insn, AArch64reg regNO, bool &isOutOfRange); + void InsertCallerSave(Insn &insn, Operand &opnd, bool isDef); + uint32 GetRegFromSet(MapleSet &set, regno_t offset, LiveInterval &li, regno_t forcedReg = 0); + uint32 AssignSpecialPhysRegPattern(Insn &insn, LiveInterval &li); + uint32 FindAvailablePhyReg(LiveInterval &li, Insn &insn); + RegOperand *AssignPhysRegs(Operand &opnd, Insn &insn); + void SetupIntervalRangesByOperand(Operand &opnd, const Insn &insn, uint32 blockFrom, bool isDef, bool isUse); + void BuildIntervalRangesForEachOperand(const Insn &insn, uint32 blockFrom); + void BuildIntervalRanges(); + uint32 FillInHole(LiveInterval &li); + + private: + uint32 FindAvailablePhyRegByFastAlloc(LiveInterval &li); + bool NeedSaveAcrossCall(LiveInterval &li); + uint32 FindAvailablePhyReg(LiveInterval &li, Insn &insn, bool isIntReg); + + /* Comparison function for LiveInterval */ + static constexpr uint32 kMaxSpillRegNum = 3; + static constexpr uint32 kMaxFpSpill = 2; + MapleVector liveIntervalsArray; + MapleVector lastIntParamLi; + MapleVector lastFpParamLi; + MapleQueue initialQue; + using SingleQue = MapleQueue; + MapleVector intParamQueue; + MapleVector fpParamQueue; + MapleList callList; + MapleSet active; + MapleSet::iterator itFinded; + + /* Change these into vectors so it can be added and deleted easily. */ + MapleSet intCallerRegSet; /* integer caller saved */ + MapleSet intCalleeRegSet; /* callee */ + MapleSet intParamRegSet; /* parameter */ + MapleVector intSpillRegSet; /* integer regs put aside for spills */ + /* and register */ + uint32 intCallerMask = 0; /* bit mask for all possible caller int */ + uint32 intCalleeMask = 0; /* callee */ + uint32 intParamMask = 0; /* (physical-register) parameter */ + MapleSet fpCallerRegSet; /* float caller saved */ + MapleSet fpCalleeRegSet; /* callee */ + MapleSet fpParamRegSet; /* parameter */ + MapleVector fpSpillRegSet; /* float regs put aside for spills */ + MapleVector calleeUseCnt; /* Number of time callee reg is seen */ + uint32 fpCallerMask = 0; /* bit mask for all possible caller fp */ + uint32 fpCalleeMask = 0; /* callee */ + uint32 fpParamMask = 0; /* (physical-register) parameter */ + uint32 intBBDefMask = 0; /* locally which physical reg is defined */ + uint32 fpBBDefMask = 0; + uint32 debugSpillCnt = 0; + uint32 regUsedInBBSz = 0; + uint64 *regUsedInBB = nullptr; + uint32 maxInsnNum = 0; + regno_t minVregNum = 0; + regno_t maxVregNum = 0xFFFFFFFF; + bool fastAlloc = false; + bool spillAll = false; + bool needExtraSpillReg = false; + bool isSpillZero = false; + bool shouldOptIntCallee = false; + bool shouldOptFpCallee = false; + uint64 spillCount = 0; + uint64 reloadCount = 0; + uint64 callerSaveSpillCount = 0; + uint64 callerSaveReloadCount = 0; +}; +} /* namespace maplebe */ + +#endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_LSRA_H */ diff --git a/src/mapleall/maple_be/include/cg/aarch64/aarch64_operand.h b/src/mapleall/maple_be/include/cg/aarch64/aarch64_operand.h index 8163c77a916604e0e5b58867543d89dadcfda43c..9e96d8526cf5af941d07bb5ff1661498bd8a96fa 100644 --- a/src/mapleall/maple_be/include/cg/aarch64/aarch64_operand.h +++ b/src/mapleall/maple_be/include/cg/aarch64/aarch64_operand.h @@ -498,6 +498,8 @@ class AArch64MemOperand : public MemOperand { }; /* * ARMv8-A A64 ISA Overview by Matteo Franchin @ ARM + * (presented at 64-bit Android on ARM. Sep. 2015) p.14 + * #endif // ~HUAWEI_EXTERNAL_RELEASE * o Address to load from/store to is a 64-bit base register + an optional offset * LDR X0, [X1] ; Load from address held in X1 * STR X0, [X1] ; Store to address held in X1 diff --git a/src/mapleall/maple_be/include/cg/aarch64/aarch64_proepilog.h b/src/mapleall/maple_be/include/cg/aarch64/aarch64_proepilog.h index fa9074fe1769adcbe8d44c4125b1a6297a36e2f2..918353e0498a018213f06799d8d4fff62724a5f9 100644 --- a/src/mapleall/maple_be/include/cg/aarch64/aarch64_proepilog.h +++ b/src/mapleall/maple_be/include/cg/aarch64/aarch64_proepilog.h @@ -30,10 +30,19 @@ class AArch64GenProEpilog : public GenProEpilog { explicit AArch64GenProEpilog(CGFunc &func) : GenProEpilog(func) {} ~AArch64GenProEpilog() override = default; + bool TailCallOpt() override; void Run() override; private: void GenStackGuard(BB&); BB &GenStackGuardCheckInsn(BB&); + bool HasLoop(); + bool OptimizeTailBB(BB &bb, std::set &callInsns); + void TailCallBBOpt(const BB &exitBB, std::set &callInsns); + void ForwardPropagateAndRename(Insn &mv, Insn &ld, const BB &terminateBB); + void ReplaceMachedOperand(Insn &orig, Insn &target, const RegOperand &match, bool replaceOrigSrc); + bool BackwardFindDependency(BB &ifbb, RegOperand &tgtOpnd, Insn *&ld, Insn *&mov, + Insn *&depMov, std::list &list); + BB *IsolateFastPath(BB&); AArch64MemOperand *SplitStpLdpOffsetForCalleeSavedWithAddInstruction(const AArch64MemOperand &mo, uint32 bitLen, AArch64reg baseReg = AArch64reg::kRinvalid); void AppendInstructionPushPair(AArch64reg reg0, AArch64reg reg1, RegType rty, int offset); diff --git a/src/mapleall/maple_be/include/cg/cg_option.h b/src/mapleall/maple_be/include/cg/cg_option.h index cfe1163b18d1ebb1bc9106ca78a0e6d199afd28e..e50d3860806f8fba9e0c557418a021c6ea927127 100644 --- a/src/mapleall/maple_be/include/cg/cg_option.h +++ b/src/mapleall/maple_be/include/cg/cg_option.h @@ -264,6 +264,7 @@ class CGOptions : public MapleDriverOptionBase { static bool DumpPhase(const std::string &phase); static bool FuncFilter(const std::string &name); void SplitPhases(const std::string &str, std::unordered_set &set); + void SetRange(const std::string &str, const std::string&, Range&); int32 GetOptimizeLevel() const { return optimizeLevel; @@ -400,6 +401,49 @@ class CGOptions : public MapleDriverOptionBase { static void SetDumpFunc(const std::string &func) { dumpFunc = func; } + static size_t FindIndexInProfileData(char data) { + return profileData.find(data); + } + + static void SetProfileData(const std::string &path) { + profileData = path; + } + + static std::string &GetProfileData() { + return profileData; + } + + static const std::string GetProfileDataSubStr(size_t begin, size_t end) { + return profileData.substr(begin, end); + } + + static const std::string GetProfileDataSubStr(size_t position) { + return profileData.substr(position); + } + + static bool IsProfileDataEmpty() { + return profileData.empty(); + } + + static const std::string &GetProfileFuncData() { + return profileFuncData; + } + + static bool IsProfileFuncDataEmpty() { + return profileFuncData.empty(); + } + + static void SetProfileFuncData(const std::string &data) { + profileFuncData = data; + } + + static const std::string &GetProfileClassData() { + return profileClassData; + } + + static void SetProfileClassData(const std::string &data) { + profileClassData = data; + } static const std::string &GetDuplicateAsmFile() { return duplicateAsmFile; @@ -416,6 +460,69 @@ class CGOptions : public MapleDriverOptionBase { static bool UseRange() { return range.enable; } + static const std::string &GetFastFuncsAsmFile() { + return fastFuncsAsmFile; + } + + static bool IsFastFuncsAsmFileEmpty() { + return fastFuncsAsmFile.empty(); + } + + static void SetFastFuncsAsmFile(const std::string &fileName) { + fastFuncsAsmFile = fileName; + } + + static Range &GetRange() { + return range; + } + + static uint64 GetRangeBegin() { + return range.begin; + } + + static uint64 GetRangeEnd() { + return range.end; + } + + static Range &GetSpillRanges() { + return spillRanges; + } + + static uint64 GetSpillRangesBegin() { + return spillRanges.begin; + } + + static uint64 GetSpillRangesEnd() { + return spillRanges.end; + } + + static uint64 GetLSRABBOptSize() { + return lsraBBOptSize; + } + + static void SetLSRABBOptSize(uint64 size) { + lsraBBOptSize = size; + } + + static void SetLSRAInsnOptSize(uint64 size) { + lsraInsnOptSize = size; + } + + static uint64 GetOverlapNum() { + return overlapNum; + } + + static void SetOverlapNum(uint64 num) { + overlapNum = num; + } + + static uint8 GetFastAllocMode() { + return fastAllocMode; + } + + static void SetFastAllocMode(uint8 mode) { + fastAllocMode = mode; + } static void EnableBarriersForVolatile() { useBarriersForVolatile = true; @@ -428,6 +535,13 @@ class CGOptions : public MapleDriverOptionBase { static bool UseBarriersForVolatile() { return useBarriersForVolatile; } + static void EnableFastAlloc() { + fastAlloc = true; + } + + static bool IsFastAlloc() { + return fastAlloc; + } static void EnableDumpBefore() { dumpBefore = true; @@ -536,6 +650,41 @@ class CGOptions : public MapleDriverOptionBase { static bool DoGlobalOpt() { return doGlobalOpt; } + static void EnablePreLSRAOpt() { + doPreLSRAOpt = true; + } + + static void DisablePreLSRAOpt() { + doPreLSRAOpt = false; + } + + static bool DoPreLSRAOpt() { + return doPreLSRAOpt; + } + + static void EnableLocalRefSpill() { + doLocalRefSpill = true; + } + + static void DisableLocalRefSpill() { + doLocalRefSpill = false; + } + + static bool DoLocalRefSpill() { + return doLocalRefSpill; + } + + static void EnableCalleeToSpill() { + doCalleeToSpill = true; + } + + static void DisableCalleeToSpill() { + doCalleeToSpill = false; + } + + static bool DoCalleeToSpill() { + return doCalleeToSpill; + } static void EnablePrePeephole() { doPrePeephole = true; @@ -584,6 +733,13 @@ class CGOptions : public MapleDriverOptionBase { static bool DoSchedule() { return doSchedule; } + static void EnableWriteRefFieldOpt() { + doWriteRefFieldOpt = true; + } + + static void DisableWriteRefFieldOpt() { + doWriteRefFieldOpt = false; + } static bool DoWriteRefFieldOpt() { return doWriteRefFieldOpt; } @@ -675,6 +831,17 @@ class CGOptions : public MapleDriverOptionBase { static bool IsMapleLinker() { return mapleLinker; } + static void EnableReplaceASM() { + replaceASM = true; + } + + static void DisableReplaceASM() { + replaceASM = false; + } + + static bool IsReplaceASM() { + return replaceASM; + } static void EnablePrintFunction() { printFunction = true; @@ -780,6 +947,21 @@ class CGOptions : public MapleDriverOptionBase { return simulateSched; } + static void SetABIType(const std::string &type) { + if (type == "hard") { + abiType = kABIHard; + } else if (type == "soft") { + CHECK_FATAL(false, "float-abi=soft is not supported Currently."); + } else if (type == "softfp") { + abiType = kABISoftFP; + } else { + CHECK_FATAL(false, "unexpected abi-type, only hard, soft and softfp are supported"); + } + } + + static ABIType GetABIType() { + return abiType; + } static void SetEmitFileType(const std::string &type) { if (type == "asm") { @@ -883,6 +1065,7 @@ class CGOptions : public MapleDriverOptionBase { static bool bruteForceSched; /* if true do SimulateSched */ static bool simulateSched; + static ABIType abiType; static EmitFileType emitFileType; /* if true generate adrp/ldr/blr */ static bool genLongCalls; @@ -891,6 +1074,21 @@ class CGOptions : public MapleDriverOptionBase { static bool emitBlockMarker; static Range range; static bool inRange; + static std::string profileData; + static std::string profileFuncData; + static std::string profileClassData; + static std::string fastFuncsAsmFile; + static Range spillRanges; + static uint64 lsraBBOptSize; + static uint64 lsraInsnOptSize; + static uint64 overlapNum; + static uint8 fastAllocMode; + static bool fastAlloc; + static bool doPreLSRAOpt; + static bool doLocalRefSpill; + static bool doCalleeToSpill; + static bool replaceASM; + static std::string literalProfile; }; } /* namespace maplebe */ diff --git a/src/mapleall/maple_be/include/cg/cgfunc.h b/src/mapleall/maple_be/include/cg/cgfunc.h index 6ee2d01ec4da5c470fc97e16296ebefbcc4e8fb2..7a44a88eb5891b25205b8b96ff5ee5efc187b9ff 100644 --- a/src/mapleall/maple_be/include/cg/cgfunc.h +++ b/src/mapleall/maple_be/include/cg/cgfunc.h @@ -110,7 +110,7 @@ class CGFunc { } bool HasVLAOrAlloca() const { - return false; + return hasVLAOrAlloca; } void SetRD(ReachingDefinition *paramRd) { @@ -133,6 +133,7 @@ class CGFunc { void GenerateInstruction(); bool MemBarOpt(StmtNode &membar); + void UpdateCallBBFrequency(); void HandleFunction(); void ProcessExitBBVec(); virtual void MergeReturn() = 0; @@ -736,6 +737,12 @@ class CGFunc { return newBB; } + void UpdateFrequency(StmtNode &stmt) { + bool withFreqInfo = func.HasFreqMap() && !func.GetFreqMap().empty(); + if (withFreqInfo && (func.GetFreqMap().find(stmt.GetStmtID()) != func.GetFreqMap().end())) { + frequency = func.GetFreqMap().at(stmt.GetStmtID()); + } + } BB *StartNewBBImpl(bool stmtIsCurBBLastStmt, StmtNode &stmt) { BB *newBB = CreateNewBB(); @@ -830,6 +837,7 @@ class CGFunc { int32 totalInsns = 0; int32 structCopySize; int32 maxParamStackSize; + bool hasVLAOrAlloca; bool hasProEpilogue = false; bool isVolLoad = false; bool isVolStore = false; diff --git a/src/mapleall/maple_be/include/cg/framewhitelist.def b/src/mapleall/maple_be/include/cg/framewhitelist.def new file mode 100644 index 0000000000000000000000000000000000000000..dc8aabeaa5049087cad2de275f849aea825df42d --- /dev/null +++ b/src/mapleall/maple_be/include/cg/framewhitelist.def @@ -0,0 +1,49 @@ +/* + * Copyright (c) [2020] Huawei Technologies Co.,Ltd.All rights reserved. + * + * OpenArkCompiler is licensed under the Mulan PSL v1. + * You can use this software according to the terms and conditions of the Mulan PSL v1. + * You may obtain a copy of Mulan PSL v1 at: + * + * http://license.coscl.org.cn/MulanPSL + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR + * FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v1 for more details. + */ +"Ldalvik_2Fsystem_2FVMStack_3B_7CgetStackClass1_7C_28_29Ljava_2Flang_2FClass_3B", +"Ldalvik_2Fsystem_2FVMStack_3B_7CgetStackClass2_7C_28_29Ljava_2Flang_2FClass_3B", +"Ljava_2Flang_2FClass_3B_7CnewInstance_7C_28_29Ljava_2Flang_2FObject_3B", +"Ljava_2Flang_2Freflect_2FConstructor_3B_7CnewInstance_7C_28ALjava_2Flang_2FObject_3B_29Ljava_2Flang_2FObject_3B", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetShort_7C_28Ljava_2Flang_2FObject_3B_29S", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetInt_7C_28Ljava_2Flang_2FObject_3B_29I", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetFloat_7C_28Ljava_2Flang_2FObject_3B_29F", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetDouble_7C_28Ljava_2Flang_2FObject_3B_29D", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetChar_7C_28Ljava_2Flang_2FObject_3B_29C", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetByte_7C_28Ljava_2Flang_2FObject_3B_29B", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetBoolean_7C_28Ljava_2Flang_2FObject_3B_29Z", +"Ljava_2Flang_2Freflect_2FField_3B_7CgetLong_7C_28Ljava_2Flang_2FObject_3B_29J", +"Ljava_2Flang_2Freflect_2FField_3B_7Cget_7C_28Ljava_2Flang_2FObject_3B_29Ljava_2Flang_2FObject_3B", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetBoolean_7C_28Ljava_2Flang_2FObject_3BZ_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7Cset_7C_28Ljava_2Flang_2FObject_3BLjava_2Flang_2FObject_3B_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetShort_7C_28Ljava_2Flang_2FObject_3BS_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetLong_7C_28Ljava_2Flang_2FObject_3BJ_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetInt_7C_28Ljava_2Flang_2FObject_3BI_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetFloat_7C_28Ljava_2Flang_2FObject_3BF_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetDouble_7C_28Ljava_2Flang_2FObject_3BD_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetChar_7C_28Ljava_2Flang_2FObject_3BC_29V", +"Ljava_2Flang_2Freflect_2FField_3B_7CsetByte_7C_28Ljava_2Flang_2FObject_3BB_29V", +"LThrowableNativeUncover_3B_7Cmain_7C_28ALjava_2Flang_2FString_3B_29V", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_24InterfaceWithDefault_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_247_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_249_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_2410_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_24InterfaceWithRedefinedMethods_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_2413_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_241ImplementationSuperUser_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_24InterfaceWithStatic_3B_7CstaticMethod_7C_28_29Ljava_2Flang_2FString_3B", +"Ljava_2Flang_2Freflect_2FMethod_3B_7Cinvoke_7C_28Ljava_2Flang_2FObject_3BALjava_2Flang_2FObject_3B_29Ljava_2Flang_2FObject_3B", +"Llibcore_2Fjava_2Flang_2Freflect_2FMethodTest_24OtherInterfaceWithDefault_3B_7CdefaultMethod_7C_28_29Ljava_2Flang_2FString_3B", +"LStackoverflow_3B_7CstackOverflow_7C_28_29V", +"Llibcore_2Fsun_2Fmisc_2FUnsafeTest_241_3B_7Crun_7C_28_29V" diff --git a/src/mapleall/maple_be/src/be/lower.cpp b/src/mapleall/maple_be/src/be/lower.cpp index 1a808dd71c516d023283ace67e2093da01cf1905..ea07f8115593da4991387ba1c619eefb61a5975a 100644 --- a/src/mapleall/maple_be/src/be/lower.cpp +++ b/src/mapleall/maple_be/src/be/lower.cpp @@ -55,11 +55,14 @@ struct ExtFuncDescrT { namespace { std::pair cgBuiltins[] = { - { INTRN_JAVA_ARRAY_LENGTH, "MCC_JavaArrayLength" }, - { INTRN_JAVA_ARRAY_FILL, "MCC_JavaArrayFill" }, - { INTRN_JAVA_CHECK_CAST, "MCC_JavaCheckCast" }, - { INTRN_JAVA_INSTANCE_OF, "MCC_JavaInstanceOf" }, - { INTRN_JAVA_POLYMORPHIC_CALL, "MCC_JavaPolymorphicCall" }, + { INTRN_JAVA_ARRAY_LENGTH, "MCC_DexArrayLength" }, + { INTRN_JAVA_ARRAY_FILL, "MCC_DexArrayFill" }, + { INTRN_JAVA_CHECK_CAST, "MCC_DexCheckCast" }, + { INTRN_JAVA_INSTANCE_OF, "MCC_DexInstanceOf" }, + { INTRN_JAVA_INTERFACE_CALL, "MCC_DexInterfaceCall" }, + { INTRN_JAVA_POLYMORPHIC_CALL, "MCC_DexPolymorphicCall" }, + { INTRN_MCC_DeferredFillNewArray, "MCC_DeferredFillNewArray" }, + { INTRN_MCC_DeferredInvoke, "MCC_DeferredInvoke" }, { INTRN_JAVA_CONST_CLASS, "MCC_GetReferenceToClass" }, { INTRN_JAVA_GET_CLASS, "MCC_GetClass" }, { INTRN_MPL_SET_CLASS, "MCC_SetJavaClass" }, @@ -1908,6 +1911,7 @@ BaseNode *CGLowerer::LowerExpr(const BaseNode &parent, BaseNode &expr, BlockNode return LowerIntrinsicop(parent, static_cast(expr), blkNode); case OP_alloca: { + GetCurrentFunc()->SetVlaOrAlloca(true); return &expr; } case OP_rem: diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_cgfunc.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_cgfunc.cpp index 599d62b9513ab676908c4cd63ded47e1cb6dc96f..cd80ccd99c1c15d962cd7b6c6084205c761f52df 100644 --- a/src/mapleall/maple_be/src/cg/aarch64/aarch64_cgfunc.cpp +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_cgfunc.cpp @@ -5688,6 +5688,14 @@ void AArch64CGFunc::SelectCall(CallNode &callNode) { AArch64RegOperand &extraOpnd = GetOrCreatePhysicalRegisterOperand(R9, kSizeOfPtr * kBitsPerByte, kRegTyInt); srcOpnds->PushOpnd(extraOpnd); } + const std::string &funcName = fsym->GetName(); + if (Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel2 && + funcName == "Ljava_2Flang_2FString_3B_7CindexOf_7C_28Ljava_2Flang_2FString_3B_29I") { + GStrIdx strIdx = GlobalTables::GetStrTable().GetStrIdxFromName(funcName); + MIRSymbol *st = GlobalTables::GetGsymTable().GetSymbolFromStrIdx(strIdx, true); + IntrinsifyStringIndexOf(*srcOpnds, *st); + return; + } Insn &callInsn = AppendCall(*fsym, *srcOpnds); GetCurBB()->SetHasCall(); if (retType != nullptr) { diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_emitter.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_emitter.cpp index 00a6f7e1c4902e899ee0c82eb7c282b198c962ee..c48057a64f5c937f51ded329245327deaa5095b9 100644 --- a/src/mapleall/maple_be/src/cg/aarch64/aarch64_emitter.cpp +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_emitter.cpp @@ -18,6 +18,22 @@ namespace { using namespace maple; +const std::unordered_set kJniNativeFuncList = { + "Landroid_2Fos_2FParcel_3B_7CnativeWriteString_7C_28JLjava_2Flang_2FString_3B_29V_native", + "Landroid_2Fos_2FParcel_3B_7CnativeReadString_7C_28J_29Ljava_2Flang_2FString_3B_native", + "Landroid_2Fos_2FParcel_3B_7CnativeWriteInt_7C_28JI_29V_native", + "Landroid_2Fos_2FParcel_3B_7CnativeReadInt_7C_28J_29I_native", + "Landroid_2Fos_2FParcel_3B_7CnativeWriteInterfaceToken_7C_28JLjava_2Flang_2FString_3B_29V_native", + "Landroid_2Fos_2FParcel_3B_7CnativeEnforceInterface_7C_28JLjava_2Flang_2FString_3B_29V_native" +}; +// map func name to pair +using Func2CodeInsnMap = std::unordered_map>; +Func2CodeInsnMap func2CodeInsnMap { + { "Ljava_2Flang_2FString_3B_7ChashCode_7C_28_29I", + { "maple/mrt/codetricks/arch/arm64/hashCode.s", 29 } }, + { "Ljava_2Flang_2FString_3B_7Cequals_7C_28Ljava_2Flang_2FObject_3B_29Z", + { "maple/mrt/codetricks/arch/arm64/stringEquals.s", 50 } } +}; constexpr uint32 kQuadInsnCount = 2; constexpr uint32 kInsnSize = 4; @@ -317,6 +333,31 @@ void AArch64AsmEmitter::Run(FuncEmitInfo &funcEmitInfo) { MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(cgFunc.GetFunction().GetStIdx().Idx()); const std::string &funcName = std::string(cgFunc.GetShortFuncName().c_str()); + // manually replace function with optimized assembly language + if (CGOptions::IsReplaceASM()) { + auto it = func2CodeInsnMap.find(funcSt->GetName()); + if (it != func2CodeInsnMap.end()) { + std::string optFile = it->second.first; + struct stat buffer; + if (stat(optFile.c_str(), &buffer) == 0) { + std::ifstream codetricksFd(optFile); + if (!codetricksFd.is_open()) { + ERR(kLncErr, " %s open failed!", optFile.c_str()); + LogInfo::MapleLogger() << "wrong" << '\n'; + } else { + std::string contend; + while (getline(codetricksFd, contend)) { + emitter.Emit(contend + "\n"); + } + } + } + emitter.IncreaseJavaInsnCount(it->second.second); +#ifdef EMIT_INSN_COUNT + EmitJavaInsnAddr(funcEmitInfo); +#endif /* ~EMIT_INSN_COUNT */ + return; + } + } std::string funcStName = funcSt->GetName(); if (funcSt->GetFunction()->GetAttr(FUNCATTR_weak)) { (void)emitter.Emit("\t.weak\t" + funcStName + "\n"); diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_immediate.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_immediate.cpp index f788f93e4fd586cf7e1ab46b4f0b6f10996dc079..399d293f25649757b8596ae769169da736dfa252 100644 --- a/src/mapleall/maple_be/src/cg/aarch64/aarch64_immediate.cpp +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_immediate.cpp @@ -23,6 +23,11 @@ #endif namespace maplebe { +#if DEBUG +static std::set ValidBitmaskImmSet = { +#include "valid_bitmask_imm.txt" +}; +#endif namespace { constexpr uint32 kMaxBitTableSize = 5; @@ -40,6 +45,13 @@ bool IsBitmaskImmediate(uint64 val, uint32 bitLen) { if ((bitLen == k32BitSize) && (static_cast(val) == -1)) { return false; } +#if DEBUG + uint64 val2 = val; + if (bitLen == k32BitSize) { + val2 = (val2 << k32BitSize) | (val2 & ((1ULL << k32BitSize) - 1)); + } + bool expectedOutcome = (ValidBitmaskImmSet.find(val2) != ValidBitmaskImmSet.end()); +#endif if ((val & 0x1) != 0) { /* * we want to work with @@ -60,6 +72,13 @@ bool IsBitmaskImmediate(uint64 val, uint32 bitLen) { /* now check if tmp is a power of 2 or tmpVal==0. */ tmpVal = tmpVal & (tmpVal - 1); if (tmpVal == 0) { +#if DEBUG + if (!expectedOutcome) { + LogInfo::MapleLogger() << "0x" << std::hex << std::setw(k16ByteSize) << std::setfill('0') << + static_cast(val) << "\n"; + } + ASSERT(expectedOutcome, "incorrect implementation: not valid value but returning true"); +#endif /* power of two or zero ; return true */ return true; } @@ -70,12 +89,19 @@ bool IsBitmaskImmediate(uint64 val, uint32 bitLen) { /* check if diff is a power of two; return false if not. */ if ((static_cast(diff) & (static_cast(diff) - 1)) != 0) { + ASSERT(!expectedOutcome, "incorrect implementation: valid value but returning false"); return false; } int32 logDiff = __builtin_ctzll(diff); int64 pattern = val & ((1ULL << static_cast(diff)) - 1); +#if DEBUG + bool ret = (val == pattern * bitmaskImmMultTable[kMaxBitTableSize - logDiff]); + ASSERT(expectedOutcome == ret, "incorrect implementation: return value does not match expected outcome"); + return ret; +#else return val == pattern * bitmaskImmMultTable[kMaxBitTableSize - logDiff]; +#endif } bool IsMoveWidableImmediate(uint64 val, uint32 bitLen) { diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_lsra.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_lsra.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a9adcfe540bd84f4499ddd9e7133bdb1cfeec91e --- /dev/null +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_lsra.cpp @@ -0,0 +1,2643 @@ +/* + * Copyright (c) [2020] Huawei Technologies Co.,Ltd.All rights reserved. + * + * OpenArkCompiler is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR + * FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "aarch64_lsra.h" +#include +#include +#include "aarch64_color_ra.h" +#include "aarch64_operand.h" +#include "aarch64_insn.h" +#include "aarch64_cg.h" + +namespace maplebe { +/* + * ================== + * = Linear Scan RA + * ================== + */ +#define LSRA_DUMP CG_DEBUG_FUNC(cgFunc) + +namespace { +constexpr uint32 kSpilled = 1; +constexpr uint32 kSpecialIntSpillReg = 16; +constexpr uint32 kMinLiveIntervalLength = 20; +constexpr uint32 kPrintedActiveListLength = 10; +constexpr uint32 kMinRangesSize = 2; +} + +#define IN_SPILL_RANGE \ + (cgFunc->GetName().find(CGOptions::GetDumpFunc()) != std::string::npos && ++debugSpillCnt && \ + (CGOptions::GetSpillRangesBegin() < debugSpillCnt) && (debugSpillCnt < CGOptions::GetSpillRangesEnd())) + +#undef LSRA_GRAPH + +/* + * This LSRA implementation is an interpretation of the [Poletto97] paper. + * BFS BB ordering is used to order the instructions. The live intervals are vased on + * this instruction order. All vreg defines should come before an use, else a warning is + * given. + * Live interval is traversed in order from lower instruction order to higher order. + * When encountering a live interval for the first time, it is assumed to be live and placed + * inside the 'active' structure until the vreg's last access. During the time a vreg + * is in 'active', the vreg occupies a physical register allocation and no other vreg can + * be allocated the same physical register. + */ +void LSRALinearScanRegAllocator::PrintRegSet(const MapleSet &set, const std::string &str) const { + LogInfo::MapleLogger() << str; + for (auto reg : set) { + LogInfo::MapleLogger() << " " << reg; + } + LogInfo::MapleLogger() << "\n"; +} + +bool LSRALinearScanRegAllocator::CheckForReg(Operand &opnd, Insn &insn, LiveInterval &li, regno_t regNO, + bool isDef) const { + if (!opnd.IsRegister()) { + return false; + } + auto ®Opnd = static_cast(opnd); + if (regOpnd.GetRegisterType() == kRegTyCc || regOpnd.GetRegisterType() == kRegTyVary) { + return false; + } + if (regOpnd.GetRegisterNumber() == regNO) { + LogInfo::MapleLogger() << "set object circle at " << insn.GetId() << "," << li.GetRegNO() << + " size 5 fillcolor rgb \""; + if (isDef) { + LogInfo::MapleLogger() << "black\"\n"; + } else { + LogInfo::MapleLogger() << "orange\"\n"; + } + } + return true; +} + +/* + * This is a support routine to compute the overlapping live intervals in graph form. + * The output file can be viewed by gnuplot. + * Despite the function name of LiveRanges, it is using live intervals. + */ +void LSRALinearScanRegAllocator::PrintLiveRanges() const { + /* ================= Output to plot.pg =============== */ + std::ofstream out("plot.pg"); + CHECK_FATAL(out.is_open(), "Failed to open output file: plot.pg"); + std::streambuf *coutBuf = LogInfo::MapleLogger().rdbuf(); /* old buf */ + LogInfo::MapleLogger().rdbuf(out.rdbuf()); /* new buf */ + + LogInfo::MapleLogger() << "#!/usr/bin/gnuplot\n"; + LogInfo::MapleLogger() << "#maxInsnNum " << maxInsnNum << "\n"; + LogInfo::MapleLogger() << "#minVregNum " << minVregNum << "\n"; + LogInfo::MapleLogger() << "#maxVregNum " << maxVregNum << "\n"; + LogInfo::MapleLogger() << "reset\nset terminal png\n"; + LogInfo::MapleLogger() << "set xrange [1:" << maxInsnNum << "]\n"; + LogInfo::MapleLogger() << "set grid\nset style data linespoints\n"; + LogInfo::MapleLogger() << "set datafile missing '0'\n"; + std::vector> graph; + graph.resize(maxVregNum); + for (uint32 i = 0; i < maxVregNum; ++i) { + graph[i].resize(maxInsnNum); + } + uint32 minY = 0xFFFFFFFF; + uint32 maxY = 0; + for (auto *li : liveIntervalsArray) { + if (li == nullptr || li->GetRegNO() == 0) { + continue; + } + uint32 regNO = li->GetRegNO(); + if ((li->GetLastUse() - li->GetFirstDef()) < kMinLiveIntervalLength) { + continue; + } + if (regNO < minY) { + minY = regNO; + } + if (regNO > maxY) { + maxY = regNO; + } + uint32 n; + for (n = 0; n <= (li->GetFirstDef() - 1); ++n) { + graph[regNO - minVregNum][n] = 0; + } + if (li->GetLastUse() >= n) { + for (; n <= (li->GetLastUse() - 1); ++n) { + graph[regNO - minVregNum][n] = regNO; + } + } + for (; n < maxInsnNum; ++n) { + graph[regNO - minVregNum][n] = 0; + } + + for (auto *bb : sortedBBs) { + FOR_BB_INSNS(insn, bb) { + const AArch64MD *md = &AArch64CG::kMd[static_cast(insn)->GetMachineOpcode()]; + uint32 opndNum = insn->GetOperandSize(); + for (uint32 iSecond = 0; iSecond < opndNum; ++iSecond) { + Operand &opnd = insn->GetOperand(iSecond); + if (opnd.IsList()) { + } else if (opnd.IsMemoryAccessOperand()) { + auto &memOpnd = static_cast(opnd); + Operand *base = memOpnd.GetBaseRegister(); + Operand *offset = memOpnd.GetIndexRegister(); + if (base != nullptr && !CheckForReg(*base, *insn, *li, regNO, false)) { + continue; + } + if (offset != nullptr && !CheckForReg(*offset, *insn, *li, regNO, false)) { + continue; + } + } else { + bool isDef = static_cast(md->operand[iSecond])->IsRegDef(); + (void)CheckForReg(opnd, *insn, *li, regNO, isDef); + } + } + } + } + } + LogInfo::MapleLogger() << "set yrange [" << (minY - 1) << ":" << (maxY + 1) << "]\n"; + + LogInfo::MapleLogger() << "plot \"plot.dat\" using 1:2 title \"R" << minVregNum << "\""; + for (uint32 i = 1; i < (maxVregNum - minVregNum + 1); ++i) { + LogInfo::MapleLogger() << ", \\\n\t\"\" using 1:" << (i + kDivide2) << " title \"R" << (minVregNum + i) << "\""; + } + LogInfo::MapleLogger() << ";\n"; + + /* ================= Output to plot.dat =============== */ + std::ofstream out2("plot.dat"); + CHECK_FATAL(out2.is_open(), "Failed to open output file: plot.dat"); + LogInfo::MapleLogger().rdbuf(out2.rdbuf()); /* new buf */ + LogInfo::MapleLogger() << "##reg"; + for (uint32 i = minVregNum; i <= maxVregNum; ++i) { + LogInfo::MapleLogger() << " R" << i; + } + LogInfo::MapleLogger() << "\n"; + for (uint32 n = 0; n < maxInsnNum; ++n) { + LogInfo::MapleLogger() << (n + 1); + for (uint32 i = minVregNum; i <= maxVregNum; ++i) { + LogInfo::MapleLogger() << " " << graph[i - minVregNum][n]; + } + LogInfo::MapleLogger() << "\n"; + } + LogInfo::MapleLogger().rdbuf(coutBuf); +} + +void LSRALinearScanRegAllocator::PrintLiveInterval(LiveInterval &li, const std::string &str) const { + LogInfo::MapleLogger() << str << "\n"; + if (li.GetIsCall() != nullptr) { + LogInfo::MapleLogger() << " firstDef " << li.GetFirstDef(); + LogInfo::MapleLogger() << " isCall"; + } else if (li.GetPhysUse()) { + LogInfo::MapleLogger() << "\tregNO " << li.GetRegNO(); + LogInfo::MapleLogger() << " firstDef " << li.GetFirstDef(); + LogInfo::MapleLogger() << " physUse " << li.GetPhysUse(); + LogInfo::MapleLogger() << " endByCall " << li.IsEndByCall(); + } else { + /* show regno/firstDef/lastUse with 5/8/8 width respectively */ + LogInfo::MapleLogger() << "\tregNO " << std::setw(5) << li.GetRegNO(); + LogInfo::MapleLogger() << " firstDef " << std::setw(8) << li.GetFirstDef(); + LogInfo::MapleLogger() << " lastUse " << std::setw(8) << li.GetLastUse(); + LogInfo::MapleLogger() << " assigned " << li.GetAssignedReg(); + LogInfo::MapleLogger() << " refCount " << li.GetRefCount(); + LogInfo::MapleLogger() << " priority " << li.GetPriority(); + } + LogInfo::MapleLogger() << " object_address 0x" << std::hex << &li << std::dec << "\n"; +} + +void LSRALinearScanRegAllocator::PrintParamQueue(const std::string &str) { + LogInfo::MapleLogger() << str << "\n"; + for (SingleQue &que : intParamQueue) { + if (que.empty()) { + continue; + } + LiveInterval *li = que.front(); + LiveInterval *last = que.back(); + PrintLiveInterval(*li, ""); + while (li != last) { + que.pop_front(); + que.push_back(li); + li = que.front(); + PrintLiveInterval(*li, ""); + } + que.pop_front(); + que.push_back(li); + } +} + +void LSRALinearScanRegAllocator::PrintCallQueue(const std::string &str) const { + LogInfo::MapleLogger() << str << "\n"; + for (auto *li : callList) { + PrintLiveInterval(*li, ""); + } +} + +void LSRALinearScanRegAllocator::PrintActiveList(const std::string &str, uint32 len) const { + uint32 count = 0; + LogInfo::MapleLogger() << str << " " << active.size() << "\n"; + for (auto *li : active) { + PrintLiveInterval(*li, ""); + ++count; + if ((len != 0) && (count == len)) { + break; + } + } +} + +void LSRALinearScanRegAllocator::PrintActiveListSimple() const { + for (const auto *li : active) { + uint32 assignedReg = li->GetAssignedReg(); + if (li->GetStackSlot() == kSpilled) { + assignedReg = kSpecialIntSpillReg; + } + LogInfo::MapleLogger() << li->GetRegNO() << "(" << assignedReg << ", "; + if (li->GetPhysUse()) { + LogInfo::MapleLogger() << "p) "; + } else { + LogInfo::MapleLogger() << li->GetFirstAcrossedCall(); + } + LogInfo::MapleLogger() << "<" << li->GetFirstDef() << "," << li->GetLastUse() << ">) "; + } + LogInfo::MapleLogger() << "\n"; +} + +void LSRALinearScanRegAllocator::PrintLiveIntervals() const { + for (auto *li : liveIntervalsArray) { + if (li == nullptr || li->GetRegNO() == 0) { + continue; + } + PrintLiveInterval(*li, ""); + } + LogInfo::MapleLogger() << "\n"; +} + +void LSRALinearScanRegAllocator::DebugCheckActiveList() const { + LiveInterval *prev = nullptr; + for (auto *li : active) { + if (prev != nullptr) { + if ((li->GetRegNO() <= V7) && (prev->GetRegNO() > V7)) { + if (li->GetFirstDef() < prev->GetFirstDef()) { + LogInfo::MapleLogger() << "ERRer: active list with out of order phys + vreg\n"; + PrintLiveInterval(*prev, "prev"); + PrintLiveInterval(*li, "current"); + PrintActiveList("Active", kPrintedActiveListLength); + } + } + if ((li->GetRegNO() <= V7) && (prev->GetRegNO() <= V7)) { + if (li->GetFirstDef() < prev->GetFirstDef()) { + LogInfo::MapleLogger() << "ERRer: active list with out of order phys reg use\n"; + PrintLiveInterval(*prev, "prev"); + PrintLiveInterval(*li, "current"); + PrintActiveList("Active", kPrintedActiveListLength); + } + } + } else { + prev = li; + } + } +} + +/* + * Prepare the free physical register pool for allocation. + * When a physical register is allocated, it is removed from the pool. + * The physical register is re-inserted into the pool when the associated live + * interval has ended. + */ +void LSRALinearScanRegAllocator::InitFreeRegPool() { + for (regno_t regNO = kRinvalid; regNO < kMaxRegNum; ++regNO) { + /* special handle for R9 due to MRT_CallSlowNativeExt */ + if (!AArch64Abi::IsAvailableReg(static_cast(regNO)) || regNO == R9) { + continue; + } + if (AArch64isa::IsGPRegister(static_cast(regNO))) { + /* when yieldpoint is enabled, x19 is reserved. */ + if (IsYieldPointReg(static_cast(regNO))) { + continue; + } + /* (15), 16, 17 for spill */ + if (AArch64Abi::IsSpillRegInRA(static_cast(regNO), needExtraSpillReg)) { + intSpillRegSet.push_back(regNO - R0); + continue; + } + /* 0 to 7 for parameters */ + if (AArch64Abi::IsParamReg(static_cast(regNO))) { + (void)intParamRegSet.insert(regNO - R0); + intParamMask |= 1u << (regNO - R0); + } else if (AArch64Abi::IsCalleeSavedReg(static_cast(regNO))) { + (void)intCalleeRegSet.insert(regNO - R0); + intCalleeMask |= 1u << (regNO - R0); + } else { + (void)intCallerRegSet.insert(regNO - R0); + intCallerMask |= 1u << (regNO - R0); + } + } else { + /* 30, 31 for spill */ + if (AArch64Abi::IsSpillRegInRA(static_cast(regNO), needExtraSpillReg)) { + fpSpillRegSet.push_back(regNO - V0); + continue; + } + /* 0 to 7 for parameters */ + if (AArch64Abi::IsParamReg(static_cast(regNO))) { + (void)fpParamRegSet.insert(regNO - V0); + fpParamMask |= 1u << (regNO - V0); + } else if (AArch64Abi::IsCalleeSavedReg(static_cast(regNO))) { + (void)fpCalleeRegSet.insert(regNO - V0); + fpCalleeMask |= 1u << (regNO - V0); + } else { + (void)fpCallerRegSet.insert(regNO - V0); + fpCallerMask |= 1u << (regNO - V0); + } + } + } + + if (LSRA_DUMP) { + PrintRegSet(intCallerRegSet, "ALLOCATABLE_INT_CALLER"); + PrintRegSet(intCalleeRegSet, "ALLOCATABLE_INT_CALLEE"); + PrintRegSet(intParamRegSet, "ALLOCATABLE_INT_PARAM"); + PrintRegSet(fpCallerRegSet, "ALLOCATABLE_FP_CALLER"); + PrintRegSet(fpCalleeRegSet, "ALLOCATABLE_FP_CALLEE"); + PrintRegSet(fpParamRegSet, "ALLOCATABLE_FP_PARAM"); + LogInfo::MapleLogger() << "INT_SPILL_REGS"; + for (uint32 intSpillRegNO : intSpillRegSet) { + LogInfo::MapleLogger() << " " << intSpillRegNO; + } + LogInfo::MapleLogger() << "\n"; + LogInfo::MapleLogger() << "FP_SPILL_REGS"; + for (uint32 fpSpillRegNO : fpSpillRegSet) { + LogInfo::MapleLogger() << " " << fpSpillRegNO; + } + LogInfo::MapleLogger() << "\n"; + LogInfo::MapleLogger() << std::hex; + LogInfo::MapleLogger() << "INT_CALLER_MASK " << intCallerMask << "\n"; + LogInfo::MapleLogger() << "INT_CALLEE_MASK " << intCalleeMask << "\n"; + LogInfo::MapleLogger() << "INT_PARAM_MASK " << intParamMask << "\n"; + LogInfo::MapleLogger() << "FP_CALLER_FP_MASK " << fpCallerMask << "\n"; + LogInfo::MapleLogger() << "FP_CALLEE_FP_MASK " << fpCalleeMask << "\n"; + LogInfo::MapleLogger() << "FP_PARAM_FP_MASK " << fpParamMask << "\n"; + LogInfo::MapleLogger() << std::dec; + } +} + +/* Remember calls for caller/callee allocation. */ +void LSRALinearScanRegAllocator::RecordCall(Insn &insn) { + /* Maintain call at the beginning of active list */ + auto *li = cgFunc->GetMemoryPool()->New(alloc); + li->SetFirstDef(insn.GetId()); + li->SetIsCall(insn); + callList.push_back(li); +} + +/* Handle parameter registers for live interval. */ +void LSRALinearScanRegAllocator::RecordPhysRegs(const RegOperand ®Opnd, uint32 insnNum, bool isDef) { + RegType regType = regOpnd.GetRegisterType(); + uint32 regNO = regOpnd.GetRegisterNumber(); + if (regType == kRegTyCc || regType == kRegTyVary) { + return; + } + + if (IsUntouchableReg(regNO) || regOpnd.IsConstReg()) { + return; + } + + if (regNO == R30 || regNO == R9) { + return; + } + + bool maybeParam = (regType == kRegTyInt && intParamQueue[regNO - R0].empty()) || + (regType == kRegTyFloat && fpParamQueue[regNO - V0].empty()); + + if (isDef) { + /* parameter register def is assumed to be live until a call. */ + auto *li = cgFunc->GetMemoryPool()->New(alloc); + li->SetRegNO(regNO); + li->SetRegType(regType); + li->SetStackSlot(0xFFFFFFFF); + li->SetFirstDef(insnNum); + li->SetPhysUse(insnNum); + li->SetAssignedReg(regNO); + if (regType == kRegTyInt) { + intParamQueue[regNO - R0].push_back(li); + } else { + fpParamQueue[regNO - V0].push_back(li); + } + } else if (maybeParam) { + CHECK_FATAL(false, "impossible"); + } else { + if (regType == kRegTyInt) { + if ((regNO - R0) >= intParamQueue.size()) { + CHECK_FATAL(false, "index out of range in LSRALinearScanRegAllocator::RecordPhysRegs"); + } + LiveInterval *li = intParamQueue[regNO - R0].back(); + li->SetPhysUse(insnNum); + } else { + LiveInterval *li = fpParamQueue[regNO - V0].back(); + li->SetPhysUse(insnNum); + } + } +} + +void LSRALinearScanRegAllocator::UpdateLiveIntervalState(const BB &bb, LiveInterval &li) { + if (bb.IsCatch()) { + li.SetInCatchState(); + } else { + li.SetNotInCatchState(); + } + + if (bb.GetInternalFlag1()) { + li.SetInCleanupState(); + } else { + li.SetNotInCleanupState(bb.GetId() == 1); + } +} + +/* main entry function for live interval computation. */ +void LSRALinearScanRegAllocator::SetupLiveInterval(Operand &opnd, Insn &insn, bool isDef, uint32 &nUses) { + if (!opnd.IsRegister()) { + return; + } + auto ®Opnd = static_cast(opnd); + uint32 insnNum = insn.GetId(); + if (regOpnd.IsPhysicalRegister()) { + RecordPhysRegs(regOpnd, insnNum, isDef); + return; + } + RegType regType = regOpnd.GetRegisterType(); + if (regType == kRegTyCc || regType == kRegTyVary) { + return; + } + + LiveInterval *li = nullptr; + uint32 regNO = regOpnd.GetRegisterNumber(); + if (liveIntervalsArray[regNO] == nullptr) { + li = cgFunc->GetMemoryPool()->New(alloc); + li->SetRegNO(regNO); + li->SetStackSlot(0xFFFFFFFF); + liveIntervalsArray[regNO] = li; + } else { + li = liveIntervalsArray[regNO]; + } + li->SetRegType(regType); + + BB *curBB = insn.GetBB(); + if (isDef) { + if (li->GetFirstDef() == 0) { + li->SetFirstDef(insnNum); + li->SetLastUse(insnNum + 1); + } else if (!curBB->IsUnreachable()) { + if (li->GetLastUse() < insnNum || li->IsUseBeforeDef()) { + li->SetLastUse(insnNum + 1); + } + } + /* + * try-catch related + * Not set when extending live interval with bb's livein in ComputeLiveInterval. + */ + li->SetResultCount(li->GetResultCount() + 1); + } else { + if (li->GetFirstDef() == 0) { + ASSERT(false, "SetupLiveInterval: use before def"); + } + /* + * In ComputeLiveInterval when extending live interval using + * live-out information, li created does not have a type. + */ + if (!curBB->IsUnreachable()) { + li->SetLastUse(insnNum); + } + ++nUses; + } + UpdateLiveIntervalState(*curBB, *li); + + li->SetRefCount(li->GetRefCount() + 1); + + uint32 index = regNO / (sizeof(uint64) * k8ByteSize); + uint64 bit = regNO % (sizeof(uint64) * k8ByteSize); + if ((regUsedInBB[index] & (static_cast(1) << bit)) != 0) { + li->SetMultiUseInBB(true); + } + regUsedInBB[index] |= (static_cast(1) << bit); + + if (minVregNum > regNO) { + minVregNum = regNO; + } + if (maxVregNum < regNO) { + maxVregNum = regNO; + } + + /* setup the def/use point for it */ + ASSERT(regNO < liveIntervalsArray.size(), "out of range of vector liveIntervalsArray"); +} + +/* + * Support 'hole' in LSRA. + * For a live interval, there might be multiple segments of live ranges, + * and between these segments a 'hole'. + * Some other short lived vreg can go into these 'holes'. + * + * from : starting instruction sequence id + * to : ending instruction sequence id + */ +void LSRALinearScanRegAllocator::LiveInterval::AddRange(uint32 from, uint32 to) { + if (ranges.empty()) { + ranges.push_back(std::pair(from, to)); + } else { + if (to < ranges.front().first) { + (void)ranges.insert(ranges.begin(), std::pair(from, to)); + } else if (to >= ranges.front().second && from < ranges.front().first) { + ranges.front().first = from; + ranges.front().second = to; + } else if (to >= ranges.front().first && from < ranges.front().first) { + ranges.front().first = from; + } else if (from > ranges.front().second) { + ASSERT(false, "No possible on reverse traverse."); + } + } +} + +void LSRALinearScanRegAllocator::LiveInterval::AddUsePos(uint32 pos) { + (void)usePositions.insert(pos); +} + +/* See if a vreg can fit in one of the holes of a longer live interval. */ +uint32 LSRALinearScanRegAllocator::FillInHole(LiveInterval &li) { + MapleSet::iterator it; + for (it = active.begin(); it != active.end(); ++it) { + auto *ili = static_cast(*it); + + /* + * If ili is part in cleanup, the hole info will be not correct, + * since cleanup bb do not have edge to normal func bb, and the + * live-out info will not correct. + */ + if (!ili->IsAllOutCleanup() || ili->IsInCatch()) { + continue; + } + + if (ili->GetRegType() != li.GetRegType() || ili->GetStackSlot() != 0xFFFFFFFF || ili->GetLiChild() != nullptr || + ili->GetAssignedReg() == 0) { + continue; + } + for (const auto &inner : ili->GetHoles()) { + if (inner.first <= li.GetFirstDef() && inner.second >= li.GetLastUse()) { + ili->SetLiChild(&li); + li.SetLiParent(ili); + li.SetAssignedReg(ili->GetAssignedReg()); + /* If assigned physical register is callee save register, set shouldSave false; */ + regno_t phyReg = kRinvalid; + if (li.GetRegType() == kRegTyInt || li.GetRegType() == kRegTyFloat) { + phyReg = li.GetAssignedReg(); + } else { + ASSERT(false, "FillInHole, Invalid register type"); + } + + if (AArch64Abi::IsAvailableReg(static_cast(phyReg)) && + AArch64Abi::IsCalleeSavedReg(static_cast(phyReg))) { + li.SetShouldSave(false); + } + return ili->GetAssignedReg(); + } else if (inner.first > li.GetLastUse()) { + break; + } + } + } + return 0; +} + +void LSRALinearScanRegAllocator::SetupIntervalRangesByOperand(Operand &opnd, const Insn &insn, uint32 blockFrom, + bool isDef, bool isUse) { + auto ®Opnd = static_cast(opnd); + RegType regType = regOpnd.GetRegisterType(); + if (regType != kRegTyCc && regType != kRegTyVary) { + regno_t regNO = regOpnd.GetRegisterNumber(); + if (regNO > kNArmRegisters) { + if (isDef) { + if (!liveIntervalsArray[regNO]->GetRanges().empty()) { + liveIntervalsArray[regNO]->GetRanges().front().first = insn.GetId(); + liveIntervalsArray[regNO]->UsePositionsInsert(insn.GetId()); + } + } + if (isUse) { + liveIntervalsArray[regNO]->AddRange(blockFrom, insn.GetId()); + liveIntervalsArray[regNO]->UsePositionsInsert(insn.GetId()); + } + } + } +} + +void LSRALinearScanRegAllocator::BuildIntervalRangesForEachOperand(const Insn &insn, uint32 blockFrom) { + const AArch64MD *md = &AArch64CG::kMd[static_cast(insn).GetMachineOpcode()]; + uint32 opndNum = insn.GetOperandSize(); + for (uint32 i = 0; i < opndNum; ++i) { + Operand &opnd = insn.GetOperand(i); + + if (opnd.IsMemoryAccessOperand()) { + auto &memOpnd = static_cast(opnd); + Operand *base = memOpnd.GetBaseRegister(); + Operand *offset = memOpnd.GetIndexRegister(); + if (base != nullptr && base->IsRegister()) { + SetupIntervalRangesByOperand(*base, insn, blockFrom, false, true); + } + if (offset != nullptr && offset->IsRegister()) { + SetupIntervalRangesByOperand(*offset, insn, blockFrom, false, true); + } + } else if (opnd.IsRegister()) { + bool isDef = static_cast(md->operand[i])->IsRegDef(); + bool isUse = static_cast(md->operand[i])->IsRegUse(); + SetupIntervalRangesByOperand(opnd, insn, blockFrom, isDef, isUse); + } + } +} + +/* Support finding holes by searching for ranges where holes exist. */ +void LSRALinearScanRegAllocator::BuildIntervalRanges() { + uint32 bbIdx = sortedBBs.size(); + if (bbIdx == 0) { + return; + } + + do { + --bbIdx; + BB *bb = sortedBBs[bbIdx]; + if (bb->GetFirstInsn() == nullptr || bb->GetLastInsn() == nullptr) { + continue; + } + uint32 blockFrom = bb->GetFirstInsn()->GetId(); + uint32 blockTo = bb->GetLastInsn()->GetId() + 1; + + for (auto regNO : bb->GetLiveOutRegNO()) { + if (regNO < kNArmRegisters) { + /* Do not consider physical regs. */ + continue; + } + liveIntervalsArray[regNO]->AddRange(blockFrom, blockTo); + } + + FOR_BB_INSNS_REV(insn, bb) { + BuildIntervalRangesForEachOperand(*insn, blockFrom); + } + } while (bbIdx != 0); + + /* Build holes. */ + for (uint32 i = 0; i < cgFunc->GetMaxVReg(); ++i) { + LiveInterval *li = liveIntervalsArray[i]; + if (li == nullptr) { + continue; + } + if (li->GetRangesSize() < kMinRangesSize) { + continue; + } + + auto it = li->GetRanges().begin(); + auto itPrev = it++; + for (; it != li->GetRanges().end(); ++it) { + if (((*it).first - (*itPrev).second) > kMinRangesSize) { + li->HolesPushBack((*itPrev).second, (*it).first); + } + itPrev = it; + } + } +} + +/* Extend live interval with live-in info */ +void LSRALinearScanRegAllocator::UpdateLiveIntervalByLiveIn(const BB &bb, uint32 insnNum) { + for (const auto ®NO : bb.GetLiveInRegNO()) { + if (regNO < kNArmRegisters) { + /* Do not consider physical regs. */ + continue; + } + LiveInterval *liOuter = liveIntervalsArray[regNO]; + if (liOuter != nullptr || (bb.IsEmpty() && bb.GetId() != 1)) { + continue; + } + /* + * try-catch related + * Since it is livein but not seen before, its a use before def + */ + auto *li = cgFunc->GetMemoryPool()->New(alloc); + li->SetRegNO(regNO); + li->SetStackSlot(0xFFFFFFFF); + liveIntervalsArray[regNO] = li; + li->SetFirstDef(insnNum); + li->SetUseBeforeDef(true); + if (!bb.IsUnreachable() && static_cast(cgFunc)->GetCatchRegno() != regNO) { + if (bb.GetId() != 1) { + LogInfo::MapleLogger() << "ERROR: " << regNO << " use before def in bb " << bb.GetId() << " : " << + cgFunc->GetName() << "\n"; + ASSERT(false, "There should only be [use before def in bb 1], temporarily."); + } + LogInfo::MapleLogger() << "WARNING: " << regNO << " use before def in bb " << bb.GetId() << " : " << + cgFunc->GetName() << "\n"; + } + /* + * Need to insert to active list now, as live interval is + * conservatively to start at instruction 1 + */ + (void)active.insert(li); + UpdateLiveIntervalState(bb, *li); + } +} + +/* traverse live in regNO, for each live in regNO create a new liveinterval */ +void LSRALinearScanRegAllocator::UpdateParamLiveIntervalByLiveIn(const BB &bb, uint32 insnNum) { + for (const auto ®NO : bb.GetLiveInRegNO()) { + if (!AArch64Abi::IsParamReg(static_cast(regNO))) { + continue; + } + auto *li = cgFunc->GetMemoryPool()->New(alloc); + li->SetRegNO(regNO); + li->SetStackSlot(0xFFFFFFFF); + li->SetFirstDef(insnNum); + li->SetPhysUse(insnNum); + li->SetAssignedReg(regNO); + if (AArch64isa::IsGPRegister(static_cast(regNO))) { + li->SetRegType(kRegTyInt); + intParamQueue[regNO - R0].push_back(li); + } else { + li->SetRegType(kRegTyFloat); + fpParamQueue[regNO - V0].push_back(li); + } + } +} + +void LSRALinearScanRegAllocator::ComputeLiveIn(BB &bb, uint32 insnNum) { + UpdateLiveIntervalByLiveIn(bb, insnNum); + + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "bb(" << bb.GetId() << ")LIVEOUT:"; + for (const auto &liveOutRegNO : bb.GetLiveOutRegNO()) { + LogInfo::MapleLogger() << " " << liveOutRegNO; + } + LogInfo::MapleLogger() << ".\n"; + LogInfo::MapleLogger() << "bb(" << bb.GetId() << ")LIVEIN:"; + for (const auto &liveInRegNO : bb.GetLiveInRegNO()) { + LogInfo::MapleLogger() << " " << liveInRegNO; + } + LogInfo::MapleLogger() << ".\n"; + } + + regUsedInBBSz = (cgFunc->GetMaxVReg() / (sizeof(uint64) * k8ByteSize) + 1); + regUsedInBB = new uint64[regUsedInBBSz]; + CHECK_FATAL(regUsedInBB != nullptr, "alloc regUsedInBB memory failure."); + errno_t ret = memset_s(regUsedInBB, regUsedInBBSz * sizeof(uint64), 0, regUsedInBBSz * sizeof(uint64)); + if (ret != EOK) { + CHECK_FATAL(false, "call memset_s failed in LSRALinearScanRegAllocator::ComputeLiveInterval()"); + } + + if (bb.GetFirstInsn() == nullptr) { + return; + } + if (!bb.GetEhPreds().empty()) { + bb.InsertLiveInRegNO(R0); + bb.InsertLiveInRegNO(R1); + } + UpdateParamLiveIntervalByLiveIn(bb, insnNum); + if (!bb.GetEhPreds().empty()) { + bb.EraseLiveInRegNO(R0); + bb.EraseLiveInRegNO(R1); + } +} + +void LSRALinearScanRegAllocator::ComputeLiveOut(BB &bb, uint32 insnNum) { + /* + * traverse live out regNO + * for each live out regNO if the last corresponding live interval is created within this bb + * update this lastUse of li to the end of BB + */ + for (const auto ®NO : bb.GetLiveOutRegNO()) { + if (AArch64Abi::IsParamReg(static_cast(regNO))) { + LiveInterval *liOut = nullptr; + if (AArch64isa::IsGPRegister(static_cast(regNO))) { + if (intParamQueue[regNO - R0].empty()) { + continue; + } + liOut = intParamQueue[regNO - R0].back(); + if (bb.GetFirstInsn() && liOut->GetFirstDef() >= bb.GetFirstInsn()->GetId()) { + liOut->SetPhysUse(insnNum); + } + } else { + if (fpParamQueue[regNO - V0].empty()) { + continue; + } + liOut = fpParamQueue[regNO - V0].back(); + if (bb.GetFirstInsn() && liOut->GetFirstDef() >= bb.GetFirstInsn()->GetId()) { + liOut->SetPhysUse(insnNum); + } + } + } + /* Extend live interval with live-out info */ + LiveInterval *li = liveIntervalsArray[regNO]; + if (li != nullptr && !bb.IsEmpty()) { + li->SetLastUse(bb.GetLastInsn()->GetId()); + UpdateLiveIntervalState(bb, *li); + } + } +} + +void LSRALinearScanRegAllocator::ComputeLiveIntervalForEachOperand(Insn &insn) { + uint32 numUses = 0; + const AArch64MD *md = &AArch64CG::kMd[static_cast(&insn)->GetMachineOpcode()]; + + /* + * we need to process src opnd first just in case the src/dest vreg are the same and the src vreg belongs to the + * last interval. + */ + int32 lastOpndId = insn.GetOperandSize() - 1; + for (int32 i = lastOpndId; i >= 0; --i) { + Operand &opnd = insn.GetOperand(i); + bool isDef = static_cast(md->operand[i])->IsRegDef(); + if (opnd.IsList()) { + auto &listOpnd = static_cast(opnd); + for (auto op : listOpnd.GetOperands()) { + SetupLiveInterval(*op, insn, isDef, numUses); + } + } else if (opnd.IsMemoryAccessOperand()) { + auto &memOpnd = static_cast(opnd); + Operand *base = memOpnd.GetBaseRegister(); + Operand *offset = memOpnd.GetIndexRegister(); + isDef = false; + /* + * ldr(156) (opnd0: reg:V34 class: [F]) (opnd1: Mem:literal: + * .LB_Ljava_2Fnio_2FByteBuffer_3B_7CgetDouble_7C_28_29D2) + */ + if (base != nullptr) { + SetupLiveInterval(*base, insn, isDef, numUses); + } + if (offset != nullptr) { + SetupLiveInterval(*offset, insn, isDef, numUses); + } + } else { + SetupLiveInterval(opnd, insn, isDef, numUses); + } + } + if (numUses >= AArch64Abi::kNormalUseOperandNum) { + needExtraSpillReg = true; + } +} + +/* Preference is to put bracket as 1st char of a newline */ +void LSRALinearScanRegAllocator::ComputeLiveInterval() { + /* + * initialize block ordering + * Can be either breadth first or depth first. + * To avoid use before set, we prefer breadth first + */ + calleeUseCnt.resize(kAllRegNum); + liveIntervalsArray.resize(cgFunc->GetMaxVReg()); + /* LiveInterval queue for each param register */ + lastIntParamLi.resize(AArch64Abi::kNumIntParmRegs); + lastFpParamLi.resize(AArch64Abi::kNumFloatParmRegs); + uint32 insnNum = 1; + for (BB *bb : sortedBBs) { + ComputeLiveIn(*bb, insnNum); + FOR_BB_INSNS(insn, bb) { + insn->SetId(insnNum); + + /* skip comment and debug insn */ + if (insn->IsImmaterialInsn() || !insn->IsMachineInstruction()) { + continue; + } + if (insn->GetMachineOpcode() == MOP_clinit) { + auto &dst = static_cast(insn->GetOperand(0)); + RegOperand &phyOpnd = static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( + static_cast(R30), dst.GetSize(), dst.GetRegisterType()); + insn->SetOperand(0, phyOpnd); + } + + if (insn->IsCall()) { + bool skipCall = false; + if (!insn->IsIndirectCall()) { + Operand *targetOpnd = insn->GetCallTargetOperand(); + ASSERT(targetOpnd != nullptr, "nullptr check"); + if (targetOpnd->IsFuncNameOpnd()) { + auto *target = static_cast(targetOpnd); + const MIRSymbol *funcSt = target->GetFunctionSymbol(); + ASSERT(funcSt->GetSKind() == kStFunc, "funcSt must be a kStFunc"); + if (funcSt->GetName() == "exit") { + skipCall = true; + } + } + } + + if (!skipCall) { + if (!insn->GetIsThrow() || !bb->GetEhSuccs().empty()) { + RecordCall(*insn); + } + } + } + + ComputeLiveIntervalForEachOperand(*insn); + + if (insn->IsCall()) { + auto *a64CGFunc = static_cast(cgFunc); + RegOperand ®R0 = a64CGFunc->GetOrCreatePhysicalRegisterOperand(R0, k64, kRegTyInt); + RegOperand ®V0 = a64CGFunc->GetOrCreatePhysicalRegisterOperand(V0, k64, kRegTyFloat); + /* handle return value for call insn */ + if (insn->GetRetType() == Insn::kRegInt) { + RecordPhysRegs(regR0, insnNum, true); + } else { + RecordPhysRegs(regV0, insnNum, true); + } + } + ++insnNum; + } + + ComputeLiveOut(*bb, insnNum); + + delete[] regUsedInBB; + regUsedInBB = nullptr; + maxInsnNum = insnNum - 1; /* insn_num started from 1 */ + } + + for (auto *li : liveIntervalsArray) { + if (li == nullptr || li->GetRegNO() == 0) { + continue; + } + if (li->GetIsCall() != nullptr || li->GetPhysUse()) { + continue; + } + if (li->GetLastUse() > li->GetFirstDef()) { + li->SetPriority(static_cast(li->GetRefCount()) / static_cast(li->GetLastUse() - li->GetFirstDef())); + } else { + li->SetPriority(static_cast(li->GetRefCount()) / static_cast(li->GetFirstDef() - li->GetLastUse())); + } + } + + if (LSRA_DUMP) { + PrintLiveIntervals(); + } +} + +/* + * Replace replace_reg with rename_reg. + * return true if there is a redefinition that needs to terminate the propagation. + */ +bool LSRALinearScanRegAllocator::PropagateRenameReg(Insn &nInsn, uint32 replaceReg, Operand &renameOpnd) { + uint32 renameReg = static_cast(&renameOpnd)->GetRegisterNumber(); + const AArch64MD *md = &AArch64CG::kMd[static_cast(&nInsn)->GetMachineOpcode()]; + int32 lastOpndId = nInsn.GetOperandSize() - 1; + for (int32 i = lastOpndId; i >= 0; --i) { + Operand &opnd = nInsn.GetOperand(i); + if (opnd.IsList()) { + /* call parameters */ + } else if (opnd.IsMemoryAccessOperand()) { + } else if (opnd.IsRegister()) { + bool isDef = static_cast(md->operand[i])->IsRegDef(); + auto ®Opnd = static_cast(opnd); + uint32 regCandidate = regOpnd.GetRegisterNumber(); + if (isDef) { + /* Continue if both replace_reg & rename_reg are not redefined. */ + if (regCandidate == replaceReg || regCandidate == renameReg) { + return true; + } + } else { + if (regCandidate == replaceReg) { + nInsn.SetOperand(i, renameOpnd); + } + } + } + } + return false; /* false == no redefinition */ +} + +/* + * Perform optimization. + * First propagate x0 in a bb. + * Second propagation see comment in function. + */ +void LSRALinearScanRegAllocator::PropagateX0() { + FOR_ALL_BB(bb, cgFunc) { + Insn *insn = bb->GetFirstInsn(); + while (insn != nullptr && !insn->IsMachineInstruction()) { + insn = insn->GetNext(); + } + + /* + * Propagate x0 from a call return value to a def of x0. + * This eliminates some local reloads under high register pressure, since + * the use has been replaced by x0. + */ + if (insn == nullptr || (insn->GetMachineOpcode() != MOP_xmovrr && insn->GetMachineOpcode() != MOP_wmovrr)) { + continue; + } + auto &movSrc = static_cast(insn->GetOperand(1)); + if (movSrc.GetRegisterNumber() != R0) { + continue; + } + + /* At this point the 1st insn is a mov from x0. */ + auto &movDst = static_cast(insn->GetOperand(0)); + uint32 replaceReg = movDst.GetRegisterNumber(); + + bool redefined = false; + Insn *renameInsn = nullptr; + Operand *renameOpnd = nullptr; + uint32 renameReg = 0; + for (Insn *nInsn = insn->GetNext(); nInsn != nullptr; nInsn = nInsn->GetNext()) { + if (!nInsn->IsMachineInstruction()) { + continue; + } + + if (nInsn->IsCall()) { + break; + } + + /* + * Will continue as long as the reg being replaced is not redefined. + * Does not need to check for x0 redefinition. The mov instruction src + * being replaced already defines x0 and will terminate this loop. + */ + uint32 numResults = nInsn->GetResultNum(); + const AArch64MD *md = &AArch64CG::kMd[static_cast(nInsn)->GetMachineOpcode()]; + if (numResults == 1) { + bool isDef = static_cast(md->operand[0])->IsRegDef(); + if (isDef) { + auto &opnd = static_cast(nInsn->GetOperand(0)); + if (opnd.GetRegisterNumber() == replaceReg) { + redefined = true; + } + } + } else if (numResults > 1) { + ASSERT(numResults <= nInsn->GetOperandSize(), "numResults shouldn't be more than the operands' size here"); + for (uint32 i = 0; i < numResults; ++i) { + bool isDef = static_cast(md->operand[i])->IsRegDef(); + if (isDef) { + auto &opnd = static_cast(nInsn->GetOperand(i)); + if (opnd.GetRegisterNumber() == replaceReg) { + redefined = true; + break; + } + } + } + } + + /* Look for move where src is the register equivalent to x0. */ + if (nInsn->GetMachineOpcode() != MOP_xmovrr && nInsn->GetMachineOpcode() != MOP_wmovrr) { + continue; + } + + Operand &src = nInsn->GetOperand(1); + auto &srcReg = static_cast(src); + if (srcReg.GetRegisterNumber() != replaceReg) { + if (redefined) { + break; + } + continue; + } + + /* Setup for the next optmization pattern. */ + Operand &dst = nInsn->GetOperand(0); + auto &dstReg = static_cast(dst); + if (dstReg.GetRegisterNumber() != R0) { + /* This is to set up for further propagation later. */ + if (srcReg.GetRegisterNumber() == replaceReg) { + if (renameInsn != nullptr) { + redefined = true; + break; + } else { + renameInsn = nInsn; + renameOpnd = &dst; + renameReg = dstReg.GetRegisterNumber(); + } + } + continue; + } + + if (redefined) { + break; + } + nInsn->SetOperand(1, movSrc); + break; + } + + if (redefined || renameInsn == nullptr) { + continue; + } + + /* + * Next pattern to help LSRA. Short cross bb live interval. + * Straight line code. Convert reg2 into bb local. + * bb1 + * mov reg2 <- x0 => mov reg2 <- x0 + * mov reg1 <- reg2 mov reg1 <- reg2 + * call call + * bb2 : livein< reg1 reg2 > + * use reg2 use reg1 + * .... + * reg2 not liveout + * + * Can allocate caller register for reg2. + * + * Further propagation of very short live interval cross bb reg + */ + if (renameReg < kAllRegNum) { /* dont propagate physical reg */ + continue; + } + BB *nextBB = bb->GetNext(); + if (nextBB == nullptr) { + break; + } + if (bb->GetSuccs().size() != 1 || nextBB->GetPreds().size() != 1) { + continue; + } + if (bb->GetSuccs().front() != nextBB || nextBB->GetPreds().front() != bb) { + continue; + } + if (bb->GetLiveOutRegNO().find(replaceReg) == bb->GetLiveOutRegNO().end() || + bb->GetLiveOutRegNO().find(renameReg) == bb->GetLiveOutRegNO().end() || + nextBB->GetLiveOutRegNO().find(replaceReg) != nextBB->GetLiveOutRegNO().end()) { + continue; + } + + /* Replace replace_reg by rename_reg. */ + for (Insn *nInsn = renameInsn->GetNext(); nInsn != nullptr && nInsn != bb->GetLastInsn(); + nInsn = nInsn->GetNext()) { + if (!nInsn->IsMachineInstruction()) { + continue; + } + redefined = PropagateRenameReg(*nInsn, replaceReg, *renameOpnd); + if (redefined) { + break; + } + } + + if (redefined) { + continue; + } + + for (Insn *nInsn = nextBB->GetFirstInsn(); nInsn != nextBB->GetLastInsn(); nInsn = nInsn->GetNext()) { + if (!nInsn->IsMachineInstruction()) { + continue; + } + redefined = PropagateRenameReg(*nInsn, replaceReg, *renameOpnd); + if (redefined) { + break; + } + } + } +} + +/* A physical register is freed at the end of the live interval. Return to pool. */ +void LSRALinearScanRegAllocator::ReturnPregToSet(LiveInterval &li, uint32 preg) { + if (preg == 0) { + return; + } + if (li.GetRegType() == kRegTyInt) { + preg -= R0; + } else if (li.GetRegType() == kRegTyFloat) { + preg -= V0; + } else { + ASSERT(false, "ReturnPregToSet: Invalid reg type"); + } + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "\trestoring preg " << preg << " as allocatable\n"; + } + uint32 mask = 1u << preg; + if (preg == kSpecialIntSpillReg && li.GetStackSlot() == 0xFFFFFFFF) { + /* this reg is temporary used for liveinterval which lastUse-firstDef == 1 */ + return; + } + if (li.GetRegType() == kRegTyInt) { + if (intCallerMask & mask) { + (void)intCallerRegSet.insert(preg); + } else if (intCalleeMask & mask) { + (void)intCalleeRegSet.insert(preg); + } else if (intParamMask & mask) { + (void)intParamRegSet.insert(preg); + } else { + ASSERT(false, "ReturnPregToSet: Unknown caller/callee type"); + } + } else if (fpCallerMask & mask) { + (void)fpCallerRegSet.insert(preg); + } else if (fpCalleeMask & mask) { + (void)fpCalleeRegSet.insert(preg); + } else if (fpParamMask & mask) { + (void)fpParamRegSet.insert(preg); + } else { + ASSERT(false, "ReturnPregToSet invalid physical register"); + } +} + +/* A physical register is removed from allocation as it is assigned. */ +void LSRALinearScanRegAllocator::ReleasePregToSet(LiveInterval &li, uint32 preg) { + if (preg == 0) { + return; + } + if (li.GetRegType() == kRegTyInt) { + preg -= R0; + } else if (li.GetRegType() == kRegTyFloat) { + preg -= V0; + } else { + ASSERT(false, "ReleasePregToSet: Invalid reg type"); + } + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "\treleasing preg " << preg << " as allocatable\n"; + } + uint32 mask = 1u << preg; + if (preg == kSpecialIntSpillReg && li.GetStackSlot() == 0xFFFFFFFF) { + /* this reg is temporary used for liveinterval which lastUse-firstDef == 1 */ + return; + } + if (li.GetRegType() == kRegTyInt) { + if (intCallerMask & mask) { + intCallerRegSet.erase(preg); + } else if (intCalleeMask & mask) { + intCalleeRegSet.erase(preg); + } else if (intParamMask & mask) { + intParamRegSet.erase(preg); + } else { + ASSERT(false, "ReleasePregToSet: Unknown caller/callee type"); + } + } else if (fpCallerMask & mask) { + fpCallerRegSet.erase(preg); + } else if (fpCalleeMask & mask) { + fpCalleeRegSet.erase(preg); + } else if (fpParamMask & mask) { + fpParamRegSet.erase(preg); + } else { + ASSERT(false, "ReleasePregToSet invalid physical register"); + } +} + +/* update active in retire */ +void LSRALinearScanRegAllocator::UpdateActiveAtRetirement(uint32 insnID) { + /* Retire live intervals from active list */ + MapleSet::iterator it; + for (it = active.begin(); it != active.end(); /* erase will update */) { + auto *li = static_cast(*it); + if (li->GetLastUse() > insnID) { + break; + } + /* Is it phys reg? */ + if ((li->GetRegNO() >= R0) && (li->GetRegNO() <= R7)) { + if (li->GetPhysUse() != 0 && li->GetPhysUse() <= insnID) { + it = active.erase(it); + if (li->GetPhysUse() != 0) { + ReturnPregToSet(*li, li->GetRegNO()); + } + if (LSRA_DUMP) { + PrintLiveInterval(*li, "\tRemoving phys_reg li\n"); + } + } else { + ++it; + } + continue; + } else if ((li->GetRegNO() >= V0) && (li->GetRegNO() <= V7)) { + if (li->GetPhysUse() != 0 && li->GetPhysUse() <= insnID) { + it = active.erase(it); + if (li->GetPhysUse() != 0) { + ReturnPregToSet(*li, li->GetRegNO()); + } + if (LSRA_DUMP) { + PrintLiveInterval(*li, "\tRemoving phys_reg li\n"); + } + } else { + ++it; + } + continue; + } + /* + * live interval ended for this reg in active + * release physical reg assigned to free reg pool + */ + if (li->GetLiParent() != nullptr) { + li->SetLiParentChild(nullptr); + li->SetLiParent(nullptr); + } else { + ReturnPregToSet(*li, li->GetAssignedReg()); + } + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "Removing " << "(" << li->GetAssignedReg() << ")" << "from regset\n"; + PrintLiveInterval(*li, "\tRemoving virt_reg li\n"); + } + it = active.erase(it); + } +} + +/* Remove a live interval from 'active' list. */ +void LSRALinearScanRegAllocator::RetireFromActive(const Insn &insn) { + if ((insn.GetMachineOpcode() == MOP_adrp_ldr && insn.GetNext() && + insn.GetNext()->GetMachineOpcode() == MOP_clinit_tail) || + (insn.GetMachineOpcode() == MOP_clinit_tail)) { + /* Cannot spill for clinit pair */ + } else if (spillAll) { + return; + } + uint32 insnID = insn.GetId(); + /* + * active list is sorted based on increasing lastUse + * any operand whose use is greater than current + * instruction number is still in use. + * If the use is less than or equal to instruction number + * then it is possible to retire this live interval and + * reclaim the physical register associated with it. + */ + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "RetireFromActive instr_num " << insnID << "\n"; + } + /* Retire call from call queue */ + for (auto it = callList.begin(); it != callList.end();) { + auto *li = static_cast(*it); + if (li->GetFirstDef() > insnID) { + break; + } + callList.pop_front(); + /* at here, it is invalidated */ + it = callList.begin(); + } + + for (uint32 i = 0; i < intParamQueue.size(); ++i) { + /* push back the param not yet use <- as only one is popped, just push it back again */ + if (lastIntParamLi[i] != nullptr) { + intParamQueue[i].push_front(lastIntParamLi[i]); + (void)intParamRegSet.insert(i); + lastIntParamLi[i] = nullptr; + } + if (lastFpParamLi[i] != nullptr) { + fpParamQueue[i].push_front(lastFpParamLi[i]); + (void)fpParamRegSet.insert(i); + lastFpParamLi[i] = nullptr; + } + } + + UpdateActiveAtRetirement(insnID); +} + +/* the return value is a physical reg */ +uint32 LSRALinearScanRegAllocator::GetRegFromSet(MapleSet &set, regno_t offset, LiveInterval &li, + regno_t forcedReg) { + uint32 regNO; + if (forcedReg) { + /* forced_reg is a caller save reg */ + regNO = forcedReg; + } else { + CHECK(!set.empty(), "set is null in LSRALinearScanRegAllocator::GetRegFromSet"); + regNO = *(set.begin()); + } + set.erase(regNO); + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "\tAssign " << regNO << "\n"; + } + regNO += offset; /* Mapping into Maplecg reg */ + li.SetAssignedReg(regNO); + if (LSRA_DUMP) { + PrintRegSet(set, "Reg Set AFTER"); + PrintLiveInterval(li, "LiveInterval after assignment"); + } + return regNO; +} + +/* + * Handle adrp register assignment. Use the same register for the next + * instruction. + */ +uint32 LSRALinearScanRegAllocator::AssignSpecialPhysRegPattern(Insn &insn, LiveInterval &li) { + MOperator opCode = insn.GetMachineOpcode(); + if (opCode != MOP_xadrp) { + return 0; + } + Insn *nInsn = insn.GetNext(); + if (nInsn == nullptr || !nInsn->IsMachineInstruction() || nInsn->IsDMBInsn()) { + return 0; + } + + const AArch64MD *md = &AArch64CG::kMd[static_cast(nInsn)->GetMachineOpcode()]; + bool isDef = md->GetOperand(0)->IsRegDef(); + if (!isDef) { + return 0; + } + Operand &opnd = nInsn->GetOperand(0); + if (!opnd.IsRegister()) { + return 0; + } + auto ®Opnd = static_cast(opnd); + if (!regOpnd.IsPhysicalRegister()) { + return 0; + } + uint32 regNO = regOpnd.GetRegisterNumber(); + if (!(regNO >= R0 && regNO <= R7)) { + return 0; + } + + /* next insn's dest is a physical param reg 'regNO' */ + bool match = false; + uint32 opndNum = nInsn->GetOperandSize(); + for (uint32 i = 1; i < opndNum; ++i) { + Operand &src = nInsn->GetOperand(i); + if (src.IsMemoryAccessOperand()) { + auto &memOpnd = static_cast(src); + Operand *base = memOpnd.GetBaseRegister(); + if (base != nullptr) { + auto *regSrc = static_cast(base); + uint32 srcRegNO = regSrc->GetRegisterNumber(); + if (li.GetRegNO() == srcRegNO) { + match = true; + break; + } + } + Operand *offset = memOpnd.GetIndexRegister(); + if (offset != nullptr) { + auto *regSrc = static_cast(offset); + uint32 srcRegNO = regSrc->GetRegisterNumber(); + if (li.GetRegNO() == srcRegNO) { + match = true; + break; + } + } + } else if (src.IsRegister()) { + auto ®Src = static_cast(src); + uint32 srcRegNO = regSrc.GetRegisterNumber(); + if (li.GetRegNO() == srcRegNO) { + bool srcIsDef = static_cast(md->operand[i])->IsRegDef(); + if (srcIsDef) { + break; + } + match = true; + break; + } + } + } + if (match && li.GetLastUse() > nInsn->GetId()) { + return 0; + } + /* dest of adrp is src of next insn */ + if (match) { + return GetRegFromSet(intParamRegSet, R0, li, regNO - R0); + } + return 0; +} + +uint32 LSRALinearScanRegAllocator::FindAvailablePhyRegByFastAlloc(LiveInterval &li) { + uint32 regNO = 0; + if (li.GetRegType() == kRegTyInt) { + if (!intCalleeRegSet.empty()) { + regNO = GetRegFromSet(intCalleeRegSet, R0, li); + li.SetShouldSave(false); + } else if (!intCallerRegSet.empty()) { + regNO = GetRegFromSet(intCallerRegSet, R0, li); + li.SetShouldSave(true); + } else { + li.SetShouldSave(false); + } + } else if (li.GetRegType() == kRegTyFloat) { + if (!fpCalleeRegSet.empty()) { + regNO = GetRegFromSet(fpCalleeRegSet, V0, li); + li.SetShouldSave(false); + } else if (!fpCallerRegSet.empty()) { + regNO = GetRegFromSet(fpCallerRegSet, V0, li); + li.SetShouldSave(true); + } else { + li.SetShouldSave(false); + } + } + return regNO; +} + +bool LSRALinearScanRegAllocator::NeedSaveAcrossCall(LiveInterval &li) { + bool saveAcrossCall = false; + for (const auto *cli : callList) { + if (cli->GetFirstDef() > li.GetLastUse()) { + break; + } + /* Determine if live interval crosses the call */ + if ((cli->GetFirstDef() > li.GetFirstDef()) && (cli->GetFirstDef() < li.GetLastUse())) { + li.SetShouldSave(true); + /* Need to spill/fill around this call */ + saveAcrossCall = true; + break; + } + } + return saveAcrossCall; +} + +uint32 LSRALinearScanRegAllocator::FindAvailablePhyReg(LiveInterval &li, Insn &insn, bool isIntReg) { + uint32 regNO = 0; + MapleSet &callerRegSet = isIntReg ? intCallerRegSet : fpCallerRegSet; + MapleSet &calleeRegSet = isIntReg ? intCalleeRegSet : fpCalleeRegSet; + MapleSet ¶mRegSet = isIntReg ? intParamRegSet : fpParamRegSet; + AArch64reg reg0 = isIntReg ? R0 : V0; + + /* See if register is live accross a call */ + bool saveAcrossCall = NeedSaveAcrossCall(li); + if (saveAcrossCall) { + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "\t\tlive interval crosses a call\n"; + } + if (regNO == 0) { + if (!li.IsInCatch() && li.IsAllInCleanupOrFirstBB() == false && !calleeRegSet.empty()) { + /* call in live interval, use callee if available */ + regNO = GetRegFromSet(calleeRegSet, reg0, li); + /* Since it is callee saved, no need to continue search */ + li.SetShouldSave(false); + } else if (li.IsMultiUseInBB()) { + /* + * allocate caller save if there are multiple uses in one bb + * else it is no different from spilling + */ + if (!callerRegSet.empty()) { + regNO = GetRegFromSet(callerRegSet, reg0, li); + } else if (!paramRegSet.empty()) { + regNO = GetRegFromSet(paramRegSet, reg0, li); + } + } + } + if (regNO == 0) { + /* No register left for allocation */ + regNO = FillInHole(li); + if (regNO == 0) { + li.SetShouldSave(false); + } + } + return regNO; + } else { + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "\t\tlive interval does not cross a call\n"; + } + if (isIntReg) { + regNO = AssignSpecialPhysRegPattern(insn, li); + if (regNO != 0) { + return regNO; + } + } + if (!paramRegSet.empty()) { + regNO = GetRegFromSet(paramRegSet, reg0, li); + } + if (regNO == 0) { + if (!callerRegSet.empty()) { + regNO = GetRegFromSet(callerRegSet, reg0, li); + } else if (!calleeRegSet.empty()) { + regNO = GetRegFromSet(calleeRegSet, reg0, li); + } else { + regNO = FillInHole(li); + } + } + return regNO; + } +} + +/* Return a phys register number for the live interval. */ +uint32 LSRALinearScanRegAllocator::FindAvailablePhyReg(LiveInterval &li, Insn &insn) { + if (fastAlloc) { + return FindAvailablePhyRegByFastAlloc(li); + } + uint32 regNO = 0; + if (li.GetRegType() == kRegTyInt) { + regNO = FindAvailablePhyReg(li, insn, true); + } else if (li.GetRegType() == kRegTyFloat) { + regNO = FindAvailablePhyReg(li, insn, false); + } + return regNO; +} + +/* Spill and reload for caller saved registers. */ +void LSRALinearScanRegAllocator::InsertCallerSave(Insn &insn, Operand &opnd, bool isDef) { + auto ®Opnd = static_cast(opnd); + uint32 vRegNO = regOpnd.GetRegisterNumber(); + if (vRegNO >= liveIntervalsArray.size()) { + CHECK_FATAL(false, "index out of range in LSRALinearScanRegAllocator::InsertCallerSave"); + } + LiveInterval *rli = liveIntervalsArray[vRegNO]; + RegType regType = regOpnd.GetRegisterType(); + + isSpillZero = false; + if (!isDef) { + uint32 mask; + uint32 regBase; + if (regType == kRegTyInt) { + mask = intBBDefMask; + regBase = R0; + } else { + mask = fpBBDefMask; + regBase = V0; + } + if (mask & (1u << (rli->GetAssignedReg() - regBase))) { + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "InsertCallerSave " << rli->GetAssignedReg() << " skipping due to local def\n"; + } + return; + } + } + + if (!rli->IsShouldSave()) { + return; + } + + uint32 regSize = regOpnd.GetSize(); + PrimType spType; + + if (regType == kRegTyInt) { + spType = (regSize <= k32BitSize) ? PTY_i32 : PTY_i64; + intBBDefMask |= (1u << (rli->GetAssignedReg() - R0)); + } else { + spType = (regSize <= k32BitSize) ? PTY_f32 : PTY_f64; + fpBBDefMask |= (1u << (rli->GetAssignedReg() - V0)); + } + + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "InsertCallerSave " << vRegNO << "\n"; + } + + if (!isDef && !rli->IsCallerSpilled()) { + LogInfo::MapleLogger() << "WARNING: " << vRegNO << " caller restore without spill in bb " + << insn.GetBB()->GetId() << " : " << cgFunc->GetName() << "\n"; + } + rli->SetIsCallerSpilled(true); + + if (isDef) { + MOperator opCode = insn.GetMachineOpcode(); + if (opCode == MOP_xmovri64 || opCode == MOP_xmovri32) { + Operand &opnd1 = insn.GetOperand(1); + auto &imm = static_cast(opnd1); + if (imm.IsZero()) { + isSpillZero = true; + } + } else if (opCode == MOP_wmovrr || opCode == MOP_xmovrr) { + auto &opnd1 = static_cast(insn.GetOperand(1)); + if (opnd1.IsZeroRegister()) { + isSpillZero = true; + } + } + if (isSpillZero) { + /* This has to be a caller register */ + intBBDefMask &= ~(1u << (rli->GetAssignedReg() - R0)); + } + } + + auto *a64CGFunc = static_cast(cgFunc); + CG *cg = a64CGFunc->GetCG(); + MemOperand *memOpnd = nullptr; + RegOperand *phyOpnd = nullptr; + + if (isSpillZero) { + phyOpnd = &AArch64RegOperand::GetZeroRegister(regSize); + } else { + phyOpnd = &a64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast(rli->GetAssignedReg()), regSize, + regType); + } + + std::string comment; + bool isOutOfRange = false; + if (isDef) { + memOpnd = GetSpillMem(vRegNO, true, insn, static_cast(intSpillRegSet[0] + R0), isOutOfRange); + Insn &stInsn = cg->BuildInstruction(a64CGFunc->PickStInsn(regSize, spType), *phyOpnd, *memOpnd); + comment = " SPILL for caller_save " + std::to_string(vRegNO); + ++callerSaveSpillCount; + if (rli->GetLastUse() == insn.GetId()) { + a64CGFunc->FreeSpillRegMem(vRegNO); + comment += " end"; + } + stInsn.SetComment(comment); + if (isOutOfRange) { + insn.GetBB()->InsertInsnAfter(*insn.GetNext(), stInsn); + } else { + insn.GetBB()->InsertInsnAfter(insn, stInsn); + } + } else { + memOpnd = GetSpillMem(vRegNO, false, insn, static_cast(intSpillRegSet[0] + R0), isOutOfRange); + Insn &ldInsn = cg->BuildInstruction(a64CGFunc->PickLdInsn(regSize, spType), *phyOpnd, *memOpnd); + comment = " RELOAD for caller_save " + std::to_string(vRegNO); + ++callerSaveReloadCount; + if (rli->GetLastUse() == insn.GetId()) { + a64CGFunc->FreeSpillRegMem(vRegNO); + comment += " end"; + } + ldInsn.SetComment(comment); + insn.GetBB()->InsertInsnBefore(insn, ldInsn); + } +} + +/* Shell function to find a physical register for an operand. */ +RegOperand *LSRALinearScanRegAllocator::AssignPhysRegs(Operand &opnd, Insn &insn) { + auto ®Opnd = static_cast(opnd); + uint32 vRegNO = regOpnd.GetRegisterNumber(); + RegType regType = regOpnd.GetRegisterType(); + if (vRegNO >= liveIntervalsArray.size()) { + CHECK_FATAL(false, "index out of range in LSRALinearScanRegAllocator::AssignPhysRegs"); + } + LiveInterval *li = liveIntervalsArray[vRegNO]; + + bool doNotSpill = false; + if (li->IsMustAllocate() || (insn.GetMachineOpcode() == MOP_adrp_ldr && insn.GetNext() && + insn.GetNext()->GetMachineOpcode() == MOP_clinit_tail) || + (insn.GetMachineOpcode() == MOP_clinit_tail)) { + /* Cannot spill for clinit pair */ + doNotSpill = true; + } else if (spillAll) { + return nullptr; + } else if (IN_SPILL_RANGE) { + return nullptr; + } + + if (doNotSpill) { + li->SetMustAllocate(true); + } + + /* + * if only def, no use, then should assign a new phyreg, + * otherwise, there may be conflict + */ + if (li->GetAssignedReg() != 0 && (li->GetLastUse() != 0 || li->GetPhysUse() != 0)) { + if (AArch64Abi::IsCalleeSavedReg(static_cast(li->GetAssignedReg()))) { + ++calleeUseCnt[li->GetAssignedReg()]; + } + if (li->GetStackSlot() == 0xFFFFFFFF) { + return &static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( + static_cast(li->GetAssignedReg()), opnd.GetSize(), regType); + } else { + /* need to reload */ + return nullptr; + } + } + + /* pre spilled: */ + if (li->GetStackSlot() != 0xFFFFFFFF) { + return nullptr; + } + + if (LSRA_DUMP) { + uint32 activeSz = active.size(); + LogInfo::MapleLogger() << "\tAssignPhysRegs-active_sz " << activeSz << "\n"; + } + + uint32 regNO = FindAvailablePhyReg(*li, insn); + if (regNO != 0) { + if (AArch64Abi::IsCalleeSavedReg(static_cast(regNO))) { + if (!CGOptions::DoCalleeToSpill()) { + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "\tCallee-save register for save/restore in prologue/epilogue: " << regNO << "\n"; + } + static_cast(cgFunc)->AddtoCalleeSaved(static_cast(regNO)); + } + ++calleeUseCnt[regNO]; + } + return &static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( + static_cast(li->GetAssignedReg()), opnd.GetSize(), regType); + } + + return nullptr; +} + +MemOperand *LSRALinearScanRegAllocator::GetSpillMem(uint32 vRegNO, bool isDest, Insn &insn, AArch64reg regNO, + bool &isOutOfRange) { + auto *a64CGFunc = static_cast(cgFunc); + MemOperand *memOpnd = a64CGFunc->GetOrCreatSpillMem(vRegNO); + return (a64CGFunc->AdjustMemOperandIfOffsetOutOfRange(memOpnd, vRegNO, isDest, insn, regNO, isOutOfRange)); +} + +/* Set a vreg in live interval as being marked for spill. */ +void LSRALinearScanRegAllocator::SetOperandSpill(Operand &opnd) { + auto ®Opnd = static_cast(opnd); + uint32 regNO = regOpnd.GetRegisterNumber(); + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "SetOperandSpill " << regNO; + LogInfo::MapleLogger() << "(" << liveIntervalsArray[regNO]->GetFirstAcrossedCall(); + LogInfo::MapleLogger() << ", refCount " << liveIntervalsArray[regNO]->GetRefCount() << ")\n"; + } + + ASSERT(regNO < liveIntervalsArray.size(), + "index out of vector size in LSRALinearScanRegAllocator::SetOperandSpill"); + LiveInterval *li = liveIntervalsArray[regNO]; + li->SetStackSlot(kSpilled); +} + +/* + * Generate spill/reload for an operand. + * spill_idx : one of 3 phys regs set aside for the purpose of spills. + */ +void LSRALinearScanRegAllocator::SpillOperand(Insn &insn, Operand &opnd, bool isDef, uint32 spillIdx) { + /* + * Insert spill (def) and fill (use) instructions for the operand. + * Keep track of the 'slot' (base 0). The actual slot on the stack + * will be some 'base_slot_offset' + 'slot' off FP. + * For simplification, entire 64bit register is spilled/filled. + * + * For example, a virtual register home 'slot' on the stack is location 5. + * This represents a 64bit slot (8bytes). The base_slot_offset + * from the base 'slot' determined by whoever is added, off FP. + * stack address is ( FP - (5 * 8) + base_slot_offset ) + * So the algorithm is simple, for each virtual register that is not + * allocated, it has to have a home address on the stack (a slot). + * A class variable is used, start from 0, increment by 1. + * Since LiveInterval already represent unique regNO information, + * just add a slot number to it. Subsequent reference to a regNO + * will either get an allocated physical register or a slot number + * for computing the stack location. + * + * This function will also determine the operand to be a def or use. + * For def, spill instruction(s) is appended after the insn. + * For use, spill instruction(s) is prepended before the insn. + * Use FP - (slot# *8) for now. Will recompute if base_slot_offset + * is not 0. + * + * The total number of slots used will be used to compute the stack + * frame size. This will require some interface external to LSRA. + * + * For normal instruction, two spill regs should be enough. The caller + * controls which ones to use. + * For more complex operations, need to break down the instruction. + * eg. store v1 -> [v2 + v3] // 3 regs needed + * => p1 <- v2 // address part 1 + * p2 <- v3 // address part 2 + * p1 <- p1 + p2 // freeing up p2 + * p2 <- v1 + * store p2 -> [p1] + * or we can allocate more registers to the spill register set + * For store multiple, need to break it down into two or more instr. + */ + auto ®Opnd = static_cast(opnd); + uint32 regNO = regOpnd.GetRegisterNumber(); + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "SpillOperand " << regNO << "\n"; + } + + isSpillZero = false; + + regno_t spReg; + PrimType spType; + CHECK_FATAL(regNO < liveIntervalsArray.size(), "index out of range in LSRALinearScanRegAllocator::SpillOperand"); + LiveInterval *li = liveIntervalsArray[regNO]; + ASSERT(!li->IsShouldSave(), "SpillOperand: Should not be caller"); + uint32 regSize = regOpnd.GetSize(); + auto *a64CGFunc = static_cast(cgFunc); + CG *cg = a64CGFunc->GetCG(); + RegType regType = regOpnd.GetRegisterType(); + + if (isDef) { + MOperator opCode = insn.GetMachineOpcode(); + if (opCode == MOP_xmovri64 || opCode == MOP_xmovri32) { + Operand &opnd1 = insn.GetOperand(1); + auto &imm = static_cast(opnd1); + if (imm.IsZero()) { + isSpillZero = true; + } + } else if (opCode == MOP_wmovrr || opCode == MOP_xmovrr) { + auto &opnd1 = static_cast(insn.GetOperand(1)); + if (opnd1.IsZeroRegister()) { + isSpillZero = true; + } + } + } + + if (li->GetRegType() == kRegTyInt) { + ASSERT((spillIdx < intSpillRegSet.size()), "SpillOperand: ran out int spill reg"); + spReg = intSpillRegSet[spillIdx] + R0; + spType = (regSize <= k32BitSize) ? PTY_i32 : PTY_i64; + } else if (li->GetRegType() == kRegTyFloat) { + ASSERT((spillIdx < fpSpillRegSet.size()), "SpillOperand: ran out fp spill reg"); + spReg = fpSpillRegSet[spillIdx] + V0; + spType = (regSize <= k32BitSize) ? PTY_f32 : PTY_f64; + } else { + CHECK_FATAL(false, "SpillOperand: Should be int or float type"); + } + + bool isOutOfRange = false; + RegOperand *phyOpnd = nullptr; + if (isSpillZero) { + phyOpnd = &AArch64RegOperand::GetZeroRegister(regSize); + } else { + phyOpnd = &a64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast(spReg), regSize, regType); + } + li->SetAssignedReg(phyOpnd->GetRegisterNumber()); + + MemOperand *memOpnd = nullptr; + if (isDef) { + /* + * Need to assign spReg (one of the two spill reg) to the destination of the insn. + * spill_vreg <- opn1 op opn2 + * to + * spReg <- opn1 op opn2 + * store spReg -> spillmem + */ + li->SetStackSlot(kSpilled); + + ++spillCount; + memOpnd = GetSpillMem(regNO, true, insn, static_cast(intSpillRegSet[spillIdx + 1] + R0), isOutOfRange); + Insn &stInsn = cg->BuildInstruction(a64CGFunc->PickStInsn(regSize, spType), *phyOpnd, *memOpnd); + std::string comment = " SPILL vreg:" + std::to_string(regNO); + if (li->GetLastUse() == insn.GetId()) { + a64CGFunc->FreeSpillRegMem(regNO); + comment += " end"; + } + stInsn.SetComment(comment); + if (isOutOfRange) { + insn.GetBB()->InsertInsnAfter(*insn.GetNext(), stInsn); + } else { + insn.GetBB()->InsertInsnAfter(insn, stInsn); + } + } else { + /* Here, reverse of isDef, change either opn1 or opn2 to the spReg. */ + if (li->GetStackSlot() == 0xFFFFFFFF) { + LogInfo::MapleLogger() << "WARNING: " << regNO << " assigned " << li->GetAssignedReg() << + " restore without spill in bb " << insn.GetBB()->GetId() << " : " << + cgFunc->GetName() << "\n"; + } + ++reloadCount; + memOpnd = GetSpillMem(regNO, false, insn, static_cast(intSpillRegSet[spillIdx] + R0), isOutOfRange); + Insn &ldInsn = cg->BuildInstruction(a64CGFunc->PickLdInsn(regSize, spType), *phyOpnd, *memOpnd); + std::string comment = " RELOAD vreg" + std::to_string(regNO); + if (li->GetLastUse() == insn.GetId()) { + a64CGFunc->FreeSpillRegMem(regNO); + comment += " end"; + } + ldInsn.SetComment(comment); + insn.GetBB()->InsertInsnBefore(insn, ldInsn); + } +} + +RegOperand *LSRALinearScanRegAllocator::HandleSpillForInsn(Insn &insn, Operand &opnd) { + /* choose the lowest priority li to spill */ + auto ®Opnd = static_cast(opnd); + uint32 regNO = regOpnd.GetRegisterNumber(); + ASSERT(regNO < liveIntervalsArray.size(), + "index out of range of MapleVector in LSRALinearScanRegAllocator::HandleSpillForInsn"); + LiveInterval *li = liveIntervalsArray[regNO]; + RegType regType = regOpnd.GetRegisterType(); + LiveInterval *spillLi = nullptr; + FindLowestPrioInActive(spillLi, regType, true); + + /* + * compare spill_li with current li + * spill_li is null and li->SetStackSlot(Spilled) when the li is spilled due to LiveIntervalAnalysis + */ + if (spillLi == nullptr || spillLi->GetLiParent() || spillLi->GetLiChild() || li->GetStackSlot() == kSpilled || + li->GetFirstDef() != insn.GetId() || li->GetPriority() < spillLi->GetPriority() || + li->GetRefCount() < spillLi->GetRefCount() || + !(AArch64Abi::IsCalleeSavedReg(static_cast(spillLi->GetAssignedReg())))) { + /* spill current li */ + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "Flexible Spill: still spill " << li->GetRegNO() << ".\n"; + } + SetOperandSpill(opnd); + return nullptr; + } + + ReturnPregToSet(*spillLi, spillLi->GetAssignedReg()); + RegOperand *newOpnd = AssignPhysRegs(opnd, insn); + if (newOpnd == nullptr) { + ReleasePregToSet(*spillLi, spillLi->GetAssignedReg()); + SetOperandSpill(opnd); + return nullptr; + } + + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "Flexible Spill: " << spillLi->GetRegNO() << " instead of " << li->GetRegNO() << ".\n"; + PrintLiveInterval(*spillLi, "TO spill: "); + PrintLiveInterval(*li, "Instead of: "); + } + + /* spill this live interval */ + active.erase(itFinded); + spillLi->SetStackSlot(kSpilled); + + return newOpnd; +} + +bool LSRALinearScanRegAllocator::OpndNeedAllocation(Insn &insn, Operand &opnd, bool isDef, uint32 insnNum) { + if (!opnd.IsRegister()) { + return false; + } + auto ®Opnd = static_cast(opnd); + RegType regType = regOpnd.GetRegisterType(); + uint32 regNO = regOpnd.GetRegisterNumber(); + if (regType == kRegTyCc || regType == kRegTyVary) { + return false; + } + if (IsUntouchableReg(regNO) || regOpnd.IsConstReg()) { + return false; + } + if (regOpnd.IsPhysicalRegister()) { + if ((insn.GetMachineOpcode() == MOP_adrp_ldr && insn.GetNext() && + insn.GetNext()->GetMachineOpcode() == MOP_clinit_tail) || + (insn.GetMachineOpcode() == MOP_clinit_tail)) { + /* Cannot spill for clinit pair */ + } else if (spillAll) { + return false; + } + if (isDef) { + if (regType == kRegTyInt) { + if (regNO > R7 || intParamQueue[regNO - R0].empty()) { + return false; + } + LiveInterval *li = intParamQueue[regNO - R0].front(); + /* li may have been inserted by InsertParamToActive */ + if (li->GetFirstDef() == insnNum) { + intParamRegSet.erase(regNO - R0); + (void)active.insert(li); + ASSERT((regNO - R0) < intParamQueue.size(), + "index out of range in LSRALinearScanRegAllocator::OpndNeedAllocation"); + intParamQueue[regNO - R0].pop_front(); + } + } else { + if (regNO > V7 || fpParamQueue[regNO - V0].empty()) { + return false; + } + LiveInterval *li = fpParamQueue[regNO - V0].front(); + /* li may have been inserted by InsertParamToActive */ + if (li->GetFirstDef() == insnNum) { + fpParamRegSet.erase(regNO - V0); + (void)active.insert(li); + fpParamQueue[regNO - V0].pop_front(); + } + } + } + return false; + } + /* This is a virtual register */ + return true; +} + +void LSRALinearScanRegAllocator::InsertParamToActive(Operand &opnd) { + auto ®Opnd = static_cast(opnd); + uint32 regNO = regOpnd.GetRegisterNumber(); + CHECK_FATAL(regNO < liveIntervalsArray.size(), + "index out of range in LSRALinearScanRegAllocator::InsertParamToActive"); + LiveInterval *li = liveIntervalsArray[regNO]; + /* Search for parameter registers that is in the live range to insert into queue */ + if (li->GetRegType() == kRegTyInt) { + for (uint32 i = 0; i < intParamQueue.size(); ++i) { + if (intParamQueue[i].empty()) { + continue; + } + LiveInterval *pli = intParamQueue[i].front(); + do { + if ((pli->GetFirstDef() <= li->GetFirstDef()) && (pli->GetPhysUse() <= li->GetFirstDef())) { + /* just discard it */ + intParamQueue[i].pop_front(); + if (intParamQueue[i].empty()) { + break; + } + pli = intParamQueue[i].front(); + } else { + break; + } + } while (true); + if ((pli->GetFirstDef() < li->GetLastUse()) && (pli->GetPhysUse() > li->GetFirstDef())) { + if (intParamRegSet.find(i) != intParamRegSet.end()) { + /* reserve this param register and active the its first use */ + lastIntParamLi[i] = pli; + intParamRegSet.erase(i); + intParamQueue[i].pop_front(); + } + } + } + } else { + ASSERT((li->GetRegType() == kRegTyFloat), "InsertParamToActive: Incorrect register type"); + for (uint32 i = 0; i < fpParamQueue.size(); ++i) { + if (fpParamQueue[i].empty()) { + continue; + } + LiveInterval *pli = fpParamQueue[i].front(); + do { + if ((pli->GetFirstDef() <= li->GetFirstDef()) && (pli->GetPhysUse() <= li->GetFirstDef())) { + /* just discard it */ + fpParamQueue[i].pop_front(); + if (fpParamQueue[i].empty()) { + break; + } + pli = fpParamQueue[i].front(); + } else { + break; + } + } while (true); + if ((pli->GetFirstDef() < li->GetLastUse()) && (pli->GetPhysUse() > li->GetFirstDef())) { + if (fpParamRegSet.find(i) != fpParamRegSet.end()) { + lastFpParamLi[i] = pli; + fpParamRegSet.erase(i); + fpParamQueue[i].pop_front(); + } + } + } + } +} + +/* Insert a live interval into the 'active' list. */ +void LSRALinearScanRegAllocator::InsertToActive(Operand &opnd, uint32 insnNum) { + auto ®Opnd = static_cast(opnd); + uint32 regNO = regOpnd.GetRegisterNumber(); + CHECK_FATAL(regNO < liveIntervalsArray.size(), + "index out of range in LSRALinearScanRegAllocator::InsertToActive"); + LiveInterval *li = liveIntervalsArray[regNO]; + if (li->GetLastUse() <= insnNum) { + /* insert first, and retire later, then the assigned reg can be released */ + (void)active.insert(li); + if (LSRA_DUMP) { + PrintLiveInterval(*li, "LiveInterval is skip due to past insn num --- opt to remove redunant insn"); + } + return; + } + (void)active.insert(li); +} + +/* find the lowest one and erase it from active */ +void LSRALinearScanRegAllocator::FindLowestPrioInActive(LiveInterval *&targetLi, RegType regType, bool startRA) { + float lowestPrio = 100.0; + bool found = false; + MapleSet::iterator it; + MapleSet::iterator lowestIt; + for (it = active.begin(); it != active.end(); ++it) { + auto *li = static_cast(*it); + if (startRA && li->GetPhysUse() != 0) { + continue; + } + if (li->GetPriority() < lowestPrio && li->GetRegType() == regType) { + lowestPrio = li->GetPriority(); + lowestIt = it; + found = true; + } + } + if (found) { + targetLi = *lowestIt; + itFinded = lowestIt; + } +} + +/* Calculate the weight of a live interval for pre-spill and flexible spill */ +void LSRALinearScanRegAllocator::LiveIntervalAnalysis() { + for (uint32 bbIdx = 0; bbIdx < sortedBBs.size(); ++bbIdx) { + BB *bb = sortedBBs[bbIdx]; + + /* 1. calculate live interfere */ + FOR_BB_INSNS(insn, bb) { + if (insn->IsImmaterialInsn() || !insn->IsMachineInstruction() || insn->GetId() == 0) { + /* New instruction inserted by reg alloc (ie spill) */ + continue; + } + + /* simple retire from active */ + MapleSet::iterator it; + for (it = active.begin(); it != active.end(); /* erase will update */) { + auto *li = static_cast(*it); + if (li->GetLastUse() > insn->GetId()) { + break; + } + it = active.erase(it); + } + + /* simple insert to active */ + const AArch64MD *md = &AArch64CG::kMd[static_cast(insn)->GetMachineOpcode()]; + uint32 opndNum = insn->GetOperandSize(); + for (uint32 i = 0; i < opndNum; ++i) { + Operand &opnd = insn->GetOperand(i); + bool isDef = static_cast(md->operand[i])->IsRegDef(); + if (isDef) { + auto ®Opnd = static_cast(opnd); + if (regOpnd.IsVirtualRegister() && regOpnd.GetRegisterType() != kRegTyCc) { + uint32 regNO = regOpnd.GetRegisterNumber(); + LiveInterval *li = liveIntervalsArray[regNO]; + if (li->GetFirstDef() == insn->GetId()) { + (void)active.insert(li); + } + } + } + } + + /* get interfere info */ + uint32 interNum = active.size(); + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "In insn " << insn->GetId() << ", " << interNum << " overlap live intervals.\n"; + } + + /* 2. analysis which to spill */ + while (interNum > CGOptions::GetOverlapNum()) { + LiveInterval *lowestLi = nullptr; + FindLowestPrioInActive(lowestLi); + if (lowestLi != nullptr) { + if (LSRA_DUMP) { + PrintLiveInterval(*lowestLi, "Pre spilled: "); + } + lowestLi->SetStackSlot(kSpilled); + active.erase(itFinded); + interNum = active.size(); + } else { + break; + } + } + } + } + + active.clear(); +} + +/* Iterate through the operands of an instruction for allocation. */ +void LSRALinearScanRegAllocator::AssignPhysRegsForInsn(Insn &insn) { + const AArch64MD *md = &AArch64CG::kMd[static_cast(&insn)->GetMachineOpcode()]; + + /* At the beginning of the landing pad, we handle the x1, x2 as if they are implicitly defined. */ + if (!insn.GetBB()->GetEhPreds().empty() && &insn == insn.GetBB()->GetFirstInsn()) { + if (!intParamQueue[0].empty()) { + LiveInterval *li = intParamQueue[0].front(); + if (li->GetFirstDef() == insn.GetId()) { + intParamRegSet.erase(li->GetAssignedReg() - R0); + (void)active.insert(li); + intParamQueue[0].pop_front(); + } + } + + if (!intParamQueue[1].empty()) { + LiveInterval *li = intParamQueue[1].front(); + if (li->GetFirstDef() == insn.GetId()) { + intParamRegSet.erase(li->GetAssignedReg() - R0); + (void)active.insert(li); + intParamQueue[1].pop_front(); + } + } + } + + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "active in " << insn.GetId() << " :"; + PrintActiveListSimple(); + } + uint32 opndNum = insn.GetOperandSize(); + for (uint32 i = 0; i < opndNum; ++i) { + Operand &opnd = insn.GetOperand(i); + bool isDef = static_cast(md->operand[i])->IsRegDef(); + RegOperand *newOpnd = nullptr; + if (opnd.IsList()) { + /* For arm32, not arm64 */ + } else if (opnd.IsMemoryAccessOperand()) { + auto &memOpnd = static_cast(opnd); + Operand *base = memOpnd.GetBaseRegister(); + Operand *offset = memOpnd.GetIndexRegister(); + isDef = false; + if (base != nullptr) { + if (OpndNeedAllocation(insn, *base, isDef, insn.GetId())) { + newOpnd = AssignPhysRegs(*base, insn); + if (newOpnd == nullptr) { + SetOperandSpill(*base); + } + /* add ASSERT here. */ + } + } + if (offset != nullptr) { + if (!OpndNeedAllocation(insn, *offset, isDef, insn.GetId())) { + continue; + } + newOpnd = AssignPhysRegs(*offset, insn); + if (newOpnd == nullptr) { + SetOperandSpill(*offset); + } + } + } else { + if (!OpndNeedAllocation(insn, opnd, isDef, insn.GetId())) { + continue; + } + if (isDef && !fastAlloc) { + InsertParamToActive(opnd); + } + newOpnd = AssignPhysRegs(opnd, insn); + if (newOpnd != nullptr) { + if (isDef) { + InsertToActive(opnd, insn.GetId()); + } + } else { + /* + * If dest and both src are spilled, src will use both of the + * spill registers. + * dest can use any spill reg, choose 0 + */ + if (isDef) { + newOpnd = HandleSpillForInsn(insn, opnd); + if (newOpnd != nullptr) { + InsertToActive(opnd, insn.GetId()); + } + } else { + SetOperandSpill(opnd); + } + } + } + } +} + +/* + * Create an operand with physical register assigned, or a spill register + * in the case where a physical register cannot be assigned. + */ +RegOperand *LSRALinearScanRegAllocator::GetReplaceOpnd(Insn &insn, Operand &opnd, uint32 &spillIdx, bool isDef) { + if (!opnd.IsRegister()) { + return nullptr; + } + const auto *regOpnd = static_cast(&opnd); + + uint32 vRegNO = regOpnd->GetRegisterNumber(); + RegType regType = regOpnd->GetRegisterType(); + if (regType == kRegTyCc || regType == kRegTyVary) { + return nullptr; + } + if (IsUntouchableReg(vRegNO) || regOpnd->IsConstReg()) { + return nullptr; + } + if (regOpnd->IsPhysicalRegister()) { + return nullptr; + } + + ASSERT(vRegNO < liveIntervalsArray.size(), + "index out of range of MapleVector in LSRALinearScanRegAllocator::GetReplaceOpnd"); + LiveInterval *li = liveIntervalsArray[vRegNO]; + + bool addCalleeToSaved = true; + regno_t regNO = li->GetAssignedReg(); + bool isCalleeReg = AArch64Abi::IsCalleeSavedReg(static_cast(regNO)); + if (CGOptions::DoCalleeToSpill() && + /* prolog can use stp, so try to estimate if spill callee should be done. */ + ((shouldOptIntCallee && li->GetRegType() == kRegTyInt) || + (shouldOptFpCallee && li->GetRegType() == kRegTyFloat))) { + if (isCalleeReg) { + /* Determine if it is worth keeping callee */ + const uint32 spillResult = 1; + const uint32 spillReference = 2; + if (calleeUseCnt[regNO] == kRegNum2 && li->GetResultCount() == spillResult && + li->GetRefCount() == spillReference) { + /* This callee is allocated for one def and one use */ + li->SetStackSlot(kSpilled); + li->SetAssignedReg(0); + addCalleeToSaved = false; + } + } + } + if (isCalleeReg && addCalleeToSaved) { + static_cast(cgFunc)->AddtoCalleeSaved(static_cast(regNO)); + } + + if (li->IsShouldSave()) { + if (insn.GetMachineOpcode() == MOP_adrp_ldr && insn.GetNext() && + insn.GetNext()->GetMachineOpcode() == MOP_clinit_tail) { + /* clinit pair */ + li->SetAssignedReg(R16); + } else if (insn.GetMachineOpcode() == MOP_clinit_tail && insn.GetPrev() && + insn.GetPrev()->GetMachineOpcode() == MOP_adrp_ldr) { + isDef = true; + InsertCallerSave(insn, opnd, isDef); + } else { + InsertCallerSave(insn, opnd, isDef); + } + } else if (li->GetStackSlot() == kSpilled) { + /* Determine if spill can reside in localref space */ + if ((insn.GetMachineOpcode() == MOP_adrp_ldr && insn.GetNext() && + insn.GetNext()->GetMachineOpcode() == MOP_clinit_tail)) { + /* clinit pair */ + li->SetAssignedReg(R16); + } else if (insn.GetMachineOpcode() == MOP_clinit_tail && insn.GetPrev() && + insn.GetPrev()->GetMachineOpcode() == MOP_adrp_ldr) { + isDef = true; + spillIdx = 0; + SpillOperand(insn, opnd, isDef, spillIdx); + } else { + if (isDef) { + spillIdx = 0; + } + SpillOperand(insn, opnd, isDef, spillIdx); + if (!isDef) { + ++spillIdx; + } + } + } + + RegOperand &phyOpnd = static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( + static_cast(li->GetAssignedReg()), opnd.GetSize(), regType); + + return &phyOpnd; +} + +/* Try to estimate if spill callee should be done based on even/odd for stp in prolog. */ +void LSRALinearScanRegAllocator::CheckSpillCallee() { + if (CGOptions::DoCalleeToSpill()) { + uint32 pairCnt = 0; + for (size_t idx = 0; idx < sizeof(uint32); ++idx) { + if ((intCalleeMask & (1ULL << idx)) != 0 && calleeUseCnt[idx] != 0) { + ++pairCnt; + } + } + if ((pairCnt & 0x01) != 0) { + shouldOptIntCallee = true; + } + + for (size_t idx = 0; idx < sizeof(uint32); ++idx) { + if ((fpCalleeMask & (1ULL << idx)) != 0 && calleeUseCnt[idx] != 0) { + ++pairCnt; + } + } + if ((pairCnt & 0x01) != 0) { + shouldOptFpCallee = true; + } + } +} + +/* Iterate through all instructions and change the vreg to preg. */ +void LSRALinearScanRegAllocator::FinalizeRegisters() { + CheckSpillCallee(); + for (BB *bb : sortedBBs) { + intBBDefMask = 0; + fpBBDefMask = 0; + + FOR_BB_INSNS(insn, bb) { + if (insn->IsImmaterialInsn() || !insn->IsMachineInstruction() || insn->GetId() == 0) { + continue; + } + + if (insn->IsCall()) { + intBBDefMask = 0; + fpBBDefMask = 0; + } + + uint32 spillIdx = 0; + const AArch64MD *md = &AArch64CG::kMd[static_cast(insn)->GetMachineOpcode()]; + + /* Handle source opernads first */ + uint32 opndNum = insn->GetOperandSize(); + for (uint32 i = 0; i < opndNum; ++i) { + Operand &opnd = insn->GetOperand(i); + ASSERT(md->operand[i] != nullptr, "pointer is null in LSRALinearScanRegAllocator::FinalizeRegisters"); + bool isDef = static_cast(md->operand[i])->IsRegDef(); + if (isDef) { + continue; + } + RegOperand *phyOpnd = nullptr; + if (opnd.IsList()) { + /* For arm32, not arm64 */ + } else if (opnd.IsMemoryAccessOperand()) { + auto *memOpnd = + static_cast(static_cast(opnd).Clone(*cgFunc->GetMemoryPool())); + ASSERT(memOpnd != nullptr, "memopnd is null in LSRALinearScanRegAllocator::FinalizeRegisters"); + insn->SetOperand(i, *memOpnd); + Operand *base = memOpnd->GetBaseRegister(); + Operand *offset = memOpnd->GetIndexRegister(); + if (base != nullptr) { + phyOpnd = GetReplaceOpnd(*insn, *base, spillIdx, false); + if (phyOpnd != nullptr) { + memOpnd->SetBaseRegister(*phyOpnd); + } + } + if (offset != nullptr) { + phyOpnd = GetReplaceOpnd(*insn, *offset, spillIdx, false); + if (phyOpnd != nullptr) { + memOpnd->SetIndexRegister(*phyOpnd); + } + } + } else { + phyOpnd = GetReplaceOpnd(*insn, opnd, spillIdx, false); + if (phyOpnd != nullptr) { + insn->SetOperand(i, *phyOpnd); + } + } + } + /* Handle dest opernads last */ + for (uint32 i = 0; i < opndNum; ++i) { + Operand &opnd = insn->GetOperand(i); + bool isDef = static_cast(md->operand[i])->IsRegDef(); + if (!isDef) { + continue; + } + isSpillZero = false; + RegOperand *phyOpnd = GetReplaceOpnd(*insn, opnd, spillIdx, true); + if (phyOpnd != nullptr) { + insn->SetOperand(i, *phyOpnd); + if (isSpillZero) { + insn->GetBB()->RemoveInsn(*insn); + } + } + } + } + } +} + +void LSRALinearScanRegAllocator::SetAllocMode() { + if (CGOptions::IsFastAlloc()) { + if (CGOptions::GetFastAllocMode() == 0) { + fastAlloc = true; + } else { + spillAll = true; + } + /* In-Range spill range can still be specified (only works with --dump-func=). */ + } else if (cgFunc->NumBBs() > CGOptions::GetLSRABBOptSize()) { + /* instruction size is checked in ComputeLieveInterval() */ + fastAlloc = true; + } + + if (LSRA_DUMP) { + if (fastAlloc) { + LogInfo::MapleLogger() << "fastAlloc mode on\n"; + } + if (spillAll) { + LogInfo::MapleLogger() << "spillAll mode on\n"; + } + } +} + +void LSRALinearScanRegAllocator::LinearScanRegAllocator() { + if (LSRA_DUMP) { + PrintParamQueue("Initial param queue"); + PrintCallQueue("Initial call queue"); + } + /* handle param register */ + for (auto &intParam : intParamQueue) { + if (!intParam.empty() && intParam.front()->GetFirstDef() == 0) { + LiveInterval *li = intParam.front(); + intParamRegSet.erase(li->GetAssignedReg() - R0); + (void)active.insert(li); + intParam.pop_front(); + } + } + for (auto &fpParam : fpParamQueue) { + if (!fpParam.empty() && fpParam.front()->GetFirstDef() == 0) { + LiveInterval *li = fpParam.front(); + fpParamRegSet.erase(li->GetAssignedReg() - V0); + (void)active.insert(li); + fpParam.pop_front(); + } + } + + for (BB *bb : sortedBBs) { + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "======New BB=====" << bb->GetId() << " " << std::hex << bb << std::dec << "\n"; + } + FOR_BB_INSNS(insn, bb) { + if (!insn->IsMachineInstruction()) { + continue; + } + if (insn->GetId() == 0) { + /* New instruction inserted by reg alloc (ie spill) */ + continue; + } + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "======New Insn=====" << insn->GetId() << " " << insn->GetBB()->GetId() << "\n"; + insn->Dump(); + } + RetireFromActive(*insn); +#ifdef LSRA_DEBUG + DebugCheckActiveList(); +#endif + AssignPhysRegsForInsn(*insn); + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "======After Alloc=====" << insn->GetId() << " " << insn->GetBB()->GetId() << "\n"; + insn->Dump(); + } + } + } +} + +/* Main entrance for the LSRA register allocator. */ +bool LSRALinearScanRegAllocator::AllocateRegisters() { + auto *a64CGFunc = static_cast(cgFunc); + /* + * we store both FP/LR if using FP or if not using FP, but func has a call + * Using FP, record it for saving + */ + a64CGFunc->AddtoCalleeSaved(RFP); + a64CGFunc->AddtoCalleeSaved(RLR); + a64CGFunc->NoteFPLRAddedToCalleeSavedList(); + + if (LSRA_DUMP) { + const MIRModule &mirModule = cgFunc->GetMirModule(); + DotGenerator::GenerateDot("RA", *cgFunc, mirModule); + DotGenerator::GenerateDot("RAe", *cgFunc, mirModule, true); + } + + if (CGOptions::DoPreLSRAOpt()) { + PropagateX0(); + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "******** CG IR After PreLSRA: *********" << '\n'; + cgFunc->DumpCGIR(); + } + } + + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "Entering LinearScanRegAllocator\n"; + } + + ComputeBlockOrder(); + + ComputeLiveInterval(); + +#ifdef LSRA_GRAPH + PrintLiveRanges(); +#endif + + LiveIntervalAnalysis(); + + InitFreeRegPool(); + + BuildIntervalRanges(); + + SetAllocMode(); + + LinearScanRegAllocator(); + + FinalizeRegisters(); + + if (LSRA_DUMP) { + LogInfo::MapleLogger() << "Total " << spillCount << " spillCount in " << cgFunc->GetName() << " \n"; + LogInfo::MapleLogger() << "Total " << reloadCount << " reloadCount\n"; + LogInfo::MapleLogger() << "Total " << "(" << spillCount << "+ " << callerSaveSpillCount << ") = " << + (spillCount + callerSaveSpillCount) << " SPILL\n"; + LogInfo::MapleLogger() << "Total " << "(" << reloadCount << "+ " << callerSaveReloadCount << ") = " << + (reloadCount + callerSaveReloadCount) << " RELOAD\n"; + } + + return true; +} +} /* namespace maplebe */ diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_peep.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_peep.cpp index 8f7dc6f51cce07f4396ed11fe0eebeb17b666a73..57936b3600d19938a33b24c72a3012061f7b8609 100644 --- a/src/mapleall/maple_be/src/cg/aarch64/aarch64_peep.cpp +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_peep.cpp @@ -185,6 +185,7 @@ void AArch64PrePeepHole::InitOpts() { optimizations[kReplaceCmpToCmnOpt] = optOwnMemPool->New(cgFunc); optimizations[kRemoveIncRefOpt] = optOwnMemPool->New(cgFunc); optimizations[kLongIntCompareWithZOpt] = optOwnMemPool->New(cgFunc); + optimizations[kComplexMemOperandOpt] = optOwnMemPool->New(cgFunc); optimizations[kComplexMemOperandPreOptAdd] = optOwnMemPool->New(cgFunc); optimizations[kComplexMemOperandOptLSL] = optOwnMemPool->New(cgFunc); optimizations[kComplexMemOperandOptLabel] = optOwnMemPool->New(cgFunc); @@ -221,6 +222,10 @@ void AArch64PrePeepHole::Run(BB &bb, Insn &insn) { (static_cast(optimizations[kLongIntCompareWithZOpt]))->Run(bb, insn); break; } + case MOP_xadrpl12: { + (static_cast(optimizations[kComplexMemOperandOpt]))->Run(bb, insn); + break; + } case MOP_xaddrrr: { (static_cast(optimizations[kComplexMemOperandPreOptAdd]))->Run(bb, insn); break; diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_proepilog.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_proepilog.cpp index df0cae410d6b0649658ddf951935fb5ae9e65172..94261a1e352cd3485c93b2670fd1894e089b6f66 100644 --- a/src/mapleall/maple_be/src/cg/aarch64/aarch64_proepilog.cpp +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_proepilog.cpp @@ -19,6 +19,13 @@ namespace maplebe { using namespace maple; namespace { +const std::set kFrameWhiteListFunc { +#include "framewhitelist.def" +}; + +bool IsFuncNeedFrame(const std::string &funcName) { + return kFrameWhiteListFunc.find(funcName) != kFrameWhiteListFunc.end(); +} constexpr uint32 k2BitSize = 2; constexpr int32 kSoeChckOffset = 8192; @@ -62,6 +69,155 @@ inline void AppendInstructionTo(Insn &insn, CGFunc &func) { } } +bool AArch64GenProEpilog::HasLoop() { + FOR_ALL_BB(bb, &cgFunc) { + if (bb->IsBackEdgeDest()) { + return true; + } + FOR_BB_INSNS_REV(insn, bb) { + if (!insn->IsMachineInstruction()) { + continue; + } + if (insn->HasLoop()) { + return true; + } + } + } + return false; +} + +/* + * Remove redundant mov and mark optimizable bl/blr insn in the BB. + * Return value: true if is empty bb, otherwise false. + */ +bool AArch64GenProEpilog::OptimizeTailBB(BB &bb, std::set &callInsns) { + FOR_BB_INSNS_REV_SAFE(insn, &bb, prev_insn) { + if (!insn->IsMachineInstruction()) { + continue; + } + MOperator insnMop = insn->GetMachineOpcode(); + switch (insnMop) { + case MOP_wmovrr: + case MOP_xmovrr: { + CHECK_FATAL(insn->GetOperand(0).IsRegister(), "operand0 is not register"); + CHECK_FATAL(insn->GetOperand(1).IsRegister(), "operand1 is not register"); + auto ®1 = static_cast(insn->GetOperand(0)); + auto ®2 = static_cast(insn->GetOperand(1)); + + if (reg1.GetRegisterNumber() != R0 || reg2.GetRegisterNumber() != R0) { + return false; + } + + bb.RemoveInsn(*insn); + break; + } + case MOP_xbl: + case MOP_xblr: { + (void)callInsns.insert(insn); + return false; + } + default: + return false; + } + } + + return true; +} + +/* Recursively invoke this function until exitBB's precursor not exist. */ +void AArch64GenProEpilog::TailCallBBOpt(const BB &exitBB, std::set &callInsns) { + for (auto tmpBB : exitBB.GetPreds()) { + if (tmpBB->GetSuccs().size() != 1 || !tmpBB->GetEhSuccs().empty() || tmpBB->GetKind() != BB::kBBFallthru) { + continue; + } + + if (OptimizeTailBB(*tmpBB, callInsns)) { + TailCallBBOpt(*tmpBB, callInsns); + } + } +} + +/* + * If a function without callee-saved register, and end with a function call, + * then transfer bl/blr to b/br. + * Return value: true if function do not need Prologue/Epilogue. false otherwise. + */ +bool AArch64GenProEpilog::TailCallOpt() { + auto &aarchCGFunc = static_cast(cgFunc); + BB *exitBB = nullptr; + const MapleVector ®sToRestore = aarchCGFunc.GetCalleeSavedRegs(); + + size_t calleeSavedRegSize = 2; + CHECK_FATAL(regsToRestore.size() >= calleeSavedRegSize, "Forgot FP and LR ?"); + + if (regsToRestore.size() > calleeSavedRegSize || aarchCGFunc.HasStackLoadStore() || HasLoop() || + cgFunc.GetFunction().GetAttr(FUNCATTR_callersensitive) || IsFuncNeedFrame(cgFunc.GetName())) { + return false; + } + + size_t exitBBSize = cgFunc.GetExitBBsVec().size(); + CHECK_FATAL(exitBBSize == 1, "Should not be exist multiple exits."); + + if (exitBBSize == 0) { + if (cgFunc.GetLastBB()->GetPrev()->GetFirstStmt() == cgFunc.GetCleanupLabel() && + cgFunc.GetLastBB()->GetPrev()->GetPrev() != nullptr) { + exitBB = cgFunc.GetLastBB()->GetPrev()->GetPrev(); + } else { + exitBB = cgFunc.GetLastBB()->GetPrev(); + } + } else { + exitBB = cgFunc.GetExitBBsVec().front(); + } + + CHECK_FATAL(exitBB->GetFirstInsn() == nullptr, "exit bb should be empty."); + + /* Count how many call insns in the whole function. */ + uint32 nCount = 0; + bool hasGetStackClass = false; + + FOR_ALL_BB(bb, &cgFunc) { + FOR_BB_INSNS(insn, bb) { + if (insn->IsCall()) { + if (insn->GetMachineOpcode() == MOP_xbl) { + auto &target = static_cast(insn->GetOperand(0)); + if (IsFuncNeedFrame(target.GetName())) { + hasGetStackClass = true; + } + } + ++nCount; + } + } + } + + if ((nCount > 0 && cgFunc.GetFunction().GetAttr(FUNCATTR_interface)) || hasGetStackClass) { + return false; + } + + std::set callInsns; + TailCallBBOpt(*exitBB, callInsns); + + if (nCount != callInsns.size()) { + return false; + } + /* Replace all of the call insns. */ + for (auto callInsn : callInsns) { + MOperator insnMop = callInsn->GetMachineOpcode(); + switch (insnMop) { + case MOP_xbl: { + callInsn->SetMOP(MOP_tail_call_opt_xbl); + break; + } + case MOP_xblr: { + callInsn->SetMOP(MOP_tail_call_opt_xblr); + break; + } + default: + ASSERT(false, "Internal error."); + break; + } + } + return true; +} void AArch64GenProEpilog::GenStackGuard(BB &bb) { auto &aarchCGFunc = static_cast(cgFunc); @@ -199,6 +355,447 @@ BB &AArch64GenProEpilog::GenStackGuardCheckInsn(BB &bb) { return *newBB; } +/* + * The following functions are for pattern matching for fast path + * where function has early return of spedified pattern load/test/return + */ +void AArch64GenProEpilog::ReplaceMachedOperand(Insn &orig, Insn &target, const RegOperand &matchedOpnd, + bool isFirstDst) { + auto &aarchCGFunc = static_cast(cgFunc); + for (uint32 i = 0; i < target.GetOpndNum(); ++i) { + Operand *src = target.GetOpnd(i); + CHECK_FATAL(src != nullptr, "src is null in ReplaceMachedOperand"); + if (src->IsRegister()) { + RegOperand *reg = static_cast(src); + if (reg != &matchedOpnd) { + continue; + } + if (isFirstDst) { + Operand *origSrc = orig.GetOpnd(0); + RegOperand *origPhys = static_cast(origSrc); + CHECK_FATAL(origPhys != nullptr, "pointer is null"); + regno_t regNO = origPhys->GetRegisterNumber(); + RegOperand &phys = aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(regNO), + matchedOpnd.GetSize(), kRegTyInt); + target.SetOpnd(i, phys); + } else { + /* Replace the operand with the src of inst */ + RegOperand &phys = aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(R16), + reg->GetSize(), kRegTyInt); + target.SetOpnd(i, phys); + } + return; + } + if (src->IsMemoryAccessOperand()) { + MemOperand *memOpnd = static_cast(src); + Operand *base = memOpnd->GetBaseRegister(); + Operand *offset = memOpnd->GetIndexRegister(); + if (base != nullptr && base->IsRegister()) { + RegOperand *reg = static_cast(base); + if (reg != &matchedOpnd) { + continue; + } + if (isFirstDst) { + Operand *origSrc = orig.GetOpnd(0); + RegOperand *origPhys = static_cast(origSrc); + CHECK_FATAL(origPhys != nullptr, "orig_phys cast failed"); + regno_t regNO = origPhys->GetRegisterNumber(); + RegOperand &phys = aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(regNO), + matchedOpnd.GetSize(), kRegTyInt); + memOpnd->SetBaseRegister(phys); + } else { + /* Replace the operand with the src of inst */ + RegOperand &phys = + aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(R16), base->GetSize(), kRegTyInt); + memOpnd->SetBaseRegister(phys); + } + return; + } + if (offset != nullptr && offset->IsRegister()) { + RegOperand *reg = static_cast(offset); + if (reg != &matchedOpnd) { + continue; + } + if (isFirstDst) { + Operand *origSrc = orig.GetOpnd(0); + RegOperand *origPhys = static_cast(origSrc); + CHECK_FATAL(origPhys != nullptr, "orig_phys is nullptr in ReplaceMachedOperand"); + regno_t regNO = origPhys->GetRegisterNumber(); + RegOperand &phys = aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(regNO), + matchedOpnd.GetSize(), kRegTyInt); + memOpnd->SetIndexRegister(phys); + } else { + /* Replace the operand with the src of inst */ + RegOperand &phys = aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(R16), + offset->GetSize(), kRegTyInt); + memOpnd->SetIndexRegister(phys); + } + return; + } + } + } + if (!isFirstDst) { + RegOperand &phys = aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(R16), + matchedOpnd.GetSize(), kRegTyInt); + orig.SetResult(0, phys); + } +} + +bool AArch64GenProEpilog::BackwardFindDependency(BB &ifbb, RegOperand &targetOpnd, Insn *&load, + Insn *&mov, Insn *&depMov, std::list &list) { + load = nullptr; + mov = nullptr; + depMov = nullptr; + BB *pred = &ifbb; + /* + * Pattern match, (*) instruction are moved down below branch. + * mov reg1, R0 mov reg1, R0 * + * ld Rx , [reg1, const] ld R16 , [R0, const] + * mov reg2, Rx => mov reg2, R16 <- this may exist * + * mov Rx , R16 <- replicate * + * cbr Rx, label cbr R16, label + * + * R16 is used because it is used as spill register. + * Need to modify if different register allcoation mechanism is used. + */ + do { + FOR_BB_INSNS_REV(insn, pred) { + if (insn == ifbb.GetLastInsn()) { + continue; + } + if (insn->IsImmaterialInsn()) { + continue; + } + if (!insn->IsMachineInstruction()) { + continue; + } + + bool found = false; /* allowing for only one src to be register */ + for (uint32 r = 0; r < insn->GetResultNum(); ++r) { + Operand *dst = insn->GetResult(r); + CHECK_FATAL(dst != nullptr, "pointer is null"); + if (!dst->IsRegister()) { + continue; + } + RegOperand *regOpnd = static_cast(dst); + if (regOpnd != &targetOpnd) { + continue; + } + if (load != nullptr) { + if (mov != nullptr) { + return false; + } + MOperator opCode = insn->GetMachineOpcode(); + if (opCode != MOP_xmovrr) { + return false; + } + Operand *mvSrc = insn->GetOpnd(0); + RegOperand *mvRegSrc = static_cast(mvSrc); + CHECK_FATAL(mvRegSrc != nullptr, "mv_regsrc cast failed"); + regno_t mvReg = mvRegSrc->GetRegisterNumber(); + /* make it very specific for now */ + if (mvReg != R0) { + return false; + } + Operand *mvDst = insn->GetResult(0); + RegOperand *mvRegDst = static_cast(mvDst); + CHECK_FATAL(mvRegDst != nullptr, "mv_regdst cast failed"); + mvReg = mvRegDst->GetRegisterNumber(); + if (mvReg != R20) { + return false; + } + mov = insn; + } + /* Found def, continue dep chain with src */ + for (uint32 s = 0; s < insn->GetOpndNum(); ++s) { + Operand *src = insn->GetOpnd(s); + CHECK_FATAL(src != nullptr, "src is nullptr in BackwardFindDependency"); + if (src->IsRegister()) { + if (found) { + return false; + } + RegOperand *preg = static_cast(src); + targetOpnd = *preg; + if (!preg->IsPhysicalRegister() || insn->GetMachineOpcode() != MOP_xmovrr) { + return false; + } + /* + * Skipping the start of the dependency chain because + * the the parameter reg will be propagated leaving + * the mov instruction alone to be relocated down + * to the cold path. + */ + found = false; + } else if (src->IsMemoryAccessOperand()) { + MemOperand *memOpnd = static_cast(src); + Operand *base = memOpnd->GetBaseRegister(); + Operand *offset = memOpnd->GetIndexRegister(); + if (base != nullptr && base->IsRegister()) { + if (found) { + return false; + } + load = insn; + targetOpnd = *(static_cast(base)); + found = true; + Operand *ldDst = insn->GetResult(0); + RegOperand *ldRdst = static_cast(ldDst); + CHECK_FATAL(ldRdst != nullptr, "ld_rdst is nullptr in BackwardFindDependency"); + if (ldRdst->GetRegisterNumber() != R1) { + return false; /* hard code for now. */ + } + /* Make sure instruction depending on load is mov and cond br */ + for (Insn *ni = insn->GetNext(); ni != nullptr; ni = ni->GetNext()) { + if (ni->GetMachineOpcode() == MOP_xmovrr || ni->GetMachineOpcode() == MOP_wmovrr) { + Operand *dep = ni->GetOpnd(0); + RegOperand *rdep = static_cast(dep); + if (rdep == ldRdst) { + if (depMov != nullptr) { + return false; + } + depMov = ni; + } + } + } + } + if (offset != nullptr && offset->IsRegister()) { + return false; + } + } + } + } + if (!found) { + list.push_back(insn); + } + } + if (pred->GetPreds().empty()) { + break; + } + pred = pred->GetPreds().front(); + } while (pred != nullptr); + + return true; +} + +void AArch64GenProEpilog::ForwardPropagateAndRename(Insn &mov, Insn &load, const BB &terminateBB) { + /* + * This is specialized function to work with IsolateFastPath(). + * The control flow and instruction pattern must be recognized. + */ + Insn *insn = &mov; + bool isFirstDst = true; + /* process mov and load two instructions */ + for (int32 i = 0; i < 2; ++i) { + /* Finish the bb the mov is in */ + for (Insn *target = insn->GetNext(); target != nullptr; target = target->GetNext()) { + if (target->IsImmaterialInsn()) { + continue; + } + if (!target->IsMachineInstruction()) { + continue; + } + Operand *dst = insn->GetResult(0); + RegOperand *rdst = static_cast(dst); + CHECK_FATAL(rdst != nullptr, "rdst is nullptr in ForwardPropagateAndRename"); + ReplaceMachedOperand(*insn, *target, *rdst, isFirstDst); + } + CHECK_FATAL(!insn->GetBB()->GetSuccs().empty(), "null succs check!"); + BB *bb = insn->GetBB()->GetSuccs().front(); + while (1) { + FOR_BB_INSNS(target, bb) { + if (!target->IsMachineInstruction()) { + continue; + } + Operand *dst = insn->GetResult(0); + RegOperand *rdst = static_cast(dst); + CHECK_FATAL(rdst != nullptr, "rdst is nullptr in ForwardPropagateAndRename"); + ReplaceMachedOperand(*insn, *target, *rdst, isFirstDst); + } + if (bb == &terminateBB) { + break; + } + CHECK_FATAL(!bb->GetSuccs().empty(), "null succs check!"); + bb = bb->GetSuccs().front(); + } + insn = &load; + isFirstDst = false; + } +} + +BB *AArch64GenProEpilog::IsolateFastPath(BB &bb) { + /* + * Detect "if (cond) return" fast path, and move extra instructions + * to the slow path. + * Must match the following block structure. BB1 can be a series of + * single-pred/single-succ blocks. + * BB1 ops1 cmp-br to BB3 BB1 cmp-br to BB3 + * BB2 ops2 br to retBB ==> BB2 ret + * BB3 slow path BB3 ops1 ops2 + * BB3 will be used to generate prolog stuff. + */ + if (bb.GetPrev() != nullptr) { + return nullptr; + } + BB *ifBB = nullptr; + BB *returnBB = nullptr; + BB *coldBB = nullptr; + auto &aarchCGFunc = static_cast(cgFunc); + CG *currCG = cgFunc.GetCG(); + { + BB &curBB = bb; + /* Look for straight line code */ + while (1) { + if (curBB.GetEhSuccs().empty()) { + return nullptr; + } + if (curBB.GetSuccs().size() == 1) { + if (curBB.HasCall()) { + return nullptr; + } + BB *succ = curBB.GetSuccs().front(); + if (succ->GetPreds().size() != 1 || !succ->GetEhPreds().empty()) { + return nullptr; + } + curBB = *succ; + } else if (curBB.GetKind() == BB::kBBIf) { + ifBB = &curBB; + break; + } else { + return nullptr; + } + } + } + /* targets of if bb can only be reached by if bb */ + { + CHECK_FATAL(!ifBB->GetSuccs().empty(), "null succs check!"); + BB *first = ifBB->GetSuccs().front(); + BB *second = ifBB->GetSuccs().back(); + if (first->GetPreds().size() != 1 || !first->GetEhPreds().empty()) { + return nullptr; + } + if (second->GetPreds().size() != 1 || !second->GetEhPreds().empty()) { + return nullptr; + } + + /* One target of the if bb jumps to a return bb */ + CHECK_FATAL(!first->GetSuccs().empty(), "null succs check!"); + if (first->GetKind() != BB::kBBGoto || first->GetSuccs().front()->GetKind() != BB::kBBReturn) { + return nullptr; + } + if (second->GetSuccs().empty()) { + return nullptr; + } + returnBB = first; + coldBB = second; + } + /* + * The control flow matches at this point. + * Make sure the hot bb contains atmost a + * 'mov x0, value' and 'b'. + */ + { + CHECK_FATAL(returnBB != nullptr, "null ptr check"); + const int32 twoInsnInReturnBB = 2; + if (returnBB->NumInsn() > twoInsnInReturnBB) { + return nullptr; + } + Insn *first = returnBB->GetFirstInsn(); + while (first->IsImmaterialInsn()) { + first = first->GetNext(); + } + if (first == returnBB->GetLastInsn()) { + if (!first->IsBranch()) { + return nullptr; + } + } else { + MOperator opCode = first->GetMachineOpcode(); + /* only allow mov constant */ + if (opCode != MOP_xmovri64 && opCode != MOP_xmovri32) { + return nullptr; + } + Insn *last = returnBB->GetLastInsn(); + if (!last->IsBranch()) { + return nullptr; + } + } + } + + /* + * Resolve any register usage issues. + * 1) Any use of parameter registes must be renamed + * 2) Any usage of callee saved register that needs saving in prolog + * must be able to move down into the cold path. + */ + + /* Find the branch's src register for backward propagation. */ + Insn *condBr = ifBB->GetLastInsn(); + auto &targetOpnd = static_cast(condBr->GetOperand(0)); + if (targetOpnd.GetRegisterType() != kRegTyInt) { + return nullptr; + } + + /* Search backward looking for dependencies for the cond branch */ + std::list insnList; /* instructions to be moved to coldbb */ + Insn *ld = nullptr; + Insn *mv = nullptr; + Insn *depMv = nullptr; + /* + * The mv is the 1st move using the parameter register leading to the branch + * The ld is the load using the parameter register indirectly for the branch + * The depMv is the move which preserves the result of the load but might + * destroy a parameter register which will be moved below the branch. + */ + if (!BackwardFindDependency(*ifBB, targetOpnd, ld, mv, depMv, insnList)) { + return nullptr; + } + if (ld == nullptr || mv == nullptr) { + return nullptr; + } + /* + * depMv can be live out, so duplicate it + * and set dest to output of ld and src R16 + */ + if (depMv != nullptr) { + CHECK_FATAL(depMv->GetMachineOpcode(), "return check"); + Insn &newMv = currCG->BuildInstruction( + depMv->GetMachineOpcode(), ld->GetOperand(0), + aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(R16), + depMv->GetOperand(1).GetSize(), kRegTyInt)); + insnList.push_front(&newMv); + /* temporary put it some where */ + CHECK_FATAL(coldBB != nullptr, "null ptr check"); + static_cast(coldBB->InsertInsnBegin(newMv)); + } else { + uint32 regSize = ld->GetOperand(0).GetSize(); + Insn &newMv = currCG->BuildInstruction( + (regSize <= k32BitSize) ? MOP_xmovri32 : MOP_xmovri64, ld->GetOperand(0), + aarchCGFunc.GetOrCreatePhysicalRegisterOperand(static_cast(R16), regSize, kRegTyInt)); + insnList.push_front(&newMv); + /* temporary put it some where */ + CHECK_FATAL(coldBB != nullptr, "null ptr check"); + static_cast(coldBB->InsertInsnBegin(newMv)); + } + + ForwardPropagateAndRename(*mv, *ld, *returnBB); + + for (auto *in : insnList) { + in->GetBB()->RemoveInsn(*in); + CHECK_FATAL(coldBB != nullptr, "null ptr check"); + static_cast(coldBB->InsertInsnBegin(*in)); + } + + /* All instructions are in the right place, replace branch to ret bb to just ret. */ + CHECK_FATAL(returnBB != nullptr, "null ptr check"); + returnBB->RemoveInsn(*returnBB->GetLastInsn()); + returnBB->AppendInsn(currCG->BuildInstruction(MOP_xret)); + /* bb is now a retbb and has no succ. */ + returnBB->SetKind(BB::kBBReturn); + BB *tgtBB = returnBB->GetSuccs().front(); + auto predIt = std::find(tgtBB->GetPredsBegin(), tgtBB->GetPredsEnd(), returnBB); + tgtBB->ErasePreds(predIt); + returnBB->ClearSuccs(); + + return coldBB; +} AArch64MemOperand *AArch64GenProEpilog::SplitStpLdpOffsetForCalleeSavedWithAddInstruction(const AArch64MemOperand &mo, uint32 bitLen, @@ -1113,10 +1710,14 @@ void AArch64GenProEpilog::Run() { CHECK_FATAL(cgFunc.GetFunction().GetBody()->GetFirst()->GetOpCode() == OP_label, "The first statement should be a label"); cgFunc.SetHasProEpilogue(true); - if (cgFunc.GetHasProEpilogue()) { GenStackGuard(*(cgFunc.GetFirstBB())); } + BB *proLog = nullptr; + if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel2) { + /* There are some O2 dependent assumptions made */ + proLog = IsolateFastPath(*(cgFunc.GetFirstBB())); + } if (cgFunc.IsExitBBsVecEmpty()) { if (cgFunc.GetLastBB()->GetPrev()->GetFirstStmt() == cgFunc.GetCleanupLabel() && @@ -1127,7 +1728,13 @@ void AArch64GenProEpilog::Run() { } } - GenerateProlog(*(cgFunc.GetFirstBB())); + if (proLog != nullptr) { + GenerateProlog(*proLog); + proLog->SetFastPath(true); + cgFunc.GetFirstBB()->SetFastPath(true); + } else { + GenerateProlog(*(cgFunc.GetFirstBB())); + } for (auto *exitBB : cgFunc.GetExitBBsVec()) { GenerateEpilog(*exitBB); diff --git a/src/mapleall/maple_be/src/cg/aarch64/aarch64_reg_alloc.cpp b/src/mapleall/maple_be/src/cg/aarch64/aarch64_reg_alloc.cpp index cc6318912f3a00e082d8851c86f1bfc370811e62..c1a52f83c6e235e81ab3d5ac3389dde2ea3e77a3 100644 --- a/src/mapleall/maple_be/src/cg/aarch64/aarch64_reg_alloc.cpp +++ b/src/mapleall/maple_be/src/cg/aarch64/aarch64_reg_alloc.cpp @@ -13,6 +13,7 @@ * See the Mulan PSL v2 for more details. */ #include "aarch64_reg_alloc.h" +#include "aarch64_lsra.h" #include "aarch64_color_ra.h" #include "aarch64_cg.h" #include "aarch64_live.h" @@ -666,7 +667,9 @@ AnalysisResult *CgDoRegAlloc::Run(CGFunc *cgFunc, CgFuncResultMgr *cgFuncResultM if (Globals::GetInstance()->GetOptimLevel() == 0) { regAllocator = phaseMp->New(*cgFunc, *phaseMp); } else { - if (cgFunc->GetCG()->GetCGOptions().DoColoringBasedRegisterAllocation()) { + if (cgFunc->GetCG()->GetCGOptions().DoLinearScanRegisterAllocation()) { + regAllocator = phaseMp->New(*cgFunc, *phaseMp); + } else if (cgFunc->GetCG()->GetCGOptions().DoColoringBasedRegisterAllocation()) { regAllocator = phaseMp->New(*cgFunc, *phaseMp); } else { maple::LogInfo::MapleLogger(kLlErr) << "Warning: We only support Linear Scan and GraphColor register allocation\n"; diff --git a/src/mapleall/maple_be/src/cg/aarch64/valid_bitmask_imm.txt b/src/mapleall/maple_be/src/cg/aarch64/valid_bitmask_imm.txt new file mode 100755 index 0000000000000000000000000000000000000000..53a6135b6ebd4f570d728df66a9b2584a3a677ef --- /dev/null +++ b/src/mapleall/maple_be/src/cg/aarch64/valid_bitmask_imm.txt @@ -0,0 +1,5372 @@ +0x5555555555555555, +0xaaaaaaaaaaaaaaaa, +0x1111111111111111, +0x2222222222222222, +0x4444444444444444, +0x8888888888888888, +0x3333333333333333, +0x6666666666666666, +0xcccccccccccccccc, +0x9999999999999999, +0x7777777777777777, +0xeeeeeeeeeeeeeeee, +0xdddddddddddddddd, +0xbbbbbbbbbbbbbbbb, +0x0101010101010101, +0x0202020202020202, +0x0404040404040404, +0x0808080808080808, +0x1010101010101010, +0x2020202020202020, +0x4040404040404040, +0x8080808080808080, +0x0303030303030303, +0x0606060606060606, +0x0c0c0c0c0c0c0c0c, +0x1818181818181818, +0x3030303030303030, +0x6060606060606060, +0xc0c0c0c0c0c0c0c0, +0x8181818181818181, +0x0707070707070707, +0x0e0e0e0e0e0e0e0e, +0x1c1c1c1c1c1c1c1c, +0x3838383838383838, +0x7070707070707070, +0xe0e0e0e0e0e0e0e0, +0xc1c1c1c1c1c1c1c1, +0x8383838383838383, +0x0f0f0f0f0f0f0f0f, +0x1e1e1e1e1e1e1e1e, +0x3c3c3c3c3c3c3c3c, +0x7878787878787878, +0xf0f0f0f0f0f0f0f0, +0xe1e1e1e1e1e1e1e1, +0xc3c3c3c3c3c3c3c3, +0x8787878787878787, +0x1f1f1f1f1f1f1f1f, +0x3e3e3e3e3e3e3e3e, +0x7c7c7c7c7c7c7c7c, +0xf8f8f8f8f8f8f8f8, +0xf1f1f1f1f1f1f1f1, +0xe3e3e3e3e3e3e3e3, +0xc7c7c7c7c7c7c7c7, +0x8f8f8f8f8f8f8f8f, +0x3f3f3f3f3f3f3f3f, +0x7e7e7e7e7e7e7e7e, +0xfcfcfcfcfcfcfcfc, +0xf9f9f9f9f9f9f9f9, +0xf3f3f3f3f3f3f3f3, +0xe7e7e7e7e7e7e7e7, +0xcfcfcfcfcfcfcfcf, +0x9f9f9f9f9f9f9f9f, +0x7f7f7f7f7f7f7f7f, +0xfefefefefefefefe, +0xfdfdfdfdfdfdfdfd, +0xfbfbfbfbfbfbfbfb, +0xf7f7f7f7f7f7f7f7, +0xefefefefefefefef, +0xdfdfdfdfdfdfdfdf, +0xbfbfbfbfbfbfbfbf, +0x0001000100010001, +0x0002000200020002, +0x0004000400040004, +0x0008000800080008, +0x0010001000100010, +0x0020002000200020, +0x0040004000400040, +0x0080008000800080, +0x0100010001000100, +0x0200020002000200, +0x0400040004000400, +0x0800080008000800, +0x1000100010001000, +0x2000200020002000, +0x4000400040004000, +0x8000800080008000, +0x0003000300030003, +0x0006000600060006, +0x000c000c000c000c, +0x0018001800180018, +0x0030003000300030, +0x0060006000600060, +0x00c000c000c000c0, +0x0180018001800180, +0x0300030003000300, +0x0600060006000600, +0x0c000c000c000c00, +0x1800180018001800, +0x3000300030003000, +0x6000600060006000, +0xc000c000c000c000, +0x8001800180018001, +0x0007000700070007, +0x000e000e000e000e, +0x001c001c001c001c, +0x0038003800380038, +0x0070007000700070, +0x00e000e000e000e0, +0x01c001c001c001c0, +0x0380038003800380, +0x0700070007000700, +0x0e000e000e000e00, +0x1c001c001c001c00, +0x3800380038003800, +0x7000700070007000, +0xe000e000e000e000, +0xc001c001c001c001, +0x8003800380038003, +0x000f000f000f000f, +0x001e001e001e001e, +0x003c003c003c003c, +0x0078007800780078, +0x00f000f000f000f0, +0x01e001e001e001e0, +0x03c003c003c003c0, +0x0780078007800780, +0x0f000f000f000f00, +0x1e001e001e001e00, +0x3c003c003c003c00, +0x7800780078007800, +0xf000f000f000f000, +0xe001e001e001e001, +0xc003c003c003c003, +0x8007800780078007, +0x001f001f001f001f, +0x003e003e003e003e, +0x007c007c007c007c, +0x00f800f800f800f8, +0x01f001f001f001f0, +0x03e003e003e003e0, +0x07c007c007c007c0, +0x0f800f800f800f80, +0x1f001f001f001f00, +0x3e003e003e003e00, +0x7c007c007c007c00, +0xf800f800f800f800, +0xf001f001f001f001, +0xe003e003e003e003, +0xc007c007c007c007, +0x800f800f800f800f, +0x003f003f003f003f, +0x007e007e007e007e, +0x00fc00fc00fc00fc, +0x01f801f801f801f8, +0x03f003f003f003f0, +0x07e007e007e007e0, +0x0fc00fc00fc00fc0, +0x1f801f801f801f80, +0x3f003f003f003f00, +0x7e007e007e007e00, +0xfc00fc00fc00fc00, +0xf801f801f801f801, +0xf003f003f003f003, +0xe007e007e007e007, +0xc00fc00fc00fc00f, +0x801f801f801f801f, +0x007f007f007f007f, +0x00fe00fe00fe00fe, +0x01fc01fc01fc01fc, +0x03f803f803f803f8, +0x07f007f007f007f0, +0x0fe00fe00fe00fe0, +0x1fc01fc01fc01fc0, +0x3f803f803f803f80, +0x7f007f007f007f00, +0xfe00fe00fe00fe00, +0xfc01fc01fc01fc01, +0xf803f803f803f803, +0xf007f007f007f007, +0xe00fe00fe00fe00f, +0xc01fc01fc01fc01f, +0x803f803f803f803f, +0x00ff00ff00ff00ff, +0x01fe01fe01fe01fe, +0x03fc03fc03fc03fc, +0x07f807f807f807f8, +0x0ff00ff00ff00ff0, +0x1fe01fe01fe01fe0, +0x3fc03fc03fc03fc0, +0x7f807f807f807f80, +0xff00ff00ff00ff00, +0xfe01fe01fe01fe01, +0xfc03fc03fc03fc03, +0xf807f807f807f807, +0xf00ff00ff00ff00f, +0xe01fe01fe01fe01f, +0xc03fc03fc03fc03f, +0x807f807f807f807f, +0x01ff01ff01ff01ff, +0x03fe03fe03fe03fe, +0x07fc07fc07fc07fc, +0x0ff80ff80ff80ff8, +0x1ff01ff01ff01ff0, +0x3fe03fe03fe03fe0, +0x7fc07fc07fc07fc0, +0xff80ff80ff80ff80, +0xff01ff01ff01ff01, +0xfe03fe03fe03fe03, +0xfc07fc07fc07fc07, +0xf80ff80ff80ff80f, +0xf01ff01ff01ff01f, +0xe03fe03fe03fe03f, +0xc07fc07fc07fc07f, +0x80ff80ff80ff80ff, +0x03ff03ff03ff03ff, +0x07fe07fe07fe07fe, +0x0ffc0ffc0ffc0ffc, +0x1ff81ff81ff81ff8, +0x3ff03ff03ff03ff0, +0x7fe07fe07fe07fe0, +0xffc0ffc0ffc0ffc0, +0xff81ff81ff81ff81, +0xff03ff03ff03ff03, +0xfe07fe07fe07fe07, +0xfc0ffc0ffc0ffc0f, +0xf81ff81ff81ff81f, +0xf03ff03ff03ff03f, +0xe07fe07fe07fe07f, +0xc0ffc0ffc0ffc0ff, +0x81ff81ff81ff81ff, +0x07ff07ff07ff07ff, +0x0ffe0ffe0ffe0ffe, +0x1ffc1ffc1ffc1ffc, +0x3ff83ff83ff83ff8, +0x7ff07ff07ff07ff0, +0xffe0ffe0ffe0ffe0, +0xffc1ffc1ffc1ffc1, +0xff83ff83ff83ff83, +0xff07ff07ff07ff07, +0xfe0ffe0ffe0ffe0f, +0xfc1ffc1ffc1ffc1f, +0xf83ff83ff83ff83f, +0xf07ff07ff07ff07f, +0xe0ffe0ffe0ffe0ff, +0xc1ffc1ffc1ffc1ff, +0x83ff83ff83ff83ff, +0x0fff0fff0fff0fff, +0x1ffe1ffe1ffe1ffe, +0x3ffc3ffc3ffc3ffc, +0x7ff87ff87ff87ff8, +0xfff0fff0fff0fff0, +0xffe1ffe1ffe1ffe1, +0xffc3ffc3ffc3ffc3, +0xff87ff87ff87ff87, +0xff0fff0fff0fff0f, +0xfe1ffe1ffe1ffe1f, +0xfc3ffc3ffc3ffc3f, +0xf87ff87ff87ff87f, +0xf0fff0fff0fff0ff, +0xe1ffe1ffe1ffe1ff, +0xc3ffc3ffc3ffc3ff, +0x87ff87ff87ff87ff, +0x1fff1fff1fff1fff, +0x3ffe3ffe3ffe3ffe, +0x7ffc7ffc7ffc7ffc, +0xfff8fff8fff8fff8, +0xfff1fff1fff1fff1, +0xffe3ffe3ffe3ffe3, +0xffc7ffc7ffc7ffc7, +0xff8fff8fff8fff8f, +0xff1fff1fff1fff1f, +0xfe3ffe3ffe3ffe3f, +0xfc7ffc7ffc7ffc7f, +0xf8fff8fff8fff8ff, +0xf1fff1fff1fff1ff, +0xe3ffe3ffe3ffe3ff, +0xc7ffc7ffc7ffc7ff, +0x8fff8fff8fff8fff, +0x3fff3fff3fff3fff, +0x7ffe7ffe7ffe7ffe, +0xfffcfffcfffcfffc, +0xfff9fff9fff9fff9, +0xfff3fff3fff3fff3, +0xffe7ffe7ffe7ffe7, +0xffcfffcfffcfffcf, +0xff9fff9fff9fff9f, +0xff3fff3fff3fff3f, +0xfe7ffe7ffe7ffe7f, +0xfcfffcfffcfffcff, +0xf9fff9fff9fff9ff, +0xf3fff3fff3fff3ff, +0xe7ffe7ffe7ffe7ff, +0xcfffcfffcfffcfff, +0x9fff9fff9fff9fff, +0x7fff7fff7fff7fff, +0xfffefffefffefffe, +0xfffdfffdfffdfffd, +0xfffbfffbfffbfffb, +0xfff7fff7fff7fff7, +0xffefffefffefffef, +0xffdfffdfffdfffdf, +0xffbfffbfffbfffbf, +0xff7fff7fff7fff7f, +0xfefffefffefffeff, +0xfdfffdfffdfffdff, +0xfbfffbfffbfffbff, +0xf7fff7fff7fff7ff, +0xefffefffefffefff, +0xdfffdfffdfffdfff, +0xbfffbfffbfffbfff, +0x0000000100000001, +0x0000000200000002, +0x0000000400000004, +0x0000000800000008, +0x0000001000000010, +0x0000002000000020, +0x0000004000000040, +0x0000008000000080, +0x0000010000000100, +0x0000020000000200, +0x0000040000000400, +0x0000080000000800, +0x0000100000001000, +0x0000200000002000, +0x0000400000004000, +0x0000800000008000, +0x0001000000010000, +0x0002000000020000, +0x0004000000040000, +0x0008000000080000, +0x0010000000100000, +0x0020000000200000, +0x0040000000400000, +0x0080000000800000, +0x0100000001000000, +0x0200000002000000, +0x0400000004000000, +0x0800000008000000, +0x1000000010000000, +0x2000000020000000, +0x4000000040000000, +0x8000000080000000, +0x0000000300000003, +0x0000000600000006, +0x0000000c0000000c, +0x0000001800000018, +0x0000003000000030, +0x0000006000000060, +0x000000c0000000c0, +0x0000018000000180, +0x0000030000000300, +0x0000060000000600, +0x00000c0000000c00, +0x0000180000001800, +0x0000300000003000, +0x0000600000006000, +0x0000c0000000c000, +0x0001800000018000, +0x0003000000030000, +0x0006000000060000, +0x000c0000000c0000, +0x0018000000180000, +0x0030000000300000, +0x0060000000600000, +0x00c0000000c00000, +0x0180000001800000, +0x0300000003000000, +0x0600000006000000, +0x0c0000000c000000, +0x1800000018000000, +0x3000000030000000, +0x6000000060000000, +0xc0000000c0000000, +0x8000000180000001, +0x0000000700000007, +0x0000000e0000000e, +0x0000001c0000001c, +0x0000003800000038, +0x0000007000000070, +0x000000e0000000e0, +0x000001c0000001c0, +0x0000038000000380, +0x0000070000000700, +0x00000e0000000e00, +0x00001c0000001c00, +0x0000380000003800, +0x0000700000007000, +0x0000e0000000e000, +0x0001c0000001c000, +0x0003800000038000, +0x0007000000070000, +0x000e0000000e0000, +0x001c0000001c0000, +0x0038000000380000, +0x0070000000700000, +0x00e0000000e00000, +0x01c0000001c00000, +0x0380000003800000, +0x0700000007000000, +0x0e0000000e000000, +0x1c0000001c000000, +0x3800000038000000, +0x7000000070000000, +0xe0000000e0000000, +0xc0000001c0000001, +0x8000000380000003, +0x0000000f0000000f, +0x0000001e0000001e, +0x0000003c0000003c, +0x0000007800000078, +0x000000f0000000f0, +0x000001e0000001e0, +0x000003c0000003c0, +0x0000078000000780, +0x00000f0000000f00, +0x00001e0000001e00, +0x00003c0000003c00, +0x0000780000007800, +0x0000f0000000f000, +0x0001e0000001e000, +0x0003c0000003c000, +0x0007800000078000, +0x000f0000000f0000, +0x001e0000001e0000, +0x003c0000003c0000, +0x0078000000780000, +0x00f0000000f00000, +0x01e0000001e00000, +0x03c0000003c00000, +0x0780000007800000, +0x0f0000000f000000, +0x1e0000001e000000, +0x3c0000003c000000, +0x7800000078000000, +0xf0000000f0000000, +0xe0000001e0000001, +0xc0000003c0000003, +0x8000000780000007, +0x0000001f0000001f, +0x0000003e0000003e, +0x0000007c0000007c, +0x000000f8000000f8, +0x000001f0000001f0, +0x000003e0000003e0, +0x000007c0000007c0, +0x00000f8000000f80, +0x00001f0000001f00, +0x00003e0000003e00, +0x00007c0000007c00, +0x0000f8000000f800, +0x0001f0000001f000, +0x0003e0000003e000, +0x0007c0000007c000, +0x000f8000000f8000, +0x001f0000001f0000, +0x003e0000003e0000, +0x007c0000007c0000, +0x00f8000000f80000, +0x01f0000001f00000, +0x03e0000003e00000, +0x07c0000007c00000, +0x0f8000000f800000, +0x1f0000001f000000, +0x3e0000003e000000, +0x7c0000007c000000, +0xf8000000f8000000, +0xf0000001f0000001, +0xe0000003e0000003, +0xc0000007c0000007, +0x8000000f8000000f, +0x0000003f0000003f, +0x0000007e0000007e, +0x000000fc000000fc, +0x000001f8000001f8, +0x000003f0000003f0, +0x000007e0000007e0, +0x00000fc000000fc0, +0x00001f8000001f80, +0x00003f0000003f00, +0x00007e0000007e00, +0x0000fc000000fc00, +0x0001f8000001f800, +0x0003f0000003f000, +0x0007e0000007e000, +0x000fc000000fc000, +0x001f8000001f8000, +0x003f0000003f0000, +0x007e0000007e0000, +0x00fc000000fc0000, +0x01f8000001f80000, +0x03f0000003f00000, +0x07e0000007e00000, +0x0fc000000fc00000, +0x1f8000001f800000, +0x3f0000003f000000, +0x7e0000007e000000, +0xfc000000fc000000, +0xf8000001f8000001, +0xf0000003f0000003, +0xe0000007e0000007, +0xc000000fc000000f, +0x8000001f8000001f, +0x0000007f0000007f, +0x000000fe000000fe, +0x000001fc000001fc, +0x000003f8000003f8, +0x000007f0000007f0, +0x00000fe000000fe0, +0x00001fc000001fc0, +0x00003f8000003f80, +0x00007f0000007f00, +0x0000fe000000fe00, +0x0001fc000001fc00, +0x0003f8000003f800, +0x0007f0000007f000, +0x000fe000000fe000, +0x001fc000001fc000, +0x003f8000003f8000, +0x007f0000007f0000, +0x00fe000000fe0000, +0x01fc000001fc0000, +0x03f8000003f80000, +0x07f0000007f00000, +0x0fe000000fe00000, +0x1fc000001fc00000, +0x3f8000003f800000, +0x7f0000007f000000, +0xfe000000fe000000, +0xfc000001fc000001, +0xf8000003f8000003, +0xf0000007f0000007, +0xe000000fe000000f, +0xc000001fc000001f, +0x8000003f8000003f, +0x000000ff000000ff, +0x000001fe000001fe, +0x000003fc000003fc, +0x000007f8000007f8, +0x00000ff000000ff0, +0x00001fe000001fe0, +0x00003fc000003fc0, +0x00007f8000007f80, +0x0000ff000000ff00, +0x0001fe000001fe00, +0x0003fc000003fc00, +0x0007f8000007f800, +0x000ff000000ff000, +0x001fe000001fe000, +0x003fc000003fc000, +0x007f8000007f8000, +0x00ff000000ff0000, +0x01fe000001fe0000, +0x03fc000003fc0000, +0x07f8000007f80000, +0x0ff000000ff00000, +0x1fe000001fe00000, +0x3fc000003fc00000, +0x7f8000007f800000, +0xff000000ff000000, +0xfe000001fe000001, +0xfc000003fc000003, +0xf8000007f8000007, +0xf000000ff000000f, +0xe000001fe000001f, +0xc000003fc000003f, +0x8000007f8000007f, +0x000001ff000001ff, +0x000003fe000003fe, +0x000007fc000007fc, +0x00000ff800000ff8, +0x00001ff000001ff0, +0x00003fe000003fe0, +0x00007fc000007fc0, +0x0000ff800000ff80, +0x0001ff000001ff00, +0x0003fe000003fe00, +0x0007fc000007fc00, +0x000ff800000ff800, +0x001ff000001ff000, +0x003fe000003fe000, +0x007fc000007fc000, +0x00ff800000ff8000, +0x01ff000001ff0000, +0x03fe000003fe0000, +0x07fc000007fc0000, +0x0ff800000ff80000, +0x1ff000001ff00000, +0x3fe000003fe00000, +0x7fc000007fc00000, +0xff800000ff800000, +0xff000001ff000001, +0xfe000003fe000003, +0xfc000007fc000007, +0xf800000ff800000f, +0xf000001ff000001f, +0xe000003fe000003f, +0xc000007fc000007f, +0x800000ff800000ff, +0x000003ff000003ff, +0x000007fe000007fe, +0x00000ffc00000ffc, +0x00001ff800001ff8, +0x00003ff000003ff0, +0x00007fe000007fe0, +0x0000ffc00000ffc0, +0x0001ff800001ff80, +0x0003ff000003ff00, +0x0007fe000007fe00, +0x000ffc00000ffc00, +0x001ff800001ff800, +0x003ff000003ff000, +0x007fe000007fe000, +0x00ffc00000ffc000, +0x01ff800001ff8000, +0x03ff000003ff0000, +0x07fe000007fe0000, +0x0ffc00000ffc0000, +0x1ff800001ff80000, +0x3ff000003ff00000, +0x7fe000007fe00000, +0xffc00000ffc00000, +0xff800001ff800001, +0xff000003ff000003, +0xfe000007fe000007, +0xfc00000ffc00000f, +0xf800001ff800001f, +0xf000003ff000003f, +0xe000007fe000007f, +0xc00000ffc00000ff, +0x800001ff800001ff, +0x000007ff000007ff, +0x00000ffe00000ffe, +0x00001ffc00001ffc, +0x00003ff800003ff8, +0x00007ff000007ff0, +0x0000ffe00000ffe0, +0x0001ffc00001ffc0, +0x0003ff800003ff80, +0x0007ff000007ff00, +0x000ffe00000ffe00, +0x001ffc00001ffc00, +0x003ff800003ff800, +0x007ff000007ff000, +0x00ffe00000ffe000, +0x01ffc00001ffc000, +0x03ff800003ff8000, +0x07ff000007ff0000, +0x0ffe00000ffe0000, +0x1ffc00001ffc0000, +0x3ff800003ff80000, +0x7ff000007ff00000, +0xffe00000ffe00000, +0xffc00001ffc00001, +0xff800003ff800003, +0xff000007ff000007, +0xfe00000ffe00000f, +0xfc00001ffc00001f, +0xf800003ff800003f, +0xf000007ff000007f, +0xe00000ffe00000ff, +0xc00001ffc00001ff, +0x800003ff800003ff, +0x00000fff00000fff, +0x00001ffe00001ffe, +0x00003ffc00003ffc, +0x00007ff800007ff8, +0x0000fff00000fff0, +0x0001ffe00001ffe0, +0x0003ffc00003ffc0, +0x0007ff800007ff80, +0x000fff00000fff00, +0x001ffe00001ffe00, +0x003ffc00003ffc00, +0x007ff800007ff800, +0x00fff00000fff000, +0x01ffe00001ffe000, +0x03ffc00003ffc000, +0x07ff800007ff8000, +0x0fff00000fff0000, +0x1ffe00001ffe0000, +0x3ffc00003ffc0000, +0x7ff800007ff80000, +0xfff00000fff00000, +0xffe00001ffe00001, +0xffc00003ffc00003, +0xff800007ff800007, +0xff00000fff00000f, +0xfe00001ffe00001f, +0xfc00003ffc00003f, +0xf800007ff800007f, +0xf00000fff00000ff, +0xe00001ffe00001ff, +0xc00003ffc00003ff, +0x800007ff800007ff, +0x00001fff00001fff, +0x00003ffe00003ffe, +0x00007ffc00007ffc, +0x0000fff80000fff8, +0x0001fff00001fff0, +0x0003ffe00003ffe0, +0x0007ffc00007ffc0, +0x000fff80000fff80, +0x001fff00001fff00, +0x003ffe00003ffe00, +0x007ffc00007ffc00, +0x00fff80000fff800, +0x01fff00001fff000, +0x03ffe00003ffe000, +0x07ffc00007ffc000, +0x0fff80000fff8000, +0x1fff00001fff0000, +0x3ffe00003ffe0000, +0x7ffc00007ffc0000, +0xfff80000fff80000, +0xfff00001fff00001, +0xffe00003ffe00003, +0xffc00007ffc00007, +0xff80000fff80000f, +0xff00001fff00001f, +0xfe00003ffe00003f, +0xfc00007ffc00007f, +0xf80000fff80000ff, +0xf00001fff00001ff, +0xe00003ffe00003ff, +0xc00007ffc00007ff, +0x80000fff80000fff, +0x00003fff00003fff, +0x00007ffe00007ffe, +0x0000fffc0000fffc, +0x0001fff80001fff8, +0x0003fff00003fff0, +0x0007ffe00007ffe0, +0x000fffc0000fffc0, +0x001fff80001fff80, +0x003fff00003fff00, +0x007ffe00007ffe00, +0x00fffc0000fffc00, +0x01fff80001fff800, +0x03fff00003fff000, +0x07ffe00007ffe000, +0x0fffc0000fffc000, +0x1fff80001fff8000, +0x3fff00003fff0000, +0x7ffe00007ffe0000, +0xfffc0000fffc0000, +0xfff80001fff80001, +0xfff00003fff00003, +0xffe00007ffe00007, +0xffc0000fffc0000f, +0xff80001fff80001f, +0xff00003fff00003f, +0xfe00007ffe00007f, +0xfc0000fffc0000ff, +0xf80001fff80001ff, +0xf00003fff00003ff, +0xe00007ffe00007ff, +0xc0000fffc0000fff, +0x80001fff80001fff, +0x00007fff00007fff, +0x0000fffe0000fffe, +0x0001fffc0001fffc, +0x0003fff80003fff8, +0x0007fff00007fff0, +0x000fffe0000fffe0, +0x001fffc0001fffc0, +0x003fff80003fff80, +0x007fff00007fff00, +0x00fffe0000fffe00, +0x01fffc0001fffc00, +0x03fff80003fff800, +0x07fff00007fff000, +0x0fffe0000fffe000, +0x1fffc0001fffc000, +0x3fff80003fff8000, +0x7fff00007fff0000, +0xfffe0000fffe0000, +0xfffc0001fffc0001, +0xfff80003fff80003, +0xfff00007fff00007, +0xffe0000fffe0000f, +0xffc0001fffc0001f, +0xff80003fff80003f, +0xff00007fff00007f, +0xfe0000fffe0000ff, +0xfc0001fffc0001ff, +0xf80003fff80003ff, +0xf00007fff00007ff, +0xe0000fffe0000fff, +0xc0001fffc0001fff, +0x80003fff80003fff, +0x0000ffff0000ffff, +0x0001fffe0001fffe, +0x0003fffc0003fffc, +0x0007fff80007fff8, +0x000ffff0000ffff0, +0x001fffe0001fffe0, +0x003fffc0003fffc0, +0x007fff80007fff80, +0x00ffff0000ffff00, +0x01fffe0001fffe00, +0x03fffc0003fffc00, +0x07fff80007fff800, +0x0ffff0000ffff000, +0x1fffe0001fffe000, +0x3fffc0003fffc000, +0x7fff80007fff8000, +0xffff0000ffff0000, +0xfffe0001fffe0001, +0xfffc0003fffc0003, +0xfff80007fff80007, +0xfff0000ffff0000f, +0xffe0001fffe0001f, +0xffc0003fffc0003f, +0xff80007fff80007f, +0xff0000ffff0000ff, +0xfe0001fffe0001ff, +0xfc0003fffc0003ff, +0xf80007fff80007ff, +0xf0000ffff0000fff, +0xe0001fffe0001fff, +0xc0003fffc0003fff, +0x80007fff80007fff, +0x0001ffff0001ffff, +0x0003fffe0003fffe, +0x0007fffc0007fffc, +0x000ffff8000ffff8, +0x001ffff0001ffff0, +0x003fffe0003fffe0, +0x007fffc0007fffc0, +0x00ffff8000ffff80, +0x01ffff0001ffff00, +0x03fffe0003fffe00, +0x07fffc0007fffc00, +0x0ffff8000ffff800, +0x1ffff0001ffff000, +0x3fffe0003fffe000, +0x7fffc0007fffc000, +0xffff8000ffff8000, +0xffff0001ffff0001, +0xfffe0003fffe0003, +0xfffc0007fffc0007, +0xfff8000ffff8000f, +0xfff0001ffff0001f, +0xffe0003fffe0003f, +0xffc0007fffc0007f, +0xff8000ffff8000ff, +0xff0001ffff0001ff, +0xfe0003fffe0003ff, +0xfc0007fffc0007ff, +0xf8000ffff8000fff, +0xf0001ffff0001fff, +0xe0003fffe0003fff, +0xc0007fffc0007fff, +0x8000ffff8000ffff, +0x0003ffff0003ffff, +0x0007fffe0007fffe, +0x000ffffc000ffffc, +0x001ffff8001ffff8, +0x003ffff0003ffff0, +0x007fffe0007fffe0, +0x00ffffc000ffffc0, +0x01ffff8001ffff80, +0x03ffff0003ffff00, +0x07fffe0007fffe00, +0x0ffffc000ffffc00, +0x1ffff8001ffff800, +0x3ffff0003ffff000, +0x7fffe0007fffe000, +0xffffc000ffffc000, +0xffff8001ffff8001, +0xffff0003ffff0003, +0xfffe0007fffe0007, +0xfffc000ffffc000f, +0xfff8001ffff8001f, +0xfff0003ffff0003f, +0xffe0007fffe0007f, +0xffc000ffffc000ff, +0xff8001ffff8001ff, +0xff0003ffff0003ff, +0xfe0007fffe0007ff, +0xfc000ffffc000fff, +0xf8001ffff8001fff, +0xf0003ffff0003fff, +0xe0007fffe0007fff, +0xc000ffffc000ffff, +0x8001ffff8001ffff, +0x0007ffff0007ffff, +0x000ffffe000ffffe, +0x001ffffc001ffffc, +0x003ffff8003ffff8, +0x007ffff0007ffff0, +0x00ffffe000ffffe0, +0x01ffffc001ffffc0, +0x03ffff8003ffff80, +0x07ffff0007ffff00, +0x0ffffe000ffffe00, +0x1ffffc001ffffc00, +0x3ffff8003ffff800, +0x7ffff0007ffff000, +0xffffe000ffffe000, +0xffffc001ffffc001, +0xffff8003ffff8003, +0xffff0007ffff0007, +0xfffe000ffffe000f, +0xfffc001ffffc001f, +0xfff8003ffff8003f, +0xfff0007ffff0007f, +0xffe000ffffe000ff, +0xffc001ffffc001ff, +0xff8003ffff8003ff, +0xff0007ffff0007ff, +0xfe000ffffe000fff, +0xfc001ffffc001fff, +0xf8003ffff8003fff, +0xf0007ffff0007fff, +0xe000ffffe000ffff, +0xc001ffffc001ffff, +0x8003ffff8003ffff, +0x000fffff000fffff, +0x001ffffe001ffffe, +0x003ffffc003ffffc, +0x007ffff8007ffff8, +0x00fffff000fffff0, +0x01ffffe001ffffe0, +0x03ffffc003ffffc0, +0x07ffff8007ffff80, +0x0fffff000fffff00, +0x1ffffe001ffffe00, +0x3ffffc003ffffc00, +0x7ffff8007ffff800, +0xfffff000fffff000, +0xffffe001ffffe001, +0xffffc003ffffc003, +0xffff8007ffff8007, +0xffff000fffff000f, +0xfffe001ffffe001f, +0xfffc003ffffc003f, +0xfff8007ffff8007f, +0xfff000fffff000ff, +0xffe001ffffe001ff, +0xffc003ffffc003ff, +0xff8007ffff8007ff, +0xff000fffff000fff, +0xfe001ffffe001fff, +0xfc003ffffc003fff, +0xf8007ffff8007fff, +0xf000fffff000ffff, +0xe001ffffe001ffff, +0xc003ffffc003ffff, +0x8007ffff8007ffff, +0x001fffff001fffff, +0x003ffffe003ffffe, +0x007ffffc007ffffc, +0x00fffff800fffff8, +0x01fffff001fffff0, +0x03ffffe003ffffe0, +0x07ffffc007ffffc0, +0x0fffff800fffff80, +0x1fffff001fffff00, +0x3ffffe003ffffe00, +0x7ffffc007ffffc00, +0xfffff800fffff800, +0xfffff001fffff001, +0xffffe003ffffe003, +0xffffc007ffffc007, +0xffff800fffff800f, +0xffff001fffff001f, +0xfffe003ffffe003f, +0xfffc007ffffc007f, +0xfff800fffff800ff, +0xfff001fffff001ff, +0xffe003ffffe003ff, +0xffc007ffffc007ff, +0xff800fffff800fff, +0xff001fffff001fff, +0xfe003ffffe003fff, +0xfc007ffffc007fff, +0xf800fffff800ffff, +0xf001fffff001ffff, +0xe003ffffe003ffff, +0xc007ffffc007ffff, +0x800fffff800fffff, +0x003fffff003fffff, +0x007ffffe007ffffe, +0x00fffffc00fffffc, +0x01fffff801fffff8, +0x03fffff003fffff0, +0x07ffffe007ffffe0, +0x0fffffc00fffffc0, +0x1fffff801fffff80, +0x3fffff003fffff00, +0x7ffffe007ffffe00, +0xfffffc00fffffc00, +0xfffff801fffff801, +0xfffff003fffff003, +0xffffe007ffffe007, +0xffffc00fffffc00f, +0xffff801fffff801f, +0xffff003fffff003f, +0xfffe007ffffe007f, +0xfffc00fffffc00ff, +0xfff801fffff801ff, +0xfff003fffff003ff, +0xffe007ffffe007ff, +0xffc00fffffc00fff, +0xff801fffff801fff, +0xff003fffff003fff, +0xfe007ffffe007fff, +0xfc00fffffc00ffff, +0xf801fffff801ffff, +0xf003fffff003ffff, +0xe007ffffe007ffff, +0xc00fffffc00fffff, +0x801fffff801fffff, +0x007fffff007fffff, +0x00fffffe00fffffe, +0x01fffffc01fffffc, +0x03fffff803fffff8, +0x07fffff007fffff0, +0x0fffffe00fffffe0, +0x1fffffc01fffffc0, +0x3fffff803fffff80, +0x7fffff007fffff00, +0xfffffe00fffffe00, +0xfffffc01fffffc01, +0xfffff803fffff803, +0xfffff007fffff007, +0xffffe00fffffe00f, +0xffffc01fffffc01f, +0xffff803fffff803f, +0xffff007fffff007f, +0xfffe00fffffe00ff, +0xfffc01fffffc01ff, +0xfff803fffff803ff, +0xfff007fffff007ff, +0xffe00fffffe00fff, +0xffc01fffffc01fff, +0xff803fffff803fff, +0xff007fffff007fff, +0xfe00fffffe00ffff, +0xfc01fffffc01ffff, +0xf803fffff803ffff, +0xf007fffff007ffff, +0xe00fffffe00fffff, +0xc01fffffc01fffff, +0x803fffff803fffff, +0x00ffffff00ffffff, +0x01fffffe01fffffe, +0x03fffffc03fffffc, +0x07fffff807fffff8, +0x0ffffff00ffffff0, +0x1fffffe01fffffe0, +0x3fffffc03fffffc0, +0x7fffff807fffff80, +0xffffff00ffffff00, +0xfffffe01fffffe01, +0xfffffc03fffffc03, +0xfffff807fffff807, +0xfffff00ffffff00f, +0xffffe01fffffe01f, +0xffffc03fffffc03f, +0xffff807fffff807f, +0xffff00ffffff00ff, +0xfffe01fffffe01ff, +0xfffc03fffffc03ff, +0xfff807fffff807ff, +0xfff00ffffff00fff, +0xffe01fffffe01fff, +0xffc03fffffc03fff, +0xff807fffff807fff, +0xff00ffffff00ffff, +0xfe01fffffe01ffff, +0xfc03fffffc03ffff, +0xf807fffff807ffff, +0xf00ffffff00fffff, +0xe01fffffe01fffff, +0xc03fffffc03fffff, +0x807fffff807fffff, +0x01ffffff01ffffff, +0x03fffffe03fffffe, +0x07fffffc07fffffc, +0x0ffffff80ffffff8, +0x1ffffff01ffffff0, +0x3fffffe03fffffe0, +0x7fffffc07fffffc0, +0xffffff80ffffff80, +0xffffff01ffffff01, +0xfffffe03fffffe03, +0xfffffc07fffffc07, +0xfffff80ffffff80f, +0xfffff01ffffff01f, +0xffffe03fffffe03f, +0xffffc07fffffc07f, +0xffff80ffffff80ff, +0xffff01ffffff01ff, +0xfffe03fffffe03ff, +0xfffc07fffffc07ff, +0xfff80ffffff80fff, +0xfff01ffffff01fff, +0xffe03fffffe03fff, +0xffc07fffffc07fff, +0xff80ffffff80ffff, +0xff01ffffff01ffff, +0xfe03fffffe03ffff, +0xfc07fffffc07ffff, +0xf80ffffff80fffff, +0xf01ffffff01fffff, +0xe03fffffe03fffff, +0xc07fffffc07fffff, +0x80ffffff80ffffff, +0x03ffffff03ffffff, +0x07fffffe07fffffe, +0x0ffffffc0ffffffc, +0x1ffffff81ffffff8, +0x3ffffff03ffffff0, +0x7fffffe07fffffe0, +0xffffffc0ffffffc0, +0xffffff81ffffff81, +0xffffff03ffffff03, +0xfffffe07fffffe07, +0xfffffc0ffffffc0f, +0xfffff81ffffff81f, +0xfffff03ffffff03f, +0xffffe07fffffe07f, +0xffffc0ffffffc0ff, +0xffff81ffffff81ff, +0xffff03ffffff03ff, +0xfffe07fffffe07ff, +0xfffc0ffffffc0fff, +0xfff81ffffff81fff, +0xfff03ffffff03fff, +0xffe07fffffe07fff, +0xffc0ffffffc0ffff, +0xff81ffffff81ffff, +0xff03ffffff03ffff, +0xfe07fffffe07ffff, +0xfc0ffffffc0fffff, +0xf81ffffff81fffff, +0xf03ffffff03fffff, +0xe07fffffe07fffff, +0xc0ffffffc0ffffff, +0x81ffffff81ffffff, +0x07ffffff07ffffff, +0x0ffffffe0ffffffe, +0x1ffffffc1ffffffc, +0x3ffffff83ffffff8, +0x7ffffff07ffffff0, +0xffffffe0ffffffe0, +0xffffffc1ffffffc1, +0xffffff83ffffff83, +0xffffff07ffffff07, +0xfffffe0ffffffe0f, +0xfffffc1ffffffc1f, +0xfffff83ffffff83f, +0xfffff07ffffff07f, +0xffffe0ffffffe0ff, +0xffffc1ffffffc1ff, +0xffff83ffffff83ff, +0xffff07ffffff07ff, +0xfffe0ffffffe0fff, +0xfffc1ffffffc1fff, +0xfff83ffffff83fff, +0xfff07ffffff07fff, +0xffe0ffffffe0ffff, +0xffc1ffffffc1ffff, +0xff83ffffff83ffff, +0xff07ffffff07ffff, +0xfe0ffffffe0fffff, +0xfc1ffffffc1fffff, +0xf83ffffff83fffff, +0xf07ffffff07fffff, +0xe0ffffffe0ffffff, +0xc1ffffffc1ffffff, +0x83ffffff83ffffff, +0x0fffffff0fffffff, +0x1ffffffe1ffffffe, +0x3ffffffc3ffffffc, +0x7ffffff87ffffff8, +0xfffffff0fffffff0, +0xffffffe1ffffffe1, +0xffffffc3ffffffc3, +0xffffff87ffffff87, +0xffffff0fffffff0f, +0xfffffe1ffffffe1f, +0xfffffc3ffffffc3f, +0xfffff87ffffff87f, +0xfffff0fffffff0ff, +0xffffe1ffffffe1ff, +0xffffc3ffffffc3ff, +0xffff87ffffff87ff, +0xffff0fffffff0fff, +0xfffe1ffffffe1fff, +0xfffc3ffffffc3fff, +0xfff87ffffff87fff, +0xfff0fffffff0ffff, +0xffe1ffffffe1ffff, +0xffc3ffffffc3ffff, +0xff87ffffff87ffff, +0xff0fffffff0fffff, +0xfe1ffffffe1fffff, +0xfc3ffffffc3fffff, +0xf87ffffff87fffff, +0xf0fffffff0ffffff, +0xe1ffffffe1ffffff, +0xc3ffffffc3ffffff, +0x87ffffff87ffffff, +0x1fffffff1fffffff, +0x3ffffffe3ffffffe, +0x7ffffffc7ffffffc, +0xfffffff8fffffff8, +0xfffffff1fffffff1, +0xffffffe3ffffffe3, +0xffffffc7ffffffc7, +0xffffff8fffffff8f, +0xffffff1fffffff1f, +0xfffffe3ffffffe3f, +0xfffffc7ffffffc7f, +0xfffff8fffffff8ff, +0xfffff1fffffff1ff, +0xffffe3ffffffe3ff, +0xffffc7ffffffc7ff, +0xffff8fffffff8fff, +0xffff1fffffff1fff, +0xfffe3ffffffe3fff, +0xfffc7ffffffc7fff, +0xfff8fffffff8ffff, +0xfff1fffffff1ffff, +0xffe3ffffffe3ffff, +0xffc7ffffffc7ffff, +0xff8fffffff8fffff, +0xff1fffffff1fffff, +0xfe3ffffffe3fffff, +0xfc7ffffffc7fffff, +0xf8fffffff8ffffff, +0xf1fffffff1ffffff, +0xe3ffffffe3ffffff, +0xc7ffffffc7ffffff, +0x8fffffff8fffffff, +0x3fffffff3fffffff, +0x7ffffffe7ffffffe, +0xfffffffcfffffffc, +0xfffffff9fffffff9, +0xfffffff3fffffff3, +0xffffffe7ffffffe7, +0xffffffcfffffffcf, +0xffffff9fffffff9f, +0xffffff3fffffff3f, +0xfffffe7ffffffe7f, +0xfffffcfffffffcff, +0xfffff9fffffff9ff, +0xfffff3fffffff3ff, +0xffffe7ffffffe7ff, +0xffffcfffffffcfff, +0xffff9fffffff9fff, +0xffff3fffffff3fff, +0xfffe7ffffffe7fff, +0xfffcfffffffcffff, +0xfff9fffffff9ffff, +0xfff3fffffff3ffff, +0xffe7ffffffe7ffff, +0xffcfffffffcfffff, +0xff9fffffff9fffff, +0xff3fffffff3fffff, +0xfe7ffffffe7fffff, +0xfcfffffffcffffff, +0xf9fffffff9ffffff, +0xf3fffffff3ffffff, +0xe7ffffffe7ffffff, +0xcfffffffcfffffff, +0x9fffffff9fffffff, +0x7fffffff7fffffff, +0xfffffffefffffffe, +0xfffffffdfffffffd, +0xfffffffbfffffffb, +0xfffffff7fffffff7, +0xffffffefffffffef, +0xffffffdfffffffdf, +0xffffffbfffffffbf, +0xffffff7fffffff7f, +0xfffffefffffffeff, +0xfffffdfffffffdff, +0xfffffbfffffffbff, +0xfffff7fffffff7ff, +0xffffefffffffefff, +0xffffdfffffffdfff, +0xffffbfffffffbfff, +0xffff7fffffff7fff, +0xfffefffffffeffff, +0xfffdfffffffdffff, +0xfffbfffffffbffff, +0xfff7fffffff7ffff, +0xffefffffffefffff, +0xffdfffffffdfffff, +0xffbfffffffbfffff, +0xff7fffffff7fffff, +0xfefffffffeffffff, +0xfdfffffffdffffff, +0xfbfffffffbffffff, +0xf7fffffff7ffffff, +0xefffffffefffffff, +0xdfffffffdfffffff, +0xbfffffffbfffffff, +0x0000000000000001, +0x0000000000000002, +0x0000000000000004, +0x0000000000000008, +0x0000000000000010, +0x0000000000000020, +0x0000000000000040, +0x0000000000000080, +0x0000000000000100, +0x0000000000000200, +0x0000000000000400, +0x0000000000000800, +0x0000000000001000, +0x0000000000002000, +0x0000000000004000, +0x0000000000008000, +0x0000000000010000, +0x0000000000020000, +0x0000000000040000, +0x0000000000080000, +0x0000000000100000, +0x0000000000200000, +0x0000000000400000, +0x0000000000800000, +0x0000000001000000, +0x0000000002000000, +0x0000000004000000, +0x0000000008000000, +0x0000000010000000, +0x0000000020000000, +0x0000000040000000, +0x0000000080000000, +0x0000000100000000, +0x0000000200000000, +0x0000000400000000, +0x0000000800000000, +0x0000001000000000, +0x0000002000000000, +0x0000004000000000, +0x0000008000000000, +0x0000010000000000, +0x0000020000000000, +0x0000040000000000, +0x0000080000000000, +0x0000100000000000, +0x0000200000000000, +0x0000400000000000, +0x0000800000000000, +0x0001000000000000, +0x0002000000000000, +0x0004000000000000, +0x0008000000000000, +0x0010000000000000, +0x0020000000000000, +0x0040000000000000, +0x0080000000000000, +0x0100000000000000, +0x0200000000000000, +0x0400000000000000, +0x0800000000000000, +0x1000000000000000, +0x2000000000000000, +0x4000000000000000, +0x8000000000000000, +0x0000000000000003, +0x0000000000000006, +0x000000000000000c, +0x0000000000000018, +0x0000000000000030, +0x0000000000000060, +0x00000000000000c0, +0x0000000000000180, +0x0000000000000300, +0x0000000000000600, +0x0000000000000c00, +0x0000000000001800, +0x0000000000003000, +0x0000000000006000, +0x000000000000c000, +0x0000000000018000, +0x0000000000030000, +0x0000000000060000, +0x00000000000c0000, +0x0000000000180000, +0x0000000000300000, +0x0000000000600000, +0x0000000000c00000, +0x0000000001800000, +0x0000000003000000, +0x0000000006000000, +0x000000000c000000, +0x0000000018000000, +0x0000000030000000, +0x0000000060000000, +0x00000000c0000000, +0x0000000180000000, +0x0000000300000000, +0x0000000600000000, +0x0000000c00000000, +0x0000001800000000, +0x0000003000000000, +0x0000006000000000, +0x000000c000000000, +0x0000018000000000, +0x0000030000000000, +0x0000060000000000, +0x00000c0000000000, +0x0000180000000000, +0x0000300000000000, +0x0000600000000000, +0x0000c00000000000, +0x0001800000000000, +0x0003000000000000, +0x0006000000000000, +0x000c000000000000, +0x0018000000000000, +0x0030000000000000, +0x0060000000000000, +0x00c0000000000000, +0x0180000000000000, +0x0300000000000000, +0x0600000000000000, +0x0c00000000000000, +0x1800000000000000, +0x3000000000000000, +0x6000000000000000, +0xc000000000000000, +0x8000000000000001, +0x0000000000000007, +0x000000000000000e, +0x000000000000001c, +0x0000000000000038, +0x0000000000000070, +0x00000000000000e0, +0x00000000000001c0, +0x0000000000000380, +0x0000000000000700, +0x0000000000000e00, +0x0000000000001c00, +0x0000000000003800, +0x0000000000007000, +0x000000000000e000, +0x000000000001c000, +0x0000000000038000, +0x0000000000070000, +0x00000000000e0000, +0x00000000001c0000, +0x0000000000380000, +0x0000000000700000, +0x0000000000e00000, +0x0000000001c00000, +0x0000000003800000, +0x0000000007000000, +0x000000000e000000, +0x000000001c000000, +0x0000000038000000, +0x0000000070000000, +0x00000000e0000000, +0x00000001c0000000, +0x0000000380000000, +0x0000000700000000, +0x0000000e00000000, +0x0000001c00000000, +0x0000003800000000, +0x0000007000000000, +0x000000e000000000, +0x000001c000000000, +0x0000038000000000, +0x0000070000000000, +0x00000e0000000000, +0x00001c0000000000, +0x0000380000000000, +0x0000700000000000, +0x0000e00000000000, +0x0001c00000000000, +0x0003800000000000, +0x0007000000000000, +0x000e000000000000, +0x001c000000000000, +0x0038000000000000, +0x0070000000000000, +0x00e0000000000000, +0x01c0000000000000, +0x0380000000000000, +0x0700000000000000, +0x0e00000000000000, +0x1c00000000000000, +0x3800000000000000, +0x7000000000000000, +0xe000000000000000, +0xc000000000000001, +0x8000000000000003, +0x000000000000000f, +0x000000000000001e, +0x000000000000003c, +0x0000000000000078, +0x00000000000000f0, +0x00000000000001e0, +0x00000000000003c0, +0x0000000000000780, +0x0000000000000f00, +0x0000000000001e00, +0x0000000000003c00, +0x0000000000007800, +0x000000000000f000, +0x000000000001e000, +0x000000000003c000, +0x0000000000078000, +0x00000000000f0000, +0x00000000001e0000, +0x00000000003c0000, +0x0000000000780000, +0x0000000000f00000, +0x0000000001e00000, +0x0000000003c00000, +0x0000000007800000, +0x000000000f000000, +0x000000001e000000, +0x000000003c000000, +0x0000000078000000, +0x00000000f0000000, +0x00000001e0000000, +0x00000003c0000000, +0x0000000780000000, +0x0000000f00000000, +0x0000001e00000000, +0x0000003c00000000, +0x0000007800000000, +0x000000f000000000, +0x000001e000000000, +0x000003c000000000, +0x0000078000000000, +0x00000f0000000000, +0x00001e0000000000, +0x00003c0000000000, +0x0000780000000000, +0x0000f00000000000, +0x0001e00000000000, +0x0003c00000000000, +0x0007800000000000, +0x000f000000000000, +0x001e000000000000, +0x003c000000000000, +0x0078000000000000, +0x00f0000000000000, +0x01e0000000000000, +0x03c0000000000000, +0x0780000000000000, +0x0f00000000000000, +0x1e00000000000000, +0x3c00000000000000, +0x7800000000000000, +0xf000000000000000, +0xe000000000000001, +0xc000000000000003, +0x8000000000000007, +0x000000000000001f, +0x000000000000003e, +0x000000000000007c, +0x00000000000000f8, +0x00000000000001f0, +0x00000000000003e0, +0x00000000000007c0, +0x0000000000000f80, +0x0000000000001f00, +0x0000000000003e00, +0x0000000000007c00, +0x000000000000f800, +0x000000000001f000, +0x000000000003e000, +0x000000000007c000, +0x00000000000f8000, +0x00000000001f0000, +0x00000000003e0000, +0x00000000007c0000, +0x0000000000f80000, +0x0000000001f00000, +0x0000000003e00000, +0x0000000007c00000, +0x000000000f800000, +0x000000001f000000, +0x000000003e000000, +0x000000007c000000, +0x00000000f8000000, +0x00000001f0000000, +0x00000003e0000000, +0x00000007c0000000, +0x0000000f80000000, +0x0000001f00000000, +0x0000003e00000000, +0x0000007c00000000, +0x000000f800000000, +0x000001f000000000, +0x000003e000000000, +0x000007c000000000, +0x00000f8000000000, +0x00001f0000000000, +0x00003e0000000000, +0x00007c0000000000, +0x0000f80000000000, +0x0001f00000000000, +0x0003e00000000000, +0x0007c00000000000, +0x000f800000000000, +0x001f000000000000, +0x003e000000000000, +0x007c000000000000, +0x00f8000000000000, +0x01f0000000000000, +0x03e0000000000000, +0x07c0000000000000, +0x0f80000000000000, +0x1f00000000000000, +0x3e00000000000000, +0x7c00000000000000, +0xf800000000000000, +0xf000000000000001, +0xe000000000000003, +0xc000000000000007, +0x800000000000000f, +0x000000000000003f, +0x000000000000007e, +0x00000000000000fc, +0x00000000000001f8, +0x00000000000003f0, +0x00000000000007e0, +0x0000000000000fc0, +0x0000000000001f80, +0x0000000000003f00, +0x0000000000007e00, +0x000000000000fc00, +0x000000000001f800, +0x000000000003f000, +0x000000000007e000, +0x00000000000fc000, +0x00000000001f8000, +0x00000000003f0000, +0x00000000007e0000, +0x0000000000fc0000, +0x0000000001f80000, +0x0000000003f00000, +0x0000000007e00000, +0x000000000fc00000, +0x000000001f800000, +0x000000003f000000, +0x000000007e000000, +0x00000000fc000000, +0x00000001f8000000, +0x00000003f0000000, +0x00000007e0000000, +0x0000000fc0000000, +0x0000001f80000000, +0x0000003f00000000, +0x0000007e00000000, +0x000000fc00000000, +0x000001f800000000, +0x000003f000000000, +0x000007e000000000, +0x00000fc000000000, +0x00001f8000000000, +0x00003f0000000000, +0x00007e0000000000, +0x0000fc0000000000, +0x0001f80000000000, +0x0003f00000000000, +0x0007e00000000000, +0x000fc00000000000, +0x001f800000000000, +0x003f000000000000, +0x007e000000000000, +0x00fc000000000000, +0x01f8000000000000, +0x03f0000000000000, +0x07e0000000000000, +0x0fc0000000000000, +0x1f80000000000000, +0x3f00000000000000, +0x7e00000000000000, +0xfc00000000000000, +0xf800000000000001, +0xf000000000000003, +0xe000000000000007, +0xc00000000000000f, +0x800000000000001f, +0x000000000000007f, +0x00000000000000fe, +0x00000000000001fc, +0x00000000000003f8, +0x00000000000007f0, +0x0000000000000fe0, +0x0000000000001fc0, +0x0000000000003f80, +0x0000000000007f00, +0x000000000000fe00, +0x000000000001fc00, +0x000000000003f800, +0x000000000007f000, +0x00000000000fe000, +0x00000000001fc000, +0x00000000003f8000, +0x00000000007f0000, +0x0000000000fe0000, +0x0000000001fc0000, +0x0000000003f80000, +0x0000000007f00000, +0x000000000fe00000, +0x000000001fc00000, +0x000000003f800000, +0x000000007f000000, +0x00000000fe000000, +0x00000001fc000000, +0x00000003f8000000, +0x00000007f0000000, +0x0000000fe0000000, +0x0000001fc0000000, +0x0000003f80000000, +0x0000007f00000000, +0x000000fe00000000, +0x000001fc00000000, +0x000003f800000000, +0x000007f000000000, +0x00000fe000000000, +0x00001fc000000000, +0x00003f8000000000, +0x00007f0000000000, +0x0000fe0000000000, +0x0001fc0000000000, +0x0003f80000000000, +0x0007f00000000000, +0x000fe00000000000, +0x001fc00000000000, +0x003f800000000000, +0x007f000000000000, +0x00fe000000000000, +0x01fc000000000000, +0x03f8000000000000, +0x07f0000000000000, +0x0fe0000000000000, +0x1fc0000000000000, +0x3f80000000000000, +0x7f00000000000000, +0xfe00000000000000, +0xfc00000000000001, +0xf800000000000003, +0xf000000000000007, +0xe00000000000000f, +0xc00000000000001f, +0x800000000000003f, +0x00000000000000ff, +0x00000000000001fe, +0x00000000000003fc, +0x00000000000007f8, +0x0000000000000ff0, +0x0000000000001fe0, +0x0000000000003fc0, +0x0000000000007f80, +0x000000000000ff00, +0x000000000001fe00, +0x000000000003fc00, +0x000000000007f800, +0x00000000000ff000, +0x00000000001fe000, +0x00000000003fc000, +0x00000000007f8000, +0x0000000000ff0000, +0x0000000001fe0000, +0x0000000003fc0000, +0x0000000007f80000, +0x000000000ff00000, +0x000000001fe00000, +0x000000003fc00000, +0x000000007f800000, +0x00000000ff000000, +0x00000001fe000000, +0x00000003fc000000, +0x00000007f8000000, +0x0000000ff0000000, +0x0000001fe0000000, +0x0000003fc0000000, +0x0000007f80000000, +0x000000ff00000000, +0x000001fe00000000, +0x000003fc00000000, +0x000007f800000000, +0x00000ff000000000, +0x00001fe000000000, +0x00003fc000000000, +0x00007f8000000000, +0x0000ff0000000000, +0x0001fe0000000000, +0x0003fc0000000000, +0x0007f80000000000, +0x000ff00000000000, +0x001fe00000000000, +0x003fc00000000000, +0x007f800000000000, +0x00ff000000000000, +0x01fe000000000000, +0x03fc000000000000, +0x07f8000000000000, +0x0ff0000000000000, +0x1fe0000000000000, +0x3fc0000000000000, +0x7f80000000000000, +0xff00000000000000, +0xfe00000000000001, +0xfc00000000000003, +0xf800000000000007, +0xf00000000000000f, +0xe00000000000001f, +0xc00000000000003f, +0x800000000000007f, +0x00000000000001ff, +0x00000000000003fe, +0x00000000000007fc, +0x0000000000000ff8, +0x0000000000001ff0, +0x0000000000003fe0, +0x0000000000007fc0, +0x000000000000ff80, +0x000000000001ff00, +0x000000000003fe00, +0x000000000007fc00, +0x00000000000ff800, +0x00000000001ff000, +0x00000000003fe000, +0x00000000007fc000, +0x0000000000ff8000, +0x0000000001ff0000, +0x0000000003fe0000, +0x0000000007fc0000, +0x000000000ff80000, +0x000000001ff00000, +0x000000003fe00000, +0x000000007fc00000, +0x00000000ff800000, +0x00000001ff000000, +0x00000003fe000000, +0x00000007fc000000, +0x0000000ff8000000, +0x0000001ff0000000, +0x0000003fe0000000, +0x0000007fc0000000, +0x000000ff80000000, +0x000001ff00000000, +0x000003fe00000000, +0x000007fc00000000, +0x00000ff800000000, +0x00001ff000000000, +0x00003fe000000000, +0x00007fc000000000, +0x0000ff8000000000, +0x0001ff0000000000, +0x0003fe0000000000, +0x0007fc0000000000, +0x000ff80000000000, +0x001ff00000000000, +0x003fe00000000000, +0x007fc00000000000, +0x00ff800000000000, +0x01ff000000000000, +0x03fe000000000000, +0x07fc000000000000, +0x0ff8000000000000, +0x1ff0000000000000, +0x3fe0000000000000, +0x7fc0000000000000, +0xff80000000000000, +0xff00000000000001, +0xfe00000000000003, +0xfc00000000000007, +0xf80000000000000f, +0xf00000000000001f, +0xe00000000000003f, +0xc00000000000007f, +0x80000000000000ff, +0x00000000000003ff, +0x00000000000007fe, +0x0000000000000ffc, +0x0000000000001ff8, +0x0000000000003ff0, +0x0000000000007fe0, +0x000000000000ffc0, +0x000000000001ff80, +0x000000000003ff00, +0x000000000007fe00, +0x00000000000ffc00, +0x00000000001ff800, +0x00000000003ff000, +0x00000000007fe000, +0x0000000000ffc000, +0x0000000001ff8000, +0x0000000003ff0000, +0x0000000007fe0000, +0x000000000ffc0000, +0x000000001ff80000, +0x000000003ff00000, +0x000000007fe00000, +0x00000000ffc00000, +0x00000001ff800000, +0x00000003ff000000, +0x00000007fe000000, +0x0000000ffc000000, +0x0000001ff8000000, +0x0000003ff0000000, +0x0000007fe0000000, +0x000000ffc0000000, +0x000001ff80000000, +0x000003ff00000000, +0x000007fe00000000, +0x00000ffc00000000, +0x00001ff800000000, +0x00003ff000000000, +0x00007fe000000000, +0x0000ffc000000000, +0x0001ff8000000000, +0x0003ff0000000000, +0x0007fe0000000000, +0x000ffc0000000000, +0x001ff80000000000, +0x003ff00000000000, +0x007fe00000000000, +0x00ffc00000000000, +0x01ff800000000000, +0x03ff000000000000, +0x07fe000000000000, +0x0ffc000000000000, +0x1ff8000000000000, +0x3ff0000000000000, +0x7fe0000000000000, +0xffc0000000000000, +0xff80000000000001, +0xff00000000000003, +0xfe00000000000007, +0xfc0000000000000f, +0xf80000000000001f, +0xf00000000000003f, +0xe00000000000007f, +0xc0000000000000ff, +0x80000000000001ff, +0x00000000000007ff, +0x0000000000000ffe, +0x0000000000001ffc, +0x0000000000003ff8, +0x0000000000007ff0, +0x000000000000ffe0, +0x000000000001ffc0, +0x000000000003ff80, +0x000000000007ff00, +0x00000000000ffe00, +0x00000000001ffc00, +0x00000000003ff800, +0x00000000007ff000, +0x0000000000ffe000, +0x0000000001ffc000, +0x0000000003ff8000, +0x0000000007ff0000, +0x000000000ffe0000, +0x000000001ffc0000, +0x000000003ff80000, +0x000000007ff00000, +0x00000000ffe00000, +0x00000001ffc00000, +0x00000003ff800000, +0x00000007ff000000, +0x0000000ffe000000, +0x0000001ffc000000, +0x0000003ff8000000, +0x0000007ff0000000, +0x000000ffe0000000, +0x000001ffc0000000, +0x000003ff80000000, +0x000007ff00000000, +0x00000ffe00000000, +0x00001ffc00000000, +0x00003ff800000000, +0x00007ff000000000, +0x0000ffe000000000, +0x0001ffc000000000, +0x0003ff8000000000, +0x0007ff0000000000, +0x000ffe0000000000, +0x001ffc0000000000, +0x003ff80000000000, +0x007ff00000000000, +0x00ffe00000000000, +0x01ffc00000000000, +0x03ff800000000000, +0x07ff000000000000, +0x0ffe000000000000, +0x1ffc000000000000, +0x3ff8000000000000, +0x7ff0000000000000, +0xffe0000000000000, +0xffc0000000000001, +0xff80000000000003, +0xff00000000000007, +0xfe0000000000000f, +0xfc0000000000001f, +0xf80000000000003f, +0xf00000000000007f, +0xe0000000000000ff, +0xc0000000000001ff, +0x80000000000003ff, +0x0000000000000fff, +0x0000000000001ffe, +0x0000000000003ffc, +0x0000000000007ff8, +0x000000000000fff0, +0x000000000001ffe0, +0x000000000003ffc0, +0x000000000007ff80, +0x00000000000fff00, +0x00000000001ffe00, +0x00000000003ffc00, +0x00000000007ff800, +0x0000000000fff000, +0x0000000001ffe000, +0x0000000003ffc000, +0x0000000007ff8000, +0x000000000fff0000, +0x000000001ffe0000, +0x000000003ffc0000, +0x000000007ff80000, +0x00000000fff00000, +0x00000001ffe00000, +0x00000003ffc00000, +0x00000007ff800000, +0x0000000fff000000, +0x0000001ffe000000, +0x0000003ffc000000, +0x0000007ff8000000, +0x000000fff0000000, +0x000001ffe0000000, +0x000003ffc0000000, +0x000007ff80000000, +0x00000fff00000000, +0x00001ffe00000000, +0x00003ffc00000000, +0x00007ff800000000, +0x0000fff000000000, +0x0001ffe000000000, +0x0003ffc000000000, +0x0007ff8000000000, +0x000fff0000000000, +0x001ffe0000000000, +0x003ffc0000000000, +0x007ff80000000000, +0x00fff00000000000, +0x01ffe00000000000, +0x03ffc00000000000, +0x07ff800000000000, +0x0fff000000000000, +0x1ffe000000000000, +0x3ffc000000000000, +0x7ff8000000000000, +0xfff0000000000000, +0xffe0000000000001, +0xffc0000000000003, +0xff80000000000007, +0xff0000000000000f, +0xfe0000000000001f, +0xfc0000000000003f, +0xf80000000000007f, +0xf0000000000000ff, +0xe0000000000001ff, +0xc0000000000003ff, +0x80000000000007ff, +0x0000000000001fff, +0x0000000000003ffe, +0x0000000000007ffc, +0x000000000000fff8, +0x000000000001fff0, +0x000000000003ffe0, +0x000000000007ffc0, +0x00000000000fff80, +0x00000000001fff00, +0x00000000003ffe00, +0x00000000007ffc00, +0x0000000000fff800, +0x0000000001fff000, +0x0000000003ffe000, +0x0000000007ffc000, +0x000000000fff8000, +0x000000001fff0000, +0x000000003ffe0000, +0x000000007ffc0000, +0x00000000fff80000, +0x00000001fff00000, +0x00000003ffe00000, +0x00000007ffc00000, +0x0000000fff800000, +0x0000001fff000000, +0x0000003ffe000000, +0x0000007ffc000000, +0x000000fff8000000, +0x000001fff0000000, +0x000003ffe0000000, +0x000007ffc0000000, +0x00000fff80000000, +0x00001fff00000000, +0x00003ffe00000000, +0x00007ffc00000000, +0x0000fff800000000, +0x0001fff000000000, +0x0003ffe000000000, +0x0007ffc000000000, +0x000fff8000000000, +0x001fff0000000000, +0x003ffe0000000000, +0x007ffc0000000000, +0x00fff80000000000, +0x01fff00000000000, +0x03ffe00000000000, +0x07ffc00000000000, +0x0fff800000000000, +0x1fff000000000000, +0x3ffe000000000000, +0x7ffc000000000000, +0xfff8000000000000, +0xfff0000000000001, +0xffe0000000000003, +0xffc0000000000007, +0xff8000000000000f, +0xff0000000000001f, +0xfe0000000000003f, +0xfc0000000000007f, +0xf8000000000000ff, +0xf0000000000001ff, +0xe0000000000003ff, +0xc0000000000007ff, +0x8000000000000fff, +0x0000000000003fff, +0x0000000000007ffe, +0x000000000000fffc, +0x000000000001fff8, +0x000000000003fff0, +0x000000000007ffe0, +0x00000000000fffc0, +0x00000000001fff80, +0x00000000003fff00, +0x00000000007ffe00, +0x0000000000fffc00, +0x0000000001fff800, +0x0000000003fff000, +0x0000000007ffe000, +0x000000000fffc000, +0x000000001fff8000, +0x000000003fff0000, +0x000000007ffe0000, +0x00000000fffc0000, +0x00000001fff80000, +0x00000003fff00000, +0x00000007ffe00000, +0x0000000fffc00000, +0x0000001fff800000, +0x0000003fff000000, +0x0000007ffe000000, +0x000000fffc000000, +0x000001fff8000000, +0x000003fff0000000, +0x000007ffe0000000, +0x00000fffc0000000, +0x00001fff80000000, +0x00003fff00000000, +0x00007ffe00000000, +0x0000fffc00000000, +0x0001fff800000000, +0x0003fff000000000, +0x0007ffe000000000, +0x000fffc000000000, +0x001fff8000000000, +0x003fff0000000000, +0x007ffe0000000000, +0x00fffc0000000000, +0x01fff80000000000, +0x03fff00000000000, +0x07ffe00000000000, +0x0fffc00000000000, +0x1fff800000000000, +0x3fff000000000000, +0x7ffe000000000000, +0xfffc000000000000, +0xfff8000000000001, +0xfff0000000000003, +0xffe0000000000007, +0xffc000000000000f, +0xff8000000000001f, +0xff0000000000003f, +0xfe0000000000007f, +0xfc000000000000ff, +0xf8000000000001ff, +0xf0000000000003ff, +0xe0000000000007ff, +0xc000000000000fff, +0x8000000000001fff, +0x0000000000007fff, +0x000000000000fffe, +0x000000000001fffc, +0x000000000003fff8, +0x000000000007fff0, +0x00000000000fffe0, +0x00000000001fffc0, +0x00000000003fff80, +0x00000000007fff00, +0x0000000000fffe00, +0x0000000001fffc00, +0x0000000003fff800, +0x0000000007fff000, +0x000000000fffe000, +0x000000001fffc000, +0x000000003fff8000, +0x000000007fff0000, +0x00000000fffe0000, +0x00000001fffc0000, +0x00000003fff80000, +0x00000007fff00000, +0x0000000fffe00000, +0x0000001fffc00000, +0x0000003fff800000, +0x0000007fff000000, +0x000000fffe000000, +0x000001fffc000000, +0x000003fff8000000, +0x000007fff0000000, +0x00000fffe0000000, +0x00001fffc0000000, +0x00003fff80000000, +0x00007fff00000000, +0x0000fffe00000000, +0x0001fffc00000000, +0x0003fff800000000, +0x0007fff000000000, +0x000fffe000000000, +0x001fffc000000000, +0x003fff8000000000, +0x007fff0000000000, +0x00fffe0000000000, +0x01fffc0000000000, +0x03fff80000000000, +0x07fff00000000000, +0x0fffe00000000000, +0x1fffc00000000000, +0x3fff800000000000, +0x7fff000000000000, +0xfffe000000000000, +0xfffc000000000001, +0xfff8000000000003, +0xfff0000000000007, +0xffe000000000000f, +0xffc000000000001f, +0xff8000000000003f, +0xff0000000000007f, +0xfe000000000000ff, +0xfc000000000001ff, +0xf8000000000003ff, +0xf0000000000007ff, +0xe000000000000fff, +0xc000000000001fff, +0x8000000000003fff, +0x000000000000ffff, +0x000000000001fffe, +0x000000000003fffc, +0x000000000007fff8, +0x00000000000ffff0, +0x00000000001fffe0, +0x00000000003fffc0, +0x00000000007fff80, +0x0000000000ffff00, +0x0000000001fffe00, +0x0000000003fffc00, +0x0000000007fff800, +0x000000000ffff000, +0x000000001fffe000, +0x000000003fffc000, +0x000000007fff8000, +0x00000000ffff0000, +0x00000001fffe0000, +0x00000003fffc0000, +0x00000007fff80000, +0x0000000ffff00000, +0x0000001fffe00000, +0x0000003fffc00000, +0x0000007fff800000, +0x000000ffff000000, +0x000001fffe000000, +0x000003fffc000000, +0x000007fff8000000, +0x00000ffff0000000, +0x00001fffe0000000, +0x00003fffc0000000, +0x00007fff80000000, +0x0000ffff00000000, +0x0001fffe00000000, +0x0003fffc00000000, +0x0007fff800000000, +0x000ffff000000000, +0x001fffe000000000, +0x003fffc000000000, +0x007fff8000000000, +0x00ffff0000000000, +0x01fffe0000000000, +0x03fffc0000000000, +0x07fff80000000000, +0x0ffff00000000000, +0x1fffe00000000000, +0x3fffc00000000000, +0x7fff800000000000, +0xffff000000000000, +0xfffe000000000001, +0xfffc000000000003, +0xfff8000000000007, +0xfff000000000000f, +0xffe000000000001f, +0xffc000000000003f, +0xff8000000000007f, +0xff000000000000ff, +0xfe000000000001ff, +0xfc000000000003ff, +0xf8000000000007ff, +0xf000000000000fff, +0xe000000000001fff, +0xc000000000003fff, +0x8000000000007fff, +0x000000000001ffff, +0x000000000003fffe, +0x000000000007fffc, +0x00000000000ffff8, +0x00000000001ffff0, +0x00000000003fffe0, +0x00000000007fffc0, +0x0000000000ffff80, +0x0000000001ffff00, +0x0000000003fffe00, +0x0000000007fffc00, +0x000000000ffff800, +0x000000001ffff000, +0x000000003fffe000, +0x000000007fffc000, +0x00000000ffff8000, +0x00000001ffff0000, +0x00000003fffe0000, +0x00000007fffc0000, +0x0000000ffff80000, +0x0000001ffff00000, +0x0000003fffe00000, +0x0000007fffc00000, +0x000000ffff800000, +0x000001ffff000000, +0x000003fffe000000, +0x000007fffc000000, +0x00000ffff8000000, +0x00001ffff0000000, +0x00003fffe0000000, +0x00007fffc0000000, +0x0000ffff80000000, +0x0001ffff00000000, +0x0003fffe00000000, +0x0007fffc00000000, +0x000ffff800000000, +0x001ffff000000000, +0x003fffe000000000, +0x007fffc000000000, +0x00ffff8000000000, +0x01ffff0000000000, +0x03fffe0000000000, +0x07fffc0000000000, +0x0ffff80000000000, +0x1ffff00000000000, +0x3fffe00000000000, +0x7fffc00000000000, +0xffff800000000000, +0xffff000000000001, +0xfffe000000000003, +0xfffc000000000007, +0xfff800000000000f, +0xfff000000000001f, +0xffe000000000003f, +0xffc000000000007f, +0xff800000000000ff, +0xff000000000001ff, +0xfe000000000003ff, +0xfc000000000007ff, +0xf800000000000fff, +0xf000000000001fff, +0xe000000000003fff, +0xc000000000007fff, +0x800000000000ffff, +0x000000000003ffff, +0x000000000007fffe, +0x00000000000ffffc, +0x00000000001ffff8, +0x00000000003ffff0, +0x00000000007fffe0, +0x0000000000ffffc0, +0x0000000001ffff80, +0x0000000003ffff00, +0x0000000007fffe00, +0x000000000ffffc00, +0x000000001ffff800, +0x000000003ffff000, +0x000000007fffe000, +0x00000000ffffc000, +0x00000001ffff8000, +0x00000003ffff0000, +0x00000007fffe0000, +0x0000000ffffc0000, +0x0000001ffff80000, +0x0000003ffff00000, +0x0000007fffe00000, +0x000000ffffc00000, +0x000001ffff800000, +0x000003ffff000000, +0x000007fffe000000, +0x00000ffffc000000, +0x00001ffff8000000, +0x00003ffff0000000, +0x00007fffe0000000, +0x0000ffffc0000000, +0x0001ffff80000000, +0x0003ffff00000000, +0x0007fffe00000000, +0x000ffffc00000000, +0x001ffff800000000, +0x003ffff000000000, +0x007fffe000000000, +0x00ffffc000000000, +0x01ffff8000000000, +0x03ffff0000000000, +0x07fffe0000000000, +0x0ffffc0000000000, +0x1ffff80000000000, +0x3ffff00000000000, +0x7fffe00000000000, +0xffffc00000000000, +0xffff800000000001, +0xffff000000000003, +0xfffe000000000007, +0xfffc00000000000f, +0xfff800000000001f, +0xfff000000000003f, +0xffe000000000007f, +0xffc00000000000ff, +0xff800000000001ff, +0xff000000000003ff, +0xfe000000000007ff, +0xfc00000000000fff, +0xf800000000001fff, +0xf000000000003fff, +0xe000000000007fff, +0xc00000000000ffff, +0x800000000001ffff, +0x000000000007ffff, +0x00000000000ffffe, +0x00000000001ffffc, +0x00000000003ffff8, +0x00000000007ffff0, +0x0000000000ffffe0, +0x0000000001ffffc0, +0x0000000003ffff80, +0x0000000007ffff00, +0x000000000ffffe00, +0x000000001ffffc00, +0x000000003ffff800, +0x000000007ffff000, +0x00000000ffffe000, +0x00000001ffffc000, +0x00000003ffff8000, +0x00000007ffff0000, +0x0000000ffffe0000, +0x0000001ffffc0000, +0x0000003ffff80000, +0x0000007ffff00000, +0x000000ffffe00000, +0x000001ffffc00000, +0x000003ffff800000, +0x000007ffff000000, +0x00000ffffe000000, +0x00001ffffc000000, +0x00003ffff8000000, +0x00007ffff0000000, +0x0000ffffe0000000, +0x0001ffffc0000000, +0x0003ffff80000000, +0x0007ffff00000000, +0x000ffffe00000000, +0x001ffffc00000000, +0x003ffff800000000, +0x007ffff000000000, +0x00ffffe000000000, +0x01ffffc000000000, +0x03ffff8000000000, +0x07ffff0000000000, +0x0ffffe0000000000, +0x1ffffc0000000000, +0x3ffff80000000000, +0x7ffff00000000000, +0xffffe00000000000, +0xffffc00000000001, +0xffff800000000003, +0xffff000000000007, +0xfffe00000000000f, +0xfffc00000000001f, +0xfff800000000003f, +0xfff000000000007f, +0xffe00000000000ff, +0xffc00000000001ff, +0xff800000000003ff, +0xff000000000007ff, +0xfe00000000000fff, +0xfc00000000001fff, +0xf800000000003fff, +0xf000000000007fff, +0xe00000000000ffff, +0xc00000000001ffff, +0x800000000003ffff, +0x00000000000fffff, +0x00000000001ffffe, +0x00000000003ffffc, +0x00000000007ffff8, +0x0000000000fffff0, +0x0000000001ffffe0, +0x0000000003ffffc0, +0x0000000007ffff80, +0x000000000fffff00, +0x000000001ffffe00, +0x000000003ffffc00, +0x000000007ffff800, +0x00000000fffff000, +0x00000001ffffe000, +0x00000003ffffc000, +0x00000007ffff8000, +0x0000000fffff0000, +0x0000001ffffe0000, +0x0000003ffffc0000, +0x0000007ffff80000, +0x000000fffff00000, +0x000001ffffe00000, +0x000003ffffc00000, +0x000007ffff800000, +0x00000fffff000000, +0x00001ffffe000000, +0x00003ffffc000000, +0x00007ffff8000000, +0x0000fffff0000000, +0x0001ffffe0000000, +0x0003ffffc0000000, +0x0007ffff80000000, +0x000fffff00000000, +0x001ffffe00000000, +0x003ffffc00000000, +0x007ffff800000000, +0x00fffff000000000, +0x01ffffe000000000, +0x03ffffc000000000, +0x07ffff8000000000, +0x0fffff0000000000, +0x1ffffe0000000000, +0x3ffffc0000000000, +0x7ffff80000000000, +0xfffff00000000000, +0xffffe00000000001, +0xffffc00000000003, +0xffff800000000007, +0xffff00000000000f, +0xfffe00000000001f, +0xfffc00000000003f, +0xfff800000000007f, +0xfff00000000000ff, +0xffe00000000001ff, +0xffc00000000003ff, +0xff800000000007ff, +0xff00000000000fff, +0xfe00000000001fff, +0xfc00000000003fff, +0xf800000000007fff, +0xf00000000000ffff, +0xe00000000001ffff, +0xc00000000003ffff, +0x800000000007ffff, +0x00000000001fffff, +0x00000000003ffffe, +0x00000000007ffffc, +0x0000000000fffff8, +0x0000000001fffff0, +0x0000000003ffffe0, +0x0000000007ffffc0, +0x000000000fffff80, +0x000000001fffff00, +0x000000003ffffe00, +0x000000007ffffc00, +0x00000000fffff800, +0x00000001fffff000, +0x00000003ffffe000, +0x00000007ffffc000, +0x0000000fffff8000, +0x0000001fffff0000, +0x0000003ffffe0000, +0x0000007ffffc0000, +0x000000fffff80000, +0x000001fffff00000, +0x000003ffffe00000, +0x000007ffffc00000, +0x00000fffff800000, +0x00001fffff000000, +0x00003ffffe000000, +0x00007ffffc000000, +0x0000fffff8000000, +0x0001fffff0000000, +0x0003ffffe0000000, +0x0007ffffc0000000, +0x000fffff80000000, +0x001fffff00000000, +0x003ffffe00000000, +0x007ffffc00000000, +0x00fffff800000000, +0x01fffff000000000, +0x03ffffe000000000, +0x07ffffc000000000, +0x0fffff8000000000, +0x1fffff0000000000, +0x3ffffe0000000000, +0x7ffffc0000000000, +0xfffff80000000000, +0xfffff00000000001, +0xffffe00000000003, +0xffffc00000000007, +0xffff80000000000f, +0xffff00000000001f, +0xfffe00000000003f, +0xfffc00000000007f, +0xfff80000000000ff, +0xfff00000000001ff, +0xffe00000000003ff, +0xffc00000000007ff, +0xff80000000000fff, +0xff00000000001fff, +0xfe00000000003fff, +0xfc00000000007fff, +0xf80000000000ffff, +0xf00000000001ffff, +0xe00000000003ffff, +0xc00000000007ffff, +0x80000000000fffff, +0x00000000003fffff, +0x00000000007ffffe, +0x0000000000fffffc, +0x0000000001fffff8, +0x0000000003fffff0, +0x0000000007ffffe0, +0x000000000fffffc0, +0x000000001fffff80, +0x000000003fffff00, +0x000000007ffffe00, +0x00000000fffffc00, +0x00000001fffff800, +0x00000003fffff000, +0x00000007ffffe000, +0x0000000fffffc000, +0x0000001fffff8000, +0x0000003fffff0000, +0x0000007ffffe0000, +0x000000fffffc0000, +0x000001fffff80000, +0x000003fffff00000, +0x000007ffffe00000, +0x00000fffffc00000, +0x00001fffff800000, +0x00003fffff000000, +0x00007ffffe000000, +0x0000fffffc000000, +0x0001fffff8000000, +0x0003fffff0000000, +0x0007ffffe0000000, +0x000fffffc0000000, +0x001fffff80000000, +0x003fffff00000000, +0x007ffffe00000000, +0x00fffffc00000000, +0x01fffff800000000, +0x03fffff000000000, +0x07ffffe000000000, +0x0fffffc000000000, +0x1fffff8000000000, +0x3fffff0000000000, +0x7ffffe0000000000, +0xfffffc0000000000, +0xfffff80000000001, +0xfffff00000000003, +0xffffe00000000007, +0xffffc0000000000f, +0xffff80000000001f, +0xffff00000000003f, +0xfffe00000000007f, +0xfffc0000000000ff, +0xfff80000000001ff, +0xfff00000000003ff, +0xffe00000000007ff, +0xffc0000000000fff, +0xff80000000001fff, +0xff00000000003fff, +0xfe00000000007fff, +0xfc0000000000ffff, +0xf80000000001ffff, +0xf00000000003ffff, +0xe00000000007ffff, +0xc0000000000fffff, +0x80000000001fffff, +0x00000000007fffff, +0x0000000000fffffe, +0x0000000001fffffc, +0x0000000003fffff8, +0x0000000007fffff0, +0x000000000fffffe0, +0x000000001fffffc0, +0x000000003fffff80, +0x000000007fffff00, +0x00000000fffffe00, +0x00000001fffffc00, +0x00000003fffff800, +0x00000007fffff000, +0x0000000fffffe000, +0x0000001fffffc000, +0x0000003fffff8000, +0x0000007fffff0000, +0x000000fffffe0000, +0x000001fffffc0000, +0x000003fffff80000, +0x000007fffff00000, +0x00000fffffe00000, +0x00001fffffc00000, +0x00003fffff800000, +0x00007fffff000000, +0x0000fffffe000000, +0x0001fffffc000000, +0x0003fffff8000000, +0x0007fffff0000000, +0x000fffffe0000000, +0x001fffffc0000000, +0x003fffff80000000, +0x007fffff00000000, +0x00fffffe00000000, +0x01fffffc00000000, +0x03fffff800000000, +0x07fffff000000000, +0x0fffffe000000000, +0x1fffffc000000000, +0x3fffff8000000000, +0x7fffff0000000000, +0xfffffe0000000000, +0xfffffc0000000001, +0xfffff80000000003, +0xfffff00000000007, +0xffffe0000000000f, +0xffffc0000000001f, +0xffff80000000003f, +0xffff00000000007f, +0xfffe0000000000ff, +0xfffc0000000001ff, +0xfff80000000003ff, +0xfff00000000007ff, +0xffe0000000000fff, +0xffc0000000001fff, +0xff80000000003fff, +0xff00000000007fff, +0xfe0000000000ffff, +0xfc0000000001ffff, +0xf80000000003ffff, +0xf00000000007ffff, +0xe0000000000fffff, +0xc0000000001fffff, +0x80000000003fffff, +0x0000000000ffffff, +0x0000000001fffffe, +0x0000000003fffffc, +0x0000000007fffff8, +0x000000000ffffff0, +0x000000001fffffe0, +0x000000003fffffc0, +0x000000007fffff80, +0x00000000ffffff00, +0x00000001fffffe00, +0x00000003fffffc00, +0x00000007fffff800, +0x0000000ffffff000, +0x0000001fffffe000, +0x0000003fffffc000, +0x0000007fffff8000, +0x000000ffffff0000, +0x000001fffffe0000, +0x000003fffffc0000, +0x000007fffff80000, +0x00000ffffff00000, +0x00001fffffe00000, +0x00003fffffc00000, +0x00007fffff800000, +0x0000ffffff000000, +0x0001fffffe000000, +0x0003fffffc000000, +0x0007fffff8000000, +0x000ffffff0000000, +0x001fffffe0000000, +0x003fffffc0000000, +0x007fffff80000000, +0x00ffffff00000000, +0x01fffffe00000000, +0x03fffffc00000000, +0x07fffff800000000, +0x0ffffff000000000, +0x1fffffe000000000, +0x3fffffc000000000, +0x7fffff8000000000, +0xffffff0000000000, +0xfffffe0000000001, +0xfffffc0000000003, +0xfffff80000000007, +0xfffff0000000000f, +0xffffe0000000001f, +0xffffc0000000003f, +0xffff80000000007f, +0xffff0000000000ff, +0xfffe0000000001ff, +0xfffc0000000003ff, +0xfff80000000007ff, +0xfff0000000000fff, +0xffe0000000001fff, +0xffc0000000003fff, +0xff80000000007fff, +0xff0000000000ffff, +0xfe0000000001ffff, +0xfc0000000003ffff, +0xf80000000007ffff, +0xf0000000000fffff, +0xe0000000001fffff, +0xc0000000003fffff, +0x80000000007fffff, +0x0000000001ffffff, +0x0000000003fffffe, +0x0000000007fffffc, +0x000000000ffffff8, +0x000000001ffffff0, +0x000000003fffffe0, +0x000000007fffffc0, +0x00000000ffffff80, +0x00000001ffffff00, +0x00000003fffffe00, +0x00000007fffffc00, +0x0000000ffffff800, +0x0000001ffffff000, +0x0000003fffffe000, +0x0000007fffffc000, +0x000000ffffff8000, +0x000001ffffff0000, +0x000003fffffe0000, +0x000007fffffc0000, +0x00000ffffff80000, +0x00001ffffff00000, +0x00003fffffe00000, +0x00007fffffc00000, +0x0000ffffff800000, +0x0001ffffff000000, +0x0003fffffe000000, +0x0007fffffc000000, +0x000ffffff8000000, +0x001ffffff0000000, +0x003fffffe0000000, +0x007fffffc0000000, +0x00ffffff80000000, +0x01ffffff00000000, +0x03fffffe00000000, +0x07fffffc00000000, +0x0ffffff800000000, +0x1ffffff000000000, +0x3fffffe000000000, +0x7fffffc000000000, +0xffffff8000000000, +0xffffff0000000001, +0xfffffe0000000003, +0xfffffc0000000007, +0xfffff8000000000f, +0xfffff0000000001f, +0xffffe0000000003f, +0xffffc0000000007f, +0xffff8000000000ff, +0xffff0000000001ff, +0xfffe0000000003ff, +0xfffc0000000007ff, +0xfff8000000000fff, +0xfff0000000001fff, +0xffe0000000003fff, +0xffc0000000007fff, +0xff8000000000ffff, +0xff0000000001ffff, +0xfe0000000003ffff, +0xfc0000000007ffff, +0xf8000000000fffff, +0xf0000000001fffff, +0xe0000000003fffff, +0xc0000000007fffff, +0x8000000000ffffff, +0x0000000003ffffff, +0x0000000007fffffe, +0x000000000ffffffc, +0x000000001ffffff8, +0x000000003ffffff0, +0x000000007fffffe0, +0x00000000ffffffc0, +0x00000001ffffff80, +0x00000003ffffff00, +0x00000007fffffe00, +0x0000000ffffffc00, +0x0000001ffffff800, +0x0000003ffffff000, +0x0000007fffffe000, +0x000000ffffffc000, +0x000001ffffff8000, +0x000003ffffff0000, +0x000007fffffe0000, +0x00000ffffffc0000, +0x00001ffffff80000, +0x00003ffffff00000, +0x00007fffffe00000, +0x0000ffffffc00000, +0x0001ffffff800000, +0x0003ffffff000000, +0x0007fffffe000000, +0x000ffffffc000000, +0x001ffffff8000000, +0x003ffffff0000000, +0x007fffffe0000000, +0x00ffffffc0000000, +0x01ffffff80000000, +0x03ffffff00000000, +0x07fffffe00000000, +0x0ffffffc00000000, +0x1ffffff800000000, +0x3ffffff000000000, +0x7fffffe000000000, +0xffffffc000000000, +0xffffff8000000001, +0xffffff0000000003, +0xfffffe0000000007, +0xfffffc000000000f, +0xfffff8000000001f, +0xfffff0000000003f, +0xffffe0000000007f, +0xffffc000000000ff, +0xffff8000000001ff, +0xffff0000000003ff, +0xfffe0000000007ff, +0xfffc000000000fff, +0xfff8000000001fff, +0xfff0000000003fff, +0xffe0000000007fff, +0xffc000000000ffff, +0xff8000000001ffff, +0xff0000000003ffff, +0xfe0000000007ffff, +0xfc000000000fffff, +0xf8000000001fffff, +0xf0000000003fffff, +0xe0000000007fffff, +0xc000000000ffffff, +0x8000000001ffffff, +0x0000000007ffffff, +0x000000000ffffffe, +0x000000001ffffffc, +0x000000003ffffff8, +0x000000007ffffff0, +0x00000000ffffffe0, +0x00000001ffffffc0, +0x00000003ffffff80, +0x00000007ffffff00, +0x0000000ffffffe00, +0x0000001ffffffc00, +0x0000003ffffff800, +0x0000007ffffff000, +0x000000ffffffe000, +0x000001ffffffc000, +0x000003ffffff8000, +0x000007ffffff0000, +0x00000ffffffe0000, +0x00001ffffffc0000, +0x00003ffffff80000, +0x00007ffffff00000, +0x0000ffffffe00000, +0x0001ffffffc00000, +0x0003ffffff800000, +0x0007ffffff000000, +0x000ffffffe000000, +0x001ffffffc000000, +0x003ffffff8000000, +0x007ffffff0000000, +0x00ffffffe0000000, +0x01ffffffc0000000, +0x03ffffff80000000, +0x07ffffff00000000, +0x0ffffffe00000000, +0x1ffffffc00000000, +0x3ffffff800000000, +0x7ffffff000000000, +0xffffffe000000000, +0xffffffc000000001, +0xffffff8000000003, +0xffffff0000000007, +0xfffffe000000000f, +0xfffffc000000001f, +0xfffff8000000003f, +0xfffff0000000007f, +0xffffe000000000ff, +0xffffc000000001ff, +0xffff8000000003ff, +0xffff0000000007ff, +0xfffe000000000fff, +0xfffc000000001fff, +0xfff8000000003fff, +0xfff0000000007fff, +0xffe000000000ffff, +0xffc000000001ffff, +0xff8000000003ffff, +0xff0000000007ffff, +0xfe000000000fffff, +0xfc000000001fffff, +0xf8000000003fffff, +0xf0000000007fffff, +0xe000000000ffffff, +0xc000000001ffffff, +0x8000000003ffffff, +0x000000000fffffff, +0x000000001ffffffe, +0x000000003ffffffc, +0x000000007ffffff8, +0x00000000fffffff0, +0x00000001ffffffe0, +0x00000003ffffffc0, +0x00000007ffffff80, +0x0000000fffffff00, +0x0000001ffffffe00, +0x0000003ffffffc00, +0x0000007ffffff800, +0x000000fffffff000, +0x000001ffffffe000, +0x000003ffffffc000, +0x000007ffffff8000, +0x00000fffffff0000, +0x00001ffffffe0000, +0x00003ffffffc0000, +0x00007ffffff80000, +0x0000fffffff00000, +0x0001ffffffe00000, +0x0003ffffffc00000, +0x0007ffffff800000, +0x000fffffff000000, +0x001ffffffe000000, +0x003ffffffc000000, +0x007ffffff8000000, +0x00fffffff0000000, +0x01ffffffe0000000, +0x03ffffffc0000000, +0x07ffffff80000000, +0x0fffffff00000000, +0x1ffffffe00000000, +0x3ffffffc00000000, +0x7ffffff800000000, +0xfffffff000000000, +0xffffffe000000001, +0xffffffc000000003, +0xffffff8000000007, +0xffffff000000000f, +0xfffffe000000001f, +0xfffffc000000003f, +0xfffff8000000007f, +0xfffff000000000ff, +0xffffe000000001ff, +0xffffc000000003ff, +0xffff8000000007ff, +0xffff000000000fff, +0xfffe000000001fff, +0xfffc000000003fff, +0xfff8000000007fff, +0xfff000000000ffff, +0xffe000000001ffff, +0xffc000000003ffff, +0xff8000000007ffff, +0xff000000000fffff, +0xfe000000001fffff, +0xfc000000003fffff, +0xf8000000007fffff, +0xf000000000ffffff, +0xe000000001ffffff, +0xc000000003ffffff, +0x8000000007ffffff, +0x000000001fffffff, +0x000000003ffffffe, +0x000000007ffffffc, +0x00000000fffffff8, +0x00000001fffffff0, +0x00000003ffffffe0, +0x00000007ffffffc0, +0x0000000fffffff80, +0x0000001fffffff00, +0x0000003ffffffe00, +0x0000007ffffffc00, +0x000000fffffff800, +0x000001fffffff000, +0x000003ffffffe000, +0x000007ffffffc000, +0x00000fffffff8000, +0x00001fffffff0000, +0x00003ffffffe0000, +0x00007ffffffc0000, +0x0000fffffff80000, +0x0001fffffff00000, +0x0003ffffffe00000, +0x0007ffffffc00000, +0x000fffffff800000, +0x001fffffff000000, +0x003ffffffe000000, +0x007ffffffc000000, +0x00fffffff8000000, +0x01fffffff0000000, +0x03ffffffe0000000, +0x07ffffffc0000000, +0x0fffffff80000000, +0x1fffffff00000000, +0x3ffffffe00000000, +0x7ffffffc00000000, +0xfffffff800000000, +0xfffffff000000001, +0xffffffe000000003, +0xffffffc000000007, +0xffffff800000000f, +0xffffff000000001f, +0xfffffe000000003f, +0xfffffc000000007f, +0xfffff800000000ff, +0xfffff000000001ff, +0xffffe000000003ff, +0xffffc000000007ff, +0xffff800000000fff, +0xffff000000001fff, +0xfffe000000003fff, +0xfffc000000007fff, +0xfff800000000ffff, +0xfff000000001ffff, +0xffe000000003ffff, +0xffc000000007ffff, +0xff800000000fffff, +0xff000000001fffff, +0xfe000000003fffff, +0xfc000000007fffff, +0xf800000000ffffff, +0xf000000001ffffff, +0xe000000003ffffff, +0xc000000007ffffff, +0x800000000fffffff, +0x000000003fffffff, +0x000000007ffffffe, +0x00000000fffffffc, +0x00000001fffffff8, +0x00000003fffffff0, +0x00000007ffffffe0, +0x0000000fffffffc0, +0x0000001fffffff80, +0x0000003fffffff00, +0x0000007ffffffe00, +0x000000fffffffc00, +0x000001fffffff800, +0x000003fffffff000, +0x000007ffffffe000, +0x00000fffffffc000, +0x00001fffffff8000, +0x00003fffffff0000, +0x00007ffffffe0000, +0x0000fffffffc0000, +0x0001fffffff80000, +0x0003fffffff00000, +0x0007ffffffe00000, +0x000fffffffc00000, +0x001fffffff800000, +0x003fffffff000000, +0x007ffffffe000000, +0x00fffffffc000000, +0x01fffffff8000000, +0x03fffffff0000000, +0x07ffffffe0000000, +0x0fffffffc0000000, +0x1fffffff80000000, +0x3fffffff00000000, +0x7ffffffe00000000, +0xfffffffc00000000, +0xfffffff800000001, +0xfffffff000000003, +0xffffffe000000007, +0xffffffc00000000f, +0xffffff800000001f, +0xffffff000000003f, +0xfffffe000000007f, +0xfffffc00000000ff, +0xfffff800000001ff, +0xfffff000000003ff, +0xffffe000000007ff, +0xffffc00000000fff, +0xffff800000001fff, +0xffff000000003fff, +0xfffe000000007fff, +0xfffc00000000ffff, +0xfff800000001ffff, +0xfff000000003ffff, +0xffe000000007ffff, +0xffc00000000fffff, +0xff800000001fffff, +0xff000000003fffff, +0xfe000000007fffff, +0xfc00000000ffffff, +0xf800000001ffffff, +0xf000000003ffffff, +0xe000000007ffffff, +0xc00000000fffffff, +0x800000001fffffff, +0x000000007fffffff, +0x00000000fffffffe, +0x00000001fffffffc, +0x00000003fffffff8, +0x00000007fffffff0, +0x0000000fffffffe0, +0x0000001fffffffc0, +0x0000003fffffff80, +0x0000007fffffff00, +0x000000fffffffe00, +0x000001fffffffc00, +0x000003fffffff800, +0x000007fffffff000, +0x00000fffffffe000, +0x00001fffffffc000, +0x00003fffffff8000, +0x00007fffffff0000, +0x0000fffffffe0000, +0x0001fffffffc0000, +0x0003fffffff80000, +0x0007fffffff00000, +0x000fffffffe00000, +0x001fffffffc00000, +0x003fffffff800000, +0x007fffffff000000, +0x00fffffffe000000, +0x01fffffffc000000, +0x03fffffff8000000, +0x07fffffff0000000, +0x0fffffffe0000000, +0x1fffffffc0000000, +0x3fffffff80000000, +0x7fffffff00000000, +0xfffffffe00000000, +0xfffffffc00000001, +0xfffffff800000003, +0xfffffff000000007, +0xffffffe00000000f, +0xffffffc00000001f, +0xffffff800000003f, +0xffffff000000007f, +0xfffffe00000000ff, +0xfffffc00000001ff, +0xfffff800000003ff, +0xfffff000000007ff, +0xffffe00000000fff, +0xffffc00000001fff, +0xffff800000003fff, +0xffff000000007fff, +0xfffe00000000ffff, +0xfffc00000001ffff, +0xfff800000003ffff, +0xfff000000007ffff, +0xffe00000000fffff, +0xffc00000001fffff, +0xff800000003fffff, +0xff000000007fffff, +0xfe00000000ffffff, +0xfc00000001ffffff, +0xf800000003ffffff, +0xf000000007ffffff, +0xe00000000fffffff, +0xc00000001fffffff, +0x800000003fffffff, +0x00000000ffffffff, +0x00000001fffffffe, +0x00000003fffffffc, +0x00000007fffffff8, +0x0000000ffffffff0, +0x0000001fffffffe0, +0x0000003fffffffc0, +0x0000007fffffff80, +0x000000ffffffff00, +0x000001fffffffe00, +0x000003fffffffc00, +0x000007fffffff800, +0x00000ffffffff000, +0x00001fffffffe000, +0x00003fffffffc000, +0x00007fffffff8000, +0x0000ffffffff0000, +0x0001fffffffe0000, +0x0003fffffffc0000, +0x0007fffffff80000, +0x000ffffffff00000, +0x001fffffffe00000, +0x003fffffffc00000, +0x007fffffff800000, +0x00ffffffff000000, +0x01fffffffe000000, +0x03fffffffc000000, +0x07fffffff8000000, +0x0ffffffff0000000, +0x1fffffffe0000000, +0x3fffffffc0000000, +0x7fffffff80000000, +0xffffffff00000000, +0xfffffffe00000001, +0xfffffffc00000003, +0xfffffff800000007, +0xfffffff00000000f, +0xffffffe00000001f, +0xffffffc00000003f, +0xffffff800000007f, +0xffffff00000000ff, +0xfffffe00000001ff, +0xfffffc00000003ff, +0xfffff800000007ff, +0xfffff00000000fff, +0xffffe00000001fff, +0xffffc00000003fff, +0xffff800000007fff, +0xffff00000000ffff, +0xfffe00000001ffff, +0xfffc00000003ffff, +0xfff800000007ffff, +0xfff00000000fffff, +0xffe00000001fffff, +0xffc00000003fffff, +0xff800000007fffff, +0xff00000000ffffff, +0xfe00000001ffffff, +0xfc00000003ffffff, +0xf800000007ffffff, +0xf00000000fffffff, +0xe00000001fffffff, +0xc00000003fffffff, +0x800000007fffffff, +0x00000001ffffffff, +0x00000003fffffffe, +0x00000007fffffffc, +0x0000000ffffffff8, +0x0000001ffffffff0, +0x0000003fffffffe0, +0x0000007fffffffc0, +0x000000ffffffff80, +0x000001ffffffff00, +0x000003fffffffe00, +0x000007fffffffc00, +0x00000ffffffff800, +0x00001ffffffff000, +0x00003fffffffe000, +0x00007fffffffc000, +0x0000ffffffff8000, +0x0001ffffffff0000, +0x0003fffffffe0000, +0x0007fffffffc0000, +0x000ffffffff80000, +0x001ffffffff00000, +0x003fffffffe00000, +0x007fffffffc00000, +0x00ffffffff800000, +0x01ffffffff000000, +0x03fffffffe000000, +0x07fffffffc000000, +0x0ffffffff8000000, +0x1ffffffff0000000, +0x3fffffffe0000000, +0x7fffffffc0000000, +0xffffffff80000000, +0xffffffff00000001, +0xfffffffe00000003, +0xfffffffc00000007, +0xfffffff80000000f, +0xfffffff00000001f, +0xffffffe00000003f, +0xffffffc00000007f, +0xffffff80000000ff, +0xffffff00000001ff, +0xfffffe00000003ff, +0xfffffc00000007ff, +0xfffff80000000fff, +0xfffff00000001fff, +0xffffe00000003fff, +0xffffc00000007fff, +0xffff80000000ffff, +0xffff00000001ffff, +0xfffe00000003ffff, +0xfffc00000007ffff, +0xfff80000000fffff, +0xfff00000001fffff, +0xffe00000003fffff, +0xffc00000007fffff, +0xff80000000ffffff, +0xff00000001ffffff, +0xfe00000003ffffff, +0xfc00000007ffffff, +0xf80000000fffffff, +0xf00000001fffffff, +0xe00000003fffffff, +0xc00000007fffffff, +0x80000000ffffffff, +0x00000003ffffffff, +0x00000007fffffffe, +0x0000000ffffffffc, +0x0000001ffffffff8, +0x0000003ffffffff0, +0x0000007fffffffe0, +0x000000ffffffffc0, +0x000001ffffffff80, +0x000003ffffffff00, +0x000007fffffffe00, +0x00000ffffffffc00, +0x00001ffffffff800, +0x00003ffffffff000, +0x00007fffffffe000, +0x0000ffffffffc000, +0x0001ffffffff8000, +0x0003ffffffff0000, +0x0007fffffffe0000, +0x000ffffffffc0000, +0x001ffffffff80000, +0x003ffffffff00000, +0x007fffffffe00000, +0x00ffffffffc00000, +0x01ffffffff800000, +0x03ffffffff000000, +0x07fffffffe000000, +0x0ffffffffc000000, +0x1ffffffff8000000, +0x3ffffffff0000000, +0x7fffffffe0000000, +0xffffffffc0000000, +0xffffffff80000001, +0xffffffff00000003, +0xfffffffe00000007, +0xfffffffc0000000f, +0xfffffff80000001f, +0xfffffff00000003f, +0xffffffe00000007f, +0xffffffc0000000ff, +0xffffff80000001ff, +0xffffff00000003ff, +0xfffffe00000007ff, +0xfffffc0000000fff, +0xfffff80000001fff, +0xfffff00000003fff, +0xffffe00000007fff, +0xffffc0000000ffff, +0xffff80000001ffff, +0xffff00000003ffff, +0xfffe00000007ffff, +0xfffc0000000fffff, +0xfff80000001fffff, +0xfff00000003fffff, +0xffe00000007fffff, +0xffc0000000ffffff, +0xff80000001ffffff, +0xff00000003ffffff, +0xfe00000007ffffff, +0xfc0000000fffffff, +0xf80000001fffffff, +0xf00000003fffffff, +0xe00000007fffffff, +0xc0000000ffffffff, +0x80000001ffffffff, +0x00000007ffffffff, +0x0000000ffffffffe, +0x0000001ffffffffc, +0x0000003ffffffff8, +0x0000007ffffffff0, +0x000000ffffffffe0, +0x000001ffffffffc0, +0x000003ffffffff80, +0x000007ffffffff00, +0x00000ffffffffe00, +0x00001ffffffffc00, +0x00003ffffffff800, +0x00007ffffffff000, +0x0000ffffffffe000, +0x0001ffffffffc000, +0x0003ffffffff8000, +0x0007ffffffff0000, +0x000ffffffffe0000, +0x001ffffffffc0000, +0x003ffffffff80000, +0x007ffffffff00000, +0x00ffffffffe00000, +0x01ffffffffc00000, +0x03ffffffff800000, +0x07ffffffff000000, +0x0ffffffffe000000, +0x1ffffffffc000000, +0x3ffffffff8000000, +0x7ffffffff0000000, +0xffffffffe0000000, +0xffffffffc0000001, +0xffffffff80000003, +0xffffffff00000007, +0xfffffffe0000000f, +0xfffffffc0000001f, +0xfffffff80000003f, +0xfffffff00000007f, +0xffffffe0000000ff, +0xffffffc0000001ff, +0xffffff80000003ff, +0xffffff00000007ff, +0xfffffe0000000fff, +0xfffffc0000001fff, +0xfffff80000003fff, +0xfffff00000007fff, +0xffffe0000000ffff, +0xffffc0000001ffff, +0xffff80000003ffff, +0xffff00000007ffff, +0xfffe0000000fffff, +0xfffc0000001fffff, +0xfff80000003fffff, +0xfff00000007fffff, +0xffe0000000ffffff, +0xffc0000001ffffff, +0xff80000003ffffff, +0xff00000007ffffff, +0xfe0000000fffffff, +0xfc0000001fffffff, +0xf80000003fffffff, +0xf00000007fffffff, +0xe0000000ffffffff, +0xc0000001ffffffff, +0x80000003ffffffff, +0x0000000fffffffff, +0x0000001ffffffffe, +0x0000003ffffffffc, +0x0000007ffffffff8, +0x000000fffffffff0, +0x000001ffffffffe0, +0x000003ffffffffc0, +0x000007ffffffff80, +0x00000fffffffff00, +0x00001ffffffffe00, +0x00003ffffffffc00, +0x00007ffffffff800, +0x0000fffffffff000, +0x0001ffffffffe000, +0x0003ffffffffc000, +0x0007ffffffff8000, +0x000fffffffff0000, +0x001ffffffffe0000, +0x003ffffffffc0000, +0x007ffffffff80000, +0x00fffffffff00000, +0x01ffffffffe00000, +0x03ffffffffc00000, +0x07ffffffff800000, +0x0fffffffff000000, +0x1ffffffffe000000, +0x3ffffffffc000000, +0x7ffffffff8000000, +0xfffffffff0000000, +0xffffffffe0000001, +0xffffffffc0000003, +0xffffffff80000007, +0xffffffff0000000f, +0xfffffffe0000001f, +0xfffffffc0000003f, +0xfffffff80000007f, +0xfffffff0000000ff, +0xffffffe0000001ff, +0xffffffc0000003ff, +0xffffff80000007ff, +0xffffff0000000fff, +0xfffffe0000001fff, +0xfffffc0000003fff, +0xfffff80000007fff, +0xfffff0000000ffff, +0xffffe0000001ffff, +0xffffc0000003ffff, +0xffff80000007ffff, +0xffff0000000fffff, +0xfffe0000001fffff, +0xfffc0000003fffff, +0xfff80000007fffff, +0xfff0000000ffffff, +0xffe0000001ffffff, +0xffc0000003ffffff, +0xff80000007ffffff, +0xff0000000fffffff, +0xfe0000001fffffff, +0xfc0000003fffffff, +0xf80000007fffffff, +0xf0000000ffffffff, +0xe0000001ffffffff, +0xc0000003ffffffff, +0x80000007ffffffff, +0x0000001fffffffff, +0x0000003ffffffffe, +0x0000007ffffffffc, +0x000000fffffffff8, +0x000001fffffffff0, +0x000003ffffffffe0, +0x000007ffffffffc0, +0x00000fffffffff80, +0x00001fffffffff00, +0x00003ffffffffe00, +0x00007ffffffffc00, +0x0000fffffffff800, +0x0001fffffffff000, +0x0003ffffffffe000, +0x0007ffffffffc000, +0x000fffffffff8000, +0x001fffffffff0000, +0x003ffffffffe0000, +0x007ffffffffc0000, +0x00fffffffff80000, +0x01fffffffff00000, +0x03ffffffffe00000, +0x07ffffffffc00000, +0x0fffffffff800000, +0x1fffffffff000000, +0x3ffffffffe000000, +0x7ffffffffc000000, +0xfffffffff8000000, +0xfffffffff0000001, +0xffffffffe0000003, +0xffffffffc0000007, +0xffffffff8000000f, +0xffffffff0000001f, +0xfffffffe0000003f, +0xfffffffc0000007f, +0xfffffff8000000ff, +0xfffffff0000001ff, +0xffffffe0000003ff, +0xffffffc0000007ff, +0xffffff8000000fff, +0xffffff0000001fff, +0xfffffe0000003fff, +0xfffffc0000007fff, +0xfffff8000000ffff, +0xfffff0000001ffff, +0xffffe0000003ffff, +0xffffc0000007ffff, +0xffff8000000fffff, +0xffff0000001fffff, +0xfffe0000003fffff, +0xfffc0000007fffff, +0xfff8000000ffffff, +0xfff0000001ffffff, +0xffe0000003ffffff, +0xffc0000007ffffff, +0xff8000000fffffff, +0xff0000001fffffff, +0xfe0000003fffffff, +0xfc0000007fffffff, +0xf8000000ffffffff, +0xf0000001ffffffff, +0xe0000003ffffffff, +0xc0000007ffffffff, +0x8000000fffffffff, +0x0000003fffffffff, +0x0000007ffffffffe, +0x000000fffffffffc, +0x000001fffffffff8, +0x000003fffffffff0, +0x000007ffffffffe0, +0x00000fffffffffc0, +0x00001fffffffff80, +0x00003fffffffff00, +0x00007ffffffffe00, +0x0000fffffffffc00, +0x0001fffffffff800, +0x0003fffffffff000, +0x0007ffffffffe000, +0x000fffffffffc000, +0x001fffffffff8000, +0x003fffffffff0000, +0x007ffffffffe0000, +0x00fffffffffc0000, +0x01fffffffff80000, +0x03fffffffff00000, +0x07ffffffffe00000, +0x0fffffffffc00000, +0x1fffffffff800000, +0x3fffffffff000000, +0x7ffffffffe000000, +0xfffffffffc000000, +0xfffffffff8000001, +0xfffffffff0000003, +0xffffffffe0000007, +0xffffffffc000000f, +0xffffffff8000001f, +0xffffffff0000003f, +0xfffffffe0000007f, +0xfffffffc000000ff, +0xfffffff8000001ff, +0xfffffff0000003ff, +0xffffffe0000007ff, +0xffffffc000000fff, +0xffffff8000001fff, +0xffffff0000003fff, +0xfffffe0000007fff, +0xfffffc000000ffff, +0xfffff8000001ffff, +0xfffff0000003ffff, +0xffffe0000007ffff, +0xffffc000000fffff, +0xffff8000001fffff, +0xffff0000003fffff, +0xfffe0000007fffff, +0xfffc000000ffffff, +0xfff8000001ffffff, +0xfff0000003ffffff, +0xffe0000007ffffff, +0xffc000000fffffff, +0xff8000001fffffff, +0xff0000003fffffff, +0xfe0000007fffffff, +0xfc000000ffffffff, +0xf8000001ffffffff, +0xf0000003ffffffff, +0xe0000007ffffffff, +0xc000000fffffffff, +0x8000001fffffffff, +0x0000007fffffffff, +0x000000fffffffffe, +0x000001fffffffffc, +0x000003fffffffff8, +0x000007fffffffff0, +0x00000fffffffffe0, +0x00001fffffffffc0, +0x00003fffffffff80, +0x00007fffffffff00, +0x0000fffffffffe00, +0x0001fffffffffc00, +0x0003fffffffff800, +0x0007fffffffff000, +0x000fffffffffe000, +0x001fffffffffc000, +0x003fffffffff8000, +0x007fffffffff0000, +0x00fffffffffe0000, +0x01fffffffffc0000, +0x03fffffffff80000, +0x07fffffffff00000, +0x0fffffffffe00000, +0x1fffffffffc00000, +0x3fffffffff800000, +0x7fffffffff000000, +0xfffffffffe000000, +0xfffffffffc000001, +0xfffffffff8000003, +0xfffffffff0000007, +0xffffffffe000000f, +0xffffffffc000001f, +0xffffffff8000003f, +0xffffffff0000007f, +0xfffffffe000000ff, +0xfffffffc000001ff, +0xfffffff8000003ff, +0xfffffff0000007ff, +0xffffffe000000fff, +0xffffffc000001fff, +0xffffff8000003fff, +0xffffff0000007fff, +0xfffffe000000ffff, +0xfffffc000001ffff, +0xfffff8000003ffff, +0xfffff0000007ffff, +0xffffe000000fffff, +0xffffc000001fffff, +0xffff8000003fffff, +0xffff0000007fffff, +0xfffe000000ffffff, +0xfffc000001ffffff, +0xfff8000003ffffff, +0xfff0000007ffffff, +0xffe000000fffffff, +0xffc000001fffffff, +0xff8000003fffffff, +0xff0000007fffffff, +0xfe000000ffffffff, +0xfc000001ffffffff, +0xf8000003ffffffff, +0xf0000007ffffffff, +0xe000000fffffffff, +0xc000001fffffffff, +0x8000003fffffffff, +0x000000ffffffffff, +0x000001fffffffffe, +0x000003fffffffffc, +0x000007fffffffff8, +0x00000ffffffffff0, +0x00001fffffffffe0, +0x00003fffffffffc0, +0x00007fffffffff80, +0x0000ffffffffff00, +0x0001fffffffffe00, +0x0003fffffffffc00, +0x0007fffffffff800, +0x000ffffffffff000, +0x001fffffffffe000, +0x003fffffffffc000, +0x007fffffffff8000, +0x00ffffffffff0000, +0x01fffffffffe0000, +0x03fffffffffc0000, +0x07fffffffff80000, +0x0ffffffffff00000, +0x1fffffffffe00000, +0x3fffffffffc00000, +0x7fffffffff800000, +0xffffffffff000000, +0xfffffffffe000001, +0xfffffffffc000003, +0xfffffffff8000007, +0xfffffffff000000f, +0xffffffffe000001f, +0xffffffffc000003f, +0xffffffff8000007f, +0xffffffff000000ff, +0xfffffffe000001ff, +0xfffffffc000003ff, +0xfffffff8000007ff, +0xfffffff000000fff, +0xffffffe000001fff, +0xffffffc000003fff, +0xffffff8000007fff, +0xffffff000000ffff, +0xfffffe000001ffff, +0xfffffc000003ffff, +0xfffff8000007ffff, +0xfffff000000fffff, +0xffffe000001fffff, +0xffffc000003fffff, +0xffff8000007fffff, +0xffff000000ffffff, +0xfffe000001ffffff, +0xfffc000003ffffff, +0xfff8000007ffffff, +0xfff000000fffffff, +0xffe000001fffffff, +0xffc000003fffffff, +0xff8000007fffffff, +0xff000000ffffffff, +0xfe000001ffffffff, +0xfc000003ffffffff, +0xf8000007ffffffff, +0xf000000fffffffff, +0xe000001fffffffff, +0xc000003fffffffff, +0x8000007fffffffff, +0x000001ffffffffff, +0x000003fffffffffe, +0x000007fffffffffc, +0x00000ffffffffff8, +0x00001ffffffffff0, +0x00003fffffffffe0, +0x00007fffffffffc0, +0x0000ffffffffff80, +0x0001ffffffffff00, +0x0003fffffffffe00, +0x0007fffffffffc00, +0x000ffffffffff800, +0x001ffffffffff000, +0x003fffffffffe000, +0x007fffffffffc000, +0x00ffffffffff8000, +0x01ffffffffff0000, +0x03fffffffffe0000, +0x07fffffffffc0000, +0x0ffffffffff80000, +0x1ffffffffff00000, +0x3fffffffffe00000, +0x7fffffffffc00000, +0xffffffffff800000, +0xffffffffff000001, +0xfffffffffe000003, +0xfffffffffc000007, +0xfffffffff800000f, +0xfffffffff000001f, +0xffffffffe000003f, +0xffffffffc000007f, +0xffffffff800000ff, +0xffffffff000001ff, +0xfffffffe000003ff, +0xfffffffc000007ff, +0xfffffff800000fff, +0xfffffff000001fff, +0xffffffe000003fff, +0xffffffc000007fff, +0xffffff800000ffff, +0xffffff000001ffff, +0xfffffe000003ffff, +0xfffffc000007ffff, +0xfffff800000fffff, +0xfffff000001fffff, +0xffffe000003fffff, +0xffffc000007fffff, +0xffff800000ffffff, +0xffff000001ffffff, +0xfffe000003ffffff, +0xfffc000007ffffff, +0xfff800000fffffff, +0xfff000001fffffff, +0xffe000003fffffff, +0xffc000007fffffff, +0xff800000ffffffff, +0xff000001ffffffff, +0xfe000003ffffffff, +0xfc000007ffffffff, +0xf800000fffffffff, +0xf000001fffffffff, +0xe000003fffffffff, +0xc000007fffffffff, +0x800000ffffffffff, +0x000003ffffffffff, +0x000007fffffffffe, +0x00000ffffffffffc, +0x00001ffffffffff8, +0x00003ffffffffff0, +0x00007fffffffffe0, +0x0000ffffffffffc0, +0x0001ffffffffff80, +0x0003ffffffffff00, +0x0007fffffffffe00, +0x000ffffffffffc00, +0x001ffffffffff800, +0x003ffffffffff000, +0x007fffffffffe000, +0x00ffffffffffc000, +0x01ffffffffff8000, +0x03ffffffffff0000, +0x07fffffffffe0000, +0x0ffffffffffc0000, +0x1ffffffffff80000, +0x3ffffffffff00000, +0x7fffffffffe00000, +0xffffffffffc00000, +0xffffffffff800001, +0xffffffffff000003, +0xfffffffffe000007, +0xfffffffffc00000f, +0xfffffffff800001f, +0xfffffffff000003f, +0xffffffffe000007f, +0xffffffffc00000ff, +0xffffffff800001ff, +0xffffffff000003ff, +0xfffffffe000007ff, +0xfffffffc00000fff, +0xfffffff800001fff, +0xfffffff000003fff, +0xffffffe000007fff, +0xffffffc00000ffff, +0xffffff800001ffff, +0xffffff000003ffff, +0xfffffe000007ffff, +0xfffffc00000fffff, +0xfffff800001fffff, +0xfffff000003fffff, +0xffffe000007fffff, +0xffffc00000ffffff, +0xffff800001ffffff, +0xffff000003ffffff, +0xfffe000007ffffff, +0xfffc00000fffffff, +0xfff800001fffffff, +0xfff000003fffffff, +0xffe000007fffffff, +0xffc00000ffffffff, +0xff800001ffffffff, +0xff000003ffffffff, +0xfe000007ffffffff, +0xfc00000fffffffff, +0xf800001fffffffff, +0xf000003fffffffff, +0xe000007fffffffff, +0xc00000ffffffffff, +0x800001ffffffffff, +0x000007ffffffffff, +0x00000ffffffffffe, +0x00001ffffffffffc, +0x00003ffffffffff8, +0x00007ffffffffff0, +0x0000ffffffffffe0, +0x0001ffffffffffc0, +0x0003ffffffffff80, +0x0007ffffffffff00, +0x000ffffffffffe00, +0x001ffffffffffc00, +0x003ffffffffff800, +0x007ffffffffff000, +0x00ffffffffffe000, +0x01ffffffffffc000, +0x03ffffffffff8000, +0x07ffffffffff0000, +0x0ffffffffffe0000, +0x1ffffffffffc0000, +0x3ffffffffff80000, +0x7ffffffffff00000, +0xffffffffffe00000, +0xffffffffffc00001, +0xffffffffff800003, +0xffffffffff000007, +0xfffffffffe00000f, +0xfffffffffc00001f, +0xfffffffff800003f, +0xfffffffff000007f, +0xffffffffe00000ff, +0xffffffffc00001ff, +0xffffffff800003ff, +0xffffffff000007ff, +0xfffffffe00000fff, +0xfffffffc00001fff, +0xfffffff800003fff, +0xfffffff000007fff, +0xffffffe00000ffff, +0xffffffc00001ffff, +0xffffff800003ffff, +0xffffff000007ffff, +0xfffffe00000fffff, +0xfffffc00001fffff, +0xfffff800003fffff, +0xfffff000007fffff, +0xffffe00000ffffff, +0xffffc00001ffffff, +0xffff800003ffffff, +0xffff000007ffffff, +0xfffe00000fffffff, +0xfffc00001fffffff, +0xfff800003fffffff, +0xfff000007fffffff, +0xffe00000ffffffff, +0xffc00001ffffffff, +0xff800003ffffffff, +0xff000007ffffffff, +0xfe00000fffffffff, +0xfc00001fffffffff, +0xf800003fffffffff, +0xf000007fffffffff, +0xe00000ffffffffff, +0xc00001ffffffffff, +0x800003ffffffffff, +0x00000fffffffffff, +0x00001ffffffffffe, +0x00003ffffffffffc, +0x00007ffffffffff8, +0x0000fffffffffff0, +0x0001ffffffffffe0, +0x0003ffffffffffc0, +0x0007ffffffffff80, +0x000fffffffffff00, +0x001ffffffffffe00, +0x003ffffffffffc00, +0x007ffffffffff800, +0x00fffffffffff000, +0x01ffffffffffe000, +0x03ffffffffffc000, +0x07ffffffffff8000, +0x0fffffffffff0000, +0x1ffffffffffe0000, +0x3ffffffffffc0000, +0x7ffffffffff80000, +0xfffffffffff00000, +0xffffffffffe00001, +0xffffffffffc00003, +0xffffffffff800007, +0xffffffffff00000f, +0xfffffffffe00001f, +0xfffffffffc00003f, +0xfffffffff800007f, +0xfffffffff00000ff, +0xffffffffe00001ff, +0xffffffffc00003ff, +0xffffffff800007ff, +0xffffffff00000fff, +0xfffffffe00001fff, +0xfffffffc00003fff, +0xfffffff800007fff, +0xfffffff00000ffff, +0xffffffe00001ffff, +0xffffffc00003ffff, +0xffffff800007ffff, +0xffffff00000fffff, +0xfffffe00001fffff, +0xfffffc00003fffff, +0xfffff800007fffff, +0xfffff00000ffffff, +0xffffe00001ffffff, +0xffffc00003ffffff, +0xffff800007ffffff, +0xffff00000fffffff, +0xfffe00001fffffff, +0xfffc00003fffffff, +0xfff800007fffffff, +0xfff00000ffffffff, +0xffe00001ffffffff, +0xffc00003ffffffff, +0xff800007ffffffff, +0xff00000fffffffff, +0xfe00001fffffffff, +0xfc00003fffffffff, +0xf800007fffffffff, +0xf00000ffffffffff, +0xe00001ffffffffff, +0xc00003ffffffffff, +0x800007ffffffffff, +0x00001fffffffffff, +0x00003ffffffffffe, +0x00007ffffffffffc, +0x0000fffffffffff8, +0x0001fffffffffff0, +0x0003ffffffffffe0, +0x0007ffffffffffc0, +0x000fffffffffff80, +0x001fffffffffff00, +0x003ffffffffffe00, +0x007ffffffffffc00, +0x00fffffffffff800, +0x01fffffffffff000, +0x03ffffffffffe000, +0x07ffffffffffc000, +0x0fffffffffff8000, +0x1fffffffffff0000, +0x3ffffffffffe0000, +0x7ffffffffffc0000, +0xfffffffffff80000, +0xfffffffffff00001, +0xffffffffffe00003, +0xffffffffffc00007, +0xffffffffff80000f, +0xffffffffff00001f, +0xfffffffffe00003f, +0xfffffffffc00007f, +0xfffffffff80000ff, +0xfffffffff00001ff, +0xffffffffe00003ff, +0xffffffffc00007ff, +0xffffffff80000fff, +0xffffffff00001fff, +0xfffffffe00003fff, +0xfffffffc00007fff, +0xfffffff80000ffff, +0xfffffff00001ffff, +0xffffffe00003ffff, +0xffffffc00007ffff, +0xffffff80000fffff, +0xffffff00001fffff, +0xfffffe00003fffff, +0xfffffc00007fffff, +0xfffff80000ffffff, +0xfffff00001ffffff, +0xffffe00003ffffff, +0xffffc00007ffffff, +0xffff80000fffffff, +0xffff00001fffffff, +0xfffe00003fffffff, +0xfffc00007fffffff, +0xfff80000ffffffff, +0xfff00001ffffffff, +0xffe00003ffffffff, +0xffc00007ffffffff, +0xff80000fffffffff, +0xff00001fffffffff, +0xfe00003fffffffff, +0xfc00007fffffffff, +0xf80000ffffffffff, +0xf00001ffffffffff, +0xe00003ffffffffff, +0xc00007ffffffffff, +0x80000fffffffffff, +0x00003fffffffffff, +0x00007ffffffffffe, +0x0000fffffffffffc, +0x0001fffffffffff8, +0x0003fffffffffff0, +0x0007ffffffffffe0, +0x000fffffffffffc0, +0x001fffffffffff80, +0x003fffffffffff00, +0x007ffffffffffe00, +0x00fffffffffffc00, +0x01fffffffffff800, +0x03fffffffffff000, +0x07ffffffffffe000, +0x0fffffffffffc000, +0x1fffffffffff8000, +0x3fffffffffff0000, +0x7ffffffffffe0000, +0xfffffffffffc0000, +0xfffffffffff80001, +0xfffffffffff00003, +0xffffffffffe00007, +0xffffffffffc0000f, +0xffffffffff80001f, +0xffffffffff00003f, +0xfffffffffe00007f, +0xfffffffffc0000ff, +0xfffffffff80001ff, +0xfffffffff00003ff, +0xffffffffe00007ff, +0xffffffffc0000fff, +0xffffffff80001fff, +0xffffffff00003fff, +0xfffffffe00007fff, +0xfffffffc0000ffff, +0xfffffff80001ffff, +0xfffffff00003ffff, +0xffffffe00007ffff, +0xffffffc0000fffff, +0xffffff80001fffff, +0xffffff00003fffff, +0xfffffe00007fffff, +0xfffffc0000ffffff, +0xfffff80001ffffff, +0xfffff00003ffffff, +0xffffe00007ffffff, +0xffffc0000fffffff, +0xffff80001fffffff, +0xffff00003fffffff, +0xfffe00007fffffff, +0xfffc0000ffffffff, +0xfff80001ffffffff, +0xfff00003ffffffff, +0xffe00007ffffffff, +0xffc0000fffffffff, +0xff80001fffffffff, +0xff00003fffffffff, +0xfe00007fffffffff, +0xfc0000ffffffffff, +0xf80001ffffffffff, +0xf00003ffffffffff, +0xe00007ffffffffff, +0xc0000fffffffffff, +0x80001fffffffffff, +0x00007fffffffffff, +0x0000fffffffffffe, +0x0001fffffffffffc, +0x0003fffffffffff8, +0x0007fffffffffff0, +0x000fffffffffffe0, +0x001fffffffffffc0, +0x003fffffffffff80, +0x007fffffffffff00, +0x00fffffffffffe00, +0x01fffffffffffc00, +0x03fffffffffff800, +0x07fffffffffff000, +0x0fffffffffffe000, +0x1fffffffffffc000, +0x3fffffffffff8000, +0x7fffffffffff0000, +0xfffffffffffe0000, +0xfffffffffffc0001, +0xfffffffffff80003, +0xfffffffffff00007, +0xffffffffffe0000f, +0xffffffffffc0001f, +0xffffffffff80003f, +0xffffffffff00007f, +0xfffffffffe0000ff, +0xfffffffffc0001ff, +0xfffffffff80003ff, +0xfffffffff00007ff, +0xffffffffe0000fff, +0xffffffffc0001fff, +0xffffffff80003fff, +0xffffffff00007fff, +0xfffffffe0000ffff, +0xfffffffc0001ffff, +0xfffffff80003ffff, +0xfffffff00007ffff, +0xffffffe0000fffff, +0xffffffc0001fffff, +0xffffff80003fffff, +0xffffff00007fffff, +0xfffffe0000ffffff, +0xfffffc0001ffffff, +0xfffff80003ffffff, +0xfffff00007ffffff, +0xffffe0000fffffff, +0xffffc0001fffffff, +0xffff80003fffffff, +0xffff00007fffffff, +0xfffe0000ffffffff, +0xfffc0001ffffffff, +0xfff80003ffffffff, +0xfff00007ffffffff, +0xffe0000fffffffff, +0xffc0001fffffffff, +0xff80003fffffffff, +0xff00007fffffffff, +0xfe0000ffffffffff, +0xfc0001ffffffffff, +0xf80003ffffffffff, +0xf00007ffffffffff, +0xe0000fffffffffff, +0xc0001fffffffffff, +0x80003fffffffffff, +0x0000ffffffffffff, +0x0001fffffffffffe, +0x0003fffffffffffc, +0x0007fffffffffff8, +0x000ffffffffffff0, +0x001fffffffffffe0, +0x003fffffffffffc0, +0x007fffffffffff80, +0x00ffffffffffff00, +0x01fffffffffffe00, +0x03fffffffffffc00, +0x07fffffffffff800, +0x0ffffffffffff000, +0x1fffffffffffe000, +0x3fffffffffffc000, +0x7fffffffffff8000, +0xffffffffffff0000, +0xfffffffffffe0001, +0xfffffffffffc0003, +0xfffffffffff80007, +0xfffffffffff0000f, +0xffffffffffe0001f, +0xffffffffffc0003f, +0xffffffffff80007f, +0xffffffffff0000ff, +0xfffffffffe0001ff, +0xfffffffffc0003ff, +0xfffffffff80007ff, +0xfffffffff0000fff, +0xffffffffe0001fff, +0xffffffffc0003fff, +0xffffffff80007fff, +0xffffffff0000ffff, +0xfffffffe0001ffff, +0xfffffffc0003ffff, +0xfffffff80007ffff, +0xfffffff0000fffff, +0xffffffe0001fffff, +0xffffffc0003fffff, +0xffffff80007fffff, +0xffffff0000ffffff, +0xfffffe0001ffffff, +0xfffffc0003ffffff, +0xfffff80007ffffff, +0xfffff0000fffffff, +0xffffe0001fffffff, +0xffffc0003fffffff, +0xffff80007fffffff, +0xffff0000ffffffff, +0xfffe0001ffffffff, +0xfffc0003ffffffff, +0xfff80007ffffffff, +0xfff0000fffffffff, +0xffe0001fffffffff, +0xffc0003fffffffff, +0xff80007fffffffff, +0xff0000ffffffffff, +0xfe0001ffffffffff, +0xfc0003ffffffffff, +0xf80007ffffffffff, +0xf0000fffffffffff, +0xe0001fffffffffff, +0xc0003fffffffffff, +0x80007fffffffffff, +0x0001ffffffffffff, +0x0003fffffffffffe, +0x0007fffffffffffc, +0x000ffffffffffff8, +0x001ffffffffffff0, +0x003fffffffffffe0, +0x007fffffffffffc0, +0x00ffffffffffff80, +0x01ffffffffffff00, +0x03fffffffffffe00, +0x07fffffffffffc00, +0x0ffffffffffff800, +0x1ffffffffffff000, +0x3fffffffffffe000, +0x7fffffffffffc000, +0xffffffffffff8000, +0xffffffffffff0001, +0xfffffffffffe0003, +0xfffffffffffc0007, +0xfffffffffff8000f, +0xfffffffffff0001f, +0xffffffffffe0003f, +0xffffffffffc0007f, +0xffffffffff8000ff, +0xffffffffff0001ff, +0xfffffffffe0003ff, +0xfffffffffc0007ff, +0xfffffffff8000fff, +0xfffffffff0001fff, +0xffffffffe0003fff, +0xffffffffc0007fff, +0xffffffff8000ffff, +0xffffffff0001ffff, +0xfffffffe0003ffff, +0xfffffffc0007ffff, +0xfffffff8000fffff, +0xfffffff0001fffff, +0xffffffe0003fffff, +0xffffffc0007fffff, +0xffffff8000ffffff, +0xffffff0001ffffff, +0xfffffe0003ffffff, +0xfffffc0007ffffff, +0xfffff8000fffffff, +0xfffff0001fffffff, +0xffffe0003fffffff, +0xffffc0007fffffff, +0xffff8000ffffffff, +0xffff0001ffffffff, +0xfffe0003ffffffff, +0xfffc0007ffffffff, +0xfff8000fffffffff, +0xfff0001fffffffff, +0xffe0003fffffffff, +0xffc0007fffffffff, +0xff8000ffffffffff, +0xff0001ffffffffff, +0xfe0003ffffffffff, +0xfc0007ffffffffff, +0xf8000fffffffffff, +0xf0001fffffffffff, +0xe0003fffffffffff, +0xc0007fffffffffff, +0x8000ffffffffffff, +0x0003ffffffffffff, +0x0007fffffffffffe, +0x000ffffffffffffc, +0x001ffffffffffff8, +0x003ffffffffffff0, +0x007fffffffffffe0, +0x00ffffffffffffc0, +0x01ffffffffffff80, +0x03ffffffffffff00, +0x07fffffffffffe00, +0x0ffffffffffffc00, +0x1ffffffffffff800, +0x3ffffffffffff000, +0x7fffffffffffe000, +0xffffffffffffc000, +0xffffffffffff8001, +0xffffffffffff0003, +0xfffffffffffe0007, +0xfffffffffffc000f, +0xfffffffffff8001f, +0xfffffffffff0003f, +0xffffffffffe0007f, +0xffffffffffc000ff, +0xffffffffff8001ff, +0xffffffffff0003ff, +0xfffffffffe0007ff, +0xfffffffffc000fff, +0xfffffffff8001fff, +0xfffffffff0003fff, +0xffffffffe0007fff, +0xffffffffc000ffff, +0xffffffff8001ffff, +0xffffffff0003ffff, +0xfffffffe0007ffff, +0xfffffffc000fffff, +0xfffffff8001fffff, +0xfffffff0003fffff, +0xffffffe0007fffff, +0xffffffc000ffffff, +0xffffff8001ffffff, +0xffffff0003ffffff, +0xfffffe0007ffffff, +0xfffffc000fffffff, +0xfffff8001fffffff, +0xfffff0003fffffff, +0xffffe0007fffffff, +0xffffc000ffffffff, +0xffff8001ffffffff, +0xffff0003ffffffff, +0xfffe0007ffffffff, +0xfffc000fffffffff, +0xfff8001fffffffff, +0xfff0003fffffffff, +0xffe0007fffffffff, +0xffc000ffffffffff, +0xff8001ffffffffff, +0xff0003ffffffffff, +0xfe0007ffffffffff, +0xfc000fffffffffff, +0xf8001fffffffffff, +0xf0003fffffffffff, +0xe0007fffffffffff, +0xc000ffffffffffff, +0x8001ffffffffffff, +0x0007ffffffffffff, +0x000ffffffffffffe, +0x001ffffffffffffc, +0x003ffffffffffff8, +0x007ffffffffffff0, +0x00ffffffffffffe0, +0x01ffffffffffffc0, +0x03ffffffffffff80, +0x07ffffffffffff00, +0x0ffffffffffffe00, +0x1ffffffffffffc00, +0x3ffffffffffff800, +0x7ffffffffffff000, +0xffffffffffffe000, +0xffffffffffffc001, +0xffffffffffff8003, +0xffffffffffff0007, +0xfffffffffffe000f, +0xfffffffffffc001f, +0xfffffffffff8003f, +0xfffffffffff0007f, +0xffffffffffe000ff, +0xffffffffffc001ff, +0xffffffffff8003ff, +0xffffffffff0007ff, +0xfffffffffe000fff, +0xfffffffffc001fff, +0xfffffffff8003fff, +0xfffffffff0007fff, +0xffffffffe000ffff, +0xffffffffc001ffff, +0xffffffff8003ffff, +0xffffffff0007ffff, +0xfffffffe000fffff, +0xfffffffc001fffff, +0xfffffff8003fffff, +0xfffffff0007fffff, +0xffffffe000ffffff, +0xffffffc001ffffff, +0xffffff8003ffffff, +0xffffff0007ffffff, +0xfffffe000fffffff, +0xfffffc001fffffff, +0xfffff8003fffffff, +0xfffff0007fffffff, +0xffffe000ffffffff, +0xffffc001ffffffff, +0xffff8003ffffffff, +0xffff0007ffffffff, +0xfffe000fffffffff, +0xfffc001fffffffff, +0xfff8003fffffffff, +0xfff0007fffffffff, +0xffe000ffffffffff, +0xffc001ffffffffff, +0xff8003ffffffffff, +0xff0007ffffffffff, +0xfe000fffffffffff, +0xfc001fffffffffff, +0xf8003fffffffffff, +0xf0007fffffffffff, +0xe000ffffffffffff, +0xc001ffffffffffff, +0x8003ffffffffffff, +0x000fffffffffffff, +0x001ffffffffffffe, +0x003ffffffffffffc, +0x007ffffffffffff8, +0x00fffffffffffff0, +0x01ffffffffffffe0, +0x03ffffffffffffc0, +0x07ffffffffffff80, +0x0fffffffffffff00, +0x1ffffffffffffe00, +0x3ffffffffffffc00, +0x7ffffffffffff800, +0xfffffffffffff000, +0xffffffffffffe001, +0xffffffffffffc003, +0xffffffffffff8007, +0xffffffffffff000f, +0xfffffffffffe001f, +0xfffffffffffc003f, +0xfffffffffff8007f, +0xfffffffffff000ff, +0xffffffffffe001ff, +0xffffffffffc003ff, +0xffffffffff8007ff, +0xffffffffff000fff, +0xfffffffffe001fff, +0xfffffffffc003fff, +0xfffffffff8007fff, +0xfffffffff000ffff, +0xffffffffe001ffff, +0xffffffffc003ffff, +0xffffffff8007ffff, +0xffffffff000fffff, +0xfffffffe001fffff, +0xfffffffc003fffff, +0xfffffff8007fffff, +0xfffffff000ffffff, +0xffffffe001ffffff, +0xffffffc003ffffff, +0xffffff8007ffffff, +0xffffff000fffffff, +0xfffffe001fffffff, +0xfffffc003fffffff, +0xfffff8007fffffff, +0xfffff000ffffffff, +0xffffe001ffffffff, +0xffffc003ffffffff, +0xffff8007ffffffff, +0xffff000fffffffff, +0xfffe001fffffffff, +0xfffc003fffffffff, +0xfff8007fffffffff, +0xfff000ffffffffff, +0xffe001ffffffffff, +0xffc003ffffffffff, +0xff8007ffffffffff, +0xff000fffffffffff, +0xfe001fffffffffff, +0xfc003fffffffffff, +0xf8007fffffffffff, +0xf000ffffffffffff, +0xe001ffffffffffff, +0xc003ffffffffffff, +0x8007ffffffffffff, +0x001fffffffffffff, +0x003ffffffffffffe, +0x007ffffffffffffc, +0x00fffffffffffff8, +0x01fffffffffffff0, +0x03ffffffffffffe0, +0x07ffffffffffffc0, +0x0fffffffffffff80, +0x1fffffffffffff00, +0x3ffffffffffffe00, +0x7ffffffffffffc00, +0xfffffffffffff800, +0xfffffffffffff001, +0xffffffffffffe003, +0xffffffffffffc007, +0xffffffffffff800f, +0xffffffffffff001f, +0xfffffffffffe003f, +0xfffffffffffc007f, +0xfffffffffff800ff, +0xfffffffffff001ff, +0xffffffffffe003ff, +0xffffffffffc007ff, +0xffffffffff800fff, +0xffffffffff001fff, +0xfffffffffe003fff, +0xfffffffffc007fff, +0xfffffffff800ffff, +0xfffffffff001ffff, +0xffffffffe003ffff, +0xffffffffc007ffff, +0xffffffff800fffff, +0xffffffff001fffff, +0xfffffffe003fffff, +0xfffffffc007fffff, +0xfffffff800ffffff, +0xfffffff001ffffff, +0xffffffe003ffffff, +0xffffffc007ffffff, +0xffffff800fffffff, +0xffffff001fffffff, +0xfffffe003fffffff, +0xfffffc007fffffff, +0xfffff800ffffffff, +0xfffff001ffffffff, +0xffffe003ffffffff, +0xffffc007ffffffff, +0xffff800fffffffff, +0xffff001fffffffff, +0xfffe003fffffffff, +0xfffc007fffffffff, +0xfff800ffffffffff, +0xfff001ffffffffff, +0xffe003ffffffffff, +0xffc007ffffffffff, +0xff800fffffffffff, +0xff001fffffffffff, +0xfe003fffffffffff, +0xfc007fffffffffff, +0xf800ffffffffffff, +0xf001ffffffffffff, +0xe003ffffffffffff, +0xc007ffffffffffff, +0x800fffffffffffff, +0x003fffffffffffff, +0x007ffffffffffffe, +0x00fffffffffffffc, +0x01fffffffffffff8, +0x03fffffffffffff0, +0x07ffffffffffffe0, +0x0fffffffffffffc0, +0x1fffffffffffff80, +0x3fffffffffffff00, +0x7ffffffffffffe00, +0xfffffffffffffc00, +0xfffffffffffff801, +0xfffffffffffff003, +0xffffffffffffe007, +0xffffffffffffc00f, +0xffffffffffff801f, +0xffffffffffff003f, +0xfffffffffffe007f, +0xfffffffffffc00ff, +0xfffffffffff801ff, +0xfffffffffff003ff, +0xffffffffffe007ff, +0xffffffffffc00fff, +0xffffffffff801fff, +0xffffffffff003fff, +0xfffffffffe007fff, +0xfffffffffc00ffff, +0xfffffffff801ffff, +0xfffffffff003ffff, +0xffffffffe007ffff, +0xffffffffc00fffff, +0xffffffff801fffff, +0xffffffff003fffff, +0xfffffffe007fffff, +0xfffffffc00ffffff, +0xfffffff801ffffff, +0xfffffff003ffffff, +0xffffffe007ffffff, +0xffffffc00fffffff, +0xffffff801fffffff, +0xffffff003fffffff, +0xfffffe007fffffff, +0xfffffc00ffffffff, +0xfffff801ffffffff, +0xfffff003ffffffff, +0xffffe007ffffffff, +0xffffc00fffffffff, +0xffff801fffffffff, +0xffff003fffffffff, +0xfffe007fffffffff, +0xfffc00ffffffffff, +0xfff801ffffffffff, +0xfff003ffffffffff, +0xffe007ffffffffff, +0xffc00fffffffffff, +0xff801fffffffffff, +0xff003fffffffffff, +0xfe007fffffffffff, +0xfc00ffffffffffff, +0xf801ffffffffffff, +0xf003ffffffffffff, +0xe007ffffffffffff, +0xc00fffffffffffff, +0x801fffffffffffff, +0x007fffffffffffff, +0x00fffffffffffffe, +0x01fffffffffffffc, +0x03fffffffffffff8, +0x07fffffffffffff0, +0x0fffffffffffffe0, +0x1fffffffffffffc0, +0x3fffffffffffff80, +0x7fffffffffffff00, +0xfffffffffffffe00, +0xfffffffffffffc01, +0xfffffffffffff803, +0xfffffffffffff007, +0xffffffffffffe00f, +0xffffffffffffc01f, +0xffffffffffff803f, +0xffffffffffff007f, +0xfffffffffffe00ff, +0xfffffffffffc01ff, +0xfffffffffff803ff, +0xfffffffffff007ff, +0xffffffffffe00fff, +0xffffffffffc01fff, +0xffffffffff803fff, +0xffffffffff007fff, +0xfffffffffe00ffff, +0xfffffffffc01ffff, +0xfffffffff803ffff, +0xfffffffff007ffff, +0xffffffffe00fffff, +0xffffffffc01fffff, +0xffffffff803fffff, +0xffffffff007fffff, +0xfffffffe00ffffff, +0xfffffffc01ffffff, +0xfffffff803ffffff, +0xfffffff007ffffff, +0xffffffe00fffffff, +0xffffffc01fffffff, +0xffffff803fffffff, +0xffffff007fffffff, +0xfffffe00ffffffff, +0xfffffc01ffffffff, +0xfffff803ffffffff, +0xfffff007ffffffff, +0xffffe00fffffffff, +0xffffc01fffffffff, +0xffff803fffffffff, +0xffff007fffffffff, +0xfffe00ffffffffff, +0xfffc01ffffffffff, +0xfff803ffffffffff, +0xfff007ffffffffff, +0xffe00fffffffffff, +0xffc01fffffffffff, +0xff803fffffffffff, +0xff007fffffffffff, +0xfe00ffffffffffff, +0xfc01ffffffffffff, +0xf803ffffffffffff, +0xf007ffffffffffff, +0xe00fffffffffffff, +0xc01fffffffffffff, +0x803fffffffffffff, +0x00ffffffffffffff, +0x01fffffffffffffe, +0x03fffffffffffffc, +0x07fffffffffffff8, +0x0ffffffffffffff0, +0x1fffffffffffffe0, +0x3fffffffffffffc0, +0x7fffffffffffff80, +0xffffffffffffff00, +0xfffffffffffffe01, +0xfffffffffffffc03, +0xfffffffffffff807, +0xfffffffffffff00f, +0xffffffffffffe01f, +0xffffffffffffc03f, +0xffffffffffff807f, +0xffffffffffff00ff, +0xfffffffffffe01ff, +0xfffffffffffc03ff, +0xfffffffffff807ff, +0xfffffffffff00fff, +0xffffffffffe01fff, +0xffffffffffc03fff, +0xffffffffff807fff, +0xffffffffff00ffff, +0xfffffffffe01ffff, +0xfffffffffc03ffff, +0xfffffffff807ffff, +0xfffffffff00fffff, +0xffffffffe01fffff, +0xffffffffc03fffff, +0xffffffff807fffff, +0xffffffff00ffffff, +0xfffffffe01ffffff, +0xfffffffc03ffffff, +0xfffffff807ffffff, +0xfffffff00fffffff, +0xffffffe01fffffff, +0xffffffc03fffffff, +0xffffff807fffffff, +0xffffff00ffffffff, +0xfffffe01ffffffff, +0xfffffc03ffffffff, +0xfffff807ffffffff, +0xfffff00fffffffff, +0xffffe01fffffffff, +0xffffc03fffffffff, +0xffff807fffffffff, +0xffff00ffffffffff, +0xfffe01ffffffffff, +0xfffc03ffffffffff, +0xfff807ffffffffff, +0xfff00fffffffffff, +0xffe01fffffffffff, +0xffc03fffffffffff, +0xff807fffffffffff, +0xff00ffffffffffff, +0xfe01ffffffffffff, +0xfc03ffffffffffff, +0xf807ffffffffffff, +0xf00fffffffffffff, +0xe01fffffffffffff, +0xc03fffffffffffff, +0x807fffffffffffff, +0x01ffffffffffffff, +0x03fffffffffffffe, +0x07fffffffffffffc, +0x0ffffffffffffff8, +0x1ffffffffffffff0, +0x3fffffffffffffe0, +0x7fffffffffffffc0, +0xffffffffffffff80, +0xffffffffffffff01, +0xfffffffffffffe03, +0xfffffffffffffc07, +0xfffffffffffff80f, +0xfffffffffffff01f, +0xffffffffffffe03f, +0xffffffffffffc07f, +0xffffffffffff80ff, +0xffffffffffff01ff, +0xfffffffffffe03ff, +0xfffffffffffc07ff, +0xfffffffffff80fff, +0xfffffffffff01fff, +0xffffffffffe03fff, +0xffffffffffc07fff, +0xffffffffff80ffff, +0xffffffffff01ffff, +0xfffffffffe03ffff, +0xfffffffffc07ffff, +0xfffffffff80fffff, +0xfffffffff01fffff, +0xffffffffe03fffff, +0xffffffffc07fffff, +0xffffffff80ffffff, +0xffffffff01ffffff, +0xfffffffe03ffffff, +0xfffffffc07ffffff, +0xfffffff80fffffff, +0xfffffff01fffffff, +0xffffffe03fffffff, +0xffffffc07fffffff, +0xffffff80ffffffff, +0xffffff01ffffffff, +0xfffffe03ffffffff, +0xfffffc07ffffffff, +0xfffff80fffffffff, +0xfffff01fffffffff, +0xffffe03fffffffff, +0xffffc07fffffffff, +0xffff80ffffffffff, +0xffff01ffffffffff, +0xfffe03ffffffffff, +0xfffc07ffffffffff, +0xfff80fffffffffff, +0xfff01fffffffffff, +0xffe03fffffffffff, +0xffc07fffffffffff, +0xff80ffffffffffff, +0xff01ffffffffffff, +0xfe03ffffffffffff, +0xfc07ffffffffffff, +0xf80fffffffffffff, +0xf01fffffffffffff, +0xe03fffffffffffff, +0xc07fffffffffffff, +0x80ffffffffffffff, +0x03ffffffffffffff, +0x07fffffffffffffe, +0x0ffffffffffffffc, +0x1ffffffffffffff8, +0x3ffffffffffffff0, +0x7fffffffffffffe0, +0xffffffffffffffc0, +0xffffffffffffff81, +0xffffffffffffff03, +0xfffffffffffffe07, +0xfffffffffffffc0f, +0xfffffffffffff81f, +0xfffffffffffff03f, +0xffffffffffffe07f, +0xffffffffffffc0ff, +0xffffffffffff81ff, +0xffffffffffff03ff, +0xfffffffffffe07ff, +0xfffffffffffc0fff, +0xfffffffffff81fff, +0xfffffffffff03fff, +0xffffffffffe07fff, +0xffffffffffc0ffff, +0xffffffffff81ffff, +0xffffffffff03ffff, +0xfffffffffe07ffff, +0xfffffffffc0fffff, +0xfffffffff81fffff, +0xfffffffff03fffff, +0xffffffffe07fffff, +0xffffffffc0ffffff, +0xffffffff81ffffff, +0xffffffff03ffffff, +0xfffffffe07ffffff, +0xfffffffc0fffffff, +0xfffffff81fffffff, +0xfffffff03fffffff, +0xffffffe07fffffff, +0xffffffc0ffffffff, +0xffffff81ffffffff, +0xffffff03ffffffff, +0xfffffe07ffffffff, +0xfffffc0fffffffff, +0xfffff81fffffffff, +0xfffff03fffffffff, +0xffffe07fffffffff, +0xffffc0ffffffffff, +0xffff81ffffffffff, +0xffff03ffffffffff, +0xfffe07ffffffffff, +0xfffc0fffffffffff, +0xfff81fffffffffff, +0xfff03fffffffffff, +0xffe07fffffffffff, +0xffc0ffffffffffff, +0xff81ffffffffffff, +0xff03ffffffffffff, +0xfe07ffffffffffff, +0xfc0fffffffffffff, +0xf81fffffffffffff, +0xf03fffffffffffff, +0xe07fffffffffffff, +0xc0ffffffffffffff, +0x81ffffffffffffff, +0x07ffffffffffffff, +0x0ffffffffffffffe, +0x1ffffffffffffffc, +0x3ffffffffffffff8, +0x7ffffffffffffff0, +0xffffffffffffffe0, +0xffffffffffffffc1, +0xffffffffffffff83, +0xffffffffffffff07, +0xfffffffffffffe0f, +0xfffffffffffffc1f, +0xfffffffffffff83f, +0xfffffffffffff07f, +0xffffffffffffe0ff, +0xffffffffffffc1ff, +0xffffffffffff83ff, +0xffffffffffff07ff, +0xfffffffffffe0fff, +0xfffffffffffc1fff, +0xfffffffffff83fff, +0xfffffffffff07fff, +0xffffffffffe0ffff, +0xffffffffffc1ffff, +0xffffffffff83ffff, +0xffffffffff07ffff, +0xfffffffffe0fffff, +0xfffffffffc1fffff, +0xfffffffff83fffff, +0xfffffffff07fffff, +0xffffffffe0ffffff, +0xffffffffc1ffffff, +0xffffffff83ffffff, +0xffffffff07ffffff, +0xfffffffe0fffffff, +0xfffffffc1fffffff, +0xfffffff83fffffff, +0xfffffff07fffffff, +0xffffffe0ffffffff, +0xffffffc1ffffffff, +0xffffff83ffffffff, +0xffffff07ffffffff, +0xfffffe0fffffffff, +0xfffffc1fffffffff, +0xfffff83fffffffff, +0xfffff07fffffffff, +0xffffe0ffffffffff, +0xffffc1ffffffffff, +0xffff83ffffffffff, +0xffff07ffffffffff, +0xfffe0fffffffffff, +0xfffc1fffffffffff, +0xfff83fffffffffff, +0xfff07fffffffffff, +0xffe0ffffffffffff, +0xffc1ffffffffffff, +0xff83ffffffffffff, +0xff07ffffffffffff, +0xfe0fffffffffffff, +0xfc1fffffffffffff, +0xf83fffffffffffff, +0xf07fffffffffffff, +0xe0ffffffffffffff, +0xc1ffffffffffffff, +0x83ffffffffffffff, +0x0fffffffffffffff, +0x1ffffffffffffffe, +0x3ffffffffffffffc, +0x7ffffffffffffff8, +0xfffffffffffffff0, +0xffffffffffffffe1, +0xffffffffffffffc3, +0xffffffffffffff87, +0xffffffffffffff0f, +0xfffffffffffffe1f, +0xfffffffffffffc3f, +0xfffffffffffff87f, +0xfffffffffffff0ff, +0xffffffffffffe1ff, +0xffffffffffffc3ff, +0xffffffffffff87ff, +0xffffffffffff0fff, +0xfffffffffffe1fff, +0xfffffffffffc3fff, +0xfffffffffff87fff, +0xfffffffffff0ffff, +0xffffffffffe1ffff, +0xffffffffffc3ffff, +0xffffffffff87ffff, +0xffffffffff0fffff, +0xfffffffffe1fffff, +0xfffffffffc3fffff, +0xfffffffff87fffff, +0xfffffffff0ffffff, +0xffffffffe1ffffff, +0xffffffffc3ffffff, +0xffffffff87ffffff, +0xffffffff0fffffff, +0xfffffffe1fffffff, +0xfffffffc3fffffff, +0xfffffff87fffffff, +0xfffffff0ffffffff, +0xffffffe1ffffffff, +0xffffffc3ffffffff, +0xffffff87ffffffff, +0xffffff0fffffffff, +0xfffffe1fffffffff, +0xfffffc3fffffffff, +0xfffff87fffffffff, +0xfffff0ffffffffff, +0xffffe1ffffffffff, +0xffffc3ffffffffff, +0xffff87ffffffffff, +0xffff0fffffffffff, +0xfffe1fffffffffff, +0xfffc3fffffffffff, +0xfff87fffffffffff, +0xfff0ffffffffffff, +0xffe1ffffffffffff, +0xffc3ffffffffffff, +0xff87ffffffffffff, +0xff0fffffffffffff, +0xfe1fffffffffffff, +0xfc3fffffffffffff, +0xf87fffffffffffff, +0xf0ffffffffffffff, +0xe1ffffffffffffff, +0xc3ffffffffffffff, +0x87ffffffffffffff, +0x1fffffffffffffff, +0x3ffffffffffffffe, +0x7ffffffffffffffc, +0xfffffffffffffff8, +0xfffffffffffffff1, +0xffffffffffffffe3, +0xffffffffffffffc7, +0xffffffffffffff8f, +0xffffffffffffff1f, +0xfffffffffffffe3f, +0xfffffffffffffc7f, +0xfffffffffffff8ff, +0xfffffffffffff1ff, +0xffffffffffffe3ff, +0xffffffffffffc7ff, +0xffffffffffff8fff, +0xffffffffffff1fff, +0xfffffffffffe3fff, +0xfffffffffffc7fff, +0xfffffffffff8ffff, +0xfffffffffff1ffff, +0xffffffffffe3ffff, +0xffffffffffc7ffff, +0xffffffffff8fffff, +0xffffffffff1fffff, +0xfffffffffe3fffff, +0xfffffffffc7fffff, +0xfffffffff8ffffff, +0xfffffffff1ffffff, +0xffffffffe3ffffff, +0xffffffffc7ffffff, +0xffffffff8fffffff, +0xffffffff1fffffff, +0xfffffffe3fffffff, +0xfffffffc7fffffff, +0xfffffff8ffffffff, +0xfffffff1ffffffff, +0xffffffe3ffffffff, +0xffffffc7ffffffff, +0xffffff8fffffffff, +0xffffff1fffffffff, +0xfffffe3fffffffff, +0xfffffc7fffffffff, +0xfffff8ffffffffff, +0xfffff1ffffffffff, +0xffffe3ffffffffff, +0xffffc7ffffffffff, +0xffff8fffffffffff, +0xffff1fffffffffff, +0xfffe3fffffffffff, +0xfffc7fffffffffff, +0xfff8ffffffffffff, +0xfff1ffffffffffff, +0xffe3ffffffffffff, +0xffc7ffffffffffff, +0xff8fffffffffffff, +0xff1fffffffffffff, +0xfe3fffffffffffff, +0xfc7fffffffffffff, +0xf8ffffffffffffff, +0xf1ffffffffffffff, +0xe3ffffffffffffff, +0xc7ffffffffffffff, +0x8fffffffffffffff, +0x3fffffffffffffff, +0x7ffffffffffffffe, +0xfffffffffffffffc, +0xfffffffffffffff9, +0xfffffffffffffff3, +0xffffffffffffffe7, +0xffffffffffffffcf, +0xffffffffffffff9f, +0xffffffffffffff3f, +0xfffffffffffffe7f, +0xfffffffffffffcff, +0xfffffffffffff9ff, +0xfffffffffffff3ff, +0xffffffffffffe7ff, +0xffffffffffffcfff, +0xffffffffffff9fff, +0xffffffffffff3fff, +0xfffffffffffe7fff, +0xfffffffffffcffff, +0xfffffffffff9ffff, +0xfffffffffff3ffff, +0xffffffffffe7ffff, +0xffffffffffcfffff, +0xffffffffff9fffff, +0xffffffffff3fffff, +0xfffffffffe7fffff, +0xfffffffffcffffff, +0xfffffffff9ffffff, +0xfffffffff3ffffff, +0xffffffffe7ffffff, +0xffffffffcfffffff, +0xffffffff9fffffff, +0xffffffff3fffffff, +0xfffffffe7fffffff, +0xfffffffcffffffff, +0xfffffff9ffffffff, +0xfffffff3ffffffff, +0xffffffe7ffffffff, +0xffffffcfffffffff, +0xffffff9fffffffff, +0xffffff3fffffffff, +0xfffffe7fffffffff, +0xfffffcffffffffff, +0xfffff9ffffffffff, +0xfffff3ffffffffff, +0xffffe7ffffffffff, +0xffffcfffffffffff, +0xffff9fffffffffff, +0xffff3fffffffffff, +0xfffe7fffffffffff, +0xfffcffffffffffff, +0xfff9ffffffffffff, +0xfff3ffffffffffff, +0xffe7ffffffffffff, +0xffcfffffffffffff, +0xff9fffffffffffff, +0xff3fffffffffffff, +0xfe7fffffffffffff, +0xfcffffffffffffff, +0xf9ffffffffffffff, +0xf3ffffffffffffff, +0xe7ffffffffffffff, +0xcfffffffffffffff, +0x9fffffffffffffff, +0x7fffffffffffffff, +0xfffffffffffffffe, +0xfffffffffffffffd, +0xfffffffffffffffb, +0xfffffffffffffff7, +0xffffffffffffffef, +0xffffffffffffffdf, +0xffffffffffffffbf, +0xffffffffffffff7f, +0xfffffffffffffeff, +0xfffffffffffffdff, +0xfffffffffffffbff, +0xfffffffffffff7ff, +0xffffffffffffefff, +0xffffffffffffdfff, +0xffffffffffffbfff, +0xffffffffffff7fff, +0xfffffffffffeffff, +0xfffffffffffdffff, +0xfffffffffffbffff, +0xfffffffffff7ffff, +0xffffffffffefffff, +0xffffffffffdfffff, +0xffffffffffbfffff, +0xffffffffff7fffff, +0xfffffffffeffffff, +0xfffffffffdffffff, +0xfffffffffbffffff, +0xfffffffff7ffffff, +0xffffffffefffffff, +0xffffffffdfffffff, +0xffffffffbfffffff, +0xffffffff7fffffff, +0xfffffffeffffffff, +0xfffffffdffffffff, +0xfffffffbffffffff, +0xfffffff7ffffffff, +0xffffffefffffffff, +0xffffffdfffffffff, +0xffffffbfffffffff, +0xffffff7fffffffff, +0xfffffeffffffffff, +0xfffffdffffffffff, +0xfffffbffffffffff, +0xfffff7ffffffffff, +0xffffefffffffffff, +0xffffdfffffffffff, +0xffffbfffffffffff, +0xffff7fffffffffff, +0xfffeffffffffffff, +0xfffdffffffffffff, +0xfffbffffffffffff, +0xfff7ffffffffffff, +0xffefffffffffffff, +0xffdfffffffffffff, +0xffbfffffffffffff, +0xff7fffffffffffff, +0xfeffffffffffffff, +0xfdffffffffffffff, +0xfbffffffffffffff, +0xf7ffffffffffffff, +0xefffffffffffffff, +0xdfffffffffffffff, +0xbfffffffffffffff, +/* +#include +#include + +// Dumps all legal bitmask immediates for ARM64 +// Total number of unique 64-bit patterns: +// 1*2 + 3*4 + 7*8 + 15*16 + 31*32 + 63*64 = 5334 + +const char *uint64_to_binary(uint64_t x) { + static char b[65]; + unsigned i; + for (i = 0; i < 64; i++, x <<= 1) + b[i] = (0x8000000000000000ULL & x)? '1' : '0'; + b[64] = '\0'; + return b; +} + +int main() { + uint64_t result; + unsigned size, length, rotation, e; + for (size = 2; size <= 64; size *= 2) + for (length = 1; length < size; ++length) { + result = 0xffffffffffffffffULL >> (64 - length); + for (e = size; e < 64; e *= 2) + result |= result << e; + for (rotation = 0; rotation < size; ++rotation) { +#if 0 + printf("0x%016llx %s (size=%u, length=%u, rotation=%u)\n", + (unsigned long long)result, uint64_to_binary(result), + size, length, rotation); +#endif + printf("0x%016llx\n", (unsigned long long)result ); + result = (result >> 63) | (result << 1); + } + } + return 0; +} +*/ diff --git a/src/mapleall/maple_be/src/cg/cg_option.cpp b/src/mapleall/maple_be/src/cg/cg_option.cpp index 70c8e346849afe67f48449e10c00936a1b7f0c5c..eb8c30e0c8facfc603cff6c401974d82cc6baeed 100644 --- a/src/mapleall/maple_be/src/cg/cg_option.cpp +++ b/src/mapleall/maple_be/src/cg/cg_option.cpp @@ -38,12 +38,22 @@ std::string CGOptions::skipFrom = ""; std::string CGOptions::skipAfter = ""; std::string CGOptions::dumpFunc = "*"; std::string CGOptions::globalVarProfile = ""; +std::string CGOptions::profileData = ""; +std::string CGOptions::profileFuncData = ""; +std::string CGOptions::profileClassData = ""; #ifdef TARGARM32 std::string CGOptions::duplicateAsmFile = ""; #else std::string CGOptions::duplicateAsmFile = "maple/mrt/codetricks/arch/arm64/duplicateFunc.s"; #endif Range CGOptions::range = Range(); +std::string CGOptions::fastFuncsAsmFile = ""; +Range CGOptions::spillRanges = Range(); +uint8 CGOptions::fastAllocMode = 0; /* 0: fast, 1: spill all */ +bool CGOptions::fastAlloc = false; +uint64 CGOptions::lsraBBOptSize = 150000; +uint64 CGOptions::lsraInsnOptSize = 200000; +uint64 CGOptions::overlapNum = 28; #if TARGAARCH64 bool CGOptions::useBarriersForVolatile = false; #else @@ -75,14 +85,19 @@ bool CGOptions::hotFix = false; bool CGOptions::debugSched = false; bool CGOptions::bruteForceSched = false; bool CGOptions::simulateSched = false; +CGOptions::ABIType CGOptions::abiType = kABIHard; CGOptions::EmitFileType CGOptions::emitFileType = kAsm; bool CGOptions::genLongCalls = false; bool CGOptions::gcOnly = false; -bool CGOptions::quiet = true; +bool CGOptions::quiet = false; bool CGOptions::doPreSchedule = false; bool CGOptions::emitBlockMarker = true; bool CGOptions::inRange = false; +bool CGOptions::doPreLSRAOpt = false; +bool CGOptions::doLocalRefSpill = false; +bool CGOptions::doCalleeToSpill = false; +bool CGOptions::replaceASM = false; enum OptionIndex : uint64 { kCGQuiet = kCommonOptionEnd + 1, @@ -97,9 +112,15 @@ enum OptionIndex : uint64 { kIco, kSlo, kGo, + kPreLSRAOpt, + kLocalrefSpill, + kOptCallee, kPrepeep, kPeep, + kPreSchedule, kSchedule, + kWriteRefFieldOpt, + kDumpOlog, kCGNativeOpt, kInsertCall, kTrace, @@ -108,6 +129,7 @@ enum OptionIndex : uint64 { kGenGctib, kCGBarrier, kGenPrimorList, + kRaLinear, kRaColor, kConstFoldOpt, kSuppressFinfo, @@ -126,15 +148,25 @@ enum OptionIndex : uint64 { kDebugAsmMix, kProfilingInfo, kProfileEnable, + kLSRABB, + kLSRAInsn, + kLSRAOverlap, kCGO0, kCGO1, kCGO2, kProepilogue, kYieldPoing, kLocalRc, + kCGRange, + kFastAlloc, + kSpillRange, + kDuplicateBB, kCalleeCFI, kCyclePatternList, kDuplicateToDelPlt, + kDuplicateToDelPlt2, + kReplaceAsm, + kEmitBlockMarker, kInsertSoe, kCheckArrayStore, kPrintFunction, @@ -147,6 +179,8 @@ enum OptionIndex : uint64 { kDebugSched, kBruteForceSched, kSimulateSched, + kCrossLoc, + kABIType, kEmitFileType, kLongCalls, }; @@ -222,6 +256,16 @@ const Descriptor kUsage[] = { " --no-cg\n", "mplcg", {} }, + { kReplaceAsm, + kEnable, + "", + "replaceasm", + kBuildTypeProduct, + kArgCheckPolicyBool, + " --replaceasm \tReplace the the assembly code\n" + " --no-replaceasm\n", + "mplcg", + {} }, { kCGLazyBinding, kEnable, "", @@ -291,6 +335,36 @@ const Descriptor kUsage[] = { " --no-globalopt\n", "mplcg", {} }, + { kPreLSRAOpt, + kEnable, + "", + "prelsra", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --prelsra \tPerform live interval simplification in LSRA\n" + " --no-prelsra\n", + "mplcg", + {} }, + { kLocalrefSpill, + kEnable, + "", + "lsra-lvarspill", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --lsra-lvarspill \tPerform LSRA spill using local ref var stack locations\n" + " --no-lsra-lvarspill\n", + "mplcg", + {} }, + { kOptCallee, + kEnable, + "", + "lsra-optcallee", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --lsra-optcallee \tSpill callee if only one def to use\n" + " --no-lsra-optcallee\n", + "mplcg", + {} }, { kPrepeep, kEnable, "", @@ -311,6 +385,16 @@ const Descriptor kUsage[] = { " --no-peep\n", "mplcg", {} }, + { kPreSchedule, + kEnable, + "", + "preschedule", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --preschedule \tPerform prescheduling\n" + " --no-preschedule\n", + "mplcg", + {} }, { kSchedule, kEnable, "", @@ -321,6 +405,26 @@ const Descriptor kUsage[] = { " --no-schedule\n", "mplcg", {} }, + { kWriteRefFieldOpt, + kEnable, + "", + "writefieldopt", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --writefieldopt \tPerform WriteRefFieldOpt\n" + " --no-writefieldopt\n", + "mplcg", + {} }, + { kDumpOlog, + kEnable, + "", + "dump-olog", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --dump-olog \tDump CFGO and ICO debug information\n" + " --no-dump-olog\n", + "mplcg", + {} }, { kCGNativeOpt, kEnable, "", @@ -389,6 +493,15 @@ const Descriptor kUsage[] = { " --add-debug-trace \tInstrument the output .s file to print call traces at runtime\n", "mplcg", {} }, + { kProfileEnable, + 0, + "", + "add-func-profile", + kBuildTypeExperimental, + kArgCheckPolicyNone, + " --add-func-profile \tInstrument the output .s file to record func at runtime\n", + "mplcg", + {} }, { kCGClassList, 0, "", @@ -477,6 +590,24 @@ const Descriptor kUsage[] = { " --gmixedasm \tComment out both original source file and mpl file for debugging\n", "mplcg", {} }, + { kProfilingInfo, + 0, + "p", + "", + kBuildTypeExperimental, + kArgCheckPolicyNone, + " -p \tGenerate profiling infomation\n", + "mplcg", + {} }, + { kRaLinear, + 0, + "", + "with-ra-linear-scan", + kBuildTypeExperimental, + kArgCheckPolicyNone, + " --with-ra-linear-scan \tDo linear-scan register allocation\n", + "mplcg", + {} }, { kRaColor, 0, "", @@ -532,6 +663,33 @@ const Descriptor kUsage[] = { " -O2 \tDo some optimization.\n", "mplcg", {} }, + { kLSRABB, + 0, + "", + "lsra-bb", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --lsra-bb=NUM \tSwitch to spill mode if number of bb in function exceeds NUM\n", + "mplcg", + {} }, + { kLSRAInsn, + 0, + "", + "lsra-insn", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --lsra-insn=NUM \tSwitch to spill mode if number of instructons in function exceeds NUM\n", + "mplcg", + {} }, + { kLSRAOverlap, + 0, + "", + "lsra-overlap", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --lsra-overlap=NUM \toverlap NUM to decide pre spill in lsra\n", + "mplcg", + {} }, { kSuppressFinfo, 0, "", @@ -636,6 +794,43 @@ const Descriptor kUsage[] = { " --no-use-barriers-for-volatile\n", "mplcg", {} }, + { kCGRange, + 0, + "", + "range", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --range=NUM0,NUM1 \tOptimize only functions in the range [NUM0, NUM1]\n", + "mplcg", + {} }, + { kFastAlloc, + 0, + "", + "fast-alloc", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --fast-alloc=[0/1] \tO2 RA fast mode, set to 1 to spill all registers\n", + "mplcg", + {} }, + { kSpillRange, + 0, + "", + "spill_range", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --spill_range=NUM0,NUM1 \tO2 RA spill registers in the range [NUM0, NUM1]\n", + "mplcg", + {} }, + { kDuplicateBB, + kEnable, + "", + "dup-bb", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --dup-bb \tAllow cfg optimizer to duplicate bb\n" + " --no-dup-bb \tDon't allow cfg optimizer to duplicate bb\n", + "mplcg", + {} }, { kCalleeCFI, kEnable, "", @@ -676,6 +871,25 @@ const Descriptor kUsage[] = { " \t--duplicate_asm_list=list_file\n", "mplcg", {} }, + { kDuplicateToDelPlt2, + 0, + "", + "duplicate_asm_list2", + kBuildTypeProduct, + kArgCheckPolicyRequired, + " --duplicate_asm_list2 \tDuplicate more asm functions to delete plt call\n" + " \t--duplicate_asm_list2=list_file\n", + "mplcg", + {} }, + { kEmitBlockMarker, + 0, + "", + "block-marker", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --block-marker \tEmit block marker symbols in emitted assembly files\n", + "mplcg", + {} }, { kInsertSoe, 0, "", @@ -725,6 +939,28 @@ const Descriptor kUsage[] = { " --no-simulate-schedule\n", "mplcg", {} }, + { kCrossLoc, + kEnable, + "", + "cross-loc", + kBuildTypeExperimental, + kArgCheckPolicyBool, + " --cross-loc \tcross loc insn schedule\n" + " --no-cross-loc\n", + "mplcg", + {} }, + { kABIType, + 0, + "", + "float-abi", + kBuildTypeExperimental, + kArgCheckPolicyRequired, + " --float-abi=name \tPrint the abi type.\n" + " \tname=hard: abi-hard (Default)\n" + " \tname=soft: abi-soft\n" + " \tname=softfp: abi-softfp\n", + "mplcg", + {} }, { kEmitFileType, 0, "", @@ -836,9 +1072,19 @@ bool CGOptions::SolveOptions(const std::vector