From a96c3e8cb3bae3d15d75fbfb486f434f517a1e1a Mon Sep 17 00:00:00 2001 From: starunvs Date: Mon, 11 Aug 2025 10:46:25 +0800 Subject: [PATCH 1/2] blue zone version, pass weekly Change-Id: I25004447ade2f205cc0ac1065a00c766a10df74e Signed-off-by: starunvs --- .../heap/allocator/alloc_buffer.h | 18 ++++--- .../heap/allocator/region_desc.h | 48 ++++++++++++++++++- .../heap/allocator/region_manager.cpp | 1 + .../heap/allocator/region_manager.h | 2 + .../heap/allocator/region_space.cpp | 42 +++++++++------- .../heap/allocator/region_space.h | 15 ++++-- .../heap/ark_collector/ark_collector.cpp | 2 +- .../heap/ark_collector/ark_collector.h | 13 ++++- .../heap/ark_collector/idle_barrier.cpp | 10 ++-- common_components/heap/space/from_space.h | 14 ++---- common_components/heap/space/to_space.cpp | 34 +++++++++++-- common_components/heap/space/to_space.h | 13 +---- common_components/heap/space/young_space.h | 5 ++ ecmascript/compiler/stub_builder.cpp | 30 ++++++++++-- ecmascript/compiler/stub_builder.h | 2 + ecmascript/mem/barriers.cpp | 8 ++-- 16 files changed, 187 insertions(+), 70 deletions(-) diff --git a/common_components/heap/allocator/alloc_buffer.h b/common_components/heap/allocator/alloc_buffer.h index 3d8a8f9af1..6565e32177 100755 --- a/common_components/heap/allocator/alloc_buffer.h +++ b/common_components/heap/allocator/alloc_buffer.h @@ -16,6 +16,7 @@ #ifndef COMMON_COMPONENTS_HEAP_ALLOC_BUFFER_H #define COMMON_COMPONENTS_HEAP_ALLOC_BUFFER_H +#include "common_components/heap/allocator/region_desc.h" #include #include "common_components/mutator/thread_local.h" @@ -38,30 +39,30 @@ public: void Init(); static AllocationBuffer* GetOrCreateAllocBuffer(); static AllocationBuffer* GetAllocBuffer(); - HeapAddress ToSpaceAllocate(size_t size); + HeapAddress ToSpaceAllocate(size_t size, RegionAge age); HeapAddress Allocate(size_t size, AllocType allocType); template RegionDesc* GetRegion() { + static_assert(type != AllocBufferType::TO, "AllocBufferType::TO is not supported in GetRegion"); + if constexpr (type == AllocBufferType::YOUNG) { return tlRegion_; } else if constexpr (type == AllocBufferType::OLD) { return tlOldRegion_; - } else if constexpr (type == AllocBufferType::TO) { - return tlToRegion_; } } template void SetRegion(RegionDesc* newRegion) { + static_assert(type != AllocBufferType::TO, "AllocBufferType::TO is not supported in GetRegion"); + if constexpr (type == AllocBufferType::YOUNG) { tlRegion_ = newRegion; } else if constexpr (type == AllocBufferType::OLD) { tlOldRegion_ = newRegion; - } else if constexpr (type == AllocBufferType::TO) { - tlToRegion_ = newRegion; } } @@ -75,7 +76,9 @@ public: } else if constexpr (type == AllocBufferType::OLD) { tlOldRegion_ = RegionDesc::NullRegion(); } else if constexpr (type == AllocBufferType::TO) { - tlToRegion_ = RegionDesc::NullRegion(); + for (int i = 0; i < REGION_AGE_COUNT; i++) { + tlToRegion_[i] = RegionDesc::NullRegion(); + } } } @@ -163,7 +166,8 @@ private: RegionDesc* tlRegion_ = RegionDesc::NullRegion(); // managed by young-space RegionDesc* tlOldRegion_ = RegionDesc::NullRegion(); // managed by old-space // only used in ToSpaceAllocate for GC copy - RegionDesc* tlToRegion_ = RegionDesc::NullRegion(); // managed by to-space + RegionDesc* tlToRegion_[REGION_AGE_COUNT] {RegionDesc::NullRegion(), RegionDesc::NullRegion(), RegionDesc::NullRegion(), + RegionDesc::NullRegion()}; // managed by to-space std::atomic preparedRegion_ = { nullptr }; // allocate objects which are exposed to runtime thus can not be moved. diff --git a/common_components/heap/allocator/region_desc.h b/common_components/heap/allocator/region_desc.h index 75ec414dbc..3f80740809 100755 --- a/common_components/heap/allocator/region_desc.h +++ b/common_components/heap/allocator/region_desc.h @@ -16,6 +16,7 @@ #ifndef COMMON_COMPONENTS_HEAP_ALLOCATOR_REGION_INFO_H #define COMMON_COMPONENTS_HEAP_ALLOCATOR_REGION_INFO_H +#include #include #include #include @@ -75,6 +76,17 @@ public: private: T fieldVal_; }; + +enum RegionAge: uint8_t { + EDEN = 0, + SURVIVOR1, + SURVIVOR2, + OLD +}; + +static constexpr RegionAge REGION_AGE_MAX = RegionAge::OLD; +static constexpr int REGION_AGE_COUNT = static_cast(REGION_AGE_MAX) + 1; + // this class is the metadata of region, it contains all the information needed to manage its corresponding memory. // Region memory is composed of several Units, described by UnitInfo. // sizeof(RegionDesc) must be equal to sizeof(UnitInfo). We rely on this fact to calculate region-related address. @@ -622,6 +634,16 @@ public: } } + RegionAge GetRegionAge() const + { + return InlinedRegionMetaData::GetInlinedRegionMetaData(this)->GetRegionAge(); + } + + void SetRegionAge(RegionAge age) + { + InlinedRegionMetaData::GetInlinedRegionMetaData(this)->SetRegionAge(age); + } + void SetMarkedRegionFlag(uint8_t flag) { metadata.regionBits.AtomicSetValue(RegionBitOffset::BIT_OFFSET_MARKED_REGION, 1, flag); @@ -747,6 +769,16 @@ public: return RegionDesc::IsInRecentSpace(type); } + bool IsRSetSource() const + { + return GetRegionAge() == RegionAge::OLD; + } + + bool IsRSetTarget() const + { + return GetRegionAge() != RegionAge::OLD; + } + bool IsInYoungSpace() const { RegionType type = GetRegionType(); @@ -1217,7 +1249,7 @@ public: // this could ONLY used in region that is ALIVE. class InlinedRegionMetaData { public: - static InlinedRegionMetaData *GetInlinedRegionMetaData(RegionDesc *region) + static InlinedRegionMetaData *GetInlinedRegionMetaData(const RegionDesc *region) { InlinedRegionMetaData *data = GetInlinedRegionMetaData(region->GetRegionStart()); DCHECK_CC(data->regionDesc_ == region); @@ -1324,10 +1356,23 @@ public: size_t offset = GetAddressOffset(static_cast(reinterpret_cast(obj))); return GetRegionRSet()->MarkCardTable(offset); } + + RegionAge GetRegionAge() const + { + + return __atomic_load_n(&age_, std::memory_order_relaxed); + } + + void SetRegionAge(RegionAge age) + { + __atomic_store_n(&age_, age, std::memory_order_relaxed); + } + private: RegionDesc *regionDesc_ {nullptr}; RegionRSet *regionRSet_ {nullptr}; RegionType regionType_ {}; + RegionAge age_ {OLD}; // fixme: inline more friend class RegionDesc; @@ -1413,6 +1458,7 @@ public: friend constexpr size_t GetMetaDataInRegionOffset(); static constexpr size_t REGION_RSET_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionRSet_); static constexpr size_t REGION_TYPE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionType_); + static constexpr size_t REGION_AGE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, age_); }; HeapAddress RegionDesc::InlinedRegionMetaData::GetRegionStart() const diff --git a/common_components/heap/allocator/region_manager.cpp b/common_components/heap/allocator/region_manager.cpp index 7469fac647..ee5a301cf6 100755 --- a/common_components/heap/allocator/region_manager.cpp +++ b/common_components/heap/allocator/region_manager.cpp @@ -880,6 +880,7 @@ uintptr_t RegionManager::AllocReadOnly(size_t size, bool allowGC) if (region == nullptr) { return 0; } + region->SetRegionAge(OLD); DLOG(REGION, "alloc read only region @0x%zx+%zu type %u", region->GetRegionStart(), region->GetRegionAllocatedSize(), region->GetRegionType()); diff --git a/common_components/heap/allocator/region_manager.h b/common_components/heap/allocator/region_manager.h index b228816d2a..443cd31d75 100755 --- a/common_components/heap/allocator/region_manager.h +++ b/common_components/heap/allocator/region_manager.h @@ -173,6 +173,7 @@ public: listMutex.unlock(); return 0; } + region->SetRegionAge(OLD); DLOG(REGION, "alloc pinned region @0x%zx+%zu type %u", region->GetRegionStart(), region->GetRegionAllocatedSize(), region->GetRegionType()); @@ -243,6 +244,7 @@ public: if (region == nullptr) { return 0; } + region->SetRegionAge(OLD); GCPhase phase = Mutator::GetMutator()->GetMutatorPhase(); if (phase == GC_PHASE_ENUM || phase == GC_PHASE_MARK || phase == GC_PHASE_REMARK_SATB || phase == GC_PHASE_POST_MARK) { diff --git a/common_components/heap/allocator/region_space.cpp b/common_components/heap/allocator/region_space.cpp index 4b797382c4..45cb2994d3 100755 --- a/common_components/heap/allocator/region_space.cpp +++ b/common_components/heap/allocator/region_space.cpp @@ -26,10 +26,12 @@ namespace common { template -RegionDesc* RegionSpace::AllocateThreadLocalRegion(bool expectPhysicalMem) +RegionDesc* RegionSpace::AllocateThreadLocalRegion(bool expectPhysicalMem, RegionAge age) { if constexpr (type == AllocBufferType::TO) { RegionDesc* region = regionManager_.TakeRegion(expectPhysicalMem, false, true); + ASSERT_LOGF(age != EDEN, "AllocBufferType::TO cannot be used with EDEN age"); + region->SetRegionAge(age); if (region != nullptr) { toSpace_.AddThreadLocalRegion(region); } @@ -50,8 +52,12 @@ RegionDesc* RegionSpace::AllocateThreadLocalRegion(bool expectPhysicalMem) if constexpr (type == AllocBufferType::YOUNG) { youngSpace_.AddThreadLocalRegion(region); + ASSERT_LOGF(age == EDEN, "AllocBufferType::YOUNG should always be EDEN"); + region->SetRegionAge(EDEN); } else if constexpr (type == AllocBufferType::OLD) { oldSpace_.AddThreadLocalRegion(region); + ASSERT_LOGF(age == OLD, "AllocBufferType::OLD should always be OLD"); + region->SetRegionAge(OLD); } } @@ -353,10 +359,13 @@ void AllocationBuffer::ClearThreadLocalRegion() heap.HandleFullThreadLocalRegion(tlOldRegion_); tlOldRegion_ = RegionDesc::NullRegion(); } - if (LIKELY_CC(tlToRegion_ != RegionDesc::NullRegion())) { - RegionSpace& heap = reinterpret_cast(Heap::GetHeap().GetAllocator()); - heap.HandleFullThreadLocalRegion(tlToRegion_); - tlToRegion_ = RegionDesc::NullRegion(); + + for (int i = 0; i < REGION_AGE_COUNT; i++) { + if (LIKELY_CC(tlToRegion_[i] != RegionDesc::NullRegion())) { + RegionSpace& heap = reinterpret_cast(Heap::GetHeap().GetAllocator()); + heap.HandleFullThreadLocalRegion(tlToRegion_[i]); + tlToRegion_[i] = RegionDesc::NullRegion(); + } } } @@ -381,26 +390,27 @@ void AllocationBuffer::Init() Heap::GetHeap().RegisterAllocBuffer(*this); } -HeapAddress AllocationBuffer::ToSpaceAllocate(size_t totalSize) +HeapAddress AllocationBuffer::ToSpaceAllocate(size_t totalSize, RegionAge age) { HeapAddress addr = 0; - if (LIKELY_CC(tlToRegion_ != RegionDesc::NullRegion())) { - addr = tlToRegion_->Alloc(totalSize); + int target = static_cast(age); + if (LIKELY_CC(tlToRegion_[target] != RegionDesc::NullRegion())) { + addr = tlToRegion_[target]->Alloc(totalSize); } if (UNLIKELY_CC(addr == 0)) { RegionSpace& heapSpace = reinterpret_cast(Heap::GetHeap().GetAllocator()); - heapSpace.HandleFullThreadLocalRegion(tlToRegion_); - tlToRegion_ = RegionDesc::NullRegion(); + heapSpace.HandleFullThreadLocalRegion(tlToRegion_[target]); + tlToRegion_[target] = RegionDesc::NullRegion(); - RegionDesc* r = heapSpace.AllocateThreadLocalRegion(false); + RegionDesc* r = heapSpace.AllocateThreadLocalRegion(false, age); if (UNLIKELY_CC(r == nullptr)) { return 0; } - tlToRegion_ = r; - addr = tlToRegion_->Alloc(totalSize); + tlToRegion_[target] = r; + addr = tlToRegion_[target]->Alloc(totalSize); } DLOG(ALLOC, "alloc to 0x%zx(%zu)", addr, totalSize); @@ -446,7 +456,7 @@ HeapAddress AllocationBuffer::AllocateImpl(size_t totalSize, AllocType allocType heapSpace.HandleFullThreadLocalRegion(tlRegion_); tlRegion_ = RegionDesc::NullRegion(); - RegionDesc* r = heapSpace.AllocateThreadLocalRegion(false); + RegionDesc* r = heapSpace.AllocateThreadLocalRegion(false, RegionAge::EDEN); if (UNLIKELY_CC(r == nullptr)) { return 0; } @@ -457,7 +467,7 @@ HeapAddress AllocationBuffer::AllocateImpl(size_t totalSize, AllocType allocType heapSpace.HandleFullThreadLocalRegion(tlOldRegion_); tlOldRegion_ = RegionDesc::NullRegion(); - RegionDesc* r = heapSpace.AllocateThreadLocalRegion(false); + RegionDesc* r = heapSpace.AllocateThreadLocalRegion(false, RegionAge::OLD); if (UNLIKELY_CC(r == nullptr)) { return 0; } @@ -510,7 +520,7 @@ void RegionSpace::FeedHungryBuffers() for (auto* buffer : hungryBuffers) { if (buffer->GetPreparedRegion() != nullptr) { continue; } if (region == nullptr) { - region = AllocateThreadLocalRegion(true); + region = AllocateThreadLocalRegion(true, EDEN); if (region == nullptr) { return; } } if (buffer->SetPreparedRegion(region)) { diff --git a/common_components/heap/allocator/region_space.h b/common_components/heap/allocator/region_space.h index be67860f26..e72659b4af 100755 --- a/common_components/heap/allocator/region_space.h +++ b/common_components/heap/allocator/region_space.h @@ -16,6 +16,7 @@ #ifndef COMMON_COMPONENTS_HEAP_ALLOCATOR_REGION_SPACE_H #define COMMON_COMPONENTS_HEAP_ALLOCATOR_REGION_SPACE_H +#include "common_components/heap/allocator/region_desc.h" #include #include #include @@ -74,7 +75,7 @@ public: void Init(const RuntimeParam ¶m) override; template - RegionDesc* AllocateThreadLocalRegion(bool expectPhysicalMem = false); + RegionDesc* AllocateThreadLocalRegion(bool expectPhysicalMem = false, RegionAge age = RegionAge::OLD); template void HandleFullThreadLocalRegion(RegionDesc* region) noexcept @@ -206,13 +207,17 @@ public: void ExemptFromSpace() { COMMON_PHASE_TIMER("ExemptFromRegions"); - fromSpace_.ExemptFromRegions(); + if (Heap::GetHeap().GetGCReason() == GC_REASON_YOUNG) { + // skip in young gc + } else { + fromSpace_.ExemptFromRegions(); + } } - BaseObject* RouteObject(BaseObject* fromObj, size_t size) + BaseObject* RouteObject(BaseObject* fromObj, size_t size, RegionAge age) { AllocationBuffer* buffer = AllocationBuffer::GetOrCreateAllocBuffer(); - uintptr_t toAddr = buffer->ToSpaceAllocate(size); + uintptr_t toAddr = buffer->ToSpaceAllocate(size, age); return reinterpret_cast(toAddr); } @@ -256,7 +261,7 @@ public: void HandlePromotion() { fromSpace_.GetPromotedTo(oldSpace_); - toSpace_.GetPromotedTo(oldSpace_); + toSpace_.GetPromotedTo(youngSpace_, oldSpace_); } void AssembleSmallGarbageCandidates() diff --git a/common_components/heap/ark_collector/ark_collector.cpp b/common_components/heap/ark_collector/ark_collector.cpp index 717d862efd..b11cdb9391 100755 --- a/common_components/heap/ark_collector/ark_collector.cpp +++ b/common_components/heap/ark_collector/ark_collector.cpp @@ -235,7 +235,7 @@ void ArkCollector::FixRefField(BaseObject* obj, RefField<>& field) const BaseObject* toObj = latest == nullptr ? targetObj : latest; RegionDesc* objRegion = RegionDesc::GetAliveRegionDescAt(reinterpret_cast((void*)obj)); RegionDesc* refRegion = RegionDesc::GetAliveRegionDescAt(reinterpret_cast((void*)toObj)); - if (!objRegion->IsInRecentSpace() && refRegion->IsInRecentSpace()) { + if (objRegion->IsRSetSource() && refRegion->IsRSetTarget()) { if (objRegion->MarkRSetCardTable(obj)) { DLOG(TRACE, "fix phase update point-out remember set of region %p, obj %p, ref: %p<%p>", objRegion, obj, toObj, toObj->GetTypeInfo()); diff --git a/common_components/heap/ark_collector/ark_collector.h b/common_components/heap/ark_collector/ark_collector.h index 34c959ab60..83fd353370 100755 --- a/common_components/heap/ark_collector/ark_collector.h +++ b/common_components/heap/ark_collector/ark_collector.h @@ -31,7 +31,8 @@ public: // if object is not relocated (forwarded or compacted), return nullptr. BaseObject* RouteObject(BaseObject* old, size_t size) { - BaseObject* toAddress = theSpace.RouteObject(old, size); + auto region = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(old)); + BaseObject* toAddress = theSpace.RouteObject(old, size, PromoteRegionAge(region->GetRegionAge())); return toAddress; } @@ -40,6 +41,16 @@ public: return old->GetForwardingPointer(); } + RegionAge PromoteRegionAge(RegionAge age) + { + RegionAge newAge = static_cast(age + 1); + if (newAge >= REGION_AGE_MAX) { + newAge = REGION_AGE_MAX; + } + + return newAge; + } + RegionSpace& theSpace; }; diff --git a/common_components/heap/ark_collector/idle_barrier.cpp b/common_components/heap/ark_collector/idle_barrier.cpp index b36155a99d..c8c0bb146b 100755 --- a/common_components/heap/ark_collector/idle_barrier.cpp +++ b/common_components/heap/ark_collector/idle_barrier.cpp @@ -90,14 +90,12 @@ bool IdleBarrier::CompareAndSwapRefField(BaseObject* obj, RefField& field, void IdleBarrier::UpdateRememberSet(BaseObject* object, BaseObject* ref) const { ASSERT(object != nullptr); - RegionDesc::InlinedRegionMetaData *objMetaRegion = RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData( - reinterpret_cast(object)); - RegionDesc::InlinedRegionMetaData *refMetaRegion = RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData( - reinterpret_cast(ref)); - if (!objMetaRegion->IsInYoungSpaceForWB() && refMetaRegion->IsInYoungSpaceForWB()) { + RegionDesc *objMetaRegion = RegionDesc::GetAliveRegionDescAt( reinterpret_cast(object)); + RegionDesc *refMetaRegion = RegionDesc::GetAliveRegionDescAt( reinterpret_cast(ref)); + if (objMetaRegion->IsRSetSource() && refMetaRegion->IsRSetTarget()) { if (objMetaRegion->MarkRSetCardTable(object)) { DLOG(BARRIER, "update point-out remember set of region %p, obj %p, ref: %p<%p>", - objMetaRegion->GetRegionDesc(), object, ref, ref->GetTypeInfo()); + objMetaRegion, object, ref, ref->GetTypeInfo()); } } } diff --git a/common_components/heap/space/from_space.h b/common_components/heap/space/from_space.h index e1569d4ffb..62f347c80f 100644 --- a/common_components/heap/space/from_space.h +++ b/common_components/heap/space/from_space.h @@ -16,17 +16,10 @@ #define COMMON_COMPONENTS_HEAP_SPACE_FROM_SPACE_H #include -#include -#include -#include -#include -#include - -#include "common_components/heap/allocator/alloc_util.h" -#include "common_components/heap/allocator/allocator.h" #include "common_components/heap/allocator/region_manager.h" -#include "common_components/mutator/mutator.h" #include "common_components/heap/allocator/fix_heap.h" +#include "common_components/heap/space/regional_space.h" + #if defined(COMMON_SANITIZER_SUPPORT) #include "common_components/base/asan_interface.h" #endif @@ -86,6 +79,7 @@ public: void ExemptFromRegion(RegionDesc* region) { + region->SetRegionAge(RegionAge::OLD); exemptedFromRegionList_.PrependRegion(region, RegionDesc::RegionType::EXEMPTED_FROM_REGION); } @@ -100,7 +94,7 @@ public: void CopyFromRegions(Taskpool* threadPool); void CopyFromRegions(); - void GetPromotedTo(OldSpace& mspace); + void GetPromotedTo(OldSpace& oldSpace); void SetExemptedRegionThreshold(double threshold) { diff --git a/common_components/heap/space/to_space.cpp b/common_components/heap/space/to_space.cpp index a3d1c6ce64..070dcbe067 100644 --- a/common_components/heap/space/to_space.cpp +++ b/common_components/heap/space/to_space.cpp @@ -15,6 +15,9 @@ #include "common_components/heap/space/to_space.h" #include "common_components/heap/space/old_space.h" +#include "heap/space/young_space.h" +#include "log/log.h" +#include "macros.h" #if defined(COMMON_SANITIZER_SUPPORT) #include "common_components/base/asan_interface.h" #endif @@ -40,7 +43,7 @@ void ToSpace::DumpRegionStats() const fullToRegions, fullToUnits, fullToSize, allocfullToSize); } -void ToSpace::GetPromotedTo(OldSpace& mspace) +void ToSpace::GetPromotedTo(YoungSpace &youngSpace, OldSpace& oldSpace) { // release thread-local to-space regions as they will promote to old-space AllocBufferVisitor visitor = [](AllocationBuffer& tlab) { @@ -48,7 +51,32 @@ void ToSpace::GetPromotedTo(OldSpace& mspace) }; Heap::GetHeap().GetAllocator().VisitAllocBuffers(visitor); - mspace.PromoteRegionList(fullToRegionList_); - mspace.PromoteRegionList(tlToRegionList_); + fullToRegionList_.MergeRegionList(tlToRegionList_, RegionDesc::RegionType::TO_REGION); + auto iter = fullToRegionList_.GetHeadRegion(); + + RegionList targets[REGION_AGE_COUNT] {"tmp-enen", "tmp-survivor1", "tmp-survivor2", "tmp-old"}; + while (iter != nullptr) { + auto current = iter; + iter = iter->GetNextRegion(); + + if (current->GetRegionAge() == EDEN) { + ASSERT_LOGF(false, "EDEN region should not be in full to-region list"); + targets[EDEN].PrependRegion(current, RegionDesc::RegionType::TO_REGION); + } else if (current->GetRegionAge() == SURVIVOR1) { + targets[SURVIVOR1].PrependRegion(current, RegionDesc::RegionType::TO_REGION); + } else if (current->GetRegionAge() == SURVIVOR2) { + targets[SURVIVOR2].PrependRegion(current, RegionDesc::RegionType::TO_REGION); + } else if (current->GetRegionAge() == OLD) { + targets[OLD].PrependRegion(current, RegionDesc::RegionType::TO_REGION); + } else { + UNREACHABLE(); + } + } + fullToRegionList_.ClearList(); + + for (int i = 0; i < REGION_AGE_COUNT - 1; i++) { + youngSpace.PromoteRegionList(targets[i]); + } + oldSpace.PromoteRegionList(targets[OLD]); } } // namespace common diff --git a/common_components/heap/space/to_space.h b/common_components/heap/space/to_space.h index bf5ae1f3ce..af33912507 100644 --- a/common_components/heap/space/to_space.h +++ b/common_components/heap/space/to_space.h @@ -15,20 +15,11 @@ #ifndef COMMON_COMPONENTS_HEAP_SPACE_TO_SPACE_H #define COMMON_COMPONENTS_HEAP_SPACE_TO_SPACE_H -#include -#include -#include -#include -#include -#include - -#include "common_components/heap/allocator/alloc_util.h" -#include "common_components/heap/allocator/allocator.h" +#include "common_components/heap/space/young_space.h" #include "common_components/heap/allocator/region_list.h" #include "common_components/heap/allocator/region_manager.h" #include "common_components/heap/space/regional_space.h" #include "common_components/heap/space/from_space.h" -#include "common_components/mutator/mutator.h" #if defined(COMMON_SANITIZER_SUPPORT) #include "common_components/base/asan_interface.h" #endif @@ -84,7 +75,7 @@ public: return tlToRegionList_.GetUnitCount() + fullToRegionList_.GetUnitCount(); } - void GetPromotedTo(OldSpace& mspace); + void GetPromotedTo(YoungSpace &youngSpace, OldSpace& oldSpace); void ClearAllGCInfo() { diff --git a/common_components/heap/space/young_space.h b/common_components/heap/space/young_space.h index 61eef6834e..d498a427ff 100644 --- a/common_components/heap/space/young_space.h +++ b/common_components/heap/space/young_space.h @@ -68,6 +68,11 @@ public: recentFullRegionList_.PrependRegion(region, RegionDesc::RegionType::RECENT_FULL_REGION); } + void PromoteRegionList(RegionList& list) + { + recentFullRegionList_.MergeRegionList(list, RegionDesc::RegionType::RECENT_FULL_REGION); + } + void AssembleGarbageCandidates(FromSpace& fromSpace) { fromSpace.AssembleGarbageCandidates(recentFullRegionList_); diff --git a/ecmascript/compiler/stub_builder.cpp b/ecmascript/compiler/stub_builder.cpp index fc3618c299..e9ead028ab 100644 --- a/ecmascript/compiler/stub_builder.cpp +++ b/ecmascript/compiler/stub_builder.cpp @@ -1924,6 +1924,25 @@ GateRef StubBuilder::GetCMCRegionType(GateRef obj) return regionDescAddr; } +GateRef StubBuilder::GetCMCRegionAge(GateRef obj) +{ + // todo: atomic? + GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); + GateRef regionDescAddr = LoadPrimitive(VariableType::INT8(), metaDataAddr, + IntPtr(common::RegionDesc::REGION_AGE_IN_INLINED_METADATA_OFFSET)); + return regionDescAddr; +} + +GateRef StubBuilder::CMCIsOldToYoungViaAge(GateRef from, GateRef to) +{ + auto env = GetEnvironment(); + GateRef isOldToYoung = LogicAndBuilder(env) + .And(Int8Equal(GetCMCRegionAge(from), Int8(static_cast(common::RegionAge::OLD)))) + .And(BoolNot(Int8Equal(GetCMCRegionAge(to), Int8(static_cast(common::RegionAge::OLD))))) + .Done(); + return isOldToYoung; +} + GateRef StubBuilder::CMCIsInYoungSpace(GateRef regionType) { auto env = GetEnvironment(); @@ -2050,7 +2069,8 @@ void StubBuilder::CMCSetValueWithBarrier(GateRef glue, GateRef obj, [[maybe_unus { GateRef objRegionType = GetCMCRegionType(obj); GateRef valueRegionType = GetCMCRegionType(value); - GateRef isOldToYoung = IsOldToYoung(objRegionType, valueRegionType); + // GateRef isOldToYoung = IsOldToYoung(objRegionType, valueRegionType); + GateRef isOldToYoung = CMCIsOldToYoungViaAge(obj, value); BRANCH_UNLIKELY(isOldToYoung, &markRSet, ¬MarkRSet); Bind(&markRSet); MarkRSetCardTable(obj, ¬MarkRSet); @@ -2095,7 +2115,8 @@ void StubBuilder::CMCArrayCopyWriteBarrier(GateRef glue, GateRef dstObj, GateRef GateRef ref = LoadPrimitive(VariableType::JS_ANY(), src, offset); BRANCH(TaggedIsHeapObject(ref), &isTaggedObject, &loopEnd); Bind(&isTaggedObject); - GateRef isOldToYoung = IsOldToYoung(objRegionType, GetCMCRegionType(ref)); + // GateRef isOldToYoung = IsOldToYoung(objRegionType, GetCMCRegionType(ref)); + GateRef isOldToYoung = CMCIsOldToYoungViaAge(dstObj, ref); BRANCH_UNLIKELY(isOldToYoung, &markRSet, &loopEnd); Bind(&markRSet); MarkRSetCardTable(dstObj, ¬MarkRSet); @@ -2127,7 +2148,7 @@ void StubBuilder::CMCArrayCopyWriteBarrierSameArray(GateRef glue, GateRef dstObj Label markRSet(env); Label notMarkRSet(env); Label notIdlePhase(env); - GateRef objRegionType = GetCMCRegionType(dstObj); + // GateRef objRegionType = GetCMCRegionType(dstObj); DEFVARIABLE(i, VariableType::INT32(), Int32(0)); GateRef gcPhase = GetGCPhase(glue); BRANCH(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_IDLE)), &exit, ¬IdlePhase); @@ -2145,7 +2166,8 @@ void StubBuilder::CMCArrayCopyWriteBarrierSameArray(GateRef glue, GateRef dstObj GateRef ref = LoadPrimitive(VariableType::JS_ANY(), src, offset); BRANCH(TaggedIsHeapObject(ref), &isTaggedObject, &loopEnd); Bind(&isTaggedObject); - GateRef isOldToYoung = IsOldToYoung(objRegionType, GetCMCRegionType(ref)); + // GateRef isOldToYoung = IsOldToYoung(objRegionType, GetCMCRegionType(ref)); + GateRef isOldToYoung = CMCIsOldToYoungViaAge(dstObj, ref); BRANCH_UNLIKELY(isOldToYoung, &markRSet, &loopEnd); Bind(&markRSet); MarkRSetCardTable(dstObj, ¬MarkRSet); diff --git a/ecmascript/compiler/stub_builder.h b/ecmascript/compiler/stub_builder.h index a909b492ca..3c9068b73d 100644 --- a/ecmascript/compiler/stub_builder.h +++ b/ecmascript/compiler/stub_builder.h @@ -822,6 +822,8 @@ public: void VerifyBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value); GateRef GetCMCRegionRSet(GateRef obj); GateRef GetCMCRegionType(GateRef obj); + GateRef GetCMCRegionAge(GateRef obj); + GateRef CMCIsOldToYoungViaAge(GateRef from, GateRef to); GateRef GetGCPhase(GateRef glue); GateRef GetGCReason(GateRef glue); GateRef CMCIsInYoungSpace(GateRef regionType); diff --git a/ecmascript/mem/barriers.cpp b/ecmascript/mem/barriers.cpp index d112ebc9a2..4095c469b7 100644 --- a/ecmascript/mem/barriers.cpp +++ b/ecmascript/mem/barriers.cpp @@ -171,16 +171,14 @@ void Barriers::CMCArrayCopyWriteBarrier(const JSThread *thread, const TaggedObje ASSERT(dstObj != nullptr); common::BaseObject* object = reinterpret_cast(const_cast(dstObj)); - common::RegionDesc::InlinedRegionMetaData *objMetaRegion = - common::RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData(reinterpret_cast(object)); + common::RegionDesc *objMetaRegion = common::RegionDesc::GetAliveRegionDescAt(reinterpret_cast(object)); JSTaggedType *srcPtr = reinterpret_cast(src); common::GCPhase gcPhase = thread->GetCMCGCPhase(); // 1. update Rememberset if (ShouldUpdateRememberSet(gcPhase)) { auto checkReference = [&](BaseObject *ref) { - common::RegionDesc::InlinedRegionMetaData *refMetaRegion = - common::RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData(reinterpret_cast(ref)); - return (!objMetaRegion->IsInYoungSpaceForWB() && refMetaRegion->IsInYoungSpaceForWB()); + common::RegionDesc *refMetaRegion = common::RegionDesc::GetAliveRegionDescAt(reinterpret_cast(ref)); + return (objMetaRegion->IsRSetSource() && refMetaRegion->IsRSetTarget()); }; for (size_t i = 0; i < count; i++) { -- Gitee From 4103d49347dc91211b7a7601647ceee13d2fbde4 Mon Sep 17 00:00:00 2001 From: starunvs Date: Tue, 12 Aug 2025 15:48:15 +0800 Subject: [PATCH 2/2] atomic get region age in stub Change-Id: Ic87e7281d983ffb269dca0de5aaa6889f313d75f --- ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp | 4 ++++ ecmascript/compiler/mcr_gate_meta_data.h | 8 +++++++- ecmascript/compiler/stub_builder-inl.h | 4 ++-- ecmascript/compiler/stub_builder.cpp | 3 +-- ecmascript/compiler/stub_builder.h | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp b/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp index 0d9475fbbb..a00f50717c 100644 --- a/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp +++ b/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp @@ -1703,6 +1703,10 @@ void LLVMIRBuilder::VisitLoad(GateRef gate, GateRef base) LLVMSetOrdering(result, LLVMAtomicOrderingRelease); break; } + case MemoryAttribute::MEMORY_ORDER_RELAXED: { + LLVMSetOrdering(result, LLVMAtomicOrderingMonotonic); + break; + } case MemoryAttribute::NOT_ATOMIC: { break; } diff --git a/ecmascript/compiler/mcr_gate_meta_data.h b/ecmascript/compiler/mcr_gate_meta_data.h index f9a8dd3e24..1add7b924b 100644 --- a/ecmascript/compiler/mcr_gate_meta_data.h +++ b/ecmascript/compiler/mcr_gate_meta_data.h @@ -431,7 +431,8 @@ public: enum Order { NOT_ATOMIC = 0, - MEMORY_ORDER_RELEASE + MEMORY_ORDER_RELEASE, + MEMORY_ORDER_RELAXED }; enum Barrier { @@ -451,6 +452,11 @@ public: return Create(NOT_ATOMIC); } + static MemoryAttribute NoBarrierAndRelaxed() + { + return Create(MEMORY_ORDER_RELAXED, NO_BARRIER); + } + static MemoryAttribute NeedBarrier() { return Create(NOT_ATOMIC, NEED_BARRIER); diff --git a/ecmascript/compiler/stub_builder-inl.h b/ecmascript/compiler/stub_builder-inl.h index 90bf40f9f4..3d536a5a79 100644 --- a/ecmascript/compiler/stub_builder-inl.h +++ b/ecmascript/compiler/stub_builder-inl.h @@ -419,12 +419,12 @@ inline GateRef StubBuilder::LoadZeroOffset(VariableType type, GateRef glue, Gate return Load(type, glue, base, IntPtr(0), mAttr); } -inline GateRef StubBuilder::LoadPrimitive(VariableType type, GateRef base, GateRef offset) +inline GateRef StubBuilder::LoadPrimitive(VariableType type, GateRef base, GateRef offset, MemoryAttribute mAttr) { if (type == VariableType::NATIVE_POINTER()) { type = env_->IsArch64Bit() ? VariableType::INT64() : VariableType::INT32(); } - return env_->GetBuilder()->LoadWithoutBarrier(type, base, offset); + return env_->GetBuilder()->LoadWithoutBarrier(type, base, offset, mAttr); } inline GateRef StubBuilder::LoadZeroOffsetPrimitive(VariableType type, GateRef base) diff --git a/ecmascript/compiler/stub_builder.cpp b/ecmascript/compiler/stub_builder.cpp index e9ead028ab..2371a915e8 100644 --- a/ecmascript/compiler/stub_builder.cpp +++ b/ecmascript/compiler/stub_builder.cpp @@ -1926,10 +1926,9 @@ GateRef StubBuilder::GetCMCRegionType(GateRef obj) GateRef StubBuilder::GetCMCRegionAge(GateRef obj) { - // todo: atomic? GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); GateRef regionDescAddr = LoadPrimitive(VariableType::INT8(), metaDataAddr, - IntPtr(common::RegionDesc::REGION_AGE_IN_INLINED_METADATA_OFFSET)); + IntPtr(common::RegionDesc::REGION_AGE_IN_INLINED_METADATA_OFFSET), MemoryAttribute::NoBarrierAndRelaxed()); return regionDescAddr; } diff --git a/ecmascript/compiler/stub_builder.h b/ecmascript/compiler/stub_builder.h index 3c9068b73d..9b268a058d 100644 --- a/ecmascript/compiler/stub_builder.h +++ b/ecmascript/compiler/stub_builder.h @@ -208,7 +208,7 @@ public: MemoryAttribute mAttr = MemoryAttribute::Default()); GateRef LoadZeroOffset(VariableType type, GateRef glue, GateRef base, MemoryAttribute mAttr = MemoryAttribute::Default()); - GateRef LoadPrimitive(VariableType type, GateRef base, GateRef offset); + GateRef LoadPrimitive(VariableType type, GateRef base, GateRef offset, MemoryAttribute mAttr = MemoryAttribute::Default()); GateRef LoadZeroOffsetPrimitive(VariableType type, GateRef base); void Store(VariableType type, GateRef glue, -- Gitee