From 5c65cde12e2848b22673bd9aeb4868d3f524cacf Mon Sep 17 00:00:00 2001 From: lukai Date: Sun, 13 Jul 2025 00:01:21 +0800 Subject: [PATCH] Reuse fixpin freeslot during gc Optimize memory by mem reuse. Issue: https://gitee.com/openharmony/arkcompiler_ets_runtime/issues/ICK7SO Signed-off-by: lukai Change-Id: I16ec2fe7d6683a87e79fe96202b395266891baf0 --- .../heap/allocator/region_desc.h | 113 +++++++++++++++++- .../heap/allocator/region_list.h | 1 + .../heap/allocator/region_manager.cpp | 52 ++++++-- .../heap/allocator/region_manager.h | 1 + .../heap/allocator/region_space.h | 13 ++ .../heap/collector/copy_data_manager.h | 4 +- 6 files changed, 172 insertions(+), 12 deletions(-) diff --git a/common_components/heap/allocator/region_desc.h b/common_components/heap/allocator/region_desc.h index 40732a494c..b4ae537621 100755 --- a/common_components/heap/allocator/region_desc.h +++ b/common_components/heap/allocator/region_desc.h @@ -240,6 +240,74 @@ public: return nullptr; } + RegionBitmap* GetAllocBitmapDruingMark() + { + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.allocBitmapDuringMark_, std::memory_order_acquire); + if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { + return nullptr; + } + return bitmap; + } + + ALWAYS_INLINE RegionBitmap* GetOrAllocBitmapDuringMark() + { + do { + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.allocBitmapDuringMark_, std::memory_order_acquire); + if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { + continue; + } + if (LIKELY_CC(bitmap != nullptr)) { + return bitmap; + } + RegionBitmap *newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + if (__atomic_compare_exchange_n(&metadata.liveInfo_.allocBitmapDuringMark_, &bitmap, newValue, false, + std::memory_order_seq_cst, std::memory_order_relaxed)) { + RegionBitmap *allocated = + HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); + __atomic_store_n(&metadata.liveInfo_.allocBitmapDuringMark_, allocated, std::memory_order_release); + DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc markbitmap %p", + this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, metadata.liveInfo_.allocBitmapDuringMark_); + return allocated; + } + } while (true); + + return nullptr; + } + + RegionBitmap* GetAllocBitmapDuringFix() + { + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.allocBitmapDuringFix_, std::memory_order_acquire); + if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { + return nullptr; + } + return bitmap; + } + + ALWAYS_INLINE RegionBitmap* GetOrAllocBitmapDuringFix() + { + do { + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.allocBitmapDuringFix_, std::memory_order_acquire); + if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { + continue; + } + if (LIKELY_CC(bitmap != nullptr)) { + return bitmap; + } + RegionBitmap *newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + if (__atomic_compare_exchange_n(&metadata.liveInfo_.allocBitmapDuringFix_, &bitmap, newValue, false, + std::memory_order_seq_cst, std::memory_order_relaxed)) { + RegionBitmap *allocated = + HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); + __atomic_store_n(&metadata.liveInfo_.allocBitmapDuringFix_, allocated, std::memory_order_release); + DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc markbitmap %p", + this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, metadata.liveInfo_.allocBitmapDuringFix_); + return allocated; + } + } while (true); + + return nullptr; + } + void ResetMarkBit() { SetMarkedRegionFlag(0); @@ -266,6 +334,24 @@ public: return marked; } + ALWAYS_INLINE bool MarkNewDuringMark(const BaseObject* obj) + { + ASSERT(GetRegionType() == RegionType::FULL_FIXED_PINNED_REGION); + size_t offset = GetAddressOffset(reinterpret_cast(obj)); + bool marked = GetOrAllocBitmapDuringMark()->MarkBits(offset); + DCHECK_CC(IsNewObjectDuringMark(obj)); + return marked; + } + + ALWAYS_INLINE bool MarkNewDuringFix(const BaseObject* obj) + { + ASSERT(GetRegionType() == RegionType::FULL_FIXED_PINNED_REGION); + size_t offset = GetAddressOffset(reinterpret_cast(obj)); + bool marked = GetOrAllocBitmapDuringFix()->MarkBits(offset); + DCHECK_CC(IsNewObjectDuringFix(obj)); + return marked; + } + bool ResurrentObject(const BaseObject* obj) { if (IsLargeRegion()) { @@ -335,6 +421,26 @@ public: return enqueBitmap->IsMarked(offset); } + bool IsNewObjectDuringMark(const BaseObject* obj) + { + RegionBitmap* bitmap = GetAllocBitmapDruingMark(); + if (bitmap == nullptr) { + return false; + } + size_t offset = GetAddressOffset(reinterpret_cast(obj)); + return bitmap->IsMarked(offset); + } + + bool IsNewObjectDuringFix(const BaseObject* obj) + { + RegionBitmap* bitmap = GetAllocBitmapDuringFix(); + if (bitmap == nullptr) { + return false; + } + size_t offset = GetAddressOffset(reinterpret_cast(obj)); + return bitmap->IsMarked(offset); + } + RegionRSet* GetRSet() { return metadata.regionRSet; @@ -1045,13 +1151,18 @@ private: markBitmap_ = nullptr; resurrectBitmap_ = nullptr; enqueueBitmap_ = nullptr; + allocBitmapDuringMark_ = nullptr; + allocBitmapDuringFix_ = nullptr; } private: RegionDesc *relatedRegion_ {nullptr}; RegionBitmap *markBitmap_ {nullptr}; RegionBitmap *resurrectBitmap_ {nullptr}; RegionBitmap *enqueueBitmap_ {nullptr}; - + // Used to record newly allocation from free slot during GC_PHASE_MARK. + RegionBitmap *allocBitmapDuringMark_ {nullptr}; + // Used to record newly allocation from free slot during GC_PHASE_FIX. + RegionBitmap *allocBitmapDuringFix_ {nullptr}; friend class RegionDesc; }; diff --git a/common_components/heap/allocator/region_list.h b/common_components/heap/allocator/region_list.h index 9548776028..f501f3a21d 100755 --- a/common_components/heap/allocator/region_list.h +++ b/common_components/heap/allocator/region_list.h @@ -165,6 +165,7 @@ protected: const char* listName_ = nullptr; private: + friend class RegionManager; void DeleteRegionLocked(RegionDesc* del); void AssignWith(const RegionList& srcList) diff --git a/common_components/heap/allocator/region_manager.cpp b/common_components/heap/allocator/region_manager.cpp index 24e7ea0be7..2d1ca348fc 100755 --- a/common_components/heap/allocator/region_manager.cpp +++ b/common_components/heap/allocator/region_manager.cpp @@ -699,19 +699,22 @@ void RegionManager::FixFixedRegionList(TraceCollector& collector, RegionList& li { size_t garbageSize = 0; RegionDesc* region = list.GetHeadRegion(); + std::lock_guard lg(list.GetListMutex()); while (region != nullptr) { auto liveBytes = region->GetLiveByteCount(); if (liveBytes == 0) { RegionDesc* del = region; region = region->GetNextRegion(); - list.DeleteRegion(del); + list.DeleteRegionLocked(del); garbageSize += CollectRegion(del); continue; } region->VisitAllObjectsWithFixedSize(cellCount, [&collector, ®ion, &cellCount, &garbageSize](BaseObject* object) { - if (collector.IsSurvivedObject(object)) { + if (RegionSpace::IsNewObjectDuringFix(object)) { + return; + } else if (collector.IsSurvivedObject(object)) { collector.FixObjectRefFields(object); } else { DLOG(ALLOC, "reclaim pinned obj %p", object); @@ -773,6 +776,19 @@ static void FixOldRegion(TraceCollector& collector, RegionDesc* region) region->VisitRememberSet(visitFunc); } +static void FixFixedRegionWithRSet(TraceCollector& collector, RegionDesc* region) +{ + auto visitFunc = [&collector, ®ion](BaseObject* object) { + if (RegionSpace::IsNewObjectDuringFix(object)) { + return; + } + DLOG(FIX, "fix: old obj %p<%p>(%zu)", object, object->GetTypeInfo(), object->GetSize()); + collector.FixObjectRefFields(object); + }; + region->VisitRememberSet(visitFunc); +} + + void RegionManager::FixOldRegionList(TraceCollector& collector, RegionList& list) { list.VisitAllRegions([&collector](RegionDesc* region) { @@ -781,6 +797,14 @@ void RegionManager::FixOldRegionList(TraceCollector& collector, RegionList& list }); } +void RegionManager::FixFixedRegionListWithRSet(TraceCollector& collector, RegionList& list) +{ + list.VisitAllRegions([&collector](RegionDesc* region) { + DLOG(REGION, "fix mature region %p@%#zx+%zu", region, region->GetRegionStart(), region->GetLiveByteCount()); + FixFixedRegionWithRSet(collector, region); + }); +} + static void FixRecentOldRegion(TraceCollector& collector, RegionDesc* region) { auto visitFunc = [&collector, ®ion](BaseObject* object) { @@ -839,7 +863,7 @@ void RegionManager::FixAllRegionLists() FixOldRegionList(collector, pinnedRegionList_); for (size_t i = 0; i < FIXED_PINNED_REGION_COUNT; i++) { FixRecentOldRegionList(collector, *recentFixedPinnedRegionList_[i]); - FixOldRegionList(collector, *fixedPinnedRegionList_[i]); + FixFixedRegionListWithRSet(collector, *fixedPinnedRegionList_[i]); } return; } @@ -1013,10 +1037,6 @@ void RegionManager::RequestForRegion(size_t size) uintptr_t RegionManager::AllocPinnedFromFreeList(size_t cellCount) { GCPhase mutatorPhase = Mutator::GetMutator()->GetMutatorPhase(); - // workaround: make sure collector doesn't fix newly allocated incomplete objects - if (mutatorPhase == GC_PHASE_MARK || mutatorPhase == GC_PHASE_FIX) { - return 0; - } RegionList* list = fixedPinnedRegionList_[cellCount]; std::lock_guard lock(list->GetListMutex()); @@ -1028,7 +1048,14 @@ uintptr_t RegionManager::AllocPinnedFromFreeList(size_t cellCount) // Mark new allocated pinned object. BaseObject* object = reinterpret_cast(allocPtr); - (reinterpret_cast(&Heap::GetHeap().GetCollector()))->MarkObject(object, cellCount); + RegionDesc* regionInfo = RegionDesc::GetRegionDescAt(allocPtr); + if (mutatorPhase == GCPhase::GC_PHASE_FIX) { + regionInfo->MarkNewDuringFix(object); + } else if (mutatorPhase == GCPhase::GC_PHASE_MARK ) { + regionInfo->MarkNewDuringMark(object); + } + regionInfo->MarkObject(object); + regionInfo->AddLiveByteCount((cellCount + 1) * sizeof(uint64_t)); return allocPtr; } @@ -1074,6 +1101,13 @@ void RegionManager::MarkRememberSet(const std::function& func auto visitFunc = [&func](RegionDesc* region) { region->VisitRememberSetBeforeTrace(func); }; + auto visitFixPinFunc = [&func](RegionDesc* region) { + region->VisitRememberSet([&func, region](BaseObject* object) { + if (!RegionSpace::IsNewObjectDuringMark(object)) { + func(object); + } + }); + }; recentPinnedRegionList_.VisitAllRegions(visitFunc); pinnedRegionList_.VisitAllRegions(visitFunc); recentLargeRegionList_.VisitAllRegions(visitFunc); @@ -1083,7 +1117,7 @@ void RegionManager::MarkRememberSet(const std::function& func for (size_t i = 0; i < FIXED_PINNED_REGION_COUNT; i++) { recentFixedPinnedRegionList_[i]->VisitAllRegions(visitFunc); - fixedPinnedRegionList_[i]->VisitAllRegions(visitFunc); + fixedPinnedRegionList_[i]->VisitAllRegions(visitFixPinFunc); } } } // namespace common diff --git a/common_components/heap/allocator/region_manager.h b/common_components/heap/allocator/region_manager.h index 7c8025df61..f2140321aa 100755 --- a/common_components/heap/allocator/region_manager.h +++ b/common_components/heap/allocator/region_manager.h @@ -77,6 +77,7 @@ public: static void FixRecentRegionList(TraceCollector& collector, RegionList& list); static void FixToRegionList(TraceCollector& collector, RegionList& list); static void FixOldRegionList(TraceCollector& collector, RegionList& list); + static void FixFixedRegionListWithRSet(TraceCollector& collector, RegionList& list); static void FixRecentOldRegionList(TraceCollector& collector, RegionList& list); void Initialize(size_t regionNum, uintptr_t regionInfoStart); diff --git a/common_components/heap/allocator/region_space.h b/common_components/heap/allocator/region_space.h index e250cc2000..7fe2cd3485 100755 --- a/common_components/heap/allocator/region_space.h +++ b/common_components/heap/allocator/region_space.h @@ -322,6 +322,19 @@ public: return regionInfo->IsEnqueuedObject(obj); } + // Newly allocatedObject from freeslot between GC_PHASE_ENUM and GC_PHASE_POST_MARK + static bool IsNewObjectDuringMark(const BaseObject* obj) + { + RegionDesc* regionInfo = RegionDesc::GetRegionDescAt(reinterpret_cast(obj)); + return regionInfo->IsNewObjectDuringMark(obj); + } + + static bool IsNewObjectDuringFix(const BaseObject* obj) + { + RegionDesc* regionInfo = RegionDesc::GetRegionDescAt(reinterpret_cast(obj)); + return regionInfo->IsNewObjectDuringFix(obj); + } + static bool IsNewObjectSinceTrace(const BaseObject* object) { RegionDesc* region = RegionDesc::GetRegionDescAt(reinterpret_cast(object)); diff --git a/common_components/heap/collector/copy_data_manager.h b/common_components/heap/collector/copy_data_manager.h index 1d1e386b49..5c7c463d71 100755 --- a/common_components/heap/collector/copy_data_manager.h +++ b/common_components/heap/collector/copy_data_manager.h @@ -159,8 +159,8 @@ private: regionUnitCount_ = unitCnt; // 64: 1 bit in bitmap marks 8bytes in region, i.e., 1 byte in bitmap marks 64 bytes in region. constexpr uint8_t bitMarksSize = 64; - // 3 bitmaps for each region: markBitmap,resurrectBitmap, enqueueBitmap. - constexpr uint8_t nBitmapTypes = 3; + // 5 bitmaps for each region: markBitmap,resurrectBitmap, enqueueBitmap. + constexpr uint8_t nBitmapTypes = 5; return unitCnt * (sizeof(RegionBitmap) + (REGION_UNIT_SIZE / bitMarksSize)) * nBitmapTypes; } -- Gitee