diff --git a/ecmascript/mem/concurrent_marker.cpp b/ecmascript/mem/concurrent_marker.cpp index 053181afc683000f3e7156f7930572283645cc52..17468bdded5c3b3d1abe39a24d7c434fb8ae3391 100644 --- a/ecmascript/mem/concurrent_marker.cpp +++ b/ecmascript/mem/concurrent_marker.cpp @@ -39,9 +39,8 @@ void ConcurrentMarker::ConcurrentMarking() ECMA_GC_LOG() << "ConcurrentMarker: Concurrent Mark Begin"; heap_->Prepare(); thread_->SetMarkStatus(MarkStatus::MARKING); - // SelectCSet - if (!heap_->IsOnlyMarkSemi()) { - ECMA_GC_LOG() << "ConcurrentMarker: not only semi, need mark all"; // select CSet in HPPGC + if (!heap_->IsOnlyMarkSemi() && heap_->GetSweeper()->IsOldSpaceSwept()) { + const_cast(heap_->GetOldSpace())->SelectCSet(); } InitializeMarking(); Platform::GetCurrentPlatform()->PostTask(std::make_unique(heap_)); @@ -97,7 +96,6 @@ void ConcurrentMarker::Reset(bool isClearCSet) } heap_->EnumerateRegions([this](Region *current) { current->SetMarking(false); - current->ResetAvailable(); current->ClearFlag(RegionFlags::IS_IN_PROMOTE_SET); }); } @@ -118,8 +116,16 @@ void ConcurrentMarker::InitializeMarking() rememberset->ClearAllBits(); } region->SetMarking(true); - region->ResetAvailable(); }); + if (heap_->IsOnlyMarkSemi()) { + heap_->EnumerateNewSpaceRegions([this](Region *current) { + current->ResetAliveObject(); + }); + } else { + heap_->EnumerateRegions([this](Region *current) { + current->ResetAliveObject(); + }); + } workList_->Initialize(TriggerGCType::OLD_GC, ParallelGCTaskPhase::CONCURRENT_HANDLE_GLOBAL_POOL_TASK); heap_->GetNonMovableMarker()->MarkRoots(0); } diff --git a/ecmascript/mem/concurrent_sweeper.cpp b/ecmascript/mem/concurrent_sweeper.cpp index 95fd49642733dd24d1ad7e5ead54079278671cd4..1039d501fec42d4167ff7a067f18dd5851137d27 100644 --- a/ecmascript/mem/concurrent_sweeper.cpp +++ b/ecmascript/mem/concurrent_sweeper.cpp @@ -56,6 +56,9 @@ void ConcurrentSweeper::SweepPhases(bool compressGC) if (!compressGC) { Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, OLD_SPACE)); + isOldSpaceSwept_ = true; + } else { + isOldSpaceSwept_ = false; } Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, NON_MOVABLE)); Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, MACHINE_CODE_SPACE)); @@ -63,6 +66,9 @@ void ConcurrentSweeper::SweepPhases(bool compressGC) if (!compressGC) { SweepSpace(OLD_SPACE, const_cast(heap_->GetOldSpace()), heap_->GetHeapManager()->GetOldSpaceAllocator()); + isOldSpaceSwept_ = true; + } else { + isOldSpaceSwept_ = false; } SweepSpace(NON_MOVABLE, const_cast(heap_->GetNonMovableSpace()), heap_->GetHeapManager()->GetNonMovableSpaceAllocator()); diff --git a/ecmascript/mem/concurrent_sweeper.h b/ecmascript/mem/concurrent_sweeper.h index 0a1252da2d7dc5c9245c234da4eb9784e2f76fb4..2823400bffdf7ba56a76c737fea654b7712b3fc9 100644 --- a/ecmascript/mem/concurrent_sweeper.h +++ b/ecmascript/mem/concurrent_sweeper.h @@ -47,6 +47,11 @@ public: return concurrentSweep_; } + bool IsOldSpaceSwept() const + { + return isOldSpaceSwept_; + } + private: class SweeperTask : public Task { public: @@ -85,8 +90,9 @@ private: std::array, FREE_LIST_NUM> sweptList_; Heap *heap_; - bool concurrentSweep_ = false; - bool isSweeping_ = false; + bool concurrentSweep_ {false}; + bool isSweeping_ {false}; + bool isOldSpaceSwept_ {false}; MemSpaceType startSpaceType_ = MemSpaceType::OLD_SPACE; }; } // namespace panda::ecmascript diff --git a/ecmascript/mem/ecma_heap_manager-inl.h b/ecmascript/mem/ecma_heap_manager-inl.h index bb8cdfeaf772d37d36aa3f9961985071dd493c01..459d4528223070cf87381a8e27c189c9c8126c72 100644 --- a/ecmascript/mem/ecma_heap_manager-inl.h +++ b/ecmascript/mem/ecma_heap_manager-inl.h @@ -171,6 +171,8 @@ TaggedObject *EcmaHeapManager::AllocateOldGenerationOrHugeObject(JSHClass *hclas } SetClass(object, hclass); heap_->OnAllocateEvent(reinterpret_cast(object)); + Region *objectRegion = Region::ObjectAddressToRange(object); + objectRegion->IncrementAliveObject(size); return object; } diff --git a/ecmascript/mem/evacuation_allocator.cpp b/ecmascript/mem/evacuation_allocator.cpp index ac9c02a2685e6b45e32988a1cc870430d8d6f6cd..da03b237bca41bd2fa9d8622cb48337510003713 100644 --- a/ecmascript/mem/evacuation_allocator.cpp +++ b/ecmascript/mem/evacuation_allocator.cpp @@ -38,6 +38,7 @@ void EvacuationAllocator::Initialize(TriggerGCType type) } else if (type == TriggerGCType::COMPRESS_FULL_GC) { heap_->InitializeCompressSpace(); auto compressSpace = const_cast(heap_->GetCompressSpace()); + compressSpace->GetCurrentRegion()->SetAliveObject(DEFAULT_REGION_SIZE - sizeof(Region)); oldSpaceAllocator_.Reset(compressSpace); } else { oldSpaceAllocator_.Swap(heapManager->GetOldSpaceAllocator()); diff --git a/ecmascript/mem/heap-inl.h b/ecmascript/mem/heap-inl.h index 33fb608e4d5e0d5bee2cc85f29703c1fe8fdd564..2594de660603b48d7c83b52402b200743ac88874 100644 --- a/ecmascript/mem/heap-inl.h +++ b/ecmascript/mem/heap-inl.h @@ -93,6 +93,9 @@ bool Heap::FillNewSpaceAndTryGC(BumpPointerAllocator *spaceAllocator, bool allow bool Heap::FillOldSpaceAndTryGC(FreeListAllocator *spaceAllocator, bool allowGc) { if (oldSpace_->Expand()) { + if (!allowGc) { + GetCompressSpace()->GetCurrentRegion()->SetAliveObject(DEFAULT_REGION_SIZE - sizeof(Region)); + } spaceAllocator->AddFree(oldSpace_->GetCurrentRegion()); return true; } diff --git a/ecmascript/mem/mix_space_collector.cpp b/ecmascript/mem/mix_space_collector.cpp index e9a47bd5925e4b788cebf03bff677735cea72001..e27e9a888b250093b1fb69b690a8d7f6a76c10a7 100644 --- a/ecmascript/mem/mix_space_collector.cpp +++ b/ecmascript/mem/mix_space_collector.cpp @@ -40,7 +40,6 @@ void MixSpaceCollector::RunPhases() ClockScope clockScope; concurrentMark_ = heap_->CheckConcurrentMark(thread); - // SelectCSet InitializePhase(); MarkingPhase(); SweepPhases(); @@ -55,6 +54,9 @@ void MixSpaceCollector::InitializePhase() { if (!concurrentMark_) { heap_->Prepare(); + if (!heap_->IsOnlyMarkSemi() && heap_->GetSweeper()->IsOldSpaceSwept()) { + const_cast(heap_->GetOldSpace())->SelectCSet(); + } heap_->EnumerateRegions([](Region *current) { // ensure mark bitmap auto bitmap = current->GetMarkBitmap(); @@ -68,8 +70,16 @@ void MixSpaceCollector::InitializePhase() rememberset->ClearAllBits(); } current->SetMarking(false); - current->ResetAvailable(); }); + if (heap_->IsOnlyMarkSemi()) { + heap_->EnumerateNewSpaceRegions([this](Region *current) { + current->ResetAliveObject(); + }); + } else { + heap_->EnumerateRegions([this](Region *current) { + current->ResetAliveObject(); + }); + } workList_->Initialize(TriggerGCType::OLD_GC, ParallelGCTaskPhase::OLD_HANDLE_GLOBAL_POOL_TASK); freeSize_ = 0; @@ -88,7 +98,6 @@ void MixSpaceCollector::FinishPhase() size_t aliveSize = 0; workList_->Finish(aliveSize); heap_->EnumerateRegions([this](Region *current) { - current->ResetAvailable(); current->ClearFlag(RegionFlags::IS_IN_PROMOTE_SET); }); } diff --git a/ecmascript/mem/parallel_evacuation-inl.h b/ecmascript/mem/parallel_evacuation-inl.h index 75c6d064c29568a910dfffc5873f8567c9be1e38..87488e284e484814b38621dfaf3623870ac9e54e 100644 --- a/ecmascript/mem/parallel_evacuation-inl.h +++ b/ecmascript/mem/parallel_evacuation-inl.h @@ -28,7 +28,7 @@ namespace panda::ecmascript { bool ParallelEvacuation::IsPromoteComplete(Region *region) { - return (static_cast(region->Available()) / DEFAULT_REGION_SIZE) > MIN_OBJECT_SURVIVAL_RATE && + return (static_cast(region->AliveObject()) / DEFAULT_REGION_SIZE) > MIN_OBJECT_SURVIVAL_RATE && !region->HasAgeMark(); } diff --git a/ecmascript/mem/parallel_work_helper.cpp b/ecmascript/mem/parallel_work_helper.cpp index b4a7f9fb5e5e2aa3b25b2c1d03c4932829f9e394..a2abb758ccc4345bc994975e3eb6c1a8a06cfa2d 100644 --- a/ecmascript/mem/parallel_work_helper.cpp +++ b/ecmascript/mem/parallel_work_helper.cpp @@ -60,7 +60,7 @@ bool WorkerHelper::Push(uint32_t threadId, TaggedObject *object, Region *region) if (Push(threadId, object)) { auto klass = object->GetClass(); auto size = klass->SizeFromJSHClass(object); - region->IncrementAvailable(size); + region->IncrementAliveObject(size); return true; } return false; diff --git a/ecmascript/mem/region.h b/ecmascript/mem/region.h index 579a97fad0a1cbf6158d6c48d204d371d9bc9c99..829d6fd6f50d4efd86619d009243e4b44ff70ef4 100644 --- a/ecmascript/mem/region.h +++ b/ecmascript/mem/region.h @@ -44,6 +44,8 @@ enum RegionFlags { IS_INVALID = 1 << 10, }; +static constexpr double MOST_OBJECT_ALIVE_THRESHOLD_PERCENT = 0.8; + class Region { public: Region(Space *space, uintptr_t allocateBase, uintptr_t begin, uintptr_t end) @@ -54,7 +56,7 @@ public: begin_(begin), end_(end), highWaterMark_(end), - available_(0) + aliveObject_(0) { } ~Region() = default; @@ -159,6 +161,11 @@ public: return reinterpret_cast(ToUintPtr(obj) & ~DEFAULT_REGION_MASK); } + static Region *ObjectAddressToRange(uintptr_t objAddress) + { + return reinterpret_cast(objAddress & ~DEFAULT_REGION_MASK); + } + bool InYoungGeneration() const { return IsFlagSet(RegionFlags::IS_IN_YOUNG_GENERATION); @@ -293,19 +300,34 @@ public: inline WorkerHelper *GetWorkList() const; - void IncrementAvailable(size_t size) + void IncrementAliveObject(size_t size) + { + aliveObject_ += size; + } + + void DecreaseAliveObject(size_t size) + { + aliveObject_ -= size; + } + + void SetAliveObject(size_t size) + { + aliveObject_ = size; + } + + void ResetAliveObject() { - available_ += size; + aliveObject_ = 0; } - void ResetAvailable() + size_t AliveObject() const { - available_ = 0; + return aliveObject_; } - size_t Available() + bool MostObjectAlive() const { - return available_; + return aliveObject_ > MOST_OBJECT_ALIVE_THRESHOLD_PERCENT * GetSize(); } private: @@ -316,7 +338,7 @@ private: uintptr_t end_; uintptr_t highWaterMark_; bool marking_ {false}; - std::atomic_size_t available_ = 0; + std::atomic_size_t aliveObject_ {0}; Region *next_ {nullptr}; Region *prev_ {nullptr}; RangeBitmap *markBitmap_ {nullptr}; diff --git a/ecmascript/mem/space.cpp b/ecmascript/mem/space.cpp index 49fd0e0c66676801dbca253b60545ecfc38eb725..40e5093a920d61cfb9d599cc14ce531d11cbd116 100644 --- a/ecmascript/mem/space.cpp +++ b/ecmascript/mem/space.cpp @@ -379,6 +379,30 @@ void OldSpace::ReclaimRegionCSet() collectRegionSet_.clear(); } +void OldSpace::SelectCSet() +{ + EnumerateRegions([this](Region *region) { + if (!region->MostObjectAlive()) { + collectRegionSet_.emplace_back(region); + } + }); + if (collectRegionSet_.size() < PARTIAL_GC_MIN_COLLECT_REGION_SIZE) { + heap_->SetOnlyMarkSemi(true); + collectRegionSet_.clear(); + return; + } + // sort + std::sort(collectRegionSet_.begin(), collectRegionSet_.end(), [](Region *first, Region *second) { + return first->AliveObject() < second->AliveObject(); + }); + if (collectRegionSet_.size() > PARTIAL_GC_MAX_COLLECT_REGION_SIZE) { + collectRegionSet_.resize(PARTIAL_GC_MAX_COLLECT_REGION_SIZE); + } + for (Region *region : collectRegionSet_) { + region->SetFlag(RegionFlags::IS_IN_COLLECT_SET); + } +} + NonMovableSpace::NonMovableSpace(Heap *heap, size_t initialCapacity, size_t maximumCapacity) : Space(heap, MemSpaceType::NON_MOVABLE, initialCapacity, maximumCapacity) { diff --git a/ecmascript/mem/space.h b/ecmascript/mem/space.h index 186244be4458ce855ab0dda31454224f70fb5865..d123a6c5cd4cf5b5c40ada14d507a6c5a39b27a4 100644 --- a/ecmascript/mem/space.h +++ b/ecmascript/mem/space.h @@ -75,6 +75,8 @@ static inline std::string ToSpaceTypeName(MemSpaceType type) } static constexpr size_t SPACE_TYPE_SIZE = helpers::ToUnderlying(MemSpaceType::SPACE_TYPE_LAST); +static constexpr size_t PARTIAL_GC_MAX_COLLECT_REGION_SIZE = 16; +static constexpr size_t PARTIAL_GC_MIN_COLLECT_REGION_SIZE = 5; class Space { public: @@ -232,9 +234,10 @@ public: void EnumerateCollectRegionSet(const Callback &cb) const; template void EnumerateNonCollectRegionSet(const Callback &cb) const; + void SelectCSet(); private: - std::vector collectRegionSet_; + CVector collectRegionSet_; }; class NonMovableSpace : public Space { diff --git a/ecmascript/mem/tlab_allocator-inl.h b/ecmascript/mem/tlab_allocator-inl.h index d5880ad069a622df25e52a77ec974281ac929575..4efa6b7d17286628b38d1887cdba0f6cf23c775c 100644 --- a/ecmascript/mem/tlab_allocator-inl.h +++ b/ecmascript/mem/tlab_allocator-inl.h @@ -46,6 +46,8 @@ inline void TlabAllocator::Finalize() } if (oldBumpPointerAllocator_.Available() != 0) { allocator_->FreeSafe(oldBumpPointerAllocator_.GetTop(), oldBumpPointerAllocator_.GetEnd()); + Region *current = Region::ObjectAddressToRange(oldBumpPointerAllocator_.GetTop()); + current->DecreaseAliveObject(oldBumpPointerAllocator_.Available()); oldBumpPointerAllocator_.Reset(); } } @@ -97,6 +99,10 @@ uintptr_t TlabAllocator::TlabAllocatorOldSpace(size_t size) oldBumpPointerAllocator_.Available()); return result; } + Region *current = Region::ObjectAddressToRange(oldBumpPointerAllocator_.GetTop()); + if (current != nullptr) { + current->DecreaseAliveObject(oldBumpPointerAllocator_.Available()); + } allocator_->FreeSafe(oldBumpPointerAllocator_.GetTop(), oldBumpPointerAllocator_.GetEnd()); if (!ExpandOld()) { return 0; @@ -137,6 +143,7 @@ bool TlabAllocator::ExpandOld() if (region == nullptr) { return false; } + region->SetAliveObject(DEFAULT_REGION_SIZE - sizeof(Region)); oldBumpPointerAllocator_.Reset(region->GetBegin(), region->GetEnd()); FreeObject::FillFreeObject(heap_->GetEcmaVM(), oldBumpPointerAllocator_.GetTop(), oldBumpPointerAllocator_.Available());