From c259d52c5f151f80ff7f6abc3908943e16fa7cba Mon Sep 17 00:00:00 2001 From: "yingguofeng@huawei.com" Date: Fri, 24 Sep 2021 14:31:54 +0800 Subject: [PATCH] Concurrent sweep Signed-off-by: yingguofeng@huawei.com Change-Id: I2a5a0172577971c96e156f99b5ccf1f6c3d0063c --- BUILD.gn | 4 + ecmascript/ecma_vm.cpp | 3 + ecmascript/hprof/heap_profiler.cpp | 2 + ecmascript/mem/allocator-inl.h | 55 ++++- ecmascript/mem/allocator.h | 15 +- ecmascript/mem/compress_collector.cpp | 63 +----- ecmascript/mem/compress_collector.h | 1 - ecmascript/mem/concurrent_sweeper.cpp | 247 +++++++++++++++++++++ ecmascript/mem/concurrent_sweeper.h | 93 ++++++++ ecmascript/mem/ecma_heap_manager-inl.h | 22 +- ecmascript/mem/ecma_heap_manager.cpp | 5 +- ecmascript/mem/ecma_heap_manager.h | 21 +- ecmascript/mem/free_object_kind.cpp | 3 + ecmascript/mem/free_object_kind.h | 5 +- ecmascript/mem/free_object_list.cpp | 114 ++++++---- ecmascript/mem/free_object_list.h | 20 +- ecmascript/mem/heap.cpp | 31 +-- ecmascript/mem/heap.h | 19 +- ecmascript/mem/old_space_collector-inl.h | 22 +- ecmascript/mem/old_space_collector.cpp | 125 ++++------- ecmascript/mem/old_space_collector.h | 21 +- ecmascript/mem/region.h | 36 ++- ecmascript/mem/semi_space_collector.cpp | 2 +- ecmascript/mem/semi_space_collector.h | 1 - ecmascript/mem/semi_space_worker.cpp | 29 +++ ecmascript/mem/semi_space_worker.h | 15 +- ecmascript/mem/space.cpp | 30 ++- ecmascript/mem/space.h | 12 +- ecmascript/mem/tlab_allocator-inl.h | 2 +- ecmascript/object_factory.cpp | 2 + ecmascript/platform/platform.cpp | 45 ++++ ecmascript/platform/platform.h | 59 +++++ ecmascript/platform/runner.cpp | 46 ++++ ecmascript/platform/runner.h | 48 ++++ ecmascript/platform/task.h | 32 +++ ecmascript/platform/task_queue.cpp | 50 +++++ ecmascript/platform/task_queue.h | 49 ++++ ecmascript/tests/BUILD.gn | 1 + ecmascript/tests/concurrent_sweep_test.cpp | 64 ++++++ 39 files changed, 1111 insertions(+), 303 deletions(-) create mode 100644 ecmascript/mem/concurrent_sweeper.cpp create mode 100644 ecmascript/mem/concurrent_sweeper.h create mode 100644 ecmascript/platform/platform.cpp create mode 100644 ecmascript/platform/platform.h create mode 100644 ecmascript/platform/runner.cpp create mode 100644 ecmascript/platform/runner.h create mode 100644 ecmascript/platform/task.h create mode 100644 ecmascript/platform/task_queue.cpp create mode 100644 ecmascript/platform/task_queue.h create mode 100644 ecmascript/tests/concurrent_sweep_test.cpp diff --git a/BUILD.gn b/BUILD.gn index 28f5d743be..4b090339b0 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -338,6 +338,7 @@ ecma_source = [ "ecmascript/mem/c_string.cpp", "ecmascript/mem/chunk.cpp", "ecmascript/mem/compress_collector.cpp", + "ecmascript/mem/concurrent_sweeper.cpp", "ecmascript/mem/ecma_heap_manager.cpp", "ecmascript/mem/free_object_kind.cpp", "ecmascript/mem/free_object_list.cpp", @@ -355,6 +356,9 @@ ecma_source = [ "ecmascript/napi/jsnapi.cpp", "ecmascript/object_factory.cpp", "ecmascript/object_operator.cpp", + "ecmascript/platform/platform.cpp", + "ecmascript/platform/runner.cpp", + "ecmascript/platform/task_queue.cpp", "ecmascript/layout_info.cpp", "ecmascript/regexp/dyn_chunk.cpp", "ecmascript/regexp/regexp_executor.cpp", diff --git a/ecmascript/ecma_vm.cpp b/ecmascript/ecma_vm.cpp index 170156ee54..e00c3723ca 100644 --- a/ecmascript/ecma_vm.cpp +++ b/ecmascript/ecma_vm.cpp @@ -36,6 +36,7 @@ #include "ecmascript/mem/heap.h" #include "ecmascript/tagged_dictionary.h" #include "ecmascript/object_factory.h" +#include "ecmascript/platform/platform.h" #include "ecmascript/regexp/regexp_parser_cache.h" #include "ecmascript/runtime_call_id.h" #include "ecmascript/runtime_trampolines.h" @@ -117,6 +118,7 @@ EcmaVM::EcmaVM(RuntimeOptions options) bool EcmaVM::Initialize() { trace::ScopedTrace scoped_trace("EcmaVM::Initialize"); + Platform::GetCurrentPlatform()->Initialize(); RuntimeTrampolines::InitializeRuntimeTrampolines(thread_); @@ -232,6 +234,7 @@ bool EcmaVM::InitializeFinish() } EcmaVM::~EcmaVM() { + Platform::GetCurrentPlatform()->Destory(); vmInitialized_ = false; ClearNativeMethodsData(); diff --git a/ecmascript/hprof/heap_profiler.cpp b/ecmascript/hprof/heap_profiler.cpp index 42a3801244..e2a8b3457a 100644 --- a/ecmascript/hprof/heap_profiler.cpp +++ b/ecmascript/hprof/heap_profiler.cpp @@ -23,6 +23,7 @@ #include "ecmascript/hprof/heap_snapshot.h" #include "ecmascript/js_thread.h" #include "ecmascript/mem/assert_scope-inl.h" +#include "ecmascript/mem/concurrent_sweeper.h" #include "ecmascript/mem/heap.h" namespace panda::ecmascript { @@ -161,6 +162,7 @@ HeapSnapShot *HeapProfiler::MakeHeapSnapShot(JSThread *thread, SampleType sample { LOG(ERROR, RUNTIME) << "HeapProfiler::MakeHeapSnapShot"; DISALLOW_GARBAGE_COLLECTION; + heap_->GetSweeper()->EnsureAllTaskFinish(); switch (sampleType) { case SampleType::ONE_SHOT: { auto *snapShot = const_cast(heap_->GetRegionFactory())->New(thread, heap_); diff --git a/ecmascript/mem/allocator-inl.h b/ecmascript/mem/allocator-inl.h index bf0d150fcb..f9ddfe7345 100644 --- a/ecmascript/mem/allocator-inl.h +++ b/ecmascript/mem/allocator-inl.h @@ -19,6 +19,7 @@ #include #include "ecmascript/mem/allocator.h" +#include "ecmascript/mem/concurrent_sweeper.h" #include "ecmascript/mem/heap-inl.h" #include "ecmascript/mem/space.h" #include "ecmascript/free_object.h" @@ -67,13 +68,13 @@ uintptr_t BumpPointerAllocator::Allocate(size_t size) return result; } -FreeListAllocator::FreeListAllocator(const Space *space) +FreeListAllocator::FreeListAllocator(const Space *space) : heap_(space->GetHeap()), type_(space->GetSpaceType()) { freeList_ = std::make_unique(); - heap_ = space->GetHeap(); uintptr_t begin = space->GetCurrentRegion()->GetBegin(); size_t size = space->GetCurrentRegion()->GetSize(); FreeObject::Cast(begin)->SetAvailable(size); + FreeObject::Cast(begin)->SetNext(nullptr); freeList_->Free(begin, size); } @@ -96,18 +97,39 @@ uintptr_t FreeListAllocator::Allocate(size_t size) } FreeObject *object = freeList_->Allocator(size); if (LIKELY(object != nullptr && !object->IsEmpty())) { - FreeBumpPoint(); - bpAllocator_.Reset(object->GetBegin(), object->GetEnd()); - ret = bpAllocator_.Allocate(size); - if (ret != 0 && bpAllocator_.Available() > 0) { - FreeObject::FillFreeObject(heap_->GetEcmaVM(), bpAllocator_.GetTop(), bpAllocator_.Available()); + return Allocate(object, size); + } + + if (sweeping_) { + // Concurrent sweep maybe sweep same region + heap_->GetSweeper()->FillSweptRegion(type_); + object = freeList_->Allocator(size); + if (LIKELY(object != nullptr && !object->IsEmpty())) { + return Allocate(object, size); + } + + // Parallel + heap_->GetSweeper()->WaitingTaskFinish(type_); + object = freeList_->Allocator(size); + if (LIKELY(object != nullptr && !object->IsEmpty())) { + return Allocate(object, size); } - return ret; } return 0; } +uintptr_t FreeListAllocator::Allocate(FreeObject *object, size_t size) +{ + FreeBumpPoint(); + bpAllocator_.Reset(object->GetBegin(), object->GetEnd()); + auto ret = bpAllocator_.Allocate(size); + if (ret != 0 && bpAllocator_.Available() > 0) { + FreeObject::FillFreeObject(heap_->GetEcmaVM(), bpAllocator_.GetTop(), bpAllocator_.Available()); + } + return ret; +} + void FreeListAllocator::FreeBumpPoint() { auto begin = bpAllocator_.GetTop(); @@ -116,7 +138,7 @@ void FreeListAllocator::FreeBumpPoint() bpAllocator_.Reset(); } -void FreeListAllocator::Free(uintptr_t begin, uintptr_t end) +void FreeListAllocator::Free(uintptr_t begin, uintptr_t end, bool isAdd) { ASSERT(heap_ != nullptr); size_t size = end - begin; @@ -125,7 +147,7 @@ void FreeListAllocator::Free(uintptr_t begin, uintptr_t end) } FreeObject::FillFreeObject(heap_->GetEcmaVM(), begin, size); - freeList_->Free(begin, static_cast(end - begin)); + freeList_->Free(begin, size, isAdd); } void FreeListAllocator::RebuildFreeList() @@ -133,5 +155,18 @@ void FreeListAllocator::RebuildFreeList() bpAllocator_.Reset(); freeList_->Rebuild(); } + +void FreeListAllocator::FillFreeList(FreeObjectKind *kind) +{ + freeList_->AddKind(kind); +} + +size_t FreeListAllocator::GetAvailableSize() const +{ + if (sweeping_) { + heap_->GetSweeper()->WaitingTaskFinish(type_); + } + return freeList_->GetFreeObjectSize(); +} } // namespace panda::ecmascript #endif // ECMASCRIPT_MEM_ALLOCATOR_INL_H diff --git a/ecmascript/mem/allocator.h b/ecmascript/mem/allocator.h index 25b3d3d4cc..0ea0cf0f08 100644 --- a/ecmascript/mem/allocator.h +++ b/ecmascript/mem/allocator.h @@ -91,28 +91,37 @@ public: inline void AddFree(Region *region); inline void RebuildFreeList(); + inline void FillFreeList(FreeObjectKind *kind); void Swap(FreeListAllocator &other) { heap_ = other.heap_; bpAllocator_.Swap(other.bpAllocator_); freeList_.swap(other.freeList_); + type_ = other.type_; + sweeping_ = other.sweeping_; } inline void FreeBumpPoint(); - inline void Free(uintptr_t begin, uintptr_t end); + inline void Free(uintptr_t begin, uintptr_t end, bool isAdd = true); inline void SplitFreeObject(FreeObject *current, size_t allocateSize); - size_t GetAvailableSize() const + inline size_t GetAvailableSize() const; + + void SetSweeping(bool sweeping) { - return freeList_->GetFreeObjectSize(); + sweeping_ = sweeping; } private: + inline uintptr_t Allocate(FreeObject *object, size_t size); + BumpPointerAllocator bpAllocator_; std::unique_ptr freeList_; Heap *heap_{nullptr}; + MemSpaceType type_ = OLD_SPACE; + bool sweeping_ = false; }; } // namespace panda::ecmascript diff --git a/ecmascript/mem/compress_collector.cpp b/ecmascript/mem/compress_collector.cpp index db3759594c..8fd14bbb39 100644 --- a/ecmascript/mem/compress_collector.cpp +++ b/ecmascript/mem/compress_collector.cpp @@ -13,9 +13,10 @@ * limitations under the License. */ +#include "ecmascript/mem/compress_collector.h" + #include "ecmascript/ecma_vm.h" #include "ecmascript/mem/clock_scope.h" -#include "ecmascript/mem/compress_collector.h" #include "ecmascript/mem/compress_gc_marker-inl.h" #include "ecmascript/mem/ecma_heap_manager.h" #include "ecmascript/mem/heap-inl.h" @@ -58,6 +59,7 @@ void CompressCollector::RunPhases() void CompressCollector::InitializePhase() { heap_->GetThreadPool()->WaitTaskFinish(); + heap_->GetSweeper()->EnsureAllTaskFinish(); auto compressSpace = const_cast(heap_->GetCompressSpace()); if (compressSpace->GetCommittedSize() == 0) { compressSpace->Initialize(); @@ -70,8 +72,6 @@ void CompressCollector::InitializePhase() FreeListAllocator compressAllocator(compressSpace); oldSpaceAllocator_.Swap(compressAllocator); fromSpaceAllocator_.Reset(fromSpace); - auto heapManager = heap_->GetHeapManager(); - nonMovableAllocator_.Swap(heapManager->GetNonMovableSpaceAllocator()); auto callback = [](Region *current) { // ensure mark bitmap @@ -87,6 +87,7 @@ void CompressCollector::InitializePhase() } }; heap_->GetNonMovableSpace()->EnumerateRegions(callback); + heap_->GetMachineCodeSpace()->EnumerateRegions(callback); heap_->GetSnapShotSpace()->EnumerateRegions(callback); heap_->GetHugeObjectSpace()->EnumerateRegions(callback); @@ -116,7 +117,6 @@ void CompressCollector::FinishPhase() workList_->Finish(youngAndOldAliveSize_); auto heapManager = heap_->GetHeapManager(); heapManager->GetOldSpaceAllocator().Swap(oldSpaceAllocator_); - heapManager->GetNonMovableSpaceAllocator().Swap(nonMovableAllocator_); heapManager->GetNewSpaceAllocator().Swap(fromSpaceAllocator_); } @@ -230,60 +230,7 @@ void CompressCollector::SweepPhases() heap_->GetEcmaVM()->GetJSThread()->IterateWeakEcmaGlobalStorage(gcUpdateWeak); heap_->GetEcmaVM()->ProcessReferences(gcUpdateWeak); - SweepSpace(const_cast(heap_->GetNonMovableSpace()), nonMovableAllocator_); - SweepSpace(const_cast(heap_->GetHugeObjectSpace())); -} - -void CompressCollector::SweepSpace(HugeObjectSpace *space) -{ - Region *currentRegion = space->GetRegionList().GetFirst(); - while (currentRegion != nullptr) { - Region *next = currentRegion->GetNext(); - auto markBitmap = currentRegion->GetMarkBitmap(); - bool isMarked = false; - markBitmap->IterateOverMarkedChunks([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; }); - if (!isMarked) { - space->GetRegionList().RemoveNode(currentRegion); - space->ClearAndFreeRegion(currentRegion); - } - currentRegion = next; - } -} - -void CompressCollector::SweepSpace(Space *space, FreeListAllocator &allocator) -{ - allocator.RebuildFreeList(); - space->EnumerateRegions([this, &allocator](Region *current) { - auto markBitmap = current->GetMarkBitmap(); - ASSERT(markBitmap != nullptr); - uintptr_t freeStart = current->GetBegin(); - markBitmap->IterateOverMarkedChunks([this, ¤t, &freeStart, &allocator](void *mem) { - ASSERT(current->InRange(ToUintPtr(mem))); - auto header = reinterpret_cast(mem); - auto klass = header->GetClass(); - JSType jsType = klass->GetObjectType(); - auto size = klass->SizeFromJSHClass(jsType, header); - size = AlignUp(size, static_cast(MemAlignment::MEM_ALIGN_OBJECT)); - - uintptr_t freeEnd = ToUintPtr(mem); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - freeStart = freeEnd + size; - }); - uintptr_t freeEnd = current->GetEnd(); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - }); -} - -void CompressCollector::FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, - uintptr_t freeEnd) -{ - allocator.Free(freeStart, freeEnd); - nonMoveSpaceFreeSize_ += freeEnd - freeStart; - heap_->ClearSlotsRange(current, freeStart, freeEnd); + heap_->GetSweeper()->SweepPhases(true); } uintptr_t CompressCollector::AllocateOld(size_t size) diff --git a/ecmascript/mem/compress_collector.h b/ecmascript/mem/compress_collector.h index 535a3f47a0..8cfb405ed1 100644 --- a/ecmascript/mem/compress_collector.h +++ b/ecmascript/mem/compress_collector.h @@ -61,7 +61,6 @@ private: os::memory::Mutex mtx_; BumpPointerAllocator fromSpaceAllocator_{}; FreeListAllocator oldSpaceAllocator_{}; - FreeListAllocator nonMovableAllocator_{}; size_t youngAndOldAliveSize_ = 0; size_t nonMoveSpaceFreeSize_ = 0; diff --git a/ecmascript/mem/concurrent_sweeper.cpp b/ecmascript/mem/concurrent_sweeper.cpp new file mode 100644 index 0000000000..59068857bb --- /dev/null +++ b/ecmascript/mem/concurrent_sweeper.cpp @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/mem/concurrent_sweeper.h" + +#include "ecmascript/js_hclass-inl.h" +#include "ecmascript/mem/allocator-inl.h" +#include "ecmascript/mem/ecma_heap_manager.h" +#include "ecmascript/mem/free_object_list.h" +#include "ecmascript/mem/heap.h" +#include "ecmascript/platform/platform.h" + +namespace panda::ecmascript { +ConcurrentSweeper::ConcurrentSweeper(Heap *heap, bool concurrentSweep) + : heap_(heap), concurrentSweep_(concurrentSweep) +{ +} + +void ConcurrentSweeper::SweepPhases(bool compressGC) +{ + if (concurrentSweep_) { + // Add all region to region list. Ensure all task finish + trace::ScopedTrace scoped_trace("ConcurrentSweeper::SweepPhases"); + if (!compressGC) { + heap_->GetOldSpace()->EnumerateRegions([this](Region *current) { AddRegion(OLD_SPACE, current); }); + } + heap_->GetNonMovableSpace()->EnumerateRegions([this](Region *current) { AddRegion(NON_MOVABLE, current); }); + heap_->GetMachineCodeSpace()->EnumerateRegions([this](Region *current) { + AddRegion(MACHINE_CODE_SPACE, current); + }); + + // Prepare + isSweeping_ = true; + startSpaceType_ = compressGC ? NON_MOVABLE : OLD_SPACE; + for (int type = startSpaceType_; type < FREE_LIST_NUM; type++) { + auto spaceType = static_cast(type); + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(spaceType); + remainderTaskNum_[type] = FREE_LIST_NUM - startSpaceType_; + allocator.SetSweeping(true); + allocator.RebuildFreeList(); + } + + if (!compressGC) { + Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, OLD_SPACE)); + } + Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, NON_MOVABLE)); + Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, MACHINE_CODE_SPACE)); + } else { + if (!compressGC) { + SweepSpace(const_cast(heap_->GetOldSpace()), heap_->GetHeapManager()->GetOldSpaceAllocator()); + } + SweepSpace(const_cast(heap_->GetNonMovableSpace()), + heap_->GetHeapManager()->GetNonMovableSpaceAllocator()); + SweepSpace(const_cast(heap_->GetMachineCodeSpace()), + heap_->GetHeapManager()->GetMachineCodeSpaceAllocator()); + } + SweepHugeSpace(); +} + +void ConcurrentSweeper::SweepSpace(MemSpaceType type, bool isMain) +{ + trace::ScopedTrace scoped_trace("Sweeper::SweepSpace"); + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(type); + Region *current = GetRegionSafe(type); + while (current != nullptr) { + FreeRegion(current, allocator, isMain); + // Main thread sweeping region is added; + if (!isMain) { + AddSweptRegionSafe(type, current); + } + current = GetRegionSafe(type); + } + + if (!isMain) { + os::memory::LockHolder holder(mutexs_[type]); + remainderTaskNum_[type]--; + if (remainderTaskNum_[type] == 0) { + cvs_[type].SignalAll(); + } + } +} + +void ConcurrentSweeper::SweepSpace(Space *space, FreeListAllocator &allocator) +{ + allocator.RebuildFreeList(); + space->EnumerateRegions([this, &allocator](Region *current) { FreeRegion(current, allocator); }); +} + +void ConcurrentSweeper::SweepHugeSpace() +{ + trace::ScopedTrace scoped_trace("SweepSpace HugeObject"); + HugeObjectSpace *space = const_cast(heap_->GetHugeObjectSpace()); + Region *currentRegion = space->GetRegionList().GetFirst(); + + while (currentRegion != nullptr) { + Region *next = currentRegion->GetNext(); + auto markBitmap = currentRegion->GetMarkBitmap(); + bool isMarked = false; + markBitmap->IterateOverMarkedChunks([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; }); + if (!isMarked) { + space->GetRegionList().RemoveNode(currentRegion); + space->ClearAndFreeRegion(currentRegion); + } + currentRegion = next; + } +} + +void ConcurrentSweeper::FreeRegion(Region *current, FreeListAllocator &allocator, bool isMain) +{ + auto markBitmap = current->GetMarkBitmap(); + ASSERT(markBitmap != nullptr); + uintptr_t freeStart = current->GetBegin(); + markBitmap->IterateOverMarkedChunks([this, ¤t, &freeStart, &allocator, isMain](void *mem) { + ASSERT(current->InRange(ToUintPtr(mem))); + auto header = reinterpret_cast(mem); + auto klass = header->GetClass(); + JSType jsType = klass->GetObjectType(); + auto size = klass->SizeFromJSHClass(jsType, header); + size = AlignUp(size, static_cast(MemAlignment::MEM_ALIGN_OBJECT)); + + uintptr_t freeEnd = ToUintPtr(mem); + if (freeStart != freeEnd) { + FreeLiveRange(allocator, current, freeStart, freeEnd, isMain); + } + freeStart = freeEnd + size; + }); + uintptr_t freeEnd = current->GetEnd(); + if (freeStart != freeEnd) { + FreeLiveRange(allocator, current, freeStart, freeEnd, isMain); + } +} + +void ConcurrentSweeper::FillSweptRegion(MemSpaceType type) +{ + trace::ScopedTrace scoped_trace("Sweeper::FillSweptRegion"); + if (sweptList_[type].empty()) { + return; + } + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(type); + Region *region = nullptr; + while ((region = GetSweptRegionSafe(type)) != nullptr) { + region->EnumerateKinds([&allocator](FreeObjectKind *kind) { + if (kind == nullptr || kind->Empty()) { + return; + } + allocator.FillFreeList(kind); + }); + } +} + +void ConcurrentSweeper::FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, + uintptr_t freeEnd, bool isMain) +{ + allocator.Free(freeStart, freeEnd, isMain); + heap_->ClearSlotsRange(current, freeStart, freeEnd); +} + +void ConcurrentSweeper::AddRegion(MemSpaceType type, Region *region) +{ + sweepingList_[type].emplace_back(region); +} + +Region *ConcurrentSweeper::GetRegionSafe(MemSpaceType type) +{ + os::memory::LockHolder holder(mutexs_[type]); + Region *region = nullptr; + if (!sweepingList_[type].empty()) { + region = sweepingList_[type].back(); + sweepingList_[type].pop_back(); + } + return region; +} + +void ConcurrentSweeper::AddSweptRegionSafe(MemSpaceType type, Region *region) +{ + os::memory::LockHolder holder(mutexs_[type]); + sweptList_[type].emplace_back(region); +} + +Region *ConcurrentSweeper::GetSweptRegionSafe(MemSpaceType type) +{ + os::memory::LockHolder holder(mutexs_[type]); + Region *region = nullptr; + if (!sweptList_[type].empty()) { + region = sweptList_[type].back(); + sweptList_[type].pop_back(); + } + return region; +} + +void ConcurrentSweeper::EnsureAllTaskFinish() +{ + if (!isSweeping_) { + return; + } + for (int i = startSpaceType_; i < FREE_LIST_NUM; i++) { + WaitingTaskFinish(static_cast(i)); + } + isSweeping_ = false; +} + +void ConcurrentSweeper::WaitingTaskFinish(MemSpaceType type) +{ + if (remainderTaskNum_[type] > 0) { + SweepSpace(type); + { + os::memory::LockHolder holder(mutexs_[type]); + while (remainderTaskNum_[type] > 0) { + cvs_[type].Wait(&mutexs_[type]); + } + } + } + FinishSweeping(type); +} + +void ConcurrentSweeper::FinishSweeping(MemSpaceType type) +{ + FillSweptRegion(type); + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(type); + allocator.SetSweeping(false); + if (type == OLD_SPACE) { + heap_->RecomputeLimits(); + } +} + +bool ConcurrentSweeper::SweeperTask::Run() +{ + int sweepTypeNum = FREE_LIST_NUM - sweeper_->startSpaceType_; + for (size_t i = sweeper_->startSpaceType_; i < FREE_LIST_NUM; i++) { + auto type = static_cast(((i + type_) % sweepTypeNum) + sweeper_->startSpaceType_); + sweeper_->SweepSpace(type, false); + } + return true; +} +} // namespace panda::ecmascript diff --git a/ecmascript/mem/concurrent_sweeper.h b/ecmascript/mem/concurrent_sweeper.h new file mode 100644 index 0000000000..b24b5a65f5 --- /dev/null +++ b/ecmascript/mem/concurrent_sweeper.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ECMASCRIPT_MEM_CONCURRENT_SWEEPER_H +#define ECMASCRIPT_MEM_CONCURRENT_SWEEPER_H + +#include +#include + +#include "ecmascript/mem/space.h" +#include "ecmascript/platform/task.h" +#include "os/mutex.h" + +namespace panda::ecmascript { +class FreeListAllocator; + +class ConcurrentSweeper { +public: + ConcurrentSweeper(Heap *heap, bool concurrentSweep); + ~ConcurrentSweeper() = default; + + NO_COPY_SEMANTIC(ConcurrentSweeper); + NO_MOVE_SEMANTIC(ConcurrentSweeper); + + void SweepPhases(bool compressGC = false); + + void EnsureAllTaskFinish(); + // Ensure task finish + void WaitingTaskFinish(MemSpaceType type); + + void FillSweptRegion(MemSpaceType type); + + bool IsConcurrentSweepEnabled() + { + return concurrentSweep_; + } + +private: + class SweeperTask : public Task { + public: + SweeperTask(ConcurrentSweeper *sweeper, MemSpaceType type) : sweeper_(sweeper), type_(type) {}; + ~SweeperTask() override = default; + bool Run() override; + + NO_COPY_SEMANTIC(SweeperTask); + NO_MOVE_SEMANTIC(SweeperTask); + + private: + ConcurrentSweeper *sweeper_; + MemSpaceType type_; + }; + + void SweepSpace(MemSpaceType type, bool isMain = true); + void SweepSpace(Space *space, FreeListAllocator &allocator); + void SweepHugeSpace(); + void FinishSweeping(MemSpaceType type); + + void FreeRegion(Region *current, FreeListAllocator &allocator, bool isMain = true); + void FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, uintptr_t freeEnd, + bool isMain); + + void AddRegion(MemSpaceType type, Region *region); + Region *GetRegionSafe(MemSpaceType type); + + void AddSweptRegionSafe(MemSpaceType type, Region *region); + Region *GetSweptRegionSafe(MemSpaceType type); + + std::array mutexs_; + std::array cvs_; + std::array remainderTaskNum_ = {0, 0, 0}; + + std::array, FREE_LIST_NUM> sweepingList_; + std::array, FREE_LIST_NUM> sweptList_; + + Heap *heap_; + bool concurrentSweep_ = false; + bool isSweeping_ = false; + MemSpaceType startSpaceType_ = MemSpaceType::OLD_SPACE; +}; +} // namespace panda::ecmascript +#endif // ECMASCRIPT_MEM_CONCURRENT_SWEEPER_H diff --git a/ecmascript/mem/ecma_heap_manager-inl.h b/ecmascript/mem/ecma_heap_manager-inl.h index 4b65d6c337..8b9f0e146b 100644 --- a/ecmascript/mem/ecma_heap_manager-inl.h +++ b/ecmascript/mem/ecma_heap_manager-inl.h @@ -83,18 +83,18 @@ TaggedObject *EcmaHeapManager::AllocateNonMovableOrHugeObject(JSHClass *hclass, if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) { return AllocateHugeObject(hclass, size); } - auto object = reinterpret_cast(nonMovableAllocator_.Allocate(size)); + auto object = reinterpret_cast(GetNonMovableSpaceAllocator().Allocate(size)); if (UNLIKELY(object == nullptr)) { if (heap_->CheckAndTriggerNonMovableGC()) { - object = reinterpret_cast(nonMovableAllocator_.Allocate(size)); + object = reinterpret_cast(GetNonMovableSpaceAllocator().Allocate(size)); } if (UNLIKELY(object == nullptr)) { // hclass must be nonmovable - if (!heap_->FillNonMovableSpaceAndTryGC(&nonMovableAllocator_)) { + if (!heap_->FillNonMovableSpaceAndTryGC(&GetNonMovableSpaceAllocator())) { LOG_ECMA_MEM(FATAL) << "OOM : extend failed"; UNREACHABLE(); } - object = reinterpret_cast(nonMovableAllocator_.Allocate(size)); + object = reinterpret_cast(GetNonMovableSpaceAllocator().Allocate(size)); if (UNLIKELY(object == nullptr)) { heap_->ThrowOutOfMemoryError(size); UNREACHABLE(); @@ -140,18 +140,18 @@ TaggedObject *EcmaHeapManager::AllocateOldGenerationOrHugeObject(JSHClass *hclas if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) { return AllocateHugeObject(hclass, size); } - auto object = reinterpret_cast(oldSpaceAllocator_.Allocate(size)); + auto object = reinterpret_cast(GetOldSpaceAllocator().Allocate(size)); if (UNLIKELY(object == nullptr)) { if (heap_->CheckAndTriggerOldGC()) { - object = reinterpret_cast(oldSpaceAllocator_.Allocate(size)); + object = reinterpret_cast(GetOldSpaceAllocator().Allocate(size)); } if (UNLIKELY(object == nullptr)) { // hclass must nonmovable - if (!heap_->FillOldSpaceAndTryGC(&oldSpaceAllocator_)) { + if (!heap_->FillOldSpaceAndTryGC(&GetOldSpaceAllocator())) { LOG_ECMA_MEM(FATAL) << "OOM : extend failed"; UNREACHABLE(); } - object = reinterpret_cast(oldSpaceAllocator_.Allocate(size)); + object = reinterpret_cast(GetOldSpaceAllocator().Allocate(size)); if (UNLIKELY(object == nullptr)) { heap_->ThrowOutOfMemoryError(size); UNREACHABLE(); @@ -186,12 +186,12 @@ TaggedObject *EcmaHeapManager::AllocateHugeObject(JSHClass *hclass, size_t size) TaggedObject *EcmaHeapManager::AllocateMachineCodeSpaceObject(JSHClass *hclass, size_t size) { - auto object = reinterpret_cast(machineCodeSpaceAllocator_.Allocate(size)); + auto object = reinterpret_cast(GetMachineCodeSpaceAllocator().Allocate(size)); if (UNLIKELY(object == nullptr)) { - if (!heap_->FillMachineCodeSpaceAndTryGC(&machineCodeSpaceAllocator_)) { + if (!heap_->FillMachineCodeSpaceAndTryGC(&GetMachineCodeSpaceAllocator())) { return nullptr; } - object = reinterpret_cast(machineCodeSpaceAllocator_.Allocate(size)); + object = reinterpret_cast(GetMachineCodeSpaceAllocator().Allocate(size)); if (UNLIKELY(object == nullptr)) { heap_->ThrowOutOfMemoryError(size); return nullptr; diff --git a/ecmascript/mem/ecma_heap_manager.cpp b/ecmascript/mem/ecma_heap_manager.cpp index ea3d0b9826..4314882672 100644 --- a/ecmascript/mem/ecma_heap_manager.cpp +++ b/ecmascript/mem/ecma_heap_manager.cpp @@ -20,9 +20,8 @@ namespace panda::ecmascript { EcmaHeapManager::EcmaHeapManager(Heap *heap) : heap_(heap), newSpaceAllocator_(heap->GetNewSpace()), - nonMovableAllocator_(heap->GetNonMovableSpace()), - oldSpaceAllocator_(heap->GetOldSpace()), - machineCodeSpaceAllocator_(heap->GetMachineCodeSpace()) + freeListAllocator_ { FreeListAllocator(heap->GetOldSpace()), FreeListAllocator(heap_->GetNonMovableSpace()), + FreeListAllocator(heap->GetMachineCodeSpace()) } { ASSERT(heap != nullptr); heap->SetHeapManager(this); diff --git a/ecmascript/mem/ecma_heap_manager.h b/ecmascript/mem/ecma_heap_manager.h index 6bfd120149..82ec7bc71d 100644 --- a/ecmascript/mem/ecma_heap_manager.h +++ b/ecmascript/mem/ecma_heap_manager.h @@ -48,9 +48,14 @@ public: return heap_; } - FreeListAllocator &GetOldSpaceAllocator() + FreeListAllocator &GetFreeListAllocator(MemSpaceType type) { - return oldSpaceAllocator_; + return freeListAllocator_[type]; + } + + inline FreeListAllocator &GetOldSpaceAllocator() + { + return freeListAllocator_[OLD_SPACE]; } BumpPointerAllocator &GetNewSpaceAllocator() @@ -58,9 +63,9 @@ public: return newSpaceAllocator_; } - FreeListAllocator &GetNonMovableSpaceAllocator() + inline FreeListAllocator &GetNonMovableSpaceAllocator() { - return nonMovableAllocator_; + return freeListAllocator_[NON_MOVABLE]; } const BumpPointerAllocator &GetSnapShotSpaceAllocator() const @@ -68,18 +73,16 @@ public: return snapshotSpaceAllocator_; } - FreeListAllocator &GetMachineCodeSpaceAllocator() + inline FreeListAllocator &GetMachineCodeSpaceAllocator() { - return machineCodeSpaceAllocator_; + return freeListAllocator_[MACHINE_CODE_SPACE]; } private: Heap *heap_{nullptr}; BumpPointerAllocator newSpaceAllocator_; - FreeListAllocator nonMovableAllocator_; - FreeListAllocator oldSpaceAllocator_; + std::array freeListAllocator_; BumpPointerAllocator snapshotSpaceAllocator_; - FreeListAllocator machineCodeSpaceAllocator_; }; } // namespace panda::ecmascript diff --git a/ecmascript/mem/free_object_kind.cpp b/ecmascript/mem/free_object_kind.cpp index 951871f252..94829d653a 100644 --- a/ecmascript/mem/free_object_kind.cpp +++ b/ecmascript/mem/free_object_kind.cpp @@ -31,6 +31,9 @@ void FreeObjectKind::Rebuild() { freeObject_ = nullptr; available_ = 0; + isAdded_ = false; + next_ = nullptr; + prev_ = nullptr; } FreeObject *FreeObjectKind::SearchSmallFreeObject(size_t size) diff --git a/ecmascript/mem/free_object_kind.h b/ecmascript/mem/free_object_kind.h index 869689c5fb..057968a16d 100644 --- a/ecmascript/mem/free_object_kind.h +++ b/ecmascript/mem/free_object_kind.h @@ -27,9 +27,9 @@ class FreeObject; class FreeObjectKind { public: - FreeObjectKind(KindType type, uintptr_t begin, size_t size) : kindType_(type) + FreeObjectKind(KindType type) : kindType_(type) { - Free(begin, size); + Rebuild(); } ~FreeObjectKind() = default; @@ -60,6 +60,7 @@ private: FreeObjectKind *prev_ = nullptr; KindType kindType_ = INVALID_KIND_TYPE; size_t available_ = 0; + bool isAdded_ = false; FreeObject *freeObject_ = nullptr; friend class FreeObjectList; diff --git a/ecmascript/mem/free_object_list.cpp b/ecmascript/mem/free_object_list.cpp index 43e220f78e..e25e2880a7 100644 --- a/ecmascript/mem/free_object_list.cpp +++ b/ecmascript/mem/free_object_list.cpp @@ -20,17 +20,16 @@ #include "ecmascript/mem/mem.h" namespace panda::ecmascript { -FreeObjectList::FreeObjectList() +FreeObjectList::FreeObjectList() : kinds_(new FreeObjectKind *[NUMBER_OF_KINDS](), NUMBER_OF_KINDS) { - kinds_ = Span(new FreeObjectKind *[NUMBER_OF_KINDS](), NUMBER_OF_KINDS); + for (int i = 0; i < NUMBER_OF_KINDS; i++) { + kinds_[i] = nullptr; + } noneEmptyKindBitMap_ = 0; } FreeObjectList::~FreeObjectList() { - for (auto it : kinds_) { - delete it; - } delete[] kinds_.data(); noneEmptyKindBitMap_ = 0; } @@ -49,33 +48,39 @@ FreeObject *FreeObjectList::Allocator(size_t size) KindType lastType = type - 1; for (type = CalcNextNoneEmptyIndex(type); type > lastType && type < NUMBER_OF_KINDS; - type = CalcNextNoneEmptyIndex(type + 1)) { + type = CalcNextNoneEmptyIndex(type + 1)) { lastType = type; - FreeObjectKind *top = kinds_[type]; - if (top == nullptr || top->Available() < size) { - continue; - } - FreeObject *current = nullptr; - if (type <= SMALL_KIND_MAX_INDEX) { - current = top->SearchSmallFreeObject(size); - } else { - current = top->SearchLargeFreeObject(size); - } - if (top->Empty()) { - RemoveKind(top); - } - if (current != nullptr) { - size_t currentSize = current->Available(); - available_ -= currentSize; - if (currentSize >= size) { - return current; + FreeObjectKind *current = kinds_[type]; + while (current != nullptr) { + if (current->Available() < size) { + current = current->next_; + continue; } + FreeObjectKind *next = nullptr; + FreeObject *object = nullptr; + if (type <= SMALL_KIND_MAX_INDEX) { + object = current->SearchSmallFreeObject(size); + } else { + next = current->next_; + object = current->SearchLargeFreeObject(size); + } + if (current->Empty()) { + RemoveKind(current); + } + if (object != nullptr) { + size_t objectSize = object->Available(); + available_ -= objectSize; + if (objectSize >= size) { + return object; + } + } + current = next; } } return nullptr; } -void FreeObjectList::Free(uintptr_t start, size_t size) +void FreeObjectList::Free(uintptr_t start, size_t size, bool isAdd) { if (start == 0 || size == 0) { return; @@ -86,26 +91,28 @@ void FreeObjectList::Free(uintptr_t start, size_t size) return; } - auto kind = kinds_[type]; + Region *region = Region::ObjectAddressToRange(reinterpret_cast(start)); + auto kind = region->GetFreeObjectKind(type); if (kind == nullptr) { - kind = new FreeObjectKind(type, start, size); - if (!AddKind(kind)) { - delete kind; - return; + LOG_ECMA(FATAL) << "The kind of region is nullptr"; + return; + } + kind->Free(start, size); + + if (isAdd) { + if (kind->isAdded_) { + available_ += size; + } else { + AddKind(kind); } - } else { - kind->Free(start, size); } - available_ += size; } void FreeObjectList::Rebuild() { - for (auto kind : kinds_) { - if (kind != nullptr) { - kind->Rebuild(); - RemoveKind(kind); - } + EnumerateKinds([](FreeObjectKind *kind) { kind->Rebuild(); }); + for (int i = 0; i < NUMBER_OF_KINDS; i++) { + kinds_[i] = nullptr; } available_ = 0; noneEmptyKindBitMap_ = 0; @@ -118,20 +125,22 @@ size_t FreeObjectList::GetFreeObjectSize() const bool FreeObjectList::AddKind(FreeObjectKind *kind) { - if (kind == nullptr || kind->Empty()) { + if (kind == nullptr || kind->Empty() || kind->isAdded_) { return false; } KindType type = kind->kindType_; FreeObjectKind *top = kinds_[type]; if (kind == top) { - return true; + return false; } if (top != nullptr) { top->prev_ = kind; } + kind->isAdded_ = true; kind->next_ = top; kinds_[type] = kind; SetNoneEmptyBit(type); + available_ += kind->Available(); return true; } @@ -151,11 +160,30 @@ void FreeObjectList::RemoveKind(FreeObjectKind *kind) if (kind->next_ != nullptr) { kind->next_->prev_ = kind->prev_; } - kind->prev_ = nullptr; - kind->next_ = nullptr; if (kinds_[type] == nullptr) { ClearNoneEmptyBit(type); } - delete kind; + available_ -= kind->Available(); + kind->Rebuild(); +} + +template +void FreeObjectList::EnumerateKinds(const Callback &cb) const +{ + for (KindType i = 0; i < NUMBER_OF_KINDS; i++) { + EnumerateKinds(i, cb); + } +} + +template +void FreeObjectList::EnumerateKinds(KindType type, const Callback &cb) const +{ + FreeObjectKind *current = kinds_[type]; + while (current != nullptr) { + // maybe reset + FreeObjectKind *next = current->next_; + cb(current); + current = next; + } } } // namespace panda::ecmascript diff --git a/ecmascript/mem/free_object_list.h b/ecmascript/mem/free_object_list.h index c7db11126e..553dd5d4c4 100644 --- a/ecmascript/mem/free_object_list.h +++ b/ecmascript/mem/free_object_list.h @@ -29,15 +29,30 @@ public: FreeObject *Allocator(size_t size); - void Free(uintptr_t start, size_t size); + void Free(uintptr_t start, size_t size, bool isAdd = true); void Rebuild(); + bool AddKind(FreeObjectKind *kind); + + void RemoveKind(FreeObjectKind *kind); + + template + void EnumerateKinds(const Callback &cb) const; + + template + void EnumerateKinds(KindType type, const Callback &cb) const; + NO_COPY_SEMANTIC(FreeObjectList); NO_MOVE_SEMANTIC(FreeObjectList); size_t GetFreeObjectSize() const; + static int NumberOfKinds() + { + return NUMBER_OF_KINDS; + } + private: static constexpr int NUMBER_OF_KINDS = 39; static constexpr size_t MIN_SIZE = 16; @@ -58,9 +73,6 @@ private: inline void ClearNoneEmptyBit(KindType type); inline size_t CalcNextNoneEmptyIndex(KindType start); - bool AddKind(FreeObjectKind *kind); - void RemoveKind(FreeObjectKind *kind); - size_t available_ = 0; uint64_t noneEmptyKindBitMap_; Span kinds_ {}; diff --git a/ecmascript/mem/heap.cpp b/ecmascript/mem/heap.cpp index 61aba40a8b..76bfc94b4c 100644 --- a/ecmascript/mem/heap.cpp +++ b/ecmascript/mem/heap.cpp @@ -20,6 +20,7 @@ #include "ecmascript/ecma_vm.h" #include "ecmascript/mem/assert_scope-inl.h" #include "ecmascript/mem/compress_collector.h" +#include "ecmascript/mem/concurrent_sweeper.h" #include "ecmascript/mem/ecma_heap_manager.h" #include "ecmascript/mem/mark_stack.h" #include "ecmascript/mem/mem_controller.h" @@ -54,8 +55,6 @@ void Heap::Initialize() machineCodeSpace_ = new MachineCodeSpace(this); machineCodeSpace_->Initialize(); hugeObjectSpace_ = new HugeObjectSpace(this); - markStack_ = new MarkStack(this); - weakProcessQueue_ = new ProcessQueue(this); bool paralledGc = ecmaVm_->GetOptions().IsEnableParalledYoungGc(); if (paralledGc) { int numOfCpuCore = get_nprocs(); @@ -63,12 +62,15 @@ void Heap::Initialize() pool_ = new ThreadPool(numThread); semiSpaceCollector_ = new SemiSpaceCollector(this, true); compressCollector_ = new CompressCollector(this, true); + oldSpaceCollector_ = new OldSpaceCollector(this, true); } else { pool_ = new ThreadPool(1); semiSpaceCollector_ = new SemiSpaceCollector(this, false); compressCollector_ = new CompressCollector(this, false); + oldSpaceCollector_ = new OldSpaceCollector(this, false); } - oldSpaceCollector_ = new OldSpaceCollector(this); + // After EcmaOptions merged, it will modified to EcmaOptions configuration + sweeper_ = new ConcurrentSweeper(this, true); } void Heap::FlipNewSpace() @@ -87,6 +89,7 @@ void Heap::FlipCompressSpace() void Heap::Destroy() { pool_->WaitTaskFinish(); + sweeper_->EnsureAllTaskFinish(); toSpace_->Destroy(); delete toSpace_; toSpace_ = nullptr; @@ -109,16 +112,10 @@ void Heap::Destroy() machineCodeSpace_->Destroy(); delete machineCodeSpace_; machineCodeSpace_ = nullptr; - markStack_->Destroy(); - delete markStack_; - markStack_ = nullptr; hugeObjectSpace_->Destroy(); delete hugeObjectSpace_; hugeObjectSpace_ = nullptr; - weakProcessQueue_->Destroy(); - delete weakProcessQueue_; - weakProcessQueue_ = nullptr; delete semiSpaceCollector_; semiSpaceCollector_ = nullptr; delete oldSpaceCollector_; @@ -130,6 +127,8 @@ void Heap::Destroy() memController_ = nullptr; delete pool_; pool_ = nullptr; + delete sweeper_; + sweeper_ = nullptr; } void Heap::CollectGarbage(TriggerGCType gcType) @@ -138,6 +137,7 @@ void Heap::CollectGarbage(TriggerGCType gcType) // pre gc heap verify { if (ecmaVm_->GetOptions().IsPreGcHeapVerifyEnabled()) { + sweeper_->EnsureAllTaskFinish(); auto failCount = Verification(this).VerifyAll(); if (failCount > 0) { LOG(FATAL, GC) << "Before gc heap corrupted and " << failCount << " corruptions"; @@ -158,18 +158,18 @@ void Heap::CollectGarbage(TriggerGCType gcType) } break; case TriggerGCType::OLD_GC: - if (oldSpace_->GetHeapObjectSize() < OLD_SPACE_LIMIT_BEGIN) { - oldSpaceCollector_->RunPhases(); - } else { - compressCollector_->RunPhases(); + oldSpaceCollector_->RunPhases(); + if (!sweeper_->IsConcurrentSweepEnabled()) { + RecomputeLimits(); } - RecomputeLimits(); break; case TriggerGCType::NON_MOVE_GC: case TriggerGCType::HUGE_GC: case TriggerGCType::MACHINE_CODE_GC: oldSpaceCollector_->RunPhases(); - RecomputeLimits(); + if (!sweeper_->IsConcurrentSweepEnabled()) { + RecomputeLimits(); + } break; case TriggerGCType::COMPRESS_FULL_GC: compressCollector_->RunPhases(); @@ -183,6 +183,7 @@ void Heap::CollectGarbage(TriggerGCType gcType) // post gc heap verify { if (ecmaVm_->GetOptions().IsPreGcHeapVerifyEnabled()) { + sweeper_->EnsureAllTaskFinish(); auto failCount = Verification(this).VerifyAll(); if (failCount > 0) { LOG(FATAL, GC) << "After gc heap corrupted and " << failCount << " corruptions"; diff --git a/ecmascript/mem/heap.h b/ecmascript/mem/heap.h index bc112f2f87..55ce680c72 100644 --- a/ecmascript/mem/heap.h +++ b/ecmascript/mem/heap.h @@ -31,6 +31,7 @@ class FreeListAllocator; class RegionFactory; class HeapTracker; class MemController; +class ConcurrentSweeper; class Heap { public: @@ -93,16 +94,6 @@ public: return machineCodeSpace_; } - MarkStack *GetMarkStack() const - { - return markStack_; - } - - ProcessQueue *GetProcessQueue() const - { - return weakProcessQueue_; - } - SemiSpaceCollector *GetSemiSpaceCollector() const { return semiSpaceCollector_; @@ -118,6 +109,11 @@ public: return compressCollector_; } + ConcurrentSweeper *GetSweeper() const + { + return sweeper_; + } + EcmaVM *GetEcmaVM() const { return ecmaVm_; @@ -271,11 +267,10 @@ private: SnapShotSpace *snapshotSpace_ {nullptr}; NonMovableSpace *nonMovableSpace_ {nullptr}; MachineCodeSpace *machineCodeSpace_ {nullptr}; - MarkStack *markStack_ {nullptr}; - ProcessQueue *weakProcessQueue_ {nullptr}; SemiSpaceCollector *semiSpaceCollector_ {nullptr}; OldSpaceCollector *oldSpaceCollector_ {nullptr}; CompressCollector *compressCollector_ {nullptr}; + ConcurrentSweeper *sweeper_ {nullptr}; EcmaHeapManager *heapManager_ {nullptr}; RegionFactory *regionFactory_ {nullptr}; HeapTracker *tracker_ {nullptr}; diff --git a/ecmascript/mem/old_space_collector-inl.h b/ecmascript/mem/old_space_collector-inl.h index a8f88de559..b6c82b7de0 100644 --- a/ecmascript/mem/old_space_collector-inl.h +++ b/ecmascript/mem/old_space_collector-inl.h @@ -24,31 +24,19 @@ #include "ecmascript/js_hclass-inl.h" namespace panda::ecmascript { -void OldSpaceCollector::MarkObject(TaggedObject *object) +void OldSpaceCollector::MarkObject(uint64_t threadId, TaggedObject *object) { Region *objectRegion = Region::ObjectAddressToRange(object); auto markBitmap = objectRegion->GetMarkBitmap(); - if (!markBitmap->Test(object)) { - markBitmap->Set(object); - markStack_.PushBack(object); + if (!markBitmap->AtomicTestAndSet(object)) { + workList_->Push(threadId, object); } } -void OldSpaceCollector::RecordWeakReference(JSTaggedType *ref) +void OldSpaceCollector::RecordWeakReference(uint64_t threadId, JSTaggedType *ref) { - Region *objectRegion = Region::ObjectAddressToRange(reinterpret_cast(ref)); - if (!objectRegion->InYoungGeneration()) { - weakProcessQueue_.PushBack(ref); - } -} - -void OldSpaceCollector::FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, - uintptr_t freeEnd) -{ - allocator.Free(freeStart, freeEnd); - freeSize_ += freeEnd - freeStart; - heap_->ClearSlotsRange(current, freeStart, freeEnd); + workList_->PushWeakReference(threadId, ref); } } // namespace panda::ecmascript diff --git a/ecmascript/mem/old_space_collector.cpp b/ecmascript/mem/old_space_collector.cpp index 3572760030..343f7c7e03 100644 --- a/ecmascript/mem/old_space_collector.cpp +++ b/ecmascript/mem/old_space_collector.cpp @@ -27,7 +27,11 @@ #include "ecmascript/vmstat/runtime_stat.h" namespace panda::ecmascript { -OldSpaceCollector::OldSpaceCollector(Heap *heap) : heap_(heap), rootManager_(heap->GetEcmaVM()) {} +OldSpaceCollector::OldSpaceCollector(Heap *heap, bool parallelGc) + : heap_(heap), rootManager_(heap->GetEcmaVM()), paralledGC_(parallelGc) +{ + workList_ = new OldGCWorker(heap_, heap_->GetThreadPool()->GetThreadNum()); +} void OldSpaceCollector::RunPhases() { @@ -45,12 +49,8 @@ void OldSpaceCollector::RunPhases() void OldSpaceCollector::InitializePhase() { - markStack_.BeginMarking(heap_, heap_->GetMarkStack()); - weakProcessQueue_.BeginMarking(heap_, heap_->GetProcessQueue()); - auto heapManager = heap_->GetHeapManager(); - oldSpaceAllocator_.Swap(heapManager->GetOldSpaceAllocator()); - nonMovableAllocator_.Swap(heapManager->GetNonMovableSpaceAllocator()); - machineCodeSpaceAllocator_.Swap(heapManager->GetMachineCodeSpaceAllocator()); + heap_->GetThreadPool()->WaitTaskFinish(); + heap_->GetSweeper()->EnsureAllTaskFinish(); heap_->EnumerateRegions([](Region *current) { // ensure mark bitmap auto bitmap = current->GetMarkBitmap(); @@ -60,6 +60,7 @@ void OldSpaceCollector::InitializePhase() bitmap->ClearAllBits(); } }); + workList_->Initialize(); freeSize_ = 0; hugeSpaceFreeSize_ = 0; oldSpaceCommitSize_ = heap_->GetOldSpace()->GetCommittedSize(); @@ -68,13 +69,8 @@ void OldSpaceCollector::InitializePhase() void OldSpaceCollector::FinishPhase() { - // swap - markStack_.FinishMarking(heap_->GetMarkStack()); - weakProcessQueue_.FinishMarking(heap_->GetProcessQueue()); - auto heapManager = heap_->GetHeapManager(); - heapManager->GetOldSpaceAllocator().Swap(oldSpaceAllocator_); - heapManager->GetNonMovableSpaceAllocator().Swap(nonMovableAllocator_); - heapManager->GetMachineCodeSpaceAllocator().Swap(machineCodeSpaceAllocator_); + size_t aliveSize = 0; + workList_->Finish(aliveSize); } void OldSpaceCollector::MarkingPhase() @@ -83,112 +79,72 @@ void OldSpaceCollector::MarkingPhase() RootVisitor gcMarkYoung = [this]([[maybe_unused]] Root type, ObjectSlot slot) { JSTaggedValue value(slot.GetTaggedType()); if (value.IsHeapObject()) { - MarkObject(value.GetTaggedObject()); + MarkObject(0, value.GetTaggedObject()); } }; RootRangeVisitor gcMarkRangeYoung = [this]([[maybe_unused]] Root type, ObjectSlot start, ObjectSlot end) { for (ObjectSlot slot = start; slot < end; slot++) { JSTaggedValue value(slot.GetTaggedType()); if (value.IsHeapObject()) { - MarkObject(value.GetTaggedObject()); + MarkObject(0, value.GetTaggedObject()); } } }; rootManager_.VisitVMRoots(gcMarkYoung, gcMarkRangeYoung); - ProcessMarkStack(); + ProcessMarkStack(0); + if (paralledGC_) { + heap_->GetThreadPool()->WaitTaskFinish(); + } } -void OldSpaceCollector::ProcessMarkStack() +void OldSpaceCollector::ProcessMarkStack(uint64_t threadId) { while (true) { - auto obj = markStack_.PopBack(); - if (UNLIKELY(obj == nullptr)) { + TaggedObject *obj = nullptr; + if (!workList_->Pop(threadId, &obj)) { break; } auto jsHclass = obj->GetClass(); // mark dynClass - MarkObject(jsHclass); + MarkObject(threadId, jsHclass); rootManager_.MarkObjectBody( - obj, jsHclass, [this]([[maybe_unused]] TaggedObject *root, ObjectSlot start, ObjectSlot end) { + obj, jsHclass, [this, &threadId]([[maybe_unused]] TaggedObject *root, ObjectSlot start, ObjectSlot end) { for (ObjectSlot slot = start; slot < end; slot++) { JSTaggedValue value(slot.GetTaggedType()); if (value.IsWeak()) { - RecordWeakReference(reinterpret_cast(slot.SlotAddress())); + RecordWeakReference(threadId, reinterpret_cast(slot.SlotAddress())); continue; } if (value.IsHeapObject()) { - MarkObject(value.GetTaggedObject()); + MarkObject(threadId, value.GetTaggedObject()); } } }); } } -void OldSpaceCollector::SweepSpace(Space *space, FreeListAllocator &allocator) -{ - allocator.RebuildFreeList(); - space->EnumerateRegions([this, &allocator](Region *current) { - auto markBitmap = current->GetMarkBitmap(); - ASSERT(markBitmap != nullptr); - uintptr_t freeStart = current->GetBegin(); - markBitmap->IterateOverMarkedChunks([this, ¤t, &freeStart, &allocator](void *mem) { - ASSERT(current->InRange(ToUintPtr(mem))); - auto header = reinterpret_cast(mem); - auto klass = header->GetClass(); - JSType jsType = klass->GetObjectType(); - auto size = klass->SizeFromJSHClass(jsType, header); - size = AlignUp(size, static_cast(MemAlignment::MEM_ALIGN_OBJECT)); - - uintptr_t freeEnd = ToUintPtr(mem); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - freeStart = freeEnd + size; - }); - uintptr_t freeEnd = current->GetEnd(); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - }); -} - -void OldSpaceCollector::SweepSpace(HugeObjectSpace *space) -{ - Region *currentRegion = space->GetRegionList().GetFirst(); - - while (currentRegion != nullptr) { - Region *next = currentRegion->GetNext(); - auto markBitmap = currentRegion->GetMarkBitmap(); - bool isMarked = false; - markBitmap->IterateOverMarkedChunks([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; }); - if (!isMarked) { - space->GetRegionList().RemoveNode(currentRegion); - hugeSpaceFreeSize_ += currentRegion->GetCapacity(); - space->ClearAndFreeRegion(currentRegion); - } - currentRegion = next; - } -} - void OldSpaceCollector::SweepPhases() { trace::ScopedTrace scoped_trace("OldSpaceCollector::SweepPhases"); // process weak reference - while (true) { - auto obj = weakProcessQueue_.PopBack(); - if (UNLIKELY(obj == nullptr)) { - break; - } - ObjectSlot slot(ToUintPtr(obj)); - JSTaggedValue value(slot.GetTaggedType()); - auto header = value.GetTaggedWeakRef(); + for (uint32_t i = 0; i < heap_->GetThreadPool()->GetThreadNum(); i++) { + ProcessQueue *queue = workList_->GetWeakReferenceQueue(i); + while (true) { + auto obj = queue->PopBack(); + if (UNLIKELY(obj == nullptr)) { + break; + } + ObjectSlot slot(ToUintPtr(obj)); + JSTaggedValue value(slot.GetTaggedType()); + auto header = value.GetTaggedWeakRef(); - Region *objectRegion = Region::ObjectAddressToRange(header); - auto markBitmap = objectRegion->GetMarkBitmap(); - if (!markBitmap->Test(header)) { - slot.Update(static_cast(JSTaggedValue::Undefined().GetRawData())); + Region *objectRegion = Region::ObjectAddressToRange(header); + auto markBitmap = objectRegion->GetMarkBitmap(); + if (!markBitmap->Test(header)) { + slot.Update(static_cast(JSTaggedValue::Undefined().GetRawData())); + } } } @@ -209,9 +165,6 @@ void OldSpaceCollector::SweepPhases() heap_->GetEcmaVM()->GetJSThread()->IterateWeakEcmaGlobalStorage(gcUpdateWeak); heap_->GetEcmaVM()->ProcessReferences(gcUpdateWeak); - SweepSpace(const_cast(heap_->GetOldSpace()), oldSpaceAllocator_); - SweepSpace(const_cast(heap_->GetNonMovableSpace()), nonMovableAllocator_); - SweepSpace(const_cast(heap_->GetHugeObjectSpace())); - SweepSpace(const_cast(heap_->GetMachineCodeSpace()), machineCodeSpaceAllocator_); + heap_->GetSweeper()->SweepPhases(); } } // namespace panda::ecmascript diff --git a/ecmascript/mem/old_space_collector.h b/ecmascript/mem/old_space_collector.h index 64940351b6..5da8b52071 100644 --- a/ecmascript/mem/old_space_collector.h +++ b/ecmascript/mem/old_space_collector.h @@ -21,6 +21,7 @@ #include "ecmascript/mem/allocator.h" #include "ecmascript/mem/mark_stack-inl.h" #include "ecmascript/mem/mark_word.h" +#include "ecmascript/mem/semi_space_worker.h" #include "ecmascript/mem/slots.h" #include "ecmascript/mem/heap_roots.h" #include "ecmascript/mem/remembered_set.h" @@ -33,7 +34,7 @@ class JSHClass; class OldSpaceCollector : public GarbageCollector { public: - explicit OldSpaceCollector(Heap *heap); + explicit OldSpaceCollector(Heap *heap, bool parallelGc); ~OldSpaceCollector() override = default; NO_COPY_SEMANTIC(OldSpaceCollector); NO_MOVE_SEMANTIC(OldSpaceCollector); @@ -50,26 +51,22 @@ private: void SweepPhases(); void FinishPhase(); - void ProcessMarkStack(); + void ProcessMarkStack(uint64_t threadId); void MarkObjectBody(TaggedObject *object, JSHClass *klass, const EcmaObjectRangeVisitor &visitor); - inline void MarkObject(TaggedObject *object); - inline void FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, uintptr_t freeEnd); - inline void RecordWeakReference(JSTaggedType *ref); - void SweepSpace(Space *space, FreeListAllocator &allocator); - void SweepSpace(HugeObjectSpace *space); // Only sweep huge space. + inline void MarkObject(uint64_t threadId, TaggedObject *object); + inline void RecordWeakReference(uint64_t threadId, JSTaggedType *ref); Heap *heap_; HeapRootManager rootManager_; - MarkStack markStack_; - ProcessQueue weakProcessQueue_; - FreeListAllocator oldSpaceAllocator_ {}; - FreeListAllocator nonMovableAllocator_ {}; - FreeListAllocator machineCodeSpaceAllocator_ {}; + bool paralledGC_{false}; + OldGCWorker *workList_{nullptr}; size_t freeSize_{0}; size_t hugeSpaceFreeSize_ = 0; size_t oldSpaceCommitSize_ = 0; size_t nonMoveSpaceCommitSize_ = 0; + + friend class OldGCWorker; }; } // namespace ecmascript } // namespace panda diff --git a/ecmascript/mem/region.h b/ecmascript/mem/region.h index 4e541d18e5..207abda5fd 100644 --- a/ecmascript/mem/region.h +++ b/ecmascript/mem/region.h @@ -16,6 +16,7 @@ #ifndef ECMASCRIPT_MEM_REGION_H #define ECMASCRIPT_MEM_REGION_H +#include "ecmascript/mem/free_object_list.h" #include "ecmascript/mem/mem.h" #include "mem/gc/bitmap.h" @@ -41,9 +42,11 @@ enum RegionFlags { // NOLINTNEXTLINE(hicpp-signed-bitwise) IS_IN_OLD_GENERATION = 1 << 6, // NOLINTNEXTLINE(hicpp-signed-bitwise) + IS_IN_NON_MOVABLE_GENERATION = 1 << 7, + // NOLINTNEXTLINE(hicpp-signed-bitwise) IS_IN_YOUNG_OR_OLD_GENERATION = IS_IN_YOUNG_GENERATION | IS_IN_OLD_GENERATION, // NOLINTNEXTLINE(hicpp-signed-bitwise) - IS_INVALID = 1 << 7, + IS_INVALID = 1 << 8, }; class Region { @@ -206,6 +209,36 @@ public: return res; } + void InitializeKind() + { + kinds_ = Span(new FreeObjectKind *[FreeObjectList::NumberOfKinds()](), + FreeObjectList::NumberOfKinds()); + for (int i = 0; i < FreeObjectList::NumberOfKinds(); i++) { + kinds_[i] = new FreeObjectKind(i); + } + } + + void DestoryKind() + { + for (auto kind : kinds_) { + delete kind; + } + delete[] kinds_.data(); + } + + FreeObjectKind *GetFreeObjectKind(KindType type) + { + return kinds_[type]; + } + + template + void EnumerateKinds(Callback cb) + { + for (auto kind : kinds_) { + cb(kind); + } + } + private: Space *space_; uintptr_t flags_; // Memory alignment, only low 32bits are used now @@ -218,6 +251,7 @@ private: RangeBitmap *markBitmap_{nullptr}; RememberedSet *crossRegionSet_{nullptr}; RememberedSet *oldToNewSet_{nullptr}; + Span kinds_; friend class SnapShot; }; } // namespace ecmascript diff --git a/ecmascript/mem/semi_space_collector.cpp b/ecmascript/mem/semi_space_collector.cpp index ef6975066f..0cc31fd8f1 100644 --- a/ecmascript/mem/semi_space_collector.cpp +++ b/ecmascript/mem/semi_space_collector.cpp @@ -59,7 +59,7 @@ void SemiSpaceCollector::RunPhases() void SemiSpaceCollector::InitializePhase() { heap_->GetThreadPool()->WaitTaskFinish(); - gcTime_++; + heap_->GetSweeper()->EnsureAllTaskFinish(); auto fromSpace = heap_->GetFromSpace(); if (fromSpace->GetCommittedSize() == 0) { heap_->InitializeFromSpace(); diff --git a/ecmascript/mem/semi_space_collector.h b/ecmascript/mem/semi_space_collector.h index 6e5a31f624..3a76149e5c 100644 --- a/ecmascript/mem/semi_space_collector.h +++ b/ecmascript/mem/semi_space_collector.h @@ -91,7 +91,6 @@ private: size_t semiCopiedSize_{0}; size_t commitSize_ = 0; uintptr_t ageMark_{0}; - size_t gcTime_{0}; friend class TlabAllocator; friend class SemiSpaceWorker; friend class SemiSpaceMarker; diff --git a/ecmascript/mem/semi_space_worker.cpp b/ecmascript/mem/semi_space_worker.cpp index dbaa62b623..0df831f442 100644 --- a/ecmascript/mem/semi_space_worker.cpp +++ b/ecmascript/mem/semi_space_worker.cpp @@ -19,6 +19,7 @@ #include "ecmascript/mem/compress_collector.h" #include "ecmascript/mem/heap.h" #include "ecmascript/mem/mark_stack.h" +#include "ecmascript/mem/old_space_collector.h" #include "ecmascript/mem/region_factory.h" #include "ecmascript/mem/tlab_allocator-inl.h" @@ -197,4 +198,32 @@ void CompressGCWorker::Initialize() holder.aliveSize_ = 0; } } + +void OldGCWorker::Initialize() +{ + spaceTop_ = markSpace_; + markSpaceEnd_ = markSpace_ + SPACE_SIZE; + for (uint32_t i = 0; i < threadNum_; i++) { + WorkNodeHolder &holder = workList_[i]; + holder.pushNode_ = AllocalWorkNode(); + holder.popNode_ = AllocalWorkNode(); + holder.weakQueue_ = new ProcessQueue(); + holder.weakQueue_->BeginMarking(heap_, continuousQueue_[i]); + } +} + +void OldGCWorker::PushWorkNodeToGlobal(uint32_t threadId) +{ + WorkNode *&pushNode = workList_[threadId].pushNode_; + if (!pushNode->IsEmpty()) { + globalWork_.Push(pushNode); + pushNode = AllocalWorkNode(); + + auto pool = heap_->GetThreadPool(); + if (pool->GetTaskCount() < pool->GetThreadNum() - 1) { + pool->Submit(std::bind(&OldSpaceCollector::ProcessMarkStack, heap_->GetOldSpaceCollector(), + std::placeholders::_1)); + } + } +} } // namespace panda::ecmascript diff --git a/ecmascript/mem/semi_space_worker.h b/ecmascript/mem/semi_space_worker.h index 2572c475e1..0a88f31f0e 100644 --- a/ecmascript/mem/semi_space_worker.h +++ b/ecmascript/mem/semi_space_worker.h @@ -233,6 +233,19 @@ public: NO_COPY_SEMANTIC(CompressGCWorker); NO_MOVE_SEMANTIC(CompressGCWorker); }; -} // namespace panda::ecmascript +class OldGCWorker : public Worker { +public: + OldGCWorker() = delete; + OldGCWorker(Heap *heap, uint32_t threadNum) : Worker(heap, threadNum) {} + + ~OldGCWorker() override = default; + + void PushWorkNodeToGlobal(uint32_t threadId) override; + void Initialize() override; + + NO_COPY_SEMANTIC(OldGCWorker); + NO_MOVE_SEMANTIC(OldGCWorker); +}; +} // namespace panda::ecmascript #endif // ECMASCRIPT_MEM_SEMI_SPACE_WORKER_H diff --git a/ecmascript/mem/space.cpp b/ecmascript/mem/space.cpp index 7a665284bf..c1156bc0ee 100644 --- a/ecmascript/mem/space.cpp +++ b/ecmascript/mem/space.cpp @@ -37,11 +37,16 @@ void Space::Initialize() } else if (spaceType_ == MemSpaceType::SNAPSHOT_SPACE) { region->SetFlag(RegionFlags::IS_IN_SNAPSHOT_GENERATION); } else if (spaceType_ == MemSpaceType::OLD_SPACE) { + region->InitializeKind(); region->SetFlag(RegionFlags::IS_IN_OLD_GENERATION); } else if (spaceType_ == MemSpaceType::MACHINE_CODE_SPACE) { + region->InitializeKind(); region->SetFlag(RegionFlags::IS_IN_OLD_GENERATION); int res = region->SetCodeExecutableAndReadable(); LOG_ECMA_MEM(DEBUG) << "Initialize SetCodeExecutableAndReadable" << res; + } else if (spaceType_ == MemSpaceType::NON_MOVABLE) { + region->InitializeKind(); + region->SetFlag(RegionFlags::IS_IN_NON_MOVABLE_GENERATION); } AddRegion(region); @@ -75,6 +80,11 @@ void Space::ClearAndFreeRegion(Region *region) delete rememberedSet; } DecrementCommitted(region->GetCapacity()); + if (spaceType_ == MemSpaceType::OLD_SPACE || + spaceType_ == MemSpaceType::NON_MOVABLE || + spaceType_ == MemSpaceType::MACHINE_CODE_SPACE) { + region->DestoryKind(); + } const_cast(heap_->GetRegionFactory())->FreeRegion(region); } @@ -177,6 +187,7 @@ bool OldSpace::Expand() Region *region = const_cast(GetHeap()->GetRegionFactory())->AllocateAlignedRegion(this, DEFAULT_REGION_SIZE); region->SetFlag(RegionFlags::IS_IN_OLD_GENERATION); + region->InitializeKind(); AddRegion(region); return true; } @@ -231,8 +242,7 @@ size_t OldSpace::GetHeapObjectSize() const { size_t result; size_t availableSize = GetHeap()->GetHeapManager()->GetOldSpaceAllocator().GetAvailableSize(); - size_t regionSize = GetRegionList().GetLength() * DEFAULT_REGION_SIZE; - result = regionSize - availableSize; + result = GetCommittedSize() - availableSize; result += GetHeap()->GetHugeObjectSpace()->GetHeapObjectSize(); return result; } @@ -250,6 +260,8 @@ bool NonMovableSpace::Expand() } Region *region = const_cast(GetHeap()->GetRegionFactory())->AllocateAlignedRegion(this, DEFAULT_REGION_SIZE); + region->SetFlag(IS_IN_NON_MOVABLE_GENERATION); + region->InitializeKind(); AddRegion(region); return true; } @@ -372,6 +384,12 @@ uintptr_t HugeObjectSpace::Allocate(size_t objectSize) return region->GetBegin(); } +void HugeObjectSpace::Free(Region *region) +{ + GetRegionList().RemoveNode(region); + ClearAndFreeRegion(region); +} + bool HugeObjectSpace::ContainObject(TaggedObject *object) const { auto region = GetRegionList().GetFirst(); @@ -391,12 +409,7 @@ bool HugeObjectSpace::IsLive(TaggedObject *object) const size_t HugeObjectSpace::GetHeapObjectSize() const { - size_t result = 0; - EnumerateRegions([&result](Region *current) { - auto obj = reinterpret_cast(current->GetBegin()); - result += obj->GetObjectSize(); - }); - return result; + return GetCommittedSize(); } void HugeObjectSpace::IterateOverObjects(const std::function &objectVisitor) const @@ -420,6 +433,7 @@ bool MachineCodeSpace::Expand() } Region *region = const_cast(GetHeap()->GetRegionFactory())->AllocateAlignedRegion(this, DEFAULT_REGION_SIZE); + region->InitializeKind(); AddRegion(region); int res = region->SetCodeExecutableAndReadable(); LOG_ECMA_MEM(DEBUG) << "MachineCodeSpace::Expand() SetCodeExecutableAndReadable" << res; diff --git a/ecmascript/mem/space.h b/ecmascript/mem/space.h index 7e6012f538..25f353e253 100644 --- a/ecmascript/mem/space.h +++ b/ecmascript/mem/space.h @@ -29,13 +29,16 @@ class Heap; class Program; enum MemSpaceType { - SEMI_SPACE, - OLD_SPACE, + OLD_SPACE = 0, NON_MOVABLE, + MACHINE_CODE_SPACE, HUGE_OBJECT_SPACE, + SEMI_SPACE, SNAPSHOT_SPACE, - MACHINE_CODE_SPACE, - SPACE_TYPE_LAST // Count of different types + COMPRESS_SPACE, + SPACE_TYPE_LAST, // Count of different types + + FREE_LIST_NUM = MACHINE_CODE_SPACE - OLD_SPACE + 1, }; enum TriggerGCType { @@ -238,6 +241,7 @@ public: NO_COPY_SEMANTIC(HugeObjectSpace); NO_MOVE_SEMANTIC(HugeObjectSpace); uintptr_t Allocate(size_t objectSize); + void Free(Region *region); size_t GetHeapObjectSize() const; bool ContainObject(TaggedObject *object) const; bool IsLive(TaggedObject *object) const; diff --git a/ecmascript/mem/tlab_allocator-inl.h b/ecmascript/mem/tlab_allocator-inl.h index 9e375449ef..32f713e199 100644 --- a/ecmascript/mem/tlab_allocator-inl.h +++ b/ecmascript/mem/tlab_allocator-inl.h @@ -24,7 +24,7 @@ #include "ecmascript/mem/tlab_allocator.h" namespace panda::ecmascript { -static constexpr size_t YOUNG_BUFFER_SIZE = 32 * 1024; +static constexpr size_t YOUNG_BUFFER_SIZE = 31 * 1024; static constexpr size_t OLD_BUFFER_SIZE = 255 * 1024; TlabAllocator::TlabAllocator(Heap *heap, TriggerGCType gcType) diff --git a/ecmascript/object_factory.cpp b/ecmascript/object_factory.cpp index d760c314c8..ba2673f875 100644 --- a/ecmascript/object_factory.cpp +++ b/ecmascript/object_factory.cpp @@ -793,10 +793,12 @@ FreeObject *ObjectFactory::FillFreeObject(uintptr_t address, size_t size, Remove if (size >= FreeObject::SIZE_OFFSET && size < FreeObject::SIZE) { object = reinterpret_cast(address); object->SetClass(freeObjectWithOneFieldClass_); + object->SetNext(nullptr); } else if (size >= FreeObject::SIZE) { object = reinterpret_cast(address); object->SetClass(freeObjectWithTwoFieldClass_); object->SetAvailable(size); + object->SetNext(nullptr); } else if (size == FreeObject::NEXT_OFFSET) { object = reinterpret_cast(address); object->SetClass(freeObjectWithNoneFieldClass_); diff --git a/ecmascript/platform/platform.cpp b/ecmascript/platform/platform.cpp new file mode 100644 index 0000000000..f589f3807e --- /dev/null +++ b/ecmascript/platform/platform.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/platform/platform.h" + +#include "sys/sysinfo.h" + +namespace panda::ecmascript { +void Platform::Initialize(int threadNum) +{ + os::memory::LockHolder lock(mutex_); + if (isInitialized_++ <= 0) { + runner_ = std::make_unique(TheMostSuitableThreadNum(threadNum)); + } +} + +void Platform::Destory() +{ + os::memory::LockHolder lock(mutex_); + if (--isInitialized_ <= 0) { + runner_->Terminate(); + } +} + +int Platform::TheMostSuitableThreadNum(int threadNum) const +{ + if (threadNum > 0) { + return std::min(threadNum, MAX_PLATFORM_THREAD_NUM); + } + int numOfCpuCore = get_nprocs() - 1; + return std::min(numOfCpuCore, MAX_PLATFORM_THREAD_NUM); +} +} // namespace panda::ecmascript diff --git a/ecmascript/platform/platform.h b/ecmascript/platform/platform.h new file mode 100644 index 0000000000..d62af797ee --- /dev/null +++ b/ecmascript/platform/platform.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ECMASCRIPT_PALTFORM_PLATFORM_H +#define ECMASCRIPT_PALTFORM_PLATFORM_H + +#include + +#include "ecmascript/platform/runner.h" +#include "os/mutex.h" + +namespace panda::ecmascript { +class Platform { +public: + static Platform *GetCurrentPlatform() + { + static Platform platform; + return &platform; + } + + Platform() = default; + ~Platform() = default; + + NO_COPY_SEMANTIC(Platform); + NO_MOVE_SEMANTIC(Platform); + + void Initialize(int threadNum = DEFAULT_PLATFORM_THREAD_NUM); + void Destory(); + + void PostTask(std::unique_ptr task) const + { + ASSERT(isInitialized_ > 0); + runner_->PostTask(std::move(task)); + } + +private: + static constexpr uint32_t MAX_PLATFORM_THREAD_NUM = 7; + static constexpr uint32_t DEFAULT_PLATFORM_THREAD_NUM = 0; + + int TheMostSuitableThreadNum(int threadNum) const; + + std::unique_ptr runner_; + int isInitialized_ = 0; + os::memory::Mutex mutex_; +}; +} // namespace panda::ecmascript +#endif // ECMASCRIPT_PALTFORM_PLATFORM_H diff --git a/ecmascript/platform/runner.cpp b/ecmascript/platform/runner.cpp new file mode 100644 index 0000000000..949607932c --- /dev/null +++ b/ecmascript/platform/runner.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/platform/runner.h" + +#include "os/thread.h" + +namespace panda::ecmascript { +Runner::Runner(int threadNum) +{ + for (int i = 0; i < threadNum; i++) { + std::unique_ptr thread = std::make_unique(&Runner::Run, this); + os::thread::SetThreadName(thread->native_handle(), "GC_WorkerThread"); + threadPool_.emplace_back(std::move(thread)); + } +} + +void Runner::Terminate() +{ + taskQueue_.Terminate(); + int threadNum = threadPool_.size(); + for (int i = 0; i < threadNum; i++) { + threadPool_.at(i)->join(); + } + threadPool_.clear(); +} + +void Runner::Run() +{ + while (std::unique_ptr task = taskQueue_.PopTask()) { + task->Run(); + } +} +} // namespace panda::ecmascript diff --git a/ecmascript/platform/runner.h b/ecmascript/platform/runner.h new file mode 100644 index 0000000000..be0ada0279 --- /dev/null +++ b/ecmascript/platform/runner.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ECMASCRIPT_PLATFORM_RUNNER_H +#define ECMASCRIPT_PLATFORM_RUNNER_H + +#include +#include +#include + +#include "ecmascript/platform/task_queue.h" + +namespace panda::ecmascript { +class Runner { +public: + explicit Runner(int threadNum); + ~Runner() = default; + + NO_COPY_SEMANTIC(Runner); + NO_MOVE_SEMANTIC(Runner); + + void PostTask(std::unique_ptr task) + { + taskQueue_.PostTask(std::move(task)); + } + + void Terminate(); + +private: + void Run(); + + std::vector> threadPool_ {}; + TaskQueue taskQueue_ {}; +}; +} // namespace panda::ecmascript +#endif // ECMASCRIPT_PLATFORM_RUNNER_H diff --git a/ecmascript/platform/task.h b/ecmascript/platform/task.h new file mode 100644 index 0000000000..71eff10111 --- /dev/null +++ b/ecmascript/platform/task.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ECMASCRIPT_PLATFORM_TASK_H +#define ECMASCRIPT_PLATFORM_TASK_H + +#include "macros.h" + +namespace panda::ecmascript { +class Task { +public: + Task() = default; + virtual ~Task() = default; + virtual bool Run() = 0; + + NO_COPY_SEMANTIC(Task); + NO_MOVE_SEMANTIC(Task); +}; +} // namespace panda::ecmascript +#endif // ECMASCRIPT_PLATFORM_TASK_H diff --git a/ecmascript/platform/task_queue.cpp b/ecmascript/platform/task_queue.cpp new file mode 100644 index 0000000000..b90c205569 --- /dev/null +++ b/ecmascript/platform/task_queue.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/platform/task_queue.h" + +namespace panda::ecmascript { +void TaskQueue::PostTask(std::unique_ptr task) +{ + os::memory::LockHolder holder(mtx_); + ASSERT(!terminate_); + tasks_.push(std::move(task)); + cv_.Signal(); +} + +std::unique_ptr TaskQueue::PopTask() +{ + os::memory::LockHolder holder(mtx_); + while (true) { + if (!tasks_.empty()) { + std::unique_ptr task = std::move(tasks_.front()); + tasks_.pop(); + return task; + } + if (terminate_) { + cv_.SignalAll(); + return nullptr; + } + cv_.Wait(&mtx_); + } +} + +void TaskQueue::Terminate() +{ + os::memory::LockHolder holder(mtx_); + terminate_ = true; + cv_.SignalAll(); +} +} // namespace panda::ecmascript diff --git a/ecmascript/platform/task_queue.h b/ecmascript/platform/task_queue.h new file mode 100644 index 0000000000..ab8d7b1f76 --- /dev/null +++ b/ecmascript/platform/task_queue.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ECMASCRIPT_PLATFORM_TASK_QUEUE_H +#define ECMASCRIPT_PLATFORM_TASK_QUEUE_H + +#include +#include +#include +#include + +#include "ecmascript/platform/task.h" +#include "os/mutex.h" + +namespace panda::ecmascript { +class TaskQueue { +public: + TaskQueue() = default; + ~TaskQueue() = default; + + NO_COPY_SEMANTIC(TaskQueue); + NO_MOVE_SEMANTIC(TaskQueue); + + void PostTask(std::unique_ptr task); + std::unique_ptr PopTask(); + + void Terminate(); + +private: + std::queue> tasks_; + + std::atomic_bool terminate_ = false; + os::memory::Mutex mtx_; + os::memory::ConditionVariable cv_; +}; +} // namespace panda::ecmascript +#endif // ECMASCRIPT_PLATFORM_TASK_QUEUE_H diff --git a/ecmascript/tests/BUILD.gn b/ecmascript/tests/BUILD.gn index f54a7db7eb..99ea6cc25a 100644 --- a/ecmascript/tests/BUILD.gn +++ b/ecmascript/tests/BUILD.gn @@ -831,6 +831,7 @@ host_unittest_action("GcTest") { sources = [ # test file + "concurrent_sweep_test.cpp", "gc_test.cpp", ] diff --git a/ecmascript/tests/concurrent_sweep_test.cpp b/ecmascript/tests/concurrent_sweep_test.cpp new file mode 100644 index 0000000000..6759428f34 --- /dev/null +++ b/ecmascript/tests/concurrent_sweep_test.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/tests/test_helper.h" + +#include "ecmascript/ecma_vm.h" +#include "ecmascript/global_env.h" +#include "ecmascript/js_handle.h" + +using namespace panda::ecmascript; + +namespace panda::test { +class ConcurrentSweepTest : public testing::Test { +public: + static void SetUpTestCase() + { + GTEST_LOG_(INFO) << "SetUpTestCase"; + } + + static void TearDownTestCase() + { + GTEST_LOG_(INFO) << "TearDownCase"; + } + + void SetUp() override + { + TestHelper::CreateEcmaVMWithScope(instance, thread, scope); + } + + void TearDown() override + { + TestHelper::DestroyEcmaVMWithScope(instance, scope); + } + + PandaVM *instance {nullptr}; + EcmaHandleScope *scope {nullptr}; + JSThread *thread; +}; + +TEST_F(ConcurrentSweepTest, ConcurrentSweep) +{ + auto vm = EcmaVM::Cast(instance); + const uint8_t *utf8 = reinterpret_cast("test"); + JSHandle test1(thread, EcmaString::CreateFromUtf8(utf8, 4, vm, false)); + if (vm->IsInitialized()) { + vm->CollectGarbage(ecmascript::TriggerGCType::OLD_GC); + } + JSHandle test2(thread, EcmaString::CreateFromUtf8(utf8, 4, vm, false)); + ASSERT_EQ(test1->GetLength(), 4); + ASSERT_NE(test1.GetTaggedValue().GetHeapObject(), test2.GetTaggedValue().GetHeapObject()); +} +} // namespace panda::test -- Gitee