diff --git a/common_components/common/work_stack-inl.h b/common_components/common/work_stack-inl.h new file mode 100644 index 0000000000000000000000000000000000000000..dbfa3e5e60345570b01bf4566c5795b49e992aeb --- /dev/null +++ b/common_components/common/work_stack-inl.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMON_COMPONENTS_COMMON_WORK_STACK_INL_H +#define COMMON_COMPONENTS_COMMON_WORK_STACK_INL_H + +#include "common_components/common/work_stack.h" + +#include + +namespace common { + +template +bool StackBase::IsEmpty() const +{ + DCHECK_CC(top_ <= capacity); + return top_ == 0; +} + +template +bool StackBase::IsFull() const +{ + DCHECK_CC(top_ <= capacity); + return top_ == capacity; +} + +template +void StackBase::Push(T *e) +{ + DCHECK_CC(top_ <= capacity); + DCHECK_CC(!IsFull()); + DCHECK_CC(e != nullptr); + data_[top_++] = e; +} + +template +void StackBase::Pop(T **e) +{ + DCHECK_CC(top_ <= capacity); + DCHECK_CC(!IsEmpty()); + T *result = data_[--top_]; + DCHECK_CC(result != nullptr); + *e = result; +} + +template +StackBase *StackBase::GetNext() const +{ + return next_; +} + +template +void StackBase::SetNext(StackBase *next) +{ + next_ = next; +} + +template +void StackList::Push(InternalStack *stack) +{ + DCHECK_CC(stack != nullptr); + DCHECK_CC(!stack->IsEmpty()); + std::lock_guard guard(mutex_); + stack->SetNext(head_); + head_ = stack; +} + +template +void StackList::Pop(InternalStack **stack) +{ + std::lock_guard guard(mutex_); + *stack = head_; + if (head_ != nullptr) { + head_ = head_->GetNext(); + } +} + +template +size_t StackList::Count() +{ + size_t cnt = 0; + std::lock_guard guard(mutex_); + InternalStack *current = head_; + while (current != nullptr) { + ++cnt; + current = current->GetNext(); + } + return cnt; +} + +namespace __work_stack_internal_impl { + +template +void LocalStackImpl::Push(T *e) +{ + DCHECK_CC(e != nullptr); + if (UNLIKELY_CC(inStack_->IsFull())) { + PushInStackToGlobal(); + } + DCHECK_CC(!inStack_->IsFull()); + inStack_->Push(e); +} + +template +bool LocalStackImpl::Pop(T **e) +{ + if (UNLIKELY_CC(outStack_->IsEmpty())) { + if (UNLIKELY_CC(!inStack_->IsEmpty())) { + std::swap(inStack_, outStack_); + } else if (!PopOutStackFromGlobal()) { + return false; + } + } + DCHECK_CC(!outStack_->IsEmpty()); + outStack_->Pop(e); + return true; +} + +template +bool LocalStackImpl::IsEmpty() const +{ + return inStack_->IsEmpty() && outStack_->IsEmpty(); +} + +template +void LocalStackImpl::Publish() +{ + if (!inStack_->IsEmpty()) { + PushInStackToGlobal(); + } + std::swap(inStack_, outStack_); + if (!inStack_->IsEmpty()) { + PushInStackToGlobal(); + } +} + +template +void LocalStackImpl::PushInStackToGlobal() +{ + DCHECK_CC(!inStack_->IsEmpty()); + globalStack_->Push(inStack_); + if constexpr (HAS_PUSH_TO_GLOBAL_NOTIFY) { + this->NotifyPushToGlobal(); + } + inStack_ = new InternalStack(); +} + +template +bool LocalStackImpl::PopOutStackFromGlobal() +{ + DCHECK_CC(outStack_->IsEmpty()); + InternalStack *newStack = nullptr; + globalStack_->Pop(&newStack); + if (LIKELY_CC(newStack != nullptr)) { + delete outStack_; + outStack_ = newStack; + return true; + } + return false; +} +} // namespace __work_stack_internal_impl +} // namespace common +#endif // COMMON_COMPONENTS_COMMON_WORK_STACK_INL_H diff --git a/common_components/common/work_stack.h b/common_components/common/work_stack.h new file mode 100644 index 0000000000000000000000000000000000000000..b42928753e1611bb2b6d87c9775909ab9fe28ff9 --- /dev/null +++ b/common_components/common/work_stack.h @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMON_COMPONENTS_COMMON_WORK_STACK_H +#define COMMON_COMPONENTS_COMMON_WORK_STACK_H + +#include +#include + +#include "common_interfaces/base/common.h" +#include "common_components/log/log.h" + +namespace common { +template +class StackBase { +public: + StackBase() = default; + ~StackBase() + { + DCHECK_CC(IsEmpty()); + } + + NO_COPY_SEMANTIC_CC(StackBase); + NO_MOVE_SEMANTIC_CC(StackBase); + + bool IsEmpty() const; + + bool IsFull() const; + + void Push(T *e); + + void Pop(T **e); + + StackBase *GetNext() const; + + void SetNext(StackBase *next); + +private: + size_t top_ {0}; + StackBase *next_ {nullptr}; + T *data_[capacity]; +}; + +namespace __work_stack_internal_impl { +template +class LocalStackImpl; +} + +template +class StackList { +public: + using InternalStack = StackBase; + StackList() = default; + ~StackList() + { + DCHECK_CC(head_ == nullptr); + } + + NO_COPY_SEMANTIC_CC(StackList); + NO_MOVE_SEMANTIC_CC(StackList); + + size_t Count(); + +private: + void Push(InternalStack *stack); + + void Pop(InternalStack **stack); + + InternalStack *head_ {nullptr}; + std::mutex mutex_; + + template + friend class __work_stack_internal_impl::LocalStackImpl; +}; + +namespace __work_stack_internal_impl { + +class LocalStackBaseWithoutNotify { +protected: + LocalStackBaseWithoutNotify() = default; + ~LocalStackBaseWithoutNotify() = default; + + NO_COPY_SEMANTIC_CC(LocalStackBaseWithoutNotify); + NO_MOVE_SEMANTIC_CC(LocalStackBaseWithoutNotify); +}; + +template +class LocalStackBaseWithNotify { +protected: + explicit LocalStackBaseWithNotify(PushToGlobalNotify *pushToGlobalNotify) + : pushToGlobalNotify_(pushToGlobalNotify) + { + DCHECK_CC(pushToGlobalNotify_ != nullptr); + } + ~LocalStackBaseWithNotify() + { + pushToGlobalNotify_ = nullptr; + } + + NO_COPY_SEMANTIC_CC(LocalStackBaseWithNotify); + NO_MOVE_SEMANTIC_CC(LocalStackBaseWithNotify); + + void NotifyPushToGlobal() + { + DCHECK_CC(pushToGlobalNotify_ != nullptr); + (*pushToGlobalNotify_)(); + } +private: + PushToGlobalNotify *pushToGlobalNotify_ {nullptr}; +}; + +struct DummyNoPushToGlobalNotify {}; + +template +class LocalStackImpl final : public std::conditional_t, + LocalStackBaseWithoutNotify, + LocalStackBaseWithNotify> { + using InternalStack = StackBase; + using GlobalStack = StackList; +private: + static constexpr bool HAS_PUSH_TO_GLOBAL_NOTIFY = !std::is_same_v; +public: + template >> + LocalStackImpl(GlobalStack *globalStack, PushToGlobalNotify *pushToGlobalNotify) + : LocalStackBaseWithNotify(pushToGlobalNotify), globalStack_(globalStack) + { + inStack_ = new InternalStack(); + outStack_ = new InternalStack(); + } + + template >> + explicit LocalStackImpl(GlobalStack *globalStack) : globalStack_(globalStack) + { + inStack_ = new InternalStack(); + outStack_ = new InternalStack(); + } + + ~LocalStackImpl() + { + DCHECK_CC(IsEmpty()); + delete inStack_; + delete outStack_; + inStack_ = nullptr; + outStack_ = nullptr; + } + + NO_COPY_SEMANTIC_CC(LocalStackImpl); + NO_MOVE_SEMANTIC_CC(LocalStackImpl); + + void Push(T *e); + + bool Pop(T **e); + + bool IsEmpty() const; + + void Publish(); + +private: + void PushInStackToGlobal(); + + bool PopOutStackFromGlobal(); + + GlobalStack *globalStack_ {nullptr}; + InternalStack *inStack_ {nullptr}; + InternalStack *outStack_ {nullptr}; +}; +} // namespace __work_stack_internal_impl + +template +using LocalStack = __work_stack_internal_impl::LocalStackImpl; +} // namespace common +#endif // COMMON_COMPONENTS_COMMON_WORK_STACK_H diff --git a/common_components/common_runtime/hooks.h b/common_components/common_runtime/hooks.h index 4fe9a9487d08fd5846725091b7d77a8d49451bb9..a59c64f5b813c6c7865b674b5e8a4bda18df6852 100644 --- a/common_components/common_runtime/hooks.h +++ b/common_components/common_runtime/hooks.h @@ -22,7 +22,7 @@ #include "common_interfaces/thread/mutator_base.h" // Visitor that iterate all `RefField`s in a TaggedObject and add them to -// `WorkStack` Should be moved to BaseRT and panda namespace later +// `LocalMarkStack` Should be moved to BaseRT and panda namespace later namespace common { // Roots in BaseRuntime PUBLIC_API void VisitBaseRoots(const RefFieldVisitor &visitor); diff --git a/common_components/heap/ark_collector/ark_collector.cpp b/common_components/heap/ark_collector/ark_collector.cpp index 019834557fbbfe88e211f4199d02d8c6091442c4..49985689be0ee104a6d9917c3539846510ad0b9a 100755 --- a/common_components/heap/ark_collector/ark_collector.cpp +++ b/common_components/heap/ark_collector/ark_collector.cpp @@ -141,9 +141,9 @@ bool ArkCollector::TryUntagRefField(BaseObject* obj, RefField<>& field, BaseObje } static void MarkingRefField(BaseObject *obj, BaseObject *targetObj, RefField<> &field, - WorkStack &workStack, RegionDesc *targetRegion); + ParallelLocalMarkStack &markStack, RegionDesc *targetRegion); // note each ref-field will not be marked twice, so each old pointer the markingr meets must come from previous gc. -static void MarkingRefField(BaseObject *obj, RefField<> &field, WorkStack &workStack, +static void MarkingRefField(BaseObject *obj, RefField<> &field, ParallelLocalMarkStack &markStack, WeakStack &weakStack, const GCReason gcReason) { RefField<> oldField(field); @@ -171,12 +171,12 @@ static void MarkingRefField(BaseObject *obj, RefField<> &field, WorkStack &workS obj, &field, targetObj, targetObj->GetTypeInfo(), targetObj->GetSize()); return; } - common::MarkingRefField(obj, targetObj, field, workStack, targetRegion); + common::MarkingRefField(obj, targetObj, field, markStack, targetRegion); } // note each ref-field will not be marked twice, so each old pointer the markingr meets must come from previous gc. static void MarkingRefField(BaseObject *obj, BaseObject *targetObj, RefField<> &field, - WorkStack &workStack, RegionDesc *targetRegion) + ParallelLocalMarkStack &markStack, RegionDesc *targetRegion) { if (targetRegion->IsNewObjectSinceMarking(targetObj)) { DLOG(TRACE, "marking: skip new obj %p<%p>(%zu)", targetObj, targetObj->GetTypeInfo(), targetObj->GetSize()); @@ -190,23 +190,23 @@ static void MarkingRefField(BaseObject *obj, BaseObject *targetObj, RefField<> & DLOG(TRACE, "marking obj %p ref@%p: %p<%p>(%zu)", obj, &field, targetObj, targetObj->GetTypeInfo(), targetObj->GetSize()); - workStack.push_back(targetObj); + markStack.Push(targetObj); } -MarkingCollector::MarkingRefFieldVisitor ArkCollector::CreateMarkingObjectRefFieldsVisitor(WorkStack *workStack, - WeakStack *weakStack) +MarkingCollector::MarkingRefFieldVisitor ArkCollector::CreateMarkingObjectRefFieldsVisitor( + ParallelLocalMarkStack &markStack, WeakStack &weakStack) { MarkingRefFieldVisitor visitor; if (gcReason_ == GCReason::GC_REASON_YOUNG) { - visitor.SetVisitor([obj = visitor.GetClosure(), workStack, weakStack](RefField<> &field) { + visitor.SetVisitor([obj = visitor.GetClosure(), &markStack, &weakStack](RefField<> &field) { const GCReason gcReason = GCReason::GC_REASON_YOUNG; - MarkingRefField(*obj, field, *workStack, *weakStack, gcReason); + MarkingRefField(*obj, field, markStack, weakStack, gcReason); }); } else { - visitor.SetVisitor([obj = visitor.GetClosure(), workStack, weakStack](RefField<> &field) { + visitor.SetVisitor([obj = visitor.GetClosure(), &markStack, &weakStack](RefField<> &field) { const GCReason gcReason = GCReason::GC_REASON_HEU; - MarkingRefField(*obj, field, *workStack, *weakStack, gcReason); + MarkingRefField(*obj, field, markStack, weakStack, gcReason); }); } return visitor; @@ -292,8 +292,8 @@ BaseObject* ArkCollector::ForwardUpdateRawRef(ObjectRef& root) class RemarkAndPreforwardVisitor { public: - RemarkAndPreforwardVisitor(WorkStack &localStack, ArkCollector *collector) - : localStack_(localStack), collector_(collector) {} + RemarkAndPreforwardVisitor(LocalCollectStack &collectStack, ArkCollector *collector) + : collectStack_(collectStack), collector_(collector) {} void operator()(RefField<> &refField) { @@ -334,7 +334,7 @@ private: void MarkObject(BaseObject *object) { if (!RegionalHeap::IsNewObjectSinceMarking(object) && !collector_->MarkObject(object)) { - localStack_.push_back(object); + collectStack_.Push(object); } } @@ -345,30 +345,33 @@ private: // No need to count oldVersion object size, as it has been copied. collector_->MarkObject(toVersion); // oldVersion don't have valid type info, cannot push it - localStack_.push_back(toVersion); + collectStack_.Push(toVersion); } } private: - WorkStack &localStack_; + LocalCollectStack &collectStack_; ArkCollector *collector_; }; class RemarkingAndPreforwardTask : public common::Task { public: - RemarkingAndPreforwardTask(ArkCollector *collector, WorkStack &localStack, TaskPackMonitor &monitor, + RemarkingAndPreforwardTask(ArkCollector *collector, GlobalMarkStack &globalMarkStack, TaskPackMonitor &monitor, std::function& next) - : Task(0), visitor_(localStack, collector), monitor_(monitor), getNextMutator_(next) + : Task(0), collector_(collector), globalMarkStack_(globalMarkStack), monitor_(monitor), getNextMutator_(next) {} bool Run([[maybe_unused]] uint32_t threadIndex) override { ThreadLocal::SetThreadType(ThreadType::GC_THREAD); + LocalCollectStack collectStack(&globalMarkStack_); + RemarkAndPreforwardVisitor visitor(collectStack, collector_); Mutator *mutator = getNextMutator_(); while (mutator != nullptr) { - VisitMutatorRoot(visitor_, *mutator); + VisitMutatorRoot(visitor, *mutator); mutator = getNextMutator_(); } + collectStack.Publish(); ThreadLocal::SetThreadType(ThreadType::ARK_PROCESSOR); ThreadLocal::ClearAllocBufferRegion(); monitor_.NotifyFinishOne(); @@ -376,12 +379,13 @@ public: } private: - RemarkAndPreforwardVisitor visitor_; + ArkCollector *collector_ {nullptr}; + GlobalMarkStack &globalMarkStack_; TaskPackMonitor &monitor_; std::function &getNextMutator_; }; -void ArkCollector::ParallelRemarkAndPreforward(WorkStack& workStack) +void ArkCollector::ParallelRemarkAndPreforward(GlobalMarkStack &globalMarkStack) { std::vector taskList; MutatorManager &mutatorManager = MutatorManager::Instance(); @@ -400,34 +404,34 @@ void ArkCollector::ParallelRemarkAndPreforward(WorkStack& workStack) const uint32_t runningWorkers = std::min(GetGCThreadCount(true), taskList.size()); uint32_t parallelCount = runningWorkers + 1; // 1 :DaemonThread TaskPackMonitor monitor(runningWorkers, runningWorkers); - WorkStack localStack[parallelCount]; for (uint32_t i = 1; i < parallelCount; ++i) { - GetThreadPool()->PostTask(std::make_unique(this, localStack[i], monitor, + GetThreadPool()->PostTask(std::make_unique(this, globalMarkStack, monitor, getNextMutator)); } // Run in daemon thread. - RemarkAndPreforwardVisitor visitor(localStack[0], this); + LocalCollectStack collectStack(&globalMarkStack); + RemarkAndPreforwardVisitor visitor(collectStack, this); VisitGlobalRoots(visitor); Mutator *mutator = getNextMutator(); while (mutator != nullptr) { VisitMutatorRoot(visitor, *mutator); mutator = getNextMutator(); } + collectStack.Publish(); monitor.WaitAllFinished(); - for (uint32_t i = 0; i < parallelCount; ++i) { - workStack.insert(localStack[i]); - } } -void ArkCollector::RemarkAndPreforwardStaticRoots(WorkStack& workStack) +void ArkCollector::RemarkAndPreforwardStaticRoots(GlobalMarkStack &globalMarkStack) { OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::RemarkAndPreforwardStaticRoots", ""); const uint32_t maxWorkers = GetGCThreadCount(true) - 1; if (maxWorkers > 0) { - ParallelRemarkAndPreforward(workStack); + ParallelRemarkAndPreforward(globalMarkStack); } else { - RemarkAndPreforwardVisitor visitor(workStack, this); + LocalCollectStack collectStack(&globalMarkStack); + RemarkAndPreforwardVisitor visitor(collectStack, this); VisitSTWRoots(visitor); + collectStack.Publish(); } } diff --git a/common_components/heap/ark_collector/ark_collector.h b/common_components/heap/ark_collector/ark_collector.h index b102df4011a561676dc6cddd25750cf705a3f77d..a80338c74441af54a7349bdae500cc39eec011ae 100755 --- a/common_components/heap/ark_collector/ark_collector.h +++ b/common_components/heap/ark_collector/ark_collector.h @@ -90,7 +90,8 @@ public: bool ShouldIgnoreRequest(GCRequest& request) override; bool MarkObject(BaseObject* obj) const override; - MarkingRefFieldVisitor CreateMarkingObjectRefFieldsVisitor(WorkStack *workStack, WeakStack *weakStack) override; + MarkingRefFieldVisitor CreateMarkingObjectRefFieldsVisitor(ParallelLocalMarkStack &workStack, + WeakStack &weakStack) override; void MarkingObjectRefFields(BaseObject *obj, MarkingRefFieldVisitor *data) override; void FixObjectRefFields(BaseObject* obj) const override; @@ -215,8 +216,8 @@ private: void MarkingHeap(const CArrayList &collectedRoots); void PostMarking(); - void RemarkAndPreforwardStaticRoots(WorkStack& workStack) override; - void ParallelRemarkAndPreforward(WorkStack& workStack); + void RemarkAndPreforwardStaticRoots(GlobalMarkStack &globalMarkStack) override; + void ParallelRemarkAndPreforward(GlobalMarkStack &globalMarkStack); void Preforward(); void ConcurrentPreforward(); diff --git a/common_components/heap/ark_collector/tests/ark_collector_test.cpp b/common_components/heap/ark_collector/tests/ark_collector_test.cpp index 6aaef3211ec5edf411c1d41350eb781f0f2211d7..dc3e611ce06d5a7ec91eca70feca13a417b97fcd 100644 --- a/common_components/heap/ark_collector/tests/ark_collector_test.cpp +++ b/common_components/heap/ark_collector/tests/ark_collector_test.cpp @@ -224,9 +224,11 @@ HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST1) BaseObject *obj = reinterpret_cast(addr | TAG_HEAP_OBJECT_MASK); RefField<> field(obj); - MarkStack workStack; + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); WeakStack weakStack; - MarkingRefField(nullptr, field, workStack, weakStack, GCReason::GC_REASON_YOUNG); + MarkingRefField(nullptr, field, markStack, weakStack, GCReason::GC_REASON_YOUNG); EXPECT_FALSE(Heap::IsTaggedObject(field.GetFieldValue())); } @@ -238,11 +240,14 @@ HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST2) BaseObject* obj = reinterpret_cast(addr); RefField field(obj); - MarkStack workStack; + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); WeakStack weakStack; - MarkingRefField(nullptr, field, workStack, weakStack, GCReason::GC_REASON_APPSPAWN); + MarkingRefField(nullptr, field, markStack, weakStack, GCReason::GC_REASON_APPSPAWN); EXPECT_FALSE(region->IsInOldSpace()); - workStack.pop_back(); + BaseObject *temp; + while (markStack.Pop(&temp)) {} } HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST3) @@ -253,11 +258,14 @@ HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST3) BaseObject* obj = reinterpret_cast(addr); RefField field(obj); - MarkStack workStack; + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); WeakStack weakStack; - MarkingRefField(nullptr, field, workStack, weakStack, GCReason::GC_REASON_APPSPAWN); + MarkingRefField(nullptr, field, markStack, weakStack, GCReason::GC_REASON_APPSPAWN); EXPECT_TRUE(region->IsInOldSpace()); - workStack.pop_back(); + BaseObject *temp; + while (markStack.Pop(&temp)) {} } HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST4) @@ -268,11 +276,14 @@ HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST4) BaseObject* obj = reinterpret_cast(addr); RefField field(obj); - MarkStack workStack; + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); WeakStack weakStack; - MarkingRefField(nullptr, field, workStack, weakStack, GCReason::GC_REASON_YOUNG); + MarkingRefField(nullptr, field, markStack, weakStack, GCReason::GC_REASON_YOUNG); EXPECT_FALSE(region->IsInOldSpace()); - workStack.pop_back(); + BaseObject *temp; + while (markStack.Pop(&temp)) {} } HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST5) @@ -283,9 +294,11 @@ HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST5) BaseObject* obj = reinterpret_cast(addr); RefField field(obj); - MarkStack workStack; + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); WeakStack weakStack; - MarkingRefField(nullptr, field, workStack, weakStack, GCReason::GC_REASON_YOUNG); + MarkingRefField(nullptr, field, markStack, weakStack, GCReason::GC_REASON_YOUNG); EXPECT_TRUE(region->IsInOldSpace()); } @@ -298,8 +311,10 @@ HWTEST_F_L0(ArkCollectorTest, MarkingRefField_TEST6) BaseObject* obj = reinterpret_cast(addr); RefField field(obj); - MarkStack workStack; - MarkingRefField(obj, obj, field, workStack, region); + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); + MarkingRefField(obj, obj, field, markStack, region); EXPECT_TRUE(region->IsNewObjectSinceMarking(obj)); } class TestCreateMarkingArkCollector : public MarkingCollector { @@ -323,7 +338,8 @@ public: BaseObject* CopyObjectAfterExclusive(BaseObject* obj) override { return nullptr; } void DoGarbageCollection() override {} bool IsCurrentPointer(RefField<>&) const override { return false; } - MarkingRefFieldVisitor CreateMarkingObjectRefFieldsVisitor(WorkStack *workStack, WeakStack *weakStack) override + MarkingRefFieldVisitor CreateMarkingObjectRefFieldsVisitor(ParallelLocalMarkStack &workStack, + WeakStack &weakStack) override { return MarkingRefFieldVisitor(); } @@ -334,11 +350,13 @@ HWTEST_F_L0(ArkCollectorTest, CreateMarkingObjectRefFieldsVisitor_TEST1) std::unique_ptr arkCollector = GetArkCollector(); ASSERT_TRUE(arkCollector != nullptr); - MarkStack workStack; + GlobalMarkStack globalMarkStack; + ParallelMarkingMonitor monitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); WeakStack weakStack; TestCreateMarkingArkCollector* collector = reinterpret_cast(arkCollector.get()); collector->SetGCReason(GCReason::GC_REASON_YOUNG); - auto visitor = arkCollector->CreateMarkingObjectRefFieldsVisitor(&workStack, &weakStack); + auto visitor = arkCollector->CreateMarkingObjectRefFieldsVisitor(markStack, weakStack); EXPECT_TRUE(visitor.GetRefFieldVisitor() != nullptr); } diff --git a/common_components/heap/collector/marking_collector.cpp b/common_components/heap/collector/marking_collector.cpp index b8dbc386eba7cdf41fc241733f65867ee4a97932..241f35a1681b566d433ae6ff8b52c66d429aa72d 100755 --- a/common_components/heap/collector/marking_collector.cpp +++ b/common_components/heap/collector/marking_collector.cpp @@ -49,41 +49,32 @@ void StaticRootTable::VisitRoots(const RefFieldVisitor& visitor) class ConcurrentMarkingTask : public common::Task { public: - ConcurrentMarkingTask(uint32_t id, MarkingCollector &tc, Taskpool *pool, TaskPackMonitor &monitor, - GlobalWorkStackQueue &globalQueue) - : Task(id), collector_(tc), threadPool_(pool), monitor_(monitor), globalQueue_(globalQueue) + ConcurrentMarkingTask(uint32_t id, MarkingCollector &tc, ParallelMarkingMonitor &monitor, + GlobalMarkStack &globalMarkStack) + : Task(id), collector_(tc), monitor_(monitor), globalMarkStack_(globalMarkStack) {} - // single work task without thread pool - ConcurrentMarkingTask(uint32_t id, MarkingCollector& tc, TaskPackMonitor &monitor, - GlobalWorkStackQueue &globalQueue) - : Task(id), collector_(tc), threadPool_(nullptr), monitor_(monitor), globalQueue_(globalQueue) - {} - - ~ConcurrentMarkingTask() override - { - threadPool_ = nullptr; - } + ~ConcurrentMarkingTask() override = default; // run concurrent marking task. bool Run([[maybe_unused]] uint32_t threadIndex) override { - while (true) { - WorkStack workStack = globalQueue_.PopWorkStack(); - if (workStack.empty()) { + ParallelLocalMarkStack markStack(&globalMarkStack_, &monitor_); + do { + if (!monitor_.TryStartStep()) { break; } - collector_.ProcessMarkStack(threadIndex, threadPool_, workStack, globalQueue_); - } + collector_.ProcessMarkStack(threadIndex, markStack); + monitor_.FinishStep(); + } while (monitor_.WaitNextStepOrFinished()); monitor_.NotifyFinishOne(); return true; } private: MarkingCollector &collector_; - Taskpool *threadPool_; - TaskPackMonitor &monitor_; - GlobalWorkStackQueue &globalQueue_; + ParallelMarkingMonitor &monitor_; + GlobalMarkStack &globalMarkStack_; }; class ClearWeakStackTask : public common::Task { @@ -125,28 +116,6 @@ private: GlobalWeakStackQueue &globalQueue_; }; -void MarkingCollector::TryForkTask(Taskpool *threadPool, WorkStack &workStack, GlobalWorkStackQueue &globalQueue) -{ - size_t size = workStack.size(); - if (size > MIN_MARKING_WORK_SIZE) { - bool doFork = false; - size_t newSize = 0; - if (size > MAX_MARKING_WORK_SIZE) { - newSize = size >> 1; // give 1/2 the stack to the thread pool as a new work task - doFork = true; - } else if (size > MIN_MARKING_WORK_SIZE) { - constexpr uint8_t shiftForEight = 3; - newSize = size >> shiftForEight; // give 1/8 the stack to the thread pool as a new work task - doFork = true; - } - - if (doFork) { - WorkStackBuf *hSplit = workStack.split(newSize); - globalQueue.AddWorkStack(WorkStack(hSplit)); - } - } -} - void MarkingCollector::ProcessWeakStack(WeakStack &weakStack) { while (!weakStack.empty()) { @@ -183,56 +152,46 @@ void MarkingCollector::ProcessWeakStack(WeakStack &weakStack) } } -void MarkingCollector::ProcessMarkStack([[maybe_unused]] uint32_t threadIndex, Taskpool *threadPool, - WorkStack &workStack, GlobalWorkStackQueue &globalQueue) +void MarkingCollector::ProcessMarkStack([[maybe_unused]] uint32_t threadIndex, ParallelLocalMarkStack &markStack) { size_t nNewlyMarked = 0; WeakStack weakStack; - auto visitor = CreateMarkingObjectRefFieldsVisitor(&workStack, &weakStack); - WorkStack remarkStack; - auto fetchFromSatbBuffer = [this, &workStack, &remarkStack]() { + auto visitor = CreateMarkingObjectRefFieldsVisitor(markStack, weakStack); + std::vector remarkStack; + auto fetchFromSatbBuffer = [this, &markStack, &remarkStack]() { SatbBuffer::Instance().TryFetchOneRetiredNode(remarkStack); + bool needProcess = false; while (!remarkStack.empty()) { BaseObject *obj = remarkStack.back(); remarkStack.pop_back(); if (Heap::IsHeapAddress(obj) && (!MarkObject(obj))) { - workStack.push_back(obj); + markStack.Push(obj); + needProcess = true; DLOG(TRACE, "tracing take from satb buffer: obj %p", obj); } } + return needProcess; }; size_t iterationCnt = 0; constexpr size_t maxIterationLoopNum = 1000; // loop until work stack empty. - do { - for (;;) { + while (true) { + BaseObject *object; + while (markStack.Pop(&object)) { ++nNewlyMarked; - if (workStack.empty()) { - break; - } - // get next object from work stack. - BaseObject *obj = workStack.back(); - workStack.pop_back(); - auto region = RegionDesc::GetAliveRegionDescAt(reinterpret_cast((void *)obj)); - region->AddLiveByteCount(obj->GetSize()); - [[maybe_unused]] auto beforeSize = workStack.count(); - MarkingObjectRefFields(obj, &visitor); - DLOG(TRACE, "[tracing] visit finished, workstack size: before=%d, after=%d, newly added=%d", beforeSize, - workStack.count(), workStack.count() - beforeSize); - // try to fork new task if needed. - if (threadPool != nullptr) { - TryForkTask(threadPool, workStack, globalQueue); - } + auto region = RegionDesc::GetAliveRegionDescAt(static_cast(reinterpret_cast(object))); + region->AddLiveByteCount(object->GetSize()); + MarkingObjectRefFields(object, &visitor); } - // Try some task from satb buffer, bound the loop to make sure it converges in time - if (++iterationCnt < maxIterationLoopNum) { - fetchFromSatbBuffer(); - if (workStack.empty()) { - fetchFromSatbBuffer(); - } + if (++iterationCnt >= maxIterationLoopNum) { + break; + } + if (!fetchFromSatbBuffer()) { + break; } - } while (!workStack.empty()); + } + DCHECK_CC(markStack.IsEmpty()); // newly marked statistics. markedObjectCount_.fetch_add(nNewlyMarked, std::memory_order_relaxed); MergeWeakStack(weakStack); @@ -292,21 +251,10 @@ private: bool worldStopped_; }; -void MarkingCollector::MergeAllocBufferRoots(WorkStack& workStack) +void MarkingCollector::TracingImpl(GlobalMarkStack &globalMarkStack, bool parallel, bool Remark) { - // hold mutator list lock to freeze mutator liveness, otherwise may access dead mutator fatally - MergeMutatorRootsScope lockScope; - theAllocator_.VisitAllocBuffers([&workStack](AllocationBuffer &buffer) { - buffer.MarkStack([&workStack](BaseObject *o) { workStack.push_back(o); }); - }); -} - -void MarkingCollector::TracingImpl(WorkStack& workStack, bool parallel, bool Remark) -{ - OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, ("CMCGC::TracingImpl_" + std::to_string(workStack.count())).c_str(), ""); - if (workStack.empty()) { - return; - } + OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, ("CMCGC::TracingImpl_" + std::to_string(globalMarkStack.Count())).c_str(), + ""); // enable parallel marking if we have thread pool. Taskpool *threadPool = GetThreadPool(); @@ -320,47 +268,30 @@ void MarkingCollector::TracingImpl(WorkStack& workStack, bool parallel, bool Rem parallelCount = GetGCThreadCount(true) - 1; } uint32_t threadCount = parallelCount + 1; - TaskPackMonitor monitor(parallelCount, parallelCount); - GlobalWorkStackQueue globalQueue; + ParallelMarkingMonitor monitor(parallelCount, parallelCount); for (uint32_t i = 0; i < parallelCount; ++i) { - threadPool->PostTask(std::make_unique(0, *this, threadPool, monitor, globalQueue)); - } - if (!AddConcurrentTracingWork(workStack, globalQueue, static_cast(threadCount))) { - ProcessMarkStack(0, threadPool, workStack, globalQueue); + threadPool->PostTask(std::make_unique(0, *this, monitor, globalMarkStack)); } - while (true) { - WorkStack stack = globalQueue.DrainAllWorkStack(); - if (stack.empty()) { + ParallelLocalMarkStack markStack(&globalMarkStack, &monitor); + do { + if (!monitor.TryStartStep()) { break; } - ProcessMarkStack(0, threadPool, stack, globalQueue); - } - globalQueue.NotifyFinish(); + ProcessMarkStack(0, markStack); + monitor.FinishStep(); + } while (monitor.WaitNextStepOrFinished()); monitor.WaitAllFinished(); } else { // serial marking with a single mark task. - GlobalWorkStackQueue globalQueue; - WorkStack stack(std::move(workStack)); - ProcessMarkStack(0, nullptr, stack, globalQueue); + // Fixme: this `ParallelLocalMarkStack` could be replaced with `SequentialLocalMarkStack`, and no need to + // use monitor, but this need to add template param to `ProcessMarkStack`. + // So for convenience just use a fake dummy parallel one. + ParallelMarkingMonitor dummyMonitor(0, 0); + ParallelLocalMarkStack markStack(&globalMarkStack, &dummyMonitor); + ProcessMarkStack(0, markStack); } } -bool MarkingCollector::AddConcurrentTracingWork(WorkStack& workStack, GlobalWorkStackQueue &globalQueue, - size_t threadCount) -{ - if (workStack.size() <= threadCount * MIN_MARKING_WORK_SIZE) { - return false; // too less init tasks, which may lead to workload imbalance, add work rejected - } - DCHECK_CC(threadCount > 0); - const size_t chunkSize = std::min(workStack.size() / threadCount + 1, MIN_MARKING_WORK_SIZE); - // Split the current work stack into work tasks. - while (!workStack.empty()) { - WorkStackBuf *hSplit = workStack.split(chunkSize); - globalQueue.AddWorkStack(WorkStack(hSplit)); - } - return true; -} - bool MarkingCollector::AddWeakStackClearWork(WeakStack &weakStack, GlobalWeakStackQueue &globalQueue, size_t threadCount) @@ -378,7 +309,7 @@ bool MarkingCollector::AddWeakStackClearWork(WeakStack &weakStack, return true; } -bool MarkingCollector::PushRootToWorkStack(RootSet *workStack, BaseObject *obj) +bool MarkingCollector::PushRootToWorkStack(LocalCollectStack &collectStack, BaseObject *obj) { RegionDesc *regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); if (gcReason_ == GCReason::GC_REASON_YOUNG && !regionInfo->IsInYoungSpace()) { @@ -392,19 +323,20 @@ bool MarkingCollector::PushRootToWorkStack(RootSet *workStack, BaseObject *obj) ASSERT(!regionInfo->IsGarbageRegion()); DLOG(TRACE, "mark obj %p<%p>(%zu) in region %p(%u)@%#zx, live %u", obj, obj->GetTypeInfo(), obj->GetSize(), regionInfo, regionInfo->GetRegionType(), regionInfo->GetRegionStart(), regionInfo->GetLiveByteCount()); - workStack->push_back(obj); + collectStack.Push(obj); return true; } else { return false; } } -void MarkingCollector::PushRootsToWorkStack(RootSet *workStack, const CArrayList &collectedRoots) +void MarkingCollector::PushRootsToWorkStack(LocalCollectStack &collectStack, + const CArrayList &collectedRoots) { OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, ("CMCGC::PushRootsToWorkStack_" + std::to_string(collectedRoots.size())).c_str(), ""); for (BaseObject *obj : collectedRoots) { - PushRootToWorkStack(workStack, obj); + PushRootToWorkStack(collectStack, obj); } } @@ -412,40 +344,45 @@ void MarkingCollector::MarkingRoots(const CArrayList &collectedRoo { OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::MarkingRoots", ""); - WorkStack workStack = NewWorkStack(); - PushRootsToWorkStack(&workStack, collectedRoots); + GlobalMarkStack globalMarkStack; + + { + LocalCollectStack collectStack(&globalMarkStack); + + PushRootsToWorkStack(collectStack, collectedRoots); + + if (Heap::GetHeap().GetGCReason() == GC_REASON_YOUNG) { + OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::PushRootInRSet", ""); + auto func = [this, &collectStack](BaseObject *object) { MarkRememberSetImpl(object, collectStack); }; + RegionalHeap &space = reinterpret_cast(Heap::GetHeap().GetAllocator()); + space.MarkRememberSet(func); + } - if (Heap::GetHeap().GetGCReason() == GC_REASON_YOUNG) { - OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::PushRootInRSet", ""); - auto func = [this, &workStack](BaseObject *object) { MarkRememberSetImpl(object, workStack); }; - RegionalHeap &space = reinterpret_cast(Heap::GetHeap().GetAllocator()); - space.MarkRememberSet(func); + collectStack.Publish(); } COMMON_PHASE_TIMER("MarkingRoots"); - VLOG(DEBUG, "roots size: %zu", workStack.size()); ASSERT_LOGF(GetThreadPool() != nullptr, "null thread pool"); // use fewer threads and lower priority for concurrent mark. const uint32_t maxWorkers = GetGCThreadCount(true) - 1; - VLOG(DEBUG, "Concurrent mark with %u threads, workStack: %zu", (maxWorkers + 1), workStack.size()); { COMMON_PHASE_TIMER("Concurrent marking"); - TracingImpl(workStack, maxWorkers > 0, false); + TracingImpl(globalMarkStack, maxWorkers > 0, false); } } void MarkingCollector::Remark() { - WorkStack workStack = NewWorkStack(); + GlobalMarkStack globalMarkStack; const uint32_t maxWorkers = GetGCThreadCount(true) - 1; OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::Remark[STW]", ""); COMMON_PHASE_TIMER("STW re-marking"); - RemarkAndPreforwardStaticRoots(workStack); - ConcurrentRemark(workStack, maxWorkers > 0); // Mark enqueue - TracingImpl(workStack, maxWorkers > 0, true); + RemarkAndPreforwardStaticRoots(globalMarkStack); + ConcurrentRemark(globalMarkStack, maxWorkers > 0); // Mark enqueue + TracingImpl(globalMarkStack, maxWorkers > 0, true); MarkAwaitingJitFort(); // Mark awaiting ClearWeakStack(maxWorkers > 0); @@ -490,13 +427,12 @@ void MarkingCollector::ClearWeakStack(bool parallel) } } - -bool MarkingCollector::MarkSatbBuffer(WorkStack& workStack) +bool MarkingCollector::MarkSatbBuffer(GlobalMarkStack &globalMarkStack) { OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::MarkSatbBuffer", ""); COMMON_PHASE_TIMER("MarkSatbBuffer"); - auto visitSatbObj = [this, &workStack]() { - WorkStack remarkStack; + auto visitSatbObj = [this, &globalMarkStack]() { + std::vector remarkStack; auto func = [&remarkStack](Mutator& mutator) { const SatbBuffer::TreapNode* node = mutator.GetSatbBufferNode(); if (node != nullptr) { @@ -506,41 +442,41 @@ bool MarkingCollector::MarkSatbBuffer(WorkStack& workStack) MutatorManager::Instance().VisitAllMutators(func); SatbBuffer::Instance().GetRetiredObjects(remarkStack); + LocalCollectStack collectStack(&globalMarkStack); while (!remarkStack.empty()) { // LCOV_EXCL_BR_LINE BaseObject* obj = remarkStack.back(); remarkStack.pop_back(); - if (Heap::IsHeapAddress(obj)) { - if (!this->MarkObject(obj)) { - workStack.push_back(obj); - DLOG(TRACE, "satb buffer add obj %p", obj); - } + if (Heap::IsHeapAddress(obj) && !this->MarkObject(obj)) { + collectStack.Push(obj); + DLOG(TRACE, "satb buffer add obj %p", obj); } } + collectStack.Publish(); }; visitSatbObj(); return true; } -void MarkingCollector::MarkRememberSetImpl(BaseObject* object, WorkStack& workStack) +void MarkingCollector::MarkRememberSetImpl(BaseObject* object, LocalCollectStack &collectStack) { - object->ForEachRefField([this, &workStack, &object](RefField<>& field) { + object->ForEachRefField([this, &collectStack, &object](RefField<>& field) { BaseObject* targetObj = field.GetTargetObject(); if (Heap::IsHeapAddress(targetObj)) { RegionDesc* region = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(targetObj)); if (region->IsInYoungSpace() && !region->IsNewObjectSinceMarking(targetObj) && !this->MarkObject(targetObj)) { - workStack.push_back(targetObj); + collectStack.Push(targetObj); DLOG(TRACE, "remember set marking obj: %p@%p, ref: %p", object, &field, targetObj); } } }); } -void MarkingCollector::ConcurrentRemark(WorkStack& remarkStack, bool parallel) +void MarkingCollector::ConcurrentRemark(GlobalMarkStack &globalMarkStack, bool parallel) { - LOGF_CHECK(MarkSatbBuffer(remarkStack)) << "not cleared\n"; + LOGF_CHECK(MarkSatbBuffer(globalMarkStack)) << "not cleared\n"; } void MarkingCollector::MarkAwaitingJitFort() diff --git a/common_components/heap/collector/marking_collector.h b/common_components/heap/collector/marking_collector.h index d15c225a879875828c2d272818c482f008980084..fd8c46c7f2b695da0d100197fb5bf3112209c181 100755 --- a/common_components/heap/collector/marking_collector.h +++ b/common_components/heap/collector/marking_collector.h @@ -22,6 +22,7 @@ #include "common_components/heap/collector/collector.h" #include "common_components/heap/collector/collector_resources.h" #include "common_components/common/mark_work_stack.h" +#include "common_components/common/work_stack-inl.h" #include "common_components/heap/allocator/regional_heap.h" #include "common_components/heap/collector/copy_data_manager.h" #include "common_components/mutator/mutator_manager.h" @@ -102,14 +103,28 @@ private: size_t totalRootsCount_; }; +class ParallelMarkingMonitor : public TaskPackMonitor { +public: + explicit ParallelMarkingMonitor(int posted, int capacity) : TaskPackMonitor(posted, capacity) {} + ~ParallelMarkingMonitor() override = default; + + void operator()() + { + WakeUpRunnerApproximately(); + } +}; + class MarkingWork; class ConcurrentMarkingWork; using RootSet = MarkStack; -using WorkStack = MarkStack; +constexpr size_t LOCAL_MARK_STACK_CAPACITY = 128; +using GlobalMarkStack = StackList; +using ParallelLocalMarkStack = LocalStack; +using SequentialLocalMarkStack = LocalStack; +using LocalCollectStack = LocalStack; using WorkStackBuf = MarkStackBuffer; using WeakStack = MarkStack*, size_t>>>; using WeakStackBuf = MarkStackBuffer*, size_t>>>; -using GlobalWorkStackQueue = GlobalStackQueue; using GlobalWeakStackQueue = GlobalStackQueue; class MarkingCollector : public Collector { @@ -164,12 +179,9 @@ public: bool ShouldIgnoreRequest(GCRequest& request) override { return request.ShouldBeIgnored(); } - void ProcessMarkStack(uint32_t threadIndex, Taskpool *threadPool, WorkStack &workStack, - GlobalWorkStackQueue &globalQueue); + void ProcessMarkStack(uint32_t threadIndex, ParallelLocalMarkStack &workStack); void ProcessWeakStack(WeakStack &weakStack); - void TryForkTask(Taskpool *threadPool, WorkStack &workStack, GlobalWorkStackQueue &globalQueue); - // live but not resurrected object. bool IsMarkedObject(const BaseObject* obj) const { return RegionalHeap::IsMarkedObject(obj); } @@ -204,7 +216,8 @@ public: common::RefFieldVisitor visitor_; std::shared_ptr closure_; }; - virtual MarkingRefFieldVisitor CreateMarkingObjectRefFieldsVisitor(WorkStack *workStack, WeakStack *weakStack) = 0; + virtual MarkingRefFieldVisitor CreateMarkingObjectRefFieldsVisitor(ParallelLocalMarkStack &workStack, + WeakStack &weakStack) = 0; virtual void MarkingObjectRefFields(BaseObject *obj, MarkingRefFieldVisitor *data) = 0; inline bool IsResurrectedObject(const BaseObject* obj) const { return RegionalHeap::IsResurrectedObject(obj); } @@ -289,12 +302,6 @@ protected: return collectorResources_.GetGCThreadCount(isConcurrent); } - inline WorkStack NewWorkStack() const - { - WorkStack workStack = WorkStack(); - return workStack; - } - inline void SetGCReason(const GCReason reason) { gcReason_ = reason; } Taskpool *GetThreadPool() const { return collectorResources_.GetThreadPool(); } @@ -304,29 +311,26 @@ protected: virtual void ProcessStringTable() {} virtual void ProcessFinalizers() {} - virtual void RemarkAndPreforwardStaticRoots(WorkStack& workStack) + virtual void RemarkAndPreforwardStaticRoots(GlobalMarkStack &globalMarkStack) { LOG_COMMON(FATAL) << "Unresolved fatal"; UNREACHABLE_CC(); } - void MergeAllocBufferRoots(WorkStack& workStack); - - bool PushRootToWorkStack(RootSet *workStack, BaseObject *obj); - void PushRootsToWorkStack(RootSet *workStack, const CArrayList &collectedRoots); + bool PushRootToWorkStack(LocalCollectStack &markStack, BaseObject *obj); + void PushRootsToWorkStack(LocalCollectStack &markStack, const CArrayList &collectedRoots); void MarkingRoots(const CArrayList &collectedRoots); void Remark(); - bool MarkSatbBuffer(WorkStack& workStack); + bool MarkSatbBuffer(GlobalMarkStack &globalMarkStack); // concurrent marking. - void TracingImpl(WorkStack& workStack, bool parallel, bool Remark); + void TracingImpl(GlobalMarkStack &globalMarkStack, bool parallel, bool Remark); - bool AddConcurrentTracingWork(WorkStack& workStack, GlobalWorkStackQueue &globalQueue, size_t threadCount); bool AddWeakStackClearWork(WeakStack& workStack, GlobalWeakStackQueue &globalQueue, size_t threadCount); private: - void MarkRememberSetImpl(BaseObject* object, WorkStack& workStack); - void ConcurrentRemark(WorkStack& remarkStack, bool parallel); + void MarkRememberSetImpl(BaseObject* object, LocalCollectStack &markStack); + void ConcurrentRemark(GlobalMarkStack &globalMarkStack, bool parallel); void MarkAwaitingJitFort(); void EnumMutatorRoot(ObjectPtr& obj, RootSet& rootSet) const; void EnumConcurrencyModelRoots(RootSet& rootSet) const; diff --git a/common_components/heap/collector/tests/marking_collector_test.cpp b/common_components/heap/collector/tests/marking_collector_test.cpp index 616de3272758066becdf2d3b15bf95ffb6a9c977..f96ff418e6b8f9789e613a2d95dc8d8f842445fd 100755 --- a/common_components/heap/collector/tests/marking_collector_test.cpp +++ b/common_components/heap/collector/tests/marking_collector_test.cpp @@ -135,22 +135,23 @@ HWTEST_F_L0(MarkingCollectorTest, PushRootToWorkStackTest) uintptr_t addr = theAllocator.AllocOldRegion(); ASSERT_NE(addr, 0); BaseObject* obj = reinterpret_cast(addr); - RootSet roots; + GlobalMarkStack globalMarkStack; + LocalCollectStack collectStack(&globalMarkStack); RegionDesc* region = RegionDesc::GetRegionDescAt(addr); region->SetRegionType(RegionDesc::RegionType::RECENT_LARGE_REGION); collector.SetGCReason(GC_REASON_NATIVE); region->MarkObject(obj); - bool result = collector.PushRootToWorkStack(&roots, obj); + bool result = collector.PushRootToWorkStack(collectStack, obj); ASSERT_FALSE(result); region->SetRegionType(RegionDesc::RegionType::RECENT_LARGE_REGION); collector.SetGCReason(GC_REASON_YOUNG); - result = collector.PushRootToWorkStack(&roots, obj); + result = collector.PushRootToWorkStack(collectStack, obj); ASSERT_FALSE(result); region->SetRegionType(RegionDesc::RegionType::OLD_REGION); collector.SetGCReason(GC_REASON_NATIVE); - result = collector.PushRootToWorkStack(&roots, obj); + result = collector.PushRootToWorkStack(collectStack, obj); ASSERT_FALSE(result); } } \ No newline at end of file diff --git a/common_components/taskpool/task.h b/common_components/taskpool/task.h index edf2c34c10ed033b1608131d860bf26c0d3718a7..102439db1b5a9594694af2c2bdcfc166ae42a15e 100644 --- a/common_components/taskpool/task.h +++ b/common_components/taskpool/task.h @@ -16,6 +16,7 @@ #ifndef COMMON_COMPONENTS_TASKPOOL_TASK_H #define COMMON_COMPONENTS_TASKPOOL_TASK_H +#include #include #include @@ -68,17 +69,17 @@ private: class TaskPackMonitor { public: - explicit TaskPackMonitor(int running, int maxRunning) : running_(running), maxRunning_(maxRunning) + explicit TaskPackMonitor(int posted, int capacity) : posted_(posted), capacity_(capacity) { - DCHECK_CC(running_ >= 0); - DCHECK_CC(running_ <= maxRunning_); + DCHECK_CC(posted_ >= 0); + DCHECK_CC(posted_ <= capacity_); } - ~TaskPackMonitor() = default; + virtual ~TaskPackMonitor() = default; void WaitAllFinished() { std::unique_lock lock(mutex_); - while (running_ > 0) { + while (posted_ > 0) { cv_.wait(lock); } } @@ -87,8 +88,8 @@ public: { std::lock_guard guard(mutex_); DCHECK_CC(running_ >= 0); - if (running_ < maxRunning_) { - ++running_; + if (posted_ < capacity_) { + ++posted_; return true; } return false; @@ -97,16 +98,63 @@ public: void NotifyFinishOne() { std::lock_guard guard(mutex_); + DCHECK_CC(posted_ >= 0); + if (--posted_ == 0) { + cv_.notify_all(); + } + } + + bool WaitNextStepOrFinished() + { + std::unique_lock lock(mutex_); + if (terminated_) { + return false; + } + cv_.wait(lock); + if (terminated_) { + return false; + } + return true; + } + + bool TryStartStep() + { + std::lock_guard guard(mutex_); + if (terminated_) { + return false; + } + ++running_; + DCHECK_CC(running_ <= capacity_ + 1); + return true; + } + + void FinishStep() + { + std::lock_guard guard(mutex_); + DCHECK_CC(!terminated_); + DCHECK_CC(running_ > 0); if (--running_ == 0) { + terminated_ = true; cv_.notify_all(); } } + void WakeUpRunnerApproximately() + { + // This check may fail because is not inside lock, but for an approximate waking up it is ok + size_t current = reinterpret_cast *>(&running_)->load(std::memory_order_relaxed); + if (UNLIKELY_CC(current < posted_)) { + cv_.notify_one(); + } + } + NO_COPY_SEMANTIC_CC(TaskPackMonitor); NO_MOVE_SEMANTIC_CC(TaskPackMonitor); private: - int running_ {0}; - int maxRunning_ {0}; + size_t running_ {0}; + size_t posted_ {0}; + size_t capacity_ {0}; + bool terminated_ {false}; std::condition_variable cv_; std::mutex mutex_; };