From e8eb248a20c067cedd98418823327c5b2abdf94e Mon Sep 17 00:00:00 2001 From: Artem Udovichenko Date: Thu, 15 Sep 2022 16:50:19 +0300 Subject: [PATCH 1/2] Refactor G1GC::RunPhasesImpl * Don't update regions in RunFulMarkAndProcessRefs * Group full gc functions * Refactor the tail of RunPhasesImpl * Remove if (is_full || is_mixed) in GetCollectibleRegions Change-Id: Iac3b497c523f6db22d31e5e701b3927df94e97f9 --- runtime/mem/gc/g1/g1-gc.cpp | 309 ++++++++++++++++++++---------------- runtime/mem/gc/g1/g1-gc.h | 8 + 2 files changed, 176 insertions(+), 141 deletions(-) diff --git a/runtime/mem/gc/g1/g1-gc.cpp b/runtime/mem/gc/g1/g1-gc.cpp index aa795fc81..cfab3c593 100644 --- a/runtime/mem/gc/g1/g1-gc.cpp +++ b/runtime/mem/gc/g1/g1-gc.cpp @@ -546,12 +546,7 @@ void G1GC::RunFullMarkAndProcessRefs(panda::GCTask &task, const { this->SetFullGC(true); LOG_DEBUG_GC << "Mark regions set size:" << collectible_regions.size(); - UpdateCollectionSet(collectible_regions); StartMarking(task); - for (auto region : collectible_regions) { - region->RmvFlag(RegionFlag::IS_COLLECTION_SET); - } - collection_set_ = CollectionSet(); // We can use pointer to vector here } template @@ -563,6 +558,19 @@ void G1GC::RunFullProcessRefsNoCollect(panda::GCTask &task) RunFullMarkAndProcessRefs(task, scan_set); } +static bool NeedToRunGC(const panda::GCTask &task) +{ + return (task.reason_ == GCTaskCause::YOUNG_GC_CAUSE) || (task.reason_ == GCTaskCause::OOM_CAUSE) || + (task.reason_ == GCTaskCause::HEAP_USAGE_THRESHOLD_CAUSE) || + (task.reason_ == GCTaskCause::STARTUP_COMPLETE_CAUSE) || (task.reason_ == GCTaskCause::EXPLICIT_CAUSE) || + (task.reason_ == GCTaskCause::NATIVE_ALLOC_CAUSE); +} + +static bool NeedFullGC(const panda::GCTask &task) +{ + return (task.reason_ == GCTaskCause::EXPLICIT_CAUSE) || (task.reason_ == GCTaskCause::OOM_CAUSE); +} + template void G1GC::RunPhasesImpl(panda::GCTask &task) { @@ -571,7 +579,6 @@ void G1GC::RunPhasesImpl(panda::GCTask &task) LOG_DEBUG_GC << "Footprint before GC: " << this->GetPandaVm()->GetMemStats()->GetFootprintHeap(); task.UpdateGCCollectionType(GCCollectionType::YOUNG); - uint64_t young_total_time {0}; this->GetTiming()->Reset(); size_t bytes_in_heap_before_move = this->GetPandaVm()->GetMemStats()->GetFootprintHeap(); { @@ -579,124 +586,35 @@ void G1GC::RunPhasesImpl(panda::GCTask &task) { GCScopedPauseStats scoped_pause_stats(this->GetPandaVm()->GetGCStats()); this->mem_stats_.Reset(); - if ((task.reason_ == GCTaskCause::YOUNG_GC_CAUSE) || (task.reason_ == GCTaskCause::OOM_CAUSE) || - (task.reason_ == GCTaskCause::HEAP_USAGE_THRESHOLD_CAUSE) || - (task.reason_ == GCTaskCause::STARTUP_COMPLETE_CAUSE) || - (task.reason_ == GCTaskCause::EXPLICIT_CAUSE) || (task.reason_ == GCTaskCause::NATIVE_ALLOC_CAUSE)) { + if (NeedToRunGC(task)) { this->GetPandaVm()->GetMemStats()->RecordGCPauseStart(); // Check there is no concurrent mark running by another thread. // Atomic with relaxed order reason: concurrent access with another thread which can running GC now ASSERT(!concurrent_marking_flag_.load(std::memory_order_relaxed)); - if ((task.reason_ == GCTaskCause::EXPLICIT_CAUSE) || (task.reason_ == GCTaskCause::OOM_CAUSE)) { - this->SetFullGC(true); - this->GetG1ObjectAllocator()->ReleaseEmptyRegions(); - } WaitForUpdateRemsetThread(); - // Atomic with acquire order reason: to see changes made by GC thread (which do concurrent marking and - // than set is_mixed_gc_required_) in mutator thread which waits for the end of concurrent marking. - auto collectible_regions = - GetCollectibleRegions(task, is_mixed_gc_required_.load(std::memory_order_acquire)); - // TODO(alovkov): add it ASSERT(!collectible_regions.empty()); - if (this->IsFullGC()) { - LOG_DEBUG_GC << "Explicit Full GC invocation due to a reason: " << task.reason_; - // Clear young first. We tried to maintain enough regions for that - // If the cause is OOM - most likely it is not true and GC will give up - // GCing Young can be done using usual young collection routines - // so no need to launch it with "full" flag. - this->SetFullGC(false); - // Issue 8183: Remove unnessesary SetYoungFullGC check after refactoring Full GC - SetYoungFullGC(true); - auto g1_allocator = this->GetG1ObjectAllocator(); - CollectionSet young_set(g1_allocator->GetYoungRegions()); - if (!young_set.empty() && HaveEnoughSpaceToMove(young_set)) { - LOG_DEBUG_GC << "Collect young-space, size:" << young_set.size(); - UpdateCollectionSet(young_set); - RunPhasesForRegions(task, young_set); - } - this->GetG1ObjectAllocator()->ReleaseEmptyRegions(); - // Issue 8183: Remove unnessesary SetYoungFullGC check after refactoring Full GC - SetYoungFullGC(false); - - // To efficiently get rid of garbage, we only work with tenured space, doing full mark - if (this->GetG1ObjectAllocator()->GetYoungRegions().empty() && HaveEnoughRegionsToMove(1)) { - RunFullForTenured(task); + if (NeedFullGC(task)) { + RunFullGC(task); + } else { + auto collectible_regions = + // Atomic with acquire order reason: to see changes made by GC thread (which do concurrent + // marking and than set is_mixed_gc_required_) in mutator thread which waits for the end of + // concurrent marking. + GetCollectibleRegions(task, is_mixed_gc_required_.load(std::memory_order_acquire)); + if (!collectible_regions.empty() && HaveEnoughSpaceToMove(collectible_regions)) { + // Ordinary collection flow + RunMixedGC(task, collectible_regions); } else { - // We cannot move or delete any garbage at this point - // However, some languages require some types of references being processed - // at OOM. That is possible since it doesn't require any free space - RunFullProcessRefsNoCollect(task); - LOG_INFO_GC << "Failed to run gc, not enough free regions"; - LOG_INFO_GC << "Accounted total object used bytes = " - << PoolManager::GetMmapMemPool()->GetObjectUsedBytes(); - } - } else if (!collectible_regions.empty() && HaveEnoughSpaceToMove(collectible_regions)) { - // Ordinary collection flow - time::Timer timer(&young_total_time, true); - LOG_DEBUG_GC << "Collect regions size:" << collectible_regions.size(); - UpdateCollectionSet(collectible_regions); - RunPhasesForRegions(task, collectible_regions); - if (!HaveEnoughSpaceToMove(collectible_regions)) { - LOG_DEBUG_GC << "Not leaving enough regions for next young/mixed collection. Proceed to " - "iterative implicit Full GC"; - // Slow path, full GC. We're short on free regions. In order to prevent OOM at the next GC, - // try to free up enough regions so we can do mixed/young once again next time. - // Here, we have a chance to compact heap so at the next GC mixed is going to have enough - // regions to move to tenured. Without this step, we won't be able to do full at any time, since - // we permanently won't have enough regions to move young to, thus not collecting anything - this->GetG1ObjectAllocator()->ReleaseEmptyRegions(); - RunFullForTenured(task); + LOG_DEBUG_GC << "Failed to run gc: " + << (collectible_regions.empty() ? "nothing to collect in movable space" + : "not enough free regions to move"); } - if (!HaveEnoughSpaceToMove(collectible_regions)) { - LOG_DEBUG_GC << "Implicit Full GC failed to free up enough space. Expect OOM GC soon"; - LOG_DEBUG_GC << "Accounted total object used bytes = " - << PoolManager::GetMmapMemPool()->GetObjectUsedBytes(); - } - if (young_total_time > 0) { - this->GetStats()->AddTimeValue(young_total_time, TimeTypeStats::YOUNG_TOTAL_TIME); - } - } else { - LOG_DEBUG_GC << "Failed to run gc: " - << (collectible_regions.empty() ? "nothing to collect in movable space" - : "not enough free regions to move"); } this->GetPandaVm()->GetMemStats()->RecordGCPauseEnd(); } } - // Atomic with acquire order reason: to see changes made by GC thread (which do concurrent marking and than set - // is_mixed_gc_required_) in mutator thread which waits for the end of concurrent marking. - if (is_mixed_gc_required_.load(std::memory_order_acquire)) { - if (!HaveGarbageRegions()) { - // Atomic with release order reason: to see changes made by GC thread (which do concurrent marking and - // than set is_mixed_gc_required_) in mutator thread which waits for the end of concurrent marking. - is_mixed_gc_required_.store(false, std::memory_order_release); - } - } else if (!interrupt_concurrent_flag_ && this->ShouldRunTenuredGC(task)) { - ASSERT(collection_set_.empty()); - // Init concurrent marking - concurrent_marking_flag_ = true; - } - if (concurrent_marking_flag_ && !interrupt_concurrent_flag_) { - StartMarking(task); - concurrent_marking_flag_ = false; - // interrupt_concurrent_flag_ may be set during concurrent marking. - if (!interrupt_concurrent_flag_) { - Remark(task); - // Enable mixed GC - if (HaveGarbageRegions()) { - // Atomic with release order reason: to see changes made by GC thread (which do concurrent marking - // and than set is_mixed_gc_required_) in mutator thread which waits for the end of concurrent - // marking. - is_mixed_gc_required_.store(true, std::memory_order_release); - } - { - ConcurrentScope concurrent_scope(this); - CollectNonRegularObjects(task); - } - } else { - ClearSatb(); - } - } + ScheduleMixedGCAndConcurrentMark(task); + RunConcurrentMarkIfNeeded(task); } // Update global and GC memstats based on generational memstats information // We will update tenured stats and record allocations, so set 'true' values @@ -706,6 +624,117 @@ void G1GC::RunPhasesImpl(panda::GCTask &task) this->SetFullGC(false); } +template +void G1GC::RunFullGC(panda::GCTask &task) +{ + LOG_DEBUG_GC << "Explicit Full GC invocation due to a reason: " << task.reason_; + auto g1_allocator = this->GetG1ObjectAllocator(); + g1_allocator->ReleaseEmptyRegions(); + // Clear young first. We tried to maintain enough regions for that + // If the cause is OOM - most likely it is not true and GC will give up + // GCing Young can be done using usual young collection routines + // so no need to launch it with "full" flag. + this->SetFullGC(false); + // Issue 8183: Remove unnessesary SetYoungFullGC check after refactoring Full GC + SetYoungFullGC(true); + CollectionSet young_set(g1_allocator->GetYoungRegions()); + if (!young_set.empty() && HaveEnoughSpaceToMove(young_set)) { + LOG_DEBUG_GC << "Collect young-space, size:" << young_set.size(); + UpdateCollectionSet(young_set); + RunPhasesForRegions(task, young_set); + } + this->GetG1ObjectAllocator()->ReleaseEmptyRegions(); + // Issue 8183: Remove unnessesary SetYoungFullGC check after refactoring Full GC + SetYoungFullGC(false); + + // To efficiently get rid of garbage, we only work with tenured space, doing full mark + if (g1_allocator->GetYoungRegions().empty() && HaveEnoughRegionsToMove(1)) { + RunFullForTenured(task); + } else { + // We cannot move or delete any garbage at this point + // However, some languages require some types of references being processed + // at OOM. That is possible since it doesn't require any free space + RunFullProcessRefsNoCollect(task); + LOG_INFO_GC << "Failed to run gc, not enough free regions"; + LOG_INFO_GC << "Accounted total object used bytes = " << PoolManager::GetMmapMemPool()->GetObjectUsedBytes(); + } +} + +template +void G1GC::RunMixedGC(panda::GCTask &task, const CollectionSet &collection_set) +{ + uint64_t young_total_time {0}; + { + time::Timer timer(&young_total_time, true); + LOG_DEBUG_GC << "Collect regions size:" << collection_set.size(); + UpdateCollectionSet(collection_set); + RunPhasesForRegions(task, collection_set); + if (!HaveEnoughSpaceToMove(collection_set)) { + LOG_DEBUG_GC << "Not leaving enough regions for next young/mixed collection. Proceed to " + "iterative implicit Full GC"; + // Slow path, full GC. We're short on free regions. In order to prevent OOM at the next GC, + // try to free up enough regions so we can do mixed/young once again next time. + // Here, we have a chance to compact heap so at the next GC mixed is going to have enough + // regions to move to tenured. Without this step, we won't be able to do full at any time, since + // we permanently won't have enough regions to move young to, thus not collecting anything + this->GetG1ObjectAllocator()->ReleaseEmptyRegions(); + RunFullForTenured(task); + } + if (!HaveEnoughSpaceToMove(collection_set)) { + LOG_DEBUG_GC << "Implicit Full GC failed to free up enough space. Expect OOM GC soon"; + LOG_DEBUG_GC << "Accounted total object used bytes = " + << PoolManager::GetMmapMemPool()->GetObjectUsedBytes(); + } + } + if (young_total_time > 0) { + this->GetStats()->AddTimeValue(young_total_time, TimeTypeStats::YOUNG_TOTAL_TIME); + } +} + +template +void G1GC::ScheduleMixedGCAndConcurrentMark(panda::GCTask &task) +{ + // Atomic with acquire order reason: to see changes made by GC thread (which do concurrent marking and than set + // is_mixed_gc_required_) in mutator thread which waits for the end of concurrent marking. + if (is_mixed_gc_required_.load(std::memory_order_acquire)) { + if (!HaveGarbageRegions()) { + // Atomic with release order reason: to see changes made by GC thread (which do concurrent marking and + // than set is_mixed_gc_required_) in mutator thread which waits for the end of concurrent marking. + is_mixed_gc_required_.store(false, std::memory_order_release); + } + } else if (!interrupt_concurrent_flag_ && this->ShouldRunTenuredGC(task)) { + ASSERT(collection_set_.empty()); + // Init concurrent marking + concurrent_marking_flag_ = true; + } +} + +template +void G1GC::RunConcurrentMarkIfNeeded(panda::GCTask &task) +{ + if (concurrent_marking_flag_ && !interrupt_concurrent_flag_) { + StartMarking(task); + concurrent_marking_flag_ = false; + // interrupt_concurrent_flag_ may be set during concurrent marking. + if (!interrupt_concurrent_flag_) { + Remark(task); + // Enable mixed GC + if (HaveGarbageRegions()) { + // Atomic with release order reason: to see changes made by GC thread (which do concurrent marking + // and than set is_mixed_gc_required_) in mutator thread which waits for the end of concurrent + // marking. + is_mixed_gc_required_.store(true, std::memory_order_release); + } + { + ConcurrentScope concurrent_scope(this); + CollectNonRegularObjects(task); + } + } else { + ClearSatb(); + } + } +} + template bool G1GC::HaveGarbageRegions() { @@ -1293,36 +1322,34 @@ CollectionSet G1GC::GetCollectibleRegions(panda::GCTask const &t LOG_DEBUG_GC << "Start GetCollectibleRegions is_mixed: " << is_mixed << " reason: " << task.reason_; CollectionSet collection_set(g1_allocator->GetYoungRegions()); bool is_full_gc = this->IsFullGC(); - if (is_mixed || is_full_gc) { - if (is_full_gc) { - auto all_movable_regions = g1_allocator->GetMovableRegions(); - LOG_DEBUG_GC << "all movable region size: " << all_movable_regions.size(); - for (const auto ®ion : all_movable_regions) { - LOG_DEBUG_GC << "region: " << *region; - if (region->HasFlag(IS_EDEN)) { - continue; - } - ASSERT(!region->HasFlag(IS_NONMOVABLE) && !region->HasFlag(IS_LARGE_OBJECT)); - ASSERT(region->HasFlag(IS_OLD)); - collection_set.AddRegion(region); + if (is_full_gc) { + auto all_movable_regions = g1_allocator->GetMovableRegions(); + LOG_DEBUG_GC << "all movable region size: " << all_movable_regions.size(); + for (const auto ®ion : all_movable_regions) { + LOG_DEBUG_GC << "region: " << *region; + if (region->HasFlag(IS_EDEN)) { + continue; } - // make new region to move objects there - g1_allocator->ClearCurrentRegion(); - } else { - auto garbage_regions = g1_allocator->template GetTopGarbageRegions(number_of_mixed_tenured_regions_); - for (auto garbage_region : garbage_regions) { - ASSERT(!garbage_region->HasFlag(IS_EDEN)); - ASSERT(is_mixed_gc_required_); // to be sure that GetLiveBytes is calculated in concurrent - double garbage_rate = static_cast(garbage_region->GetGarbageBytes()) / garbage_region->Size(); - if (garbage_rate >= region_garbage_rate_threshold_) { - LOG_DEBUG_GC << "Garbage percentage in " << std::hex << garbage_region << " region = " << std::dec - << garbage_rate << " %, add to collection set"; - collection_set.AddRegion(garbage_region); - } else { - LOG_DEBUG_GC << "Garbage percentage in " << std::hex << garbage_region << " region = " << std::dec - << garbage_rate << " %, don't add to collection set"; - break; - } + ASSERT(!region->HasFlag(IS_NONMOVABLE) && !region->HasFlag(IS_LARGE_OBJECT)); + ASSERT(region->HasFlag(IS_OLD)); + collection_set.AddRegion(region); + } + // make new region to move objects there + g1_allocator->ClearCurrentRegion(); + } else if (is_mixed) { + auto garbage_regions = g1_allocator->template GetTopGarbageRegions(number_of_mixed_tenured_regions_); + for (auto garbage_region : garbage_regions) { + ASSERT(!garbage_region->HasFlag(IS_EDEN)); + ASSERT(is_mixed_gc_required_); // to be sure that GetLiveBytes is calculated in concurrent + double garbage_rate = static_cast(garbage_region->GetGarbageBytes()) / garbage_region->Size(); + if (garbage_rate >= region_garbage_rate_threshold_) { + LOG_DEBUG_GC << "Garbage percentage in " << std::hex << garbage_region << " region = " << std::dec + << garbage_rate << " %, add to collection set"; + collection_set.AddRegion(garbage_region); + } else { + LOG_DEBUG_GC << "Garbage percentage in " << std::hex << garbage_region << " region = " << std::dec + << garbage_rate << " %, don't add to collection set"; + break; } } } diff --git a/runtime/mem/gc/g1/g1-gc.h b/runtime/mem/gc/g1/g1-gc.h index f8e31d100..32fb72ff7 100644 --- a/runtime/mem/gc/g1/g1-gc.h +++ b/runtime/mem/gc/g1/g1-gc.h @@ -148,6 +148,14 @@ private: void RunPhasesImpl(GCTask &task) override; + void RunFullGC(panda::GCTask &task); + + void RunMixedGC(panda::GCTask &task, const CollectionSet &collection_set); + + void ScheduleMixedGCAndConcurrentMark(panda::GCTask &task); + + void RunConcurrentMarkIfNeeded(panda::GCTask &task); + void RunPhasesForRegions([[maybe_unused]] panda::GCTask &task, const CollectionSet &collectible_regions); void RunFullForTenured(panda::GCTask &task); -- Gitee From 7fcd97933ff1713c56ff0afd35c00b6e266bf514 Mon Sep 17 00:00:00 2001 From: Artem Udovichenko Date: Mon, 19 Sep 2022 22:18:05 +0300 Subject: [PATCH 2/2] [G1GC] Test remset after region promotion Change-Id: Ic19a1c9fa25996f9b90dfdf37a409ff1d9ff0e16 --- runtime/tests/g1gc_test.cpp | 86 +++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/runtime/tests/g1gc_test.cpp b/runtime/tests/g1gc_test.cpp index 5acca7102..677968117 100644 --- a/runtime/tests/g1gc_test.cpp +++ b/runtime/tests/g1gc_test.cpp @@ -774,6 +774,92 @@ TEST_F(G1GCPromotionTest, TestCorrectPromotionYoungRegion) } } +class PromotionRemSetChecker : public GCListener { +public: + PromotionRemSetChecker(VMHandle *array, VMHandle *string) + : array_(array), string_(string) + { + } + + void GCPhaseStarted(GCPhase phase) override + { + if (phase != GCPhase::GC_PHASE_MARK_YOUNG) { + return; + } + // Before marking young all remsets must by actual + CheckRemSets(); + } + + bool CheckRemSets() + { + Region *ref_region = ObjectToRegion(string_->GetPtr()); + found_ = false; + ref_region->GetRemSet()->IterateOverObjects([this](ObjectHeader *obj) { + if (obj == array_->GetPtr()) { + found_ = true; + } + }); + return found_; + } + + bool IsFound() const + { + return found_; + } + +private: + VMHandle *array_; + VMHandle *string_; + bool found_ = false; +}; + +TEST_F(G1GCPromotionTest, TestPromotedRegionHasValidRemSets) +{ + MTManagedThread *thread = MTManagedThread::GetCurrent(); + Runtime *runtime = Runtime::GetCurrent(); + LanguageContext ctx = runtime->GetLanguageContext(panda_file::SourceLang::PANDA_ASSEMBLY); + GC *gc = runtime->GetPandaVM()->GetGC(); + ScopedManagedCodeThread s(thread); + [[maybe_unused]] HandleScope scope(thread); + + VMHandle string(thread, AllocString(1)); + ASSERT_TRUE(ObjectToRegion(string.GetPtr())->IsYoung()); + { + ScopedNativeCodeThread sn(thread); + // Move string to tenured + GCTask task(GCTaskCause::YOUNG_GC_CAUSE); + task.Run(*gc); + } + ASSERT_TRUE(ObjectToRegion(string.GetPtr())->HasFlag(IS_OLD)); + + // Allocate an array which ocuppies more than half region. + // This array will be promoted. + auto *array_class = runtime->GetClassLinker()->GetExtension(ctx)->GetClassRoot(ClassRoot::ARRAY_STRING); + size_t elem_size = array_class->GetComponentSize(); + size_t array_size = DEFAULT_REGION_SIZE / 2; + size_t array_length = array_size / elem_size + 1; + VMHandle array(thread, AllocArray(array_length, ClassRoot::ARRAY_STRING, false)); + ASSERT_FALSE(array->IsForwarded()); + Region *array_region = ObjectToRegion(array.GetPtr()); + ASSERT_TRUE(array_region->IsYoung()); + array->Set(0, string.GetPtr()); + + PromotionRemSetChecker listener(&array, &string); + gc->AddListener(&listener); + { + ScopedNativeCodeThread sn(thread); + // Promote array's regions to tenured + GCTask task(GCTaskCause::YOUNG_GC_CAUSE); + task.Run(*gc); + ASSERT_FALSE(listener.IsFound()); + } + // Check the array was promoted. + ASSERT_TRUE(array_region == ObjectToRegion(array.GetPtr())); + + // Check remsets + ASSERT_TRUE(listener.CheckRemSets()); +} + class InterruptGCListener : public GCListener { public: InterruptGCListener(G1GCTest *test, VMHandle *array) : test_(test), array_(array) {} -- Gitee