diff --git a/static_core/plugins/ets/tests/ets_test_suite/coroutines/CMakeLists.txt b/static_core/plugins/ets/tests/ets_test_suite/coroutines/CMakeLists.txt index 211df5eee482cd6abe95dde75741b8da011bcd70..08e8c830d04f78ad2f110c9215aafc8f8df40f80 100644 --- a/static_core/plugins/ets/tests/ets_test_suite/coroutines/CMakeLists.txt +++ b/static_core/plugins/ets/tests/ets_test_suite/coroutines/CMakeLists.txt @@ -242,7 +242,7 @@ if (NOT PANDA_TARGET_ARM32) # NOTE(26403) Profilesaver is turned off because of data race add_ets_coroutines_test(FILE launch_oom.ets SKIP_ARM32_COMPILER - OPTIONS "--gc-type=epsilon" "--profilesaver-enabled=false" "--incremental-profilesaver-enabled=false" + OPTIONS "--gc-type=epsilon" "--profilesaver-enabled=false" "--incremental-profilesaver-enabled=false" "--coroutines-stack-mem-limit=268435456" IMPL "STACKFUL" OPTION_SETS_STACKFUL "POOL" WORKERS "ONE" @@ -252,7 +252,7 @@ if (NOT PANDA_TARGET_ARM32) # NOTE(26403) Profilesaver is turned off because of data race add_ets_coroutines_test(FILE launch_oom.ets SKIP_ARM32_COMPILER - OPTIONS "--gc-type=epsilon" "--profilesaver-enabled=false" "--incremental-profilesaver-enabled=false" + OPTIONS "--gc-type=epsilon" "--profilesaver-enabled=false" "--incremental-profilesaver-enabled=false" "--coroutines-stack-mem-limit=268435456" IMPL "STACKFUL" OPTION_SETS_STACKFUL "DEFAULT" WORKERS "ONE" diff --git a/static_core/plugins/ets/tests/native/exclusive_worker/exclusive_worker_tests.cpp b/static_core/plugins/ets/tests/native/exclusive_worker/exclusive_worker_tests.cpp index ab77fb95b9f707ad9a82b3addd723b0c8e96f59e..c5e53b873994c1e36404c28b4f023e4d63135487 100644 --- a/static_core/plugins/ets/tests/native/exclusive_worker/exclusive_worker_tests.cpp +++ b/static_core/plugins/ets/tests/native/exclusive_worker/exclusive_worker_tests.cpp @@ -86,6 +86,8 @@ public: ani_option {"--ext:compiler-enable-jit", nullptr}, // Note: here set the coroutine-e-workers-limit to 2 to save system resources ani_option {"--ext:coroutine-e-workers-limit=2", nullptr}, + // Note: reduce the maximum the coroutine number here to save system resources in the limit test + ani_option {"--ext:coroutines-stack-mem-limit=268435456", nullptr}, }; } diff --git a/static_core/runtime/CMakeLists.txt b/static_core/runtime/CMakeLists.txt index b74b5d6116a2fdac7b78f97e0ca18e6f7c3de3f4..cfcb6642afed0d9b7a71a46d5889a670d5a50965 100644 --- a/static_core/runtime/CMakeLists.txt +++ b/static_core/runtime/CMakeLists.txt @@ -168,6 +168,7 @@ set(SOURCES coroutines/stackful_coroutine_manager.cpp coroutines/stackful_coroutine_worker.cpp coroutines/stackful_coroutine_state_info.cpp + coroutines/stack_manager/stack_manager.cpp coroutines/coroutine_events.cpp coroutines/coroutine_stats.cpp coroutines/priority_queue.cpp diff --git a/static_core/runtime/coroutines/coroutine.h b/static_core/runtime/coroutines/coroutine.h index e004a034a0b8053804477af0c7d7b50ca078da6c..d4cb81c86a8d723825e8359c2d35bd4edd570932 100644 --- a/static_core/runtime/coroutines/coroutine.h +++ b/static_core/runtime/coroutines/coroutine.h @@ -74,7 +74,7 @@ public: enum class Type { MUTATOR, SCHEDULER, FINALIZER }; /// Needed for object locking - static constexpr ThreadId MAX_COROUTINE_ID = MarkWord::LIGHT_LOCK_THREADID_MAX_COUNT; + static constexpr ThreadId MAX_COROUTINE_ID = MarkWord::LIGHT_LOCK_THREADID_MAX_COUNT * 4; /// A helper struct that aggregates all EP related data for a coroutine with a managed EP struct ManagedEntrypointInfo { diff --git a/static_core/runtime/coroutines/stack_manager/stack_manager.cpp b/static_core/runtime/coroutines/stack_manager/stack_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3f2207596240c64abb7ec89a0da7fe73e6e6b3d7 --- /dev/null +++ b/static_core/runtime/coroutines/stack_manager/stack_manager.cpp @@ -0,0 +1,132 @@ +#include "runtime/coroutines/stack_manager/stack_manager.h" +#include "include/object_header.h" + +namespace ark { + +StackManager::StacksHolder *StackManager::AllocHolder(size_t poolSize) +{ + Pool stackPool = PoolManager::GetMmapMemPool()->AllocPool( + AlignUp(sizeof(StacksHolder) + poolSize, PANDA_POOL_ALIGNMENT_IN_BYTES), SpaceType::SPACE_TYPE_NATIVE_STACKS, + AllocatorType::NATIVE_STACKS_ALLOCATOR); + std::cout << stackPool.GetSize() << std::endl; + return new (stackPool.GetMem()) StacksHolder(poolSize); +} + +void StackManager::FreeHolder(StacksHolder *holder, size_t poolSize) +{ + PoolManager::GetMmapMemPool()->FreePool(holder, + AlignUp(sizeof(StacksHolder) + poolSize, PANDA_POOL_ALIGNMENT_IN_BYTES)); +} + +void StackManager::Initialize(size_t stackSize) +{ + std::cout << "StackManager::Initialize" << std::endl; + os::memory::LockHolder lh(mutex_); + poolSize_ = stackSize * STACK_COUNT_IN_POOL; + first_ = AllocHolder(poolSize_); + first_->next_ = nullptr; +} + +void StackManager::Finalize() +{ + os::memory::LockHolder lh(mutex_); + if (first_ != nullptr) { + // Check if StackManager have only one holder + ASSERT(first_->next_ == nullptr); + // Check if holder is totally free + ASSERT(first_->bitset_.none()); + FreeHolder(first_, poolSize_); + } +} + +uint8_t *StackManager::AcquireStack() +{ + os::memory::LockHolder lh(mutex_); + std::cout << "StackManager::AcquireStack START" << std::endl; + StacksHolder *current = first_; + StacksHolder *prev = nullptr; + do { + auto *stack = current->AllocStack(); + if (stack != nullptr) { + return stack; + } + prev = current; + current = current->next_; + } while (current != nullptr); + prev->next_ = AllocHolder(poolSize_); + prev->next_->next_ = nullptr; + auto stack = prev->next_->AllocStack(); + std::cout << "StackManager::AcquireStack EXIT " << stack << std::endl; + return stack; +} + +void StackManager::ReleaseStack(uint8_t *stack) +{ + os::memory::LockHolder lh(mutex_); + std::cout << "StackManager::ReleaseStack" << std::endl; + StacksHolder *current = first_; + auto *next = current->next_; + if (current->TryFreeStack(stack)) { + if (current->CheckIsFree() && next != nullptr) { + first_ = next; + FreeHolder(current, poolSize_); + } + return; + } + while (next != nullptr) { + if (next->TryFreeStack(stack)) { + if (next->CheckIsFree()) { + current->next_ = next->next_; + FreeHolder(next, poolSize_); + } + return; + } + current = next; + next = next->next_; + } + UNREACHABLE(); +} + +bool StackManager::StacksHolder::CheckIsFree() const +{ + return bitset_.none(); +} + +uint8_t *StackManager::StacksHolder::AllocStack() +{ + std::cout << "AllocStack.this = " << this << std::endl; + for (int i = 0; i < STACK_COUNT_IN_POOL; i++) { + if (!bitset_[i]) { + std::cout << "alloc = " << i << std::endl; + bitset_[i] = true; + return mem_ + (size_ / STACK_COUNT_IN_POOL) * i; + } + } + return nullptr; +} + +bool StackManager::StacksHolder::TryFreeStack(uint8_t *stack) +{ + if (stack < mem_ || stack >= (mem_ + size_)) { + std::cout << "mem_ = " << mem_ << "; stack = " << stack << "; size_ = " << size_ << std::endl; + return false; + } + auto stackSize = size_ / StackManager::STACK_COUNT_IN_POOL; + auto iter = (stack - mem_) / stackSize; + ASSERT(bitset_[iter]); + bitset_[iter] = false; + std::cout << "free = " << iter << std::endl; + return true; +} + +StackManager::StacksHolder::StacksHolder(size_t size) + : size_(size), mem_(reinterpret_cast(AlignUp(sizeof(StacksHolder) + size, os::mem::GetPageSize()))) +{ +} + +StackManager::StacksHolder::~StacksHolder() +{ + ASSERT(CheckIsFree()); +} + +} // namespace ark diff --git a/static_core/runtime/coroutines/stack_manager/stack_manager.h b/static_core/runtime/coroutines/stack_manager/stack_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..e763f93d9d9606fd579b0ed052db5abb6aa8fc65 --- /dev/null +++ b/static_core/runtime/coroutines/stack_manager/stack_manager.h @@ -0,0 +1,58 @@ +#ifndef RUNTIME_COROUTINES_STACK_MANAGE_STACK_MANAGER_H +#define RUNTIME_COROUTINES_STACK_MANAGE_STACK_MANAGER_H + +#include "libpandabase/mem/mem_pool.h" +#include "libpandabase/os/mutex.h" + +#include + +namespace ark { + +class StackManager { + static constexpr int STACK_COUNT_IN_POOL = 8U; + // Check if STACK_COUNT_IN_POOL is divisible by 8 + static_assert((STACK_COUNT_IN_POOL & 0xF) == STACK_COUNT_IN_POOL); + + struct StacksHolder { + public: + StacksHolder(size_t size); + ~StacksHolder(); + NO_COPY_SEMANTIC(StacksHolder); + NO_MOVE_SEMANTIC(StacksHolder); + + uint8_t *AllocStack(); + bool TryFreeStack(uint8_t *stack); + bool CheckIsFree() const; + + StacksHolder *next_ = nullptr; + const size_t size_; + std::bitset bitset_; + uint8_t *mem_; + }; + +public: + StackManager() = default; + ~StackManager() = default; + + NO_COPY_SEMANTIC(StackManager); + NO_MOVE_SEMANTIC(StackManager); + + void Initialize(size_t stackSize); + void Finalize(); + + uint8_t *AcquireStack(); + void ReleaseStack(uint8_t *stack); + +private: + static StacksHolder *AllocHolder(size_t poolSize); + static void FreeHolder(StacksHolder *holder, size_t poolSize); + + size_t poolSize_; + + os::memory::Mutex mutex_; + StacksHolder *first_ GUARDED_BY(mutex_) = nullptr; +}; + +} // namespace ark + +#endif // RUNTIME_COROUTINES_STACK_MANAGE_STACK_MANAGER_H diff --git a/static_core/runtime/coroutines/stackful_coroutine_manager.cpp b/static_core/runtime/coroutines/stackful_coroutine_manager.cpp index 78f5df0241ea05f6a44fb6e616d3b08e6cbdf74a..777d18908deaf276b90645242dfb544b0580f447 100644 --- a/static_core/runtime/coroutines/stackful_coroutine_manager.cpp +++ b/static_core/runtime/coroutines/stackful_coroutine_manager.cpp @@ -33,15 +33,19 @@ namespace ark { uint8_t *StackfulCoroutineManager::AllocCoroutineStack() { + /* Pool stackPool = PoolManager::GetMmapMemPool()->AllocPool( coroStackSizeBytes_, SpaceType::SPACE_TYPE_NATIVE_STACKS, AllocatorType::NATIVE_STACKS_ALLOCATOR); return static_cast(stackPool.GetMem()); + */ + return stackManager.AcquireStack(); } void StackfulCoroutineManager::FreeCoroutineStack(uint8_t *stack) { if (stack != nullptr) { - PoolManager::GetMmapMemPool()->FreePool(stack, coroStackSizeBytes_); + // PoolManager::GetMmapMemPool()->FreePool(stack, coroStackSizeBytes_); + stackManager.ReleaseStack(stack); } } @@ -268,6 +272,7 @@ void StackfulCoroutineManager::Initialize(Runtime *runtime, PandaVM *vm) CalculateUserCoroutinesLimits(userCoroutineCountLimit_, Runtime::GetCurrent()->GetOptions().GetCoroutinesUserLimit()); + stackManager.Initialize(coroStackSizeBytes_); ASSERT(commonWorkersCount_ + exclusiveWorkersLimit_ <= AffinityMask::MAX_WORKERS_COUNT); InitializeWorkerIdAllocator(); { @@ -293,6 +298,7 @@ void StackfulCoroutineManager::Finalize() CoroutineManager::DestroyEntrypointfulCoroutine(co); } coroutinePool_.clear(); + stackManager.Finalize(); } void StackfulCoroutineManager::AddToRegistry(Coroutine *co) diff --git a/static_core/runtime/coroutines/stackful_coroutine_manager.h b/static_core/runtime/coroutines/stackful_coroutine_manager.h index 97370c47e279671e7571803a7ee161dceca4b770..1c76804ba67d7f2dc5ccc90e1db155cf62c93c3f 100644 --- a/static_core/runtime/coroutines/stackful_coroutine_manager.h +++ b/static_core/runtime/coroutines/stackful_coroutine_manager.h @@ -23,6 +23,8 @@ #include "runtime/coroutines/stackful_coroutine_state_info.h" +#include "runtime/coroutines/stack_manager/stack_manager.h" + namespace ark { /** @@ -312,6 +314,8 @@ private: // data members CoroutineWorkerGroup::Id generalWorkerGroup_ = CoroutineWorkerGroup::Empty(); CoroutineWorkerGroup::Id eaWorkerGroup_ = CoroutineWorkerGroup::Empty(); + + StackManager stackManager; }; } // namespace ark diff --git a/static_core/runtime/options.yaml b/static_core/runtime/options.yaml index 4e94c55c52eca436c13224843c1021a6df6b8f13..9db54f2f7217e352b247bc8d14cfbce3c1263f6b 100644 --- a/static_core/runtime/options.yaml +++ b/static_core/runtime/options.yaml @@ -561,7 +561,7 @@ options: - name: internal-memory-size-limit type: uint64_t - default: 2147483648 + default: 8589934592 description: Max internal memory used by the VM - name: frames-memory-size-limit @@ -904,7 +904,7 @@ options: - name: coroutines-stack-mem-limit type: uint64_t - default: 536870912 + default: 4194304000 description: defines the total amount of memory that can be used for stackful coroutine stacks allocation (in bytes) - name: coroutines-user-limit