diff --git a/common_components/heap/allocator/alloc_buffer.h b/common_components/heap/allocator/alloc_buffer.h index 043a91970b86e6ada31b4d52a1c469ae89093510..96ebd6d09018dd3c8fe530923e0a95f40e99fca1 100755 --- a/common_components/heap/allocator/alloc_buffer.h +++ b/common_components/heap/allocator/alloc_buffer.h @@ -66,8 +66,8 @@ public: if (tlRegion_ == RegionDesc::NullRegion()) { return; } - DLOG(REGION, "AllocBuffer clear tlRegion %p@%#zx+%zu", - tlRegion_, tlRegion_->GetRegionStart(), tlRegion_->GetRegionAllocatedSize()); + DLOG(REGION, "AllocBuffer clear tlRegion %p(base=%#zx)@%#zx+%zu", + tlRegion_, tlRegion_->GetRegionBase(), tlRegion_->GetRegionStart(), tlRegion_->GetRegionAllocatedSize()); tlRegion_ = RegionDesc::NullRegion(); } void ClearThreadLocalRegion(); diff --git a/common_components/heap/allocator/region_desc.h b/common_components/heap/allocator/region_desc.h index 03dc5d1c015ce9b9d03e69c659bbdb0ed0245940..772dbc00a9a280d1fdd64de2d232d5eac47720ac 100755 --- a/common_components/heap/allocator/region_desc.h +++ b/common_components/heap/allocator/region_desc.h @@ -91,14 +91,17 @@ public: COPYED, }; - static const size_t UNIT_SIZE; // same as system page size + // default common region unit size. + static constexpr size_t UNIT_SIZE = 256 * KB; - // regarding a object as a large object when the size is greater than 8 units. - static const size_t LARGE_OBJECT_DEFAULT_THRESHOLD; + // threshold for object to unique a region + static constexpr size_t LARGE_OBJECT_DEFAULT_THRESHOLD = UNIT_SIZE * 2 / 3; // release a large object when the size is greater than 4096KB. static constexpr size_t LARGE_OBJECT_RELEASE_THRESHOLD = 4096 * KB; + static constexpr size_t DEFAULT_REGION_UNIT_MASK = RegionDesc::UNIT_SIZE - 1; + RegionDesc() { metadata.allocPtr = reinterpret_cast(nullptr); @@ -106,6 +109,7 @@ public: metadata.forwardLine = std::numeric_limits::max(); metadata.fixLine = std::numeric_limits::max(); metadata.freeSlot = nullptr; + metadata.regionBase = reinterpret_cast(nullptr); metadata.regionStart = reinterpret_cast(nullptr); metadata.regionEnd = reinterpret_cast(nullptr); metadata.regionRSet = nullptr; @@ -119,9 +123,8 @@ public: void SetReadOnly() { constexpr int pageProtRead = 1; - DLOG(REPORT, "try to set readonly to %p, size is %ld", GetRegionStart(), GetRegionEnd() - GetRegionStart()); - if (PageProtect(reinterpret_cast(GetRegionStart()), - GetRegionEnd() - GetRegionStart(), pageProtRead) != 0) { + DLOG(REPORT, "try to set readonly to %p, size is %ld", GetRegionBase(), GetRegionBaseSize()); + if (PageProtect(reinterpret_cast(GetRegionBase()), GetRegionBaseSize(), pageProtRead) != 0) { DLOG(REPORT, "set read only fail"); } } @@ -129,131 +132,74 @@ public: void ClearReadOnly() { constexpr int pageProtReadWrite = 3; - DLOG(REPORT, "try to set read & write to %p, size is %ld", GetRegionStart(), GetRegionEnd() - GetRegionStart()); - if (PageProtect(reinterpret_cast(GetRegionStart()), - GetRegionEnd() - GetRegionStart(), pageProtReadWrite) != 0) { + DLOG(REPORT, "try to set read & write to %p, size is %ld", GetRegionBase(), GetRegionBaseSize()); + if (PageProtect(reinterpret_cast(GetRegionBase()), GetRegionBaseSize(), pageProtReadWrite) != 0) { DLOG(REPORT, "clear read only fail"); } } - RegionLiveDesc* GetLiveInfo() + RegionBitmap *GetMarkBitmap() { - RegionLiveDesc* liveInfo = __atomic_load_n(&metadata.liveInfo, std::memory_order_acquire); - if (reinterpret_cast(liveInfo) == RegionLiveDesc::TEMPORARY_PTR) { - return nullptr; - } - return liveInfo; - } - NO_INLINE RegionLiveDesc* AllocLiveInfo(RegionLiveDesc* liveInfo) - { - RegionLiveDesc* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); - if (__atomic_compare_exchange_n(&metadata.liveInfo, &liveInfo, newValue, false, std::memory_order_seq_cst, - std::memory_order_relaxed)) { - RegionLiveDesc* allocatedLiveInfo = HeapBitmapManager::GetHeapBitmapManager().AllocateRegionLiveDesc(); - allocatedLiveInfo->relatedRegion = this; - __atomic_store_n(&metadata.liveInfo, allocatedLiveInfo, std::memory_order_release); - DLOG(REGION, "region %p@%#zx alloc liveinfo %p", this, GetRegionStart(), metadata.liveInfo); - return allocatedLiveInfo; - } - return nullptr; - } - ALWAYS_INLINE RegionLiveDesc* GetOrAllocLiveInfo() - { - do { - RegionLiveDesc* liveInfo = __atomic_load_n(&metadata.liveInfo, std::memory_order_acquire); - if (UNLIKELY_CC(reinterpret_cast(liveInfo) == RegionLiveDesc::TEMPORARY_PTR)) { - continue; - } - if (LIKELY_CC(liveInfo != nullptr)) { - return liveInfo; - } - liveInfo = AllocLiveInfo(liveInfo); - if (liveInfo != nullptr) { - return liveInfo; - } - } while (true); - return nullptr; - } - - RegionBitmap* GetMarkBitmap() - { - RegionLiveDesc* liveInfo = GetLiveInfo(); - if (liveInfo == nullptr) { - return nullptr; - } - RegionBitmap* bitmap = __atomic_load_n(&liveInfo->markBitmap, std::memory_order_acquire); + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.markBitmap_, std::memory_order_acquire); if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { return nullptr; } return bitmap; } - NO_INLINE RegionBitmap* AllocMarkBitmap(RegionLiveDesc* liveInfo, RegionBitmap* bitmap) - { - RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); - if (__atomic_compare_exchange_n(&liveInfo->markBitmap, &bitmap, newValue, false, std::memory_order_seq_cst, - std::memory_order_relaxed)) { - RegionBitmap* allocated = - HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionSize()); - __atomic_store_n(&liveInfo->markBitmap, allocated, std::memory_order_release); - DLOG(REGION, "region %p@%#zx liveinfo %p alloc markbitmap %p", - this, GetRegionStart(), metadata.liveInfo, metadata.liveInfo->markBitmap); - return allocated; - } - return nullptr; - } ALWAYS_INLINE RegionBitmap* GetOrAllocMarkBitmap() { - RegionLiveDesc* liveInfo = GetOrAllocLiveInfo(); do { - RegionBitmap* bitmap = __atomic_load_n(&liveInfo->markBitmap, std::memory_order_acquire); + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.markBitmap_, std::memory_order_acquire); if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { continue; } if (LIKELY_CC(bitmap != nullptr)) { return bitmap; } - bitmap = AllocMarkBitmap(liveInfo, bitmap); - if (bitmap != nullptr) { - return bitmap; + RegionBitmap *newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + if (__atomic_compare_exchange_n(&metadata.liveInfo_.markBitmap_, &bitmap, newValue, false, + std::memory_order_seq_cst, std::memory_order_relaxed)) { + RegionBitmap *allocated = + HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); + __atomic_store_n(&metadata.liveInfo_.markBitmap_, allocated, std::memory_order_release); + DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc markbitmap %p", + this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, metadata.liveInfo_.markBitmap_); + return allocated; } } while (true); return nullptr; } - RegionBitmap* GetResurrectBitmap() + RegionBitmap *GetResurrectBitmap() { - RegionLiveDesc* liveInfo = GetLiveInfo(); - if (liveInfo == nullptr) { - return nullptr; - } - RegionBitmap* bitmap = __atomic_load_n(&liveInfo->resurrectBitmap, std::memory_order_acquire); + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.resurrectBitmap_, std::memory_order_acquire); if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { return nullptr; } return bitmap; } - RegionBitmap* GetOrAllocResurrectBitmap() + RegionBitmap *GetOrAllocResurrectBitmap() { - RegionLiveDesc* liveInfo = GetOrAllocLiveInfo(); do { - RegionBitmap* bitmap = __atomic_load_n(&liveInfo->resurrectBitmap, std::memory_order_acquire); + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.resurrectBitmap_, std::memory_order_acquire); if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { continue; } if (LIKELY_CC(bitmap != nullptr)) { return bitmap; } - RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); - if (__atomic_compare_exchange_n(&liveInfo->resurrectBitmap, &bitmap, newValue, false, + RegionBitmap *newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + if (__atomic_compare_exchange_n(&metadata.liveInfo_.resurrectBitmap_, &bitmap, newValue, false, std::memory_order_seq_cst, std::memory_order_relaxed)) { - RegionBitmap* allocated = - HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionSize()); - __atomic_store_n(&liveInfo->resurrectBitmap, allocated, std::memory_order_release); - DLOG(REGION, "region %p@%#zx liveinfo %p alloc resurrectbitmap %p", - this, GetRegionStart(), metadata.liveInfo, metadata.liveInfo->resurrectBitmap); + RegionBitmap *allocated = + HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); + __atomic_store_n(&metadata.liveInfo_.resurrectBitmap_, allocated, std::memory_order_release); + DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc resurrectbitmap %p", + this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, + metadata.liveInfo_.resurrectBitmap_); return allocated; } } while (true); @@ -263,11 +209,7 @@ public: RegionBitmap* GetEnqueueBitmap() { - RegionLiveDesc* liveInfo = GetLiveInfo(); - if (liveInfo == nullptr) { - return nullptr; - } - RegionBitmap* bitmap = __atomic_load_n(&liveInfo->enqueueBitmap, std::memory_order_acquire); + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { return nullptr; } @@ -276,23 +218,22 @@ public: RegionBitmap* GetOrAllocEnqueueBitmap() { - RegionLiveDesc* liveInfo = GetOrAllocLiveInfo(); do { - RegionBitmap* bitmap = __atomic_load_n(&liveInfo->enqueueBitmap, std::memory_order_acquire); + RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { continue; } if (LIKELY_CC(bitmap != nullptr)) { return bitmap; } - RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); - if (__atomic_compare_exchange_n(&liveInfo->enqueueBitmap, &bitmap, newValue, false, + RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); + if (__atomic_compare_exchange_n(&metadata.liveInfo_.enqueueBitmap_, &bitmap, newValue, false, std::memory_order_seq_cst, std::memory_order_relaxed)) { - RegionBitmap* allocated = - HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionSize()); - __atomic_store_n(&liveInfo->enqueueBitmap, allocated, std::memory_order_release); - DLOG(REGION, "region %p@%#zx liveinfo %p alloc enqueuebitmap %p", - this, GetRegionStart(), metadata.liveInfo, metadata.liveInfo->enqueueBitmap); + RegionBitmap *allocated = + HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); + __atomic_store_n(&metadata.liveInfo_.enqueueBitmap_, allocated, std::memory_order_release); + DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc enqueuebitmap %p", + this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, metadata.liveInfo_.enqueueBitmap_); return allocated; } } while (true); @@ -395,13 +336,6 @@ public: return enqueBitmap->IsMarked(offset); } - void CheckAndMarkObject(const BaseObject* obj) - { - if (!IsMarkedObject(obj)) { - MarkObject(obj); - } - } - RegionRSet* GetRSet() { return metadata.regionRSet; @@ -420,8 +354,8 @@ public: ALWAYS_INLINE_CC size_t GetAddressOffset(HeapAddress address) { - DCHECK_CC(GetRegionStart() <= address); - return (address - metadata.regionStart); + DCHECK_CC(GetRegionBaseFast() <= address); + return (address - GetRegionBaseFast()); } enum class UnitRole : uint8_t { @@ -438,15 +372,18 @@ public: enum class RegionType : uint8_t { FREE_REGION, + GARBAGE_REGION, + + // ************************boundary of dead region and alive region************************** THREAD_LOCAL_REGION, - THREAD_LOCAL_OLD_REGION, RECENT_FULL_REGION, FROM_REGION, - LONE_FROM_REGION, EXEMPTED_FROM_REGION, + LONE_FROM_REGION, TO_REGION, OLD_REGION, + THREAD_LOCAL_OLD_REGION, // pinned object will not be forwarded by concurrent copying gc. FULL_PINNED_REGION, @@ -464,13 +401,55 @@ public: RECENT_LARGE_REGION, LARGE_REGION, - GARBAGE_REGION, READ_ONLY_REGION, APPSPAWN_REGION, - END_OF_REGION_TYPE + END_OF_REGION_TYPE, + + ALIVE_REGION_FIRST = THREAD_LOCAL_REGION, }; + static bool IsAliveRegionType(RegionType type) + { + return static_cast(type) >= static_cast(RegionType::ALIVE_REGION_FIRST); + } + + static bool IsInRecentSpace(RegionType type) + { + // Note: THREAD_LOCAL_OLD_REGION is not included + return type == RegionType::THREAD_LOCAL_REGION || type == RegionType::RECENT_FULL_REGION; + } + + bool IsInYoungSpaceForWB() const + { + RegionType type = GetRegionType(); + return type == RegionType::THREAD_LOCAL_REGION || + type == RegionType::RECENT_FULL_REGION || + type == RegionType::FROM_REGION; + } + + static bool IsInYoungSpace(RegionType type) + { + // Note: THREAD_LOCAL_OLD_REGION is not included + return type == RegionType::THREAD_LOCAL_REGION || type == RegionType::RECENT_FULL_REGION || + type == RegionType::FROM_REGION || type == RegionType::EXEMPTED_FROM_REGION; + } + + static bool IsInFromSpace(RegionType type) + { + return type == RegionType::FROM_REGION || type == RegionType::EXEMPTED_FROM_REGION; + } + + static bool IsInToSpace(RegionType type) + { + return type == RegionType::TO_REGION; + } + + static bool IsInOldSpace(RegionType type) + { + return type == RegionType::OLD_REGION || type == RegionType::THREAD_LOCAL_OLD_REGION; + } + static void Initialize(size_t nUnit, uintptr_t regionInfoAddr, uintptr_t heapAddress) { UnitInfo::totalUnitCount = nUnit; @@ -478,15 +457,17 @@ public: UnitInfo::heapStartAddress = heapAddress; } + static uintptr_t GetHeapStartAddress() + { + return UnitInfo::heapStartAddress; + } + static RegionDesc* GetRegionDesc(uint32_t idx) { UnitInfo* unit = RegionDesc::UnitInfo::GetUnitInfo(idx); DCHECK_CC((reinterpret_cast(unit) % 8) == 0); // 8: Align with 8 - if (static_cast(unit->GetMetadata().unitRole) == UnitRole::SUBORDINATE_UNIT) { - return unit->GetMetadata().ownerRegion; - } else { - return reinterpret_cast(unit); - } + DCHECK_CC(static_cast(unit->GetMetadata().unitRole) != UnitRole::SUBORDINATE_UNIT); + return reinterpret_cast(unit); } static RegionDesc* GetRegionDescAt(uintptr_t allocAddr) @@ -495,11 +476,22 @@ public: (((allocAddr - UnitInfo::heapStartAddress) / UNIT_SIZE) + 1) * sizeof(RegionDesc)); DCHECK_CC((reinterpret_cast(unit) % 8) == 0); // 8: Align with 8 - if (static_cast(unit->GetMetadata().unitRole) == UnitRole::SUBORDINATE_UNIT) { - return unit->GetMetadata().ownerRegion; - } else { - return reinterpret_cast(unit); - } + DCHECK_CC(static_cast(unit->GetMetadata().unitRole) != UnitRole::SUBORDINATE_UNIT); + return reinterpret_cast(unit); + } + + // This could only used for surely alive region, such as from interpreter, + // because ONLY alive region have `InlinedRegionMetaData` + static RegionDesc* GetAliveRegionDescAt(uintptr_t allocAddr) + { + // only alive region have `InlinedRegionMetaData`. + DCHECK_CC(IsAliveRegionType(GetRegionDescAt(allocAddr)->GetRegionType())); + InlinedRegionMetaData *metaData = InlinedRegionMetaData::GetInlinedRegionMetaData(allocAddr); + UnitInfo *unit = reinterpret_cast(metaData->regionDesc_); + DCHECK_CC(reinterpret_cast(unit) == GetRegionDescAt(allocAddr)); + DCHECK_CC((reinterpret_cast(unit) % 8) == 0); // 8: Align with 8 + DCHECK_CC(static_cast(unit->GetMetadata().unitRole) != UnitRole::SUBORDINATE_UNIT); + return reinterpret_cast(unit); } static void InitFreeRegion(size_t unitIdx, size_t nUnit) @@ -559,21 +551,19 @@ public: #endif } - BaseObject* GetFirstObject() const { return reinterpret_cast(GetRegionStart()); } - - bool IsEmpty() const + size_t GetRegionSize() const { - ASSERT_LOGF(IsSmallRegion(), "wrong region type"); - return GetRegionAllocPtr() == GetRegionStart(); + DCHECK_CC(GetRegionEnd() > GetRegionStart()); + return GetRegionEnd() - GetRegionStart(); } - size_t GetRegionSize() const + size_t GetRegionBaseSize() const { - DCHECK_CC(metadata.regionEnd > GetRegionStart()); - return metadata.regionEnd - GetRegionStart(); + DCHECK_CC(GetRegionEnd() > GetRegionBase()); + return GetRegionEnd() - GetRegionBase(); } - size_t GetUnitCount() const { return GetRegionSize() / UNIT_SIZE; } + size_t GetUnitCount() const { return GetRegionBaseSize() / UNIT_SIZE; } size_t GetAvailableSize() const { @@ -611,29 +601,12 @@ public: } } - // the interface can only be used to clear live info after gc. - void CheckAndClearLiveInfo(RegionLiveDesc* liveInfo) - { - // Garbage region may be reused by other thread. For the sake of safety, we don't clean it here. - // We will clean it before the region is accessable. - if (IsGarbageRegion()) { - return; - } - // Check the value whether is expected, in order to avoid resetting a reused region. - if (metadata.liveInfo == liveInfo) { - metadata.liveInfo = nullptr; - metadata.liveByteCount = 0; - } - } - void ClearLiveInfo() { - if (metadata.liveInfo != nullptr) { - DCHECK_CC(metadata.liveInfo->relatedRegion == this); - metadata.liveInfo = nullptr; - DLOG(REGION, "region %p@%#zx+%zu clear liveinfo %p type %u", this, this->GetRegionStart(), - this->GetRegionSize(), metadata.liveInfo, this->GetRegionType()); - } + DCHECK_CC(metadata.liveInfo_.relatedRegion_ == this); + metadata.liveInfo_.ClearLiveInfo(); + DLOG(REGION, "region %p(base=%#zx)@%#zx+%zu clear liveinfo %p type %u", + this, GetRegionBase(), GetRegionStart(), GetRegionSize(), &metadata.liveInfo_, GetRegionType()); metadata.liveByteCount = 0; } @@ -646,6 +619,9 @@ public: { metadata.regionBits.AtomicSetValue(RegionBitOffset::BIT_OFFSET_REGION_TYPE, BITS_5, static_cast(type)); + if (IsAliveRegionType(type)) { + InlinedRegionMetaData::GetInlinedRegionMetaData(this)->SetRegionType(type); + } } void SetMarkedRegionFlag(uint8_t flag) @@ -693,7 +669,16 @@ public: size_t GetUnitIdx() const { return RegionDesc::UnitInfo::GetUnitIdx(reinterpret_cast(this)); } - HeapAddress GetRegionStart() const { return RegionDesc::GetUnitAddress(GetUnitIdx()); } + HeapAddress GetRegionBase() const { return RegionDesc::GetUnitAddress(GetUnitIdx()); } + + // This could only used to a `RegionDesc` which has been initialized + HeapAddress GetRegionBaseFast() const + { + ASSERT(metadata.regionBase == GetRegionBase()); + return metadata.regionBase; + } + + HeapAddress GetRegionStart() const { return metadata.regionStart; } HeapAddress GetRegionEnd() const { return metadata.regionEnd; } @@ -710,8 +695,8 @@ public: if (metadata.traceLine == std::numeric_limits::max()) { uintptr_t line = GetRegionAllocPtr(); metadata.traceLine = line; - DLOG(REGION, "set region %p@%#zx+%zu trace-line %#zx type %u", this, this->GetRegionStart(), - this->GetRegionSize(), this->GetTraceLine(), this->GetRegionType()); + DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu trace-line %#zx type %u", + this, GetRegionBase(), GetRegionStart(), GetRegionSize(), GetTraceLine(), GetRegionType()); } } @@ -720,8 +705,8 @@ public: if (metadata.forwardLine == std::numeric_limits::max()) { uintptr_t line = GetRegionAllocPtr(); metadata.forwardLine = line; - DLOG(REGION, "set region %p@%#zx+%zu copy-line %#zx type %u", this, this->GetRegionStart(), - this->GetRegionSize(), this->GetCopyLine(), this->GetRegionType()); + DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu copy-line %#zx type %u", + this, GetRegionBase(), GetRegionStart(), GetRegionSize(), GetCopyLine(), GetRegionType()); } } @@ -730,8 +715,8 @@ public: if (metadata.fixLine == std::numeric_limits::max()) { uintptr_t line = GetRegionAllocPtr(); metadata.fixLine = line; - DLOG(REGION, "set region %p@%#zx+%zu fix-line %#zx type %u", this, this->GetRegionStart(), - this->GetRegionSize(), this->GetFixLine(), this->GetRegionType()); + DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu fix-line %#zx type %u", + this, GetRegionBase(), GetRegionStart(), GetRegionSize(), GetFixLine(), GetRegionType()); } } @@ -775,34 +760,31 @@ public: bool IsInRecentSpace() const { RegionType type = GetRegionType(); - // Note: THREAD_LOCAL_OLD_REGION is not included - return type == RegionType::THREAD_LOCAL_REGION || type == RegionType::RECENT_FULL_REGION; + return RegionDesc::IsInRecentSpace(type); } bool IsInYoungSpace() const { RegionType type = GetRegionType(); - // Note: THREAD_LOCAL_OLD_REGION is not included - return type == RegionType::THREAD_LOCAL_REGION || type == RegionType::RECENT_FULL_REGION || - type == RegionType::FROM_REGION || type == RegionType::EXEMPTED_FROM_REGION; + return RegionDesc::IsInYoungSpace(type); } bool IsInFromSpace() const { RegionType type = GetRegionType(); - return type == RegionType::FROM_REGION || type == RegionType::EXEMPTED_FROM_REGION; + return RegionDesc::IsInFromSpace(type); } bool IsInToSpace() const { RegionType type = GetRegionType(); - return type == RegionType::TO_REGION; + return RegionDesc::IsInToSpace(type); } bool IsInOldSpace() const { RegionType type = GetRegionType(); - return type == RegionType::OLD_REGION || type == RegionType::THREAD_LOCAL_OLD_REGION; + return RegionDesc::IsInOldSpace(type); } int32_t IncRawPointerObjectCount() @@ -833,6 +815,7 @@ public: uintptr_t Alloc(size_t size) { + DCHECK_CC(size > 0); size_t limit = GetRegionEnd(); if (metadata.allocPtr + size <= limit) { uintptr_t addr = metadata.allocPtr; @@ -1054,6 +1037,37 @@ private: } }; + class RegionLiveDesc { + public: + static constexpr HeapAddress TEMPORARY_PTR = 0x1234; + + void Init(RegionDesc *region) + { + relatedRegion_ = region; + ClearLiveInfo(); + } + + void Fini() + { + relatedRegion_ = nullptr; + ClearLiveInfo(); + } + + void ClearLiveInfo() + { + markBitmap_ = nullptr; + resurrectBitmap_ = nullptr; + enqueueBitmap_ = nullptr; + } + private: + RegionDesc *relatedRegion_ {nullptr}; + RegionBitmap *markBitmap_ {nullptr}; + RegionBitmap *resurrectBitmap_ {nullptr}; + RegionBitmap *enqueueBitmap_ {nullptr}; + + friend class RegionDesc; + }; + struct UnitMetadata { struct { // basic data for RegionDesc // for fast allocation, always at the start. @@ -1065,7 +1079,16 @@ private: uintptr_t forwardLine; uintptr_t fixLine; ObjectSlot* freeSlot; + // `regionStart` is the header of the data, and `regionBase` is the header of the total region + /** + * | *********************************Region*******************************| + * | InlinedRegionMetaData | *****************Region data******************| + * ^ ^ + * | | + * regionBase regionStart + */ uintptr_t regionStart; + uintptr_t regionBase; uint32_t nextRegionIdx; uint32_t prevRegionIdx; // support fast deletion for region list. @@ -1074,10 +1097,7 @@ private: int32_t rawPointerObjectCount; }; - union { - RegionLiveDesc* liveInfo = nullptr; - RegionDesc* ownerRegion; // if unit is SUBORDINATE_UNIT - }; + RegionLiveDesc liveInfo_ {}; RegionRSet* regionRSet = nullptr;; @@ -1165,11 +1185,6 @@ private: // These interfaces are used to make sure the writing operations of value in C++ Bit Field will be atomic. void SetUnitRole(UnitRole role) { metadata_.unitBits.AtomicSetValue(0, BITS_5, static_cast(role)); } - void SetRegionType(RegionType type) - { - metadata_.regionBits.AtomicSetValue(RegionBitOffset::BIT_OFFSET_REGION_TYPE, BITS_5, - static_cast(type)); - } void SetMarkedRegionFlag(uint8_t flag) { @@ -1186,12 +1201,6 @@ private: metadata_.regionBits.AtomicSetValue(RegionBitOffset::BIT_OFFSET_RESURRECTED_REGION, 1, flag); } - void InitSubordinateUnit(RegionDesc* owner) - { - SetUnitRole(UnitRole::SUBORDINATE_UNIT); - metadata_.ownerRegion = owner; - } - void ToFreeRegion() { InitFreeRegion(GetUnitIdx(this), 1); } void ClearUnit() { ClearUnits(GetUnitIdx(this), 1); } @@ -1204,17 +1213,155 @@ private: private: UnitMetadata metadata_; + + friend class RegionDesc; }; +public: + // inline copy some data at the begin of the region data, to support fast-path in barrier or smth else. + // NOTE the data consistency between data in header and that in `RegionDesc`. + // this could ONLY used in region that is ALIVE. + class InlinedRegionMetaData { + public: + static InlinedRegionMetaData *GetInlinedRegionMetaData(RegionDesc *region) + { + InlinedRegionMetaData *data = GetInlinedRegionMetaData(region->GetRegionStart()); + DCHECK_CC(data->regionDesc_ == region); + return data; + } + static InlinedRegionMetaData *GetInlinedRegionMetaData(uintptr_t allocAddr) + { + return reinterpret_cast(allocAddr & ~DEFAULT_REGION_UNIT_MASK); + } + + explicit InlinedRegionMetaData(RegionDesc *regionDesc) + : regionDesc_(regionDesc), regionRSet_(regionDesc->GetRSet()), regionType_(regionDesc->GetRegionType()) + { + // Since this is a backup copy of `RegionDesc`, create rset at first to guarantee data consistency + DCHECK_CC(regionRSet_ != nullptr); + // Not insert to regionList and reset regionType yet + DCHECK_CC(regionType_ == RegionType::FREE_REGION); + DCHECK_CC(regionType_ == regionDesc_->GetRegionType()); + } + ~InlinedRegionMetaData() = default; + + void SetRegionType(RegionType type) + { + DCHECK_CC(RegionDesc::IsAliveRegionType(type)); + DCHECK_CC(type == regionDesc_->GetRegionType()); + regionType_ = type; + } + + RegionDesc *GetRegionDesc() const + { + return regionDesc_; + } + + RegionRSet *GetRegionRSet() const + { + return regionRSet_; + } + + RegionType GetRegionType() const + { + DCHECK_CC(RegionDesc::IsAliveRegionType(regionType_)); + return regionType_; + } + + bool IsInRecentSpace() const + { + RegionType type = GetRegionType(); + return RegionDesc::IsInRecentSpace(type); + } + + bool IsInYoungSpace() const + { + RegionType type = GetRegionType(); + return RegionDesc::IsInYoungSpace(type); + } + + bool IsInFromSpace() const + { + RegionType type = GetRegionType(); + return RegionDesc::IsInFromSpace(type); + } + + bool IsInToSpace() const + { + RegionType type = GetRegionType(); + return RegionDesc::IsInToSpace(type); + } + + bool IsInOldSpace() const + { + RegionType type = GetRegionType(); + return RegionDesc::IsInOldSpace(type); + } + + bool IsFromRegion() const + { + RegionType type = GetRegionType(); + return type == RegionType::FROM_REGION; + } + + bool IsInYoungSpaceForWB() const + { + RegionType type = GetRegionType(); + return type == RegionType::THREAD_LOCAL_REGION || + type == RegionType::RECENT_FULL_REGION || + type == RegionType::FROM_REGION; + } + + inline HeapAddress GetRegionStart() const; + + HeapAddress GetRegionBase() const + { + uintptr_t base = reinterpret_cast(this); + ASSERT(base == regionDesc_->GetRegionBaseFast()); + return static_cast(base); + } + + size_t GetAddressOffset(HeapAddress address) const + { + DCHECK_CC(GetRegionBase() <= address); + return (address - GetRegionBase()); + } + + bool MarkRSetCardTable(BaseObject *obj) + { + size_t offset = GetAddressOffset(static_cast(reinterpret_cast(obj))); + return GetRegionRSet()->MarkCardTable(offset); + } + private: + RegionDesc *regionDesc_ {nullptr}; + RegionRSet *regionRSet_ {nullptr}; + RegionType regionType_ {}; + // fixme: inline more + + friend class RegionDesc; + }; + // should keep as same as the align of BaseObject + static constexpr size_t UNIT_BEGIN_ALIGN = 8; + // default common region unit header size. + static constexpr size_t UNIT_HEADER_SIZE = AlignUp(sizeof(InlinedRegionMetaData), UNIT_BEGIN_ALIGN); + // default common region unit available size. + static constexpr size_t UNIT_AVAILABLE_SIZE = UNIT_SIZE - UNIT_HEADER_SIZE; + +private: + void InitRegionDesc(size_t nUnit, UnitRole uClass) { + DCHECK_CC(uClass != UnitRole::SUBORDINATE_UNIT); + size_t base = GetRegionBase(); + metadata.regionBase = base; + metadata.regionStart = base + RegionDesc::UNIT_HEADER_SIZE; + ASSERT(metadata.regionStart % UNIT_BEGIN_ALIGN == 0); metadata.allocPtr = GetRegionStart(); - metadata.regionStart = GetRegionStart(); - metadata.regionEnd = metadata.allocPtr + nUnit * RegionDesc::UNIT_SIZE; + metadata.regionEnd = base + nUnit * RegionDesc::UNIT_SIZE; + DCHECK_CC(GetRegionStart() < GetRegionEnd()); metadata.prevRegionIdx = NULLPTR_IDX; metadata.nextRegionIdx = NULLPTR_IDX; metadata.liveByteCount = 0; - metadata.liveInfo = nullptr; metadata.freeSlot = nullptr; if (metadata.regionRSet != nullptr) { ClearRSet(); @@ -1239,21 +1386,52 @@ private: void InitRegion(size_t nUnit, UnitRole uClass) { + DCHECK_CC(uClass != UnitRole::FREE_UNITS); //fixme: remove `UnitRole::SUBORDINATE_UNIT` + DCHECK_CC(uClass != UnitRole::SUBORDINATE_UNIT); //fixme: remove `UnitRole::SUBORDINATE_UNIT` InitRegionDesc(nUnit, uClass); + metadata.liveInfo_.Init(this); if (metadata.regionRSet == nullptr) { - metadata.regionRSet = new RegionRSet(GetRegionSize()); + metadata.regionRSet = new RegionRSet(GetRegionBaseSize()); } + HeapAddress header = GetRegionBase(); + void *ptr = reinterpret_cast(static_cast(header)); + new (ptr) InlinedRegionMetaData(this); // initialize region's subordinate units. UnitInfo* unit = reinterpret_cast(this) - (nUnit - 1); for (size_t i = 0; i < nUnit - 1; i++) { - unit[i].InitSubordinateUnit(this); + DCHECK_CC(uClass == UnitRole::LARGE_SIZED_UNITS); + unit[i].metadata_.liveInfo_.Fini(); } + std::atomic_thread_fence(std::memory_order_seq_cst); } static constexpr uint32_t NULLPTR_IDX = UnitInfo::INVALID_IDX; UnitMetadata metadata; +public: + friend constexpr size_t GetMetaDataInRegionOffset(); + static constexpr size_t METADATA_IN_UNIT_OFFSET = MEMBER_OFFSET(UnitInfo, metadata_); + static constexpr size_t UNIT_BITS_IN_METADATA_OFFSET = MEMBER_OFFSET(UnitMetadata, unitBits); + static constexpr size_t REGION_BITS_IN_METADATA_OFFSET = MEMBER_OFFSET(UnitMetadata, regionBits); + static constexpr size_t REGION_DESC_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionDesc_); + static constexpr size_t REGION_RSET_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionRSet_); + static constexpr size_t REGION_TYPE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionType_); }; + +HeapAddress RegionDesc::InlinedRegionMetaData::GetRegionStart() const +{ + HeapAddress addr = static_cast(reinterpret_cast(this) + RegionDesc::UNIT_HEADER_SIZE); + DCHECK_CC(addr == regionDesc_->GetRegionStart()); + return addr; +} + + +constexpr size_t GetMetaDataInRegionOffset() +{ + return MEMBER_OFFSET(RegionDesc, metadata); +} + +static constexpr size_t METADATA_IN_REGION_OFFSET = GetMetaDataInRegionOffset(); } // namespace common #endif // COMMON_COMPONENTS_HEAP_ALLOCATOR_REGION_INFO_H diff --git a/common_components/heap/allocator/region_manager.cpp b/common_components/heap/allocator/region_manager.cpp index 190045c9b09180cab5e87c276539488b6ffa19c4..6fcf4551ae478039f018fecf1f8ffcd9314f825f 100755 --- a/common_components/heap/allocator/region_manager.cpp +++ b/common_components/heap/allocator/region_manager.cpp @@ -70,22 +70,15 @@ static size_t GetPageSize() noexcept // System default page size const size_t COMMON_PAGE_SIZE = GetPageSize(); const size_t AllocatorUtils::ALLOC_PAGE_SIZE = COMMON_PAGE_SIZE; -// region unit size: same as system page size -const size_t RegionDesc::UNIT_SIZE = COMMON_PAGE_SIZE; -// regarding a object as a large object when the size is greater than 32KB or one page size, -// depending on the system page size. -const size_t RegionDesc::LARGE_OBJECT_DEFAULT_THRESHOLD = COMMON_PAGE_SIZE > (32 * KB) ? - COMMON_PAGE_SIZE : 32 * KB; -// max size of per region is 128KB. -const size_t RegionManager::MAX_UNIT_COUNT_PER_REGION = (128 * KB) / COMMON_PAGE_SIZE; // size of huge page is 2048KB. -const size_t RegionManager::HUGE_PAGE = (2048 * KB) / COMMON_PAGE_SIZE; +constexpr size_t HUGE_PAGE_UNIT_NUM = (2048 * KB) / RegionDesc::UNIT_SIZE; #if defined(GCINFO_DEBUG) && GCINFO_DEBUG void RegionDesc::DumpRegionDesc(LogType type) const { - DLOG(type, "Region index: %zu, type: %s, address: 0x%zx-0x%zx, allocated(B) %zu, live(B) %zu", GetUnitIdx(), - GetTypeName(), GetRegionStart(), GetRegionEnd(), GetRegionAllocatedSize(), GetLiveByteCount()); + DLOG(type, "Region index: %zu, type: %s, address: 0x%zx(start=0x%zx)-0x%zx, allocated(B) %zu, live(B) %zu", + GetUnitIdx(), GetTypeName(), GetRegionBase(), GetRegionStart(), GetRegionEnd(), GetRegionAllocatedSize(), + GetLiveByteCount()); } const char* RegionDesc::GetTypeName() const @@ -203,18 +196,18 @@ bool RegionDesc::VisitLiveObjectsUntilFalse(const std::function& func) { uintptr_t end = std::min(GetTraceLine(), GetRegionAllocPtr()); - GetRSet()->VisitAllMarkedCardBefore(func, GetRegionStart(), end); + GetRSet()->VisitAllMarkedCardBefore(func, GetRegionBaseFast(), end); } void RegionDesc::VisitRememberSetBeforeFix(const std::function& func) { uintptr_t end = std::min(GetFixLine(), GetRegionAllocPtr()); - GetRSet()->VisitAllMarkedCardBefore(func, GetRegionStart(), end); + GetRSet()->VisitAllMarkedCardBefore(func, GetRegionBaseFast(), end); } void RegionDesc::VisitRememberSet(const std::function& func) { - GetRSet()->VisitAllMarkedCardBefore(func, GetRegionStart(), GetRegionAllocPtr()); + GetRSet()->VisitAllMarkedCardBefore(func, GetRegionBaseFast(), GetRegionAllocPtr()); } void RegionList::MergeRegionList(RegionList& srcList, RegionDesc::RegionType regionType) @@ -243,6 +236,7 @@ static const char *RegionDescRegionTypeToString(RegionDesc::RegionType type) { static constexpr const char *enumStr[] = { [static_cast(RegionDesc::RegionType::FREE_REGION)] = "FREE_REGION", + [static_cast(RegionDesc::RegionType::GARBAGE_REGION)] = "GARBAGE_REGION", [static_cast(RegionDesc::RegionType::THREAD_LOCAL_REGION)] = "THREAD_LOCAL_REGION", [static_cast(RegionDesc::RegionType::THREAD_LOCAL_OLD_REGION)] = "THREAD_LOCAL_OLD_REGION", [static_cast(RegionDesc::RegionType::RECENT_FULL_REGION)] = "RECENT_FULL_REGION", @@ -259,7 +253,6 @@ static const char *RegionDescRegionTypeToString(RegionDesc::RegionType type) [static_cast(RegionDesc::RegionType::TL_RAW_POINTER_REGION)] = "TL_RAW_POINTER_REGION", [static_cast(RegionDesc::RegionType::RECENT_LARGE_REGION)] = "RECENT_LARGE_REGION", [static_cast(RegionDesc::RegionType::LARGE_REGION)] = "LARGE_REGION", - [static_cast(RegionDesc::RegionType::GARBAGE_REGION)] = "GARBAGE_REGION", [static_cast(RegionDesc::RegionType::READ_ONLY_REGION)] = "READ_ONLY_REGION", [static_cast(RegionDesc::RegionType::APPSPAWN_REGION)] = "APPSPAWN_REGION", }; @@ -279,13 +272,14 @@ void RegionList::PrependRegionLocked(RegionDesc* region, RegionDesc::RegionType return; } - DLOG(REGION, "%s (%zu, %zu)+(%zu, %zu) prepend region %p@%#zx+%zu type %u->%u", listName_, - regionCount_, unitCount_, 1llu, region->GetUnitCount(), region, region->GetRegionStart(), - region->GetRegionAllocatedSize(), region->GetRegionType(), type); + DLOG(REGION, "%s (%zu, %zu)+(%zu, %zu) prepend region %p(base=%#zx)@%#zx+%zu type %u->%u", listName_, + regionCount_, unitCount_, 1llu, region->GetUnitCount(), region, region->GetRegionBase(), + region->GetRegionStart(), region->GetRegionAllocatedSize(), region->GetRegionType(), type); region->SetRegionType(type); - os::PrctlSetVMA(reinterpret_cast(region->GetRegionStart()), region->GetRegionAllocatedSize(), + size_t totalRegionSize = region->GetRegionEnd() - region->GetRegionBase(); + os::PrctlSetVMA(reinterpret_cast(region->GetRegionBase()), totalRegionSize, (std::string("ArkTS Heap CMCGC Region ") + RegionDescRegionTypeToString(type)).c_str()); region->SetPrevRegion(nullptr); @@ -310,9 +304,9 @@ void RegionList::DeleteRegionLocked(RegionDesc* del) del->SetNextRegion(nullptr); del->SetPrevRegion(nullptr); - DLOG(REGION, "%s (%zu, %zu)-(%zu, %zu) delete region %p@%#zx+%zu type %u", listName_, + DLOG(REGION, "%s (%zu, %zu)-(%zu, %zu) delete region %p(start=%p),@%#zx+%zu type %u", listName_, regionCount_, unitCount_, 1llu, del->GetUnitCount(), - del, del->GetRegionStart(), del->GetRegionAllocatedSize(), del->GetRegionType()); + del, del->GetRegionBase(), del->GetRegionStart(), del->GetRegionAllocatedSize(), del->GetRegionType()); DecCounts(1, del->GetUnitCount()); @@ -345,8 +339,8 @@ void RegionList::DumpRegionList(const char* msg) DLOG(REGION, "dump region list %s", msg); std::lock_guard lock(listMutex_); for (RegionDesc *region = listHead_; region != nullptr; region = region->GetNextRegion()) { - DLOG(REGION, "region %p @0x%zx+%zu units [%zu+%zu, %zu) type %u prev %p next %p", region, - region->GetRegionStart(), region->GetRegionAllocatedSize(), + DLOG(REGION, "region %p @0x%zx(start=0x%zx)+%zu units [%zu+%zu, %zu) type %u prev %p next %p", region, + region->GetRegionBase(), region->GetRegionStart(), region->GetRegionAllocatedSize(), region->GetUnitIdx(), region->GetUnitCount(), region->GetUnitIdx() + region->GetUnitCount(), region->GetRegionType(), region->GetPrevRegion(), region->GetNextRegion()); } @@ -355,7 +349,7 @@ void RegionList::DumpRegionList(const char* msg) inline void RegionManager::TagHugePage(RegionDesc* region, size_t num) const { #if defined (__linux__) || defined(PANDA_TARGET_OHOS) - (void)madvise(reinterpret_cast(region->GetRegionStart()), num * RegionDesc::UNIT_SIZE, MADV_HUGEPAGE); + (void)madvise(reinterpret_cast(region->GetRegionBase()), num * RegionDesc::UNIT_SIZE, MADV_HUGEPAGE); #else (void)region; (void)num; @@ -365,7 +359,7 @@ inline void RegionManager::TagHugePage(RegionDesc* region, size_t num) const inline void RegionManager::UntagHugePage(RegionDesc* region, size_t num) const { #if defined (__linux__) || defined(PANDA_TARGET_OHOS) - (void)madvise(reinterpret_cast(region->GetRegionStart()), num * RegionDesc::UNIT_SIZE, MADV_NOHUGEPAGE); + (void)madvise(reinterpret_cast(region->GetRegionBase()), num * RegionDesc::UNIT_SIZE, MADV_NOHUGEPAGE); #else (void)region; (void)num; @@ -401,35 +395,28 @@ size_t FreeRegionManager::ReleaseGarbageRegions(size_t targetCachedSize) return releasedBytes; } -void RegionManager::SetMaxUnitCountForRegion() -{ - maxUnitCountPerRegion_ = BaseRuntime::GetInstance()->GetHeapParam().regionSize * KB / RegionDesc::UNIT_SIZE; -} - -void RegionManager::SetLargeObjectThreshold() -{ - size_t regionSize = BaseRuntime::GetInstance()->GetHeapParam().regionSize * KB; - if (regionSize < RegionDesc::LARGE_OBJECT_DEFAULT_THRESHOLD) { - largeObjectThreshold_ = regionSize; - } else { - largeObjectThreshold_ = RegionDesc::LARGE_OBJECT_DEFAULT_THRESHOLD; - } -} - void RegionManager::Initialize(size_t nRegion, uintptr_t regionInfoAddr) { size_t metadataSize = GetMetadataSize(nRegion); + size_t alignedHeapStart = RoundUp(regionInfoAddr + metadataSize, RegionDesc::UNIT_SIZE); + // align the start of region to 256KB + /*** + * |***********|<-metadataSize->|**********************| + * |**padding**|***RegionUnit***|*******Region*********| + * ^ ^ ^ + * | | | + * | reginInfoStart alignedHeapStart + * regionInfoAddr + */ + regionInfoStart_ = alignedHeapStart - metadataSize; + regionHeapStart_ = alignedHeapStart; #ifdef _WIN64 - MemoryMap::CommitMemory(reinterpret_cast(regionInfoAddr), metadataSize); + MemoryMap::CommitMemory(reinterpret_cast(regionInfoStart_), metadataSize); #endif - regionInfoStart_ = regionInfoAddr; - regionHeapStart_ = regionInfoAddr + metadataSize; regionHeapEnd_ = regionHeapStart_ + nRegion * RegionDesc::UNIT_SIZE; inactiveZone_ = regionHeapStart_; - SetMaxUnitCountForRegion(); - SetLargeObjectThreshold(); // propagate region heap layout - RegionDesc::Initialize(nRegion, regionInfoAddr, regionHeapStart_); + RegionDesc::Initialize(nRegion, regionInfoStart_, regionHeapStart_); freeRegionManager_.Initialize(nRegion); DLOG(REPORT, "region info @0x%zx+%zu, heap [0x%zx, 0x%zx), unit count %zu", regionInfoAddr, metadataSize, @@ -448,11 +435,11 @@ void RegionManager::ReclaimRegion(RegionDesc* region) { size_t num = region->GetUnitCount(); size_t unitIndex = region->GetUnitIdx(); - if (num >= HUGE_PAGE) { + if (num >= HUGE_PAGE_UNIT_NUM) { UntagHugePage(region, num); } - DLOG(REGION, "reclaim region %p@%#zx+%zu type %u", region, - region->GetRegionStart(), region->GetRegionAllocatedSize(), + DLOG(REGION, "reclaim region %p(base=%#zx)@%#zx+%zu type %u", region, + region->GetRegionBase(), region->GetRegionStart(), region->GetRegionAllocatedSize(), region->GetRegionType()); region->InitFreeUnits(); @@ -464,7 +451,7 @@ size_t RegionManager::ReleaseRegion(RegionDesc* region) size_t res = region->GetRegionSize(); size_t num = region->GetUnitCount(); size_t unitIndex = region->GetUnitIdx(); - if (num >= HUGE_PAGE) { + if (num >= HUGE_PAGE_UNIT_NUM) { UntagHugePage(region, num); } DLOG(REGION, "release region %p @%#zx+%zu type %u", region, region->GetRegionStart(), @@ -518,7 +505,8 @@ void RegionManager::ForEachObjectUnsafe(const std::function& { for (uintptr_t regionAddr = regionHeapStart_; regionAddr < inactiveZone_;) { RegionDesc* region = RegionDesc::GetRegionDescAt(regionAddr); - regionAddr = region->GetRegionEnd(); + uintptr_t next = region->GetRegionEnd(); + regionAddr = next; if (!region->IsValidRegion() || region->IsFreeRegion() || region -> IsGarbageRegion()) { continue; } @@ -574,7 +562,7 @@ RegionDesc *RegionManager::TakeRegion(size_t num, RegionDesc::UnitRole type, boo RegionDesc* region = freeRegionManager_.TakeRegion(num, type, expectPhysicalMem); if (region != nullptr) { - if (num >= HUGE_PAGE) { + if (num >= HUGE_PAGE_UNIT_NUM) { TagHugePage(region, num); } return region; @@ -593,7 +581,7 @@ RegionDesc *RegionManager::TakeRegion(size_t num, RegionDesc::UnitRole type, boo (void)idx; // eliminate compilation warning DLOG(REGION, "take inactive units [%zu+%zu, %zu) at [0x%zx, 0x%zx)", idx, num, idx + num, RegionDesc::GetUnitAddress(idx), RegionDesc::GetUnitAddress(idx + num)); - if (num >= HUGE_PAGE) { + if (num >= HUGE_PAGE_UNIT_NUM) { TagHugePage(region, num); } if (expectPhysicalMem) { @@ -815,7 +803,7 @@ size_t RegionManager::CollectLargeGarbage() region = region->GetNextRegion(); largeRegionList_.DeleteRegion(del); if (IsMachineCodeObject(reinterpret_cast(obj))) { - JitFortUnProt(del->GetRegionSize(), reinterpret_cast(obj)); + JitFortUnProt(del->GetRegionBaseSize(), reinterpret_cast(del->GetRegionBaseFast())); } if (del->GetRegionSize() > RegionDesc::LARGE_OBJECT_RELEASE_THRESHOLD) { garbageSize += ReleaseRegion(del); @@ -972,7 +960,7 @@ uintptr_t RegionManager::AllocReadOnly(size_t size, bool allowGC) } if (addr == 0) { RegionDesc* region = - TakeRegion(maxUnitCountPerRegion_, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, allowGC); + TakeRegion(1, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, allowGC); if (region == nullptr) { return 0; } diff --git a/common_components/heap/allocator/region_manager.h b/common_components/heap/allocator/region_manager.h index 287a1735c1070c0fd4fefa32fefac1a5a3d7c923..60e2e035f43eceecdd4e75b4fef7f3ee78f4a404 100755 --- a/common_components/heap/allocator/region_manager.h +++ b/common_components/heap/allocator/region_manager.h @@ -43,14 +43,17 @@ public: constexpr static size_t FIXED_PINNED_REGION_COUNT = 128; constexpr static size_t FIXED_PINNED_THRESHOLD = sizeof(uint64_t) * FIXED_PINNED_REGION_COUNT; /* region memory layout: - 1. region info for each region, part of heap metadata - 2. region space for allocation, i.e., the heap + 1. some paddings memory to aligned + 2. region info for each region, part of heap metadata + 3. region space for allocation, i.e., the heap --- start address is aligend to `RegionDesc::UNIT_SIZE` */ static size_t GetHeapMemorySize(size_t heapSize) { size_t regionNum = GetHeapUnitCount(heapSize); size_t metadataSize = GetMetadataSize(regionNum); - size_t totalSize = metadataSize + RoundUp(heapSize, RegionDesc::UNIT_SIZE); + // Add one more `RegionDesc::UNIT_SIZE` totalSize, because we need the region address is aligned to + // `RegionDesc::UNIT_SIZE`, this need some paddings + size_t totalSize = metadataSize + RoundUp(heapSize, RegionDesc::UNIT_SIZE) + RegionDesc::UNIT_SIZE; return totalSize; } @@ -134,7 +137,7 @@ public: RegionDesc* TakeRegion(bool expectPhysicalMem, bool allowgc) { - return TakeRegion(maxUnitCountPerRegion_, RegionDesc::UnitRole::SMALL_SIZED_UNITS, expectPhysicalMem, allowgc); + return TakeRegion(1, RegionDesc::UnitRole::SMALL_SIZED_UNITS, expectPhysicalMem, allowgc); } void AddRecentPinnedRegion(RegionDesc* region) @@ -167,7 +170,7 @@ public: } if (addr == 0) { RegionDesc* region = - TakeRegion(maxUnitCountPerRegion_, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, allowGC); + TakeRegion(1, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, allowGC); if (region == nullptr) { listMutex.unlock(); return 0; @@ -210,7 +213,7 @@ public: } if (addr == 0) { RegionDesc* region = - TakeRegion(maxUnitCountPerRegion_, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, allowGC); + TakeRegion(1, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, allowGC); if (region == nullptr) { return 0; } @@ -244,7 +247,8 @@ public: // caller assures size is truely large (> region size) uintptr_t AllocLarge(size_t size, bool allowGC = true) { - size_t regionCount = (size + RegionDesc::UNIT_SIZE - 1) / RegionDesc::UNIT_SIZE; + size_t alignedSize = AlignUp(size + RegionDesc::UNIT_HEADER_SIZE, RegionDesc::UNIT_SIZE); + size_t regionCount = alignedSize / RegionDesc::UNIT_SIZE; RegionDesc* region = TakeRegion(regionCount, RegionDesc::UnitRole::LARGE_SIZED_UNITS, false, allowGC); if (region == nullptr) { return 0; @@ -265,6 +269,7 @@ public: DLOG(REGION, "alloc large region @0x%zx+%zu type %u", region->GetRegionStart(), region->GetRegionSize(), region->GetRegionType()); uintptr_t addr = region->Alloc(size); + ASSERT(addr > 0); recentLargeRegionList_.PrependRegion(region, RegionDesc::RegionType::RECENT_LARGE_REGION); return addr; } @@ -388,15 +393,10 @@ public: return nullptr; } - size_t GetLargeObjectThreshold() const { return largeObjectThreshold_; } - // this method checks whether allocation is permitted for now, otherwise, it is suspened // until allocation does no harm to gc. void RequestForRegion(size_t size); - void SetMaxUnitCountForRegion(); - void SetLargeObjectThreshold(); - void PrepareTrace() { AllocBufferVisitor visitor = [](AllocationBuffer& regionBuffer) { @@ -526,8 +526,6 @@ public: } private: - static const size_t MAX_UNIT_COUNT_PER_REGION; - static const size_t HUGE_PAGE; inline void TagHugePage(RegionDesc* region, size_t num) const; inline void UntagHugePage(RegionDesc* region, size_t num) const; @@ -592,8 +590,6 @@ private: // heap space not allocated yet for even once. this value should not be decreased. std::atomic inactiveZone_ = { 0 }; - size_t maxUnitCountPerRegion_ = MAX_UNIT_COUNT_PER_REGION; // max units count for threadLocal buffer. - size_t largeObjectThreshold_; // Awaiting JitFort object has no references from other objects, // but we need to keep them as live untill jit compilation has finished installing. std::set awaitingJitFort_; diff --git a/common_components/heap/allocator/region_space.cpp b/common_components/heap/allocator/region_space.cpp index f72db06eead90b5ce26c362bb627f352b5ee1344..0da76bb5ecc627dddec45b051da71ed9223b0955 100755 --- a/common_components/heap/allocator/region_space.cpp +++ b/common_components/heap/allocator/region_space.cpp @@ -85,7 +85,7 @@ HeapAddress RegionSpace::TryAllocateOnce(size_t allocSize, AllocType allocType) if (UNLIKELY_CC(allocType == AllocType::READ_ONLY_OBJECT)) { return regionManager_.AllocReadOnly(allocSize); } - if (UNLIKELY_CC(allocSize >= regionManager_.GetLargeObjectThreshold())) { + if (UNLIKELY_CC(allocSize >= RegionDesc::LARGE_OBJECT_DEFAULT_THRESHOLD)) { return regionManager_.AllocLarge(allocSize); } if (UNLIKELY_CC(allocType == AllocType::PINNED_OBJECT)) { @@ -153,7 +153,7 @@ uintptr_t RegionSpace::AllocRegion() youngSpace_.AddFullRegion(region); uintptr_t start = region->GetRegionStart(); - uintptr_t addr = region->Alloc(region->GetRegionSize()); + uintptr_t addr = region->Alloc(region->GetRegionEnd() - region->GetRegionAllocPtr()); ASSERT(addr != 0); return start; @@ -183,7 +183,7 @@ uintptr_t RegionSpace::AllocPinnedRegion() regionManager_.AddRecentPinnedRegion(region); uintptr_t start = region->GetRegionStart(); - uintptr_t addr = region->Alloc(region->GetRegionSize()); + uintptr_t addr = region->Alloc(region->GetRegionEnd() - region->GetRegionAllocPtr()); ASSERT(addr != 0); return start; diff --git a/common_components/heap/allocator/tests/BUILD.gn b/common_components/heap/allocator/tests/BUILD.gn index c8bac8a5a59903e6d9fe1df94bc579a1430d5745..a388033a486692e3e6f504372180fbfeaa027641 100755 --- a/common_components/heap/allocator/tests/BUILD.gn +++ b/common_components/heap/allocator/tests/BUILD.gn @@ -110,6 +110,29 @@ host_unittest_action("Region_Space_Test") { ] } +host_unittest_action("Heap_Allocator_Test") { + module_out_path = module_output_path + + sources = [ + # test file + "heap_allocator_test.cpp", + ] + + configs = [ + "//arkcompiler/ets_runtime/common_components:common_components_test_config", + "//arkcompiler/ets_runtime:icu_path_test_config", + ] + + deps = [ "//arkcompiler/ets_runtime/common_components:libark_common_components_test" ] + + # hiviewdfx libraries + external_deps = [ + "icu:shared_icui18n", + "icu:shared_icuuc", + "zlib:libz", + ] +} + group("unittest") { testonly = true @@ -119,6 +142,7 @@ group("unittest") { ":Region_Manager_Test", ":Treap_Test", ":Region_Space_Test", + ":Heap_Allocator_Test", ] } @@ -131,5 +155,6 @@ group("host_unittest") { ":Region_Manager_TestAction", ":Treap_TestAction", ":Region_Space_TestAction", + ":Heap_Allocator_Test", ] } diff --git a/common_components/heap/allocator/tests/heap_allocator_test.cpp b/common_components/heap/allocator/tests/heap_allocator_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7f8c1372b245f9548dbb020a6d50287fa3e91d7a --- /dev/null +++ b/common_components/heap/allocator/tests/heap_allocator_test.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common_interfaces/heap/heap_allocator.h" +#include "common_components/heap/allocator/region_space.h" +#include "common_components/heap/collector/trace_collector.h" +#include "common_components/heap/heap.cpp" +#include "common_components/base_runtime/base_runtime_param.h" +#include "common_components/heap/heap_manager.h" +#include "common_components/tests/test_helper.h" + +using namespace common; + +namespace common::test { +class HeapAllocatorTest : public BaseTestWithScope { + void SetUp() override + { + BaseRuntime::GetInstance()->Init(); + holder_ = ThreadHolder::CreateAndRegisterNewThreadHolder(nullptr); + scope_ = new ThreadHolder::TryBindMutatorScope(holder_); + } + + void TearDown() override + { + delete scope_; + BaseRuntime::GetInstance()->Fini(); + } + + ThreadHolder *holder_ {nullptr}; + ThreadHolder::TryBindMutatorScope *scope_ {nullptr}; +}; + +HWTEST_F_L0(HeapAllocatorTest, AllocLargeObject) +{ + uintptr_t addr = common::HeapAllocator::Allocate(Heap::NORMAL_UNIT_SIZE, common::LanguageType::DYNAMIC); + ASSERT(addr > 0); + RegionDesc* region = RegionDesc::GetAliveRegionDescAt(addr); + ASSERT(region->IsLargeRegion()); +} + +HWTEST_F_L0(HeapAllocatorTest, AllocLargeRegion) +{ + uintptr_t addr = common::HeapAllocator::AllocateLargeRegion(Heap::NORMAL_UNIT_SIZE); + ASSERT(addr > 0); + RegionDesc *region = RegionDesc::GetAliveRegionDescAt(addr); + ASSERT(region->IsLargeRegion()); +} +} \ No newline at end of file diff --git a/common_components/heap/allocator/tests/region_manager_test.cpp b/common_components/heap/allocator/tests/region_manager_test.cpp index 65314efbd733e2ff506c48123b149b0eebe4cc5c..6d1661bde23af81f0c5a3ea7d7fa2427411f0c31 100755 --- a/common_components/heap/allocator/tests/region_manager_test.cpp +++ b/common_components/heap/allocator/tests/region_manager_test.cpp @@ -134,8 +134,8 @@ HWTEST_F_L0(RegionManagerTest, VisitLiveObjectsUntilFalse_IsLargeRegion) HWTEST_F_L0(RegionManagerTest, VisitLiveObjectsUntilFalse) { size_t unitIdx = 0; - size_t nUnit = 4; - RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::FREE_UNITS); + size_t nUnit = 1; + RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::SMALL_SIZED_UNITS); ASSERT_NE(region, nullptr); region->AddLiveByteCount(SIZE_SIXTEEN); bool callbackCalled = false; @@ -169,8 +169,9 @@ HWTEST_F_L0(RegionManagerTest, VisitAllObjectsBeforeFix1) HWTEST_F_L0(RegionManagerTest, VisitAllObjectsBeforeFix2) { size_t unitIdx = 0; - size_t nUnit = 4; - RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::FREE_UNITS); + size_t nUnit = 1; + RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::SMALL_SIZED_UNITS); + RegionDesc::InitFreeRegion(unitIdx, nUnit); ASSERT_NE(region, nullptr); uintptr_t start = region->GetRegionStart(); @@ -203,9 +204,9 @@ HWTEST_F_L0(RegionManagerTest, VisitAllObjectsBeforeFix3) HWTEST_F_L0(RegionManagerTest, VisitAllObjectsBeforeFix4) { size_t unitIdx = 0; - size_t nUnit = 4; + size_t nUnit = 1; - RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::FREE_UNITS); + RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::SMALL_SIZED_UNITS); ASSERT_NE(region, nullptr); region->SetFixLine(); bool callbackCalled = false; @@ -236,27 +237,17 @@ HWTEST_F_L0(RegionManagerTest, ReleaseGarbageRegions) EXPECT_GT(released, 0); } -HWTEST_F_L0(RegionManagerTest, SetLargeObjectThreshold) -{ - auto& heapParam = BaseRuntime::GetInstance()->GetHeapParam(); - heapParam.regionSize = SIZE_SIXTEEN; - RegionManager rm; - rm.SetLargeObjectThreshold(); - EXPECT_LT(BaseRuntime::GetInstance()->GetHeapParam().regionSize * KB, - RegionDesc::LARGE_OBJECT_DEFAULT_THRESHOLD); -} - HWTEST_F_L0(RegionManagerTest, ReclaimRegion1) { size_t huge_page = (2048 * KB) / getpagesize(); size_t nUnit = 1; size_t unitIdx = 0; + RegionManager manager; + manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::SMALL_SIZED_UNITS); ASSERT_NE(region, nullptr); EXPECT_FALSE(region->IsLargeRegion()); - RegionManager manager; - manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); manager.ReclaimRegion(region); EXPECT_GT(manager.GetDirtyUnitCount(), 0); } @@ -266,11 +257,11 @@ HWTEST_F_L0(RegionManagerTest, ReclaimRegion2) size_t huge_page = (2048 * KB) / getpagesize(); size_t nUnit = huge_page; size_t unitIdx = 0; + RegionManager manager; + manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::LARGE_SIZED_UNITS); ASSERT_NE(region, nullptr); - RegionManager manager; - manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); manager.ReclaimRegion(region); EXPECT_GT(manager.GetDirtyUnitCount(), 0); } @@ -280,11 +271,11 @@ HWTEST_F_L0(RegionManagerTest, ReleaseRegion) size_t huge_page = (2048 * KB) / getpagesize(); size_t nUnit = 1; size_t unitIdx = 0; + RegionManager manager; + manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); RegionDesc* region = RegionDesc::InitRegion(unitIdx, nUnit, RegionDesc::UnitRole::LARGE_SIZED_UNITS); ASSERT_NE(region, nullptr); - RegionManager manager; - manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); auto ret = manager.ReleaseRegion(region); EXPECT_EQ(ret, region->GetRegionSize()); } @@ -294,12 +285,12 @@ HWTEST_F_L0(RegionManagerTest, TakeRegion1) ASSERT_NE(mutator_, nullptr); mutator_->SetMutatorPhase(GCPhase::GC_PHASE_ENUM); RegionManager manager; - size_t nUnit = 1; manager.Initialize(SIZE_MAX_TEST, reinterpret_cast(regionMemory_)); + size_t nUnit = 4; RegionDesc* garbageRegion = RegionDesc::InitRegion(SIZE_HALF_MAX_TEST, nUnit, - RegionDesc::UnitRole::SMALL_SIZED_UNITS); + RegionDesc::UnitRole::LARGE_SIZED_UNITS); auto size = manager.CollectRegion(garbageRegion); - RegionDesc* region = manager.TakeRegion(16, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, false); + RegionDesc* region = manager.TakeRegion(1, RegionDesc::UnitRole::SMALL_SIZED_UNITS, false, false); EXPECT_GT(manager.GetDirtyUnitCount(), 0); } diff --git a/common_components/heap/collector/copy_data_manager.h b/common_components/heap/collector/copy_data_manager.h index 466de3e4ddf3f1486e1d42a5cd4f5c22ac9cfa6b..1d1e386b497a1f52972ecb9af6775ee152c24cb1 100755 --- a/common_components/heap/collector/copy_data_manager.h +++ b/common_components/heap/collector/copy_data_manager.h @@ -43,7 +43,6 @@ class HeapBitmapManager { public: struct Zone { enum ZoneType : size_t { - LIVE_INFO, BIT_MAP, ZONE_TYPE_CNT, }; @@ -60,12 +59,6 @@ class HeapBitmapManager { void InitZones(size_t unitCount) { uintptr_t start = startAddress_; - allocZone_[Zone::ZoneType::LIVE_INFO].zoneStartAddress = start; - allocZone_[Zone::ZoneType::LIVE_INFO].zonePosition = start; -#if defined(_WIN64) - lastCommitEndAddr[Zone::ZoneType::LIVE_INFO].store(start); -#endif - start += unitCount * sizeof(RegionLiveDesc); allocZone_[Zone::ZoneType::BIT_MAP].zoneStartAddress = start; allocZone_[Zone::ZoneType::BIT_MAP].zonePosition = start; #if defined(_WIN64) @@ -157,12 +150,6 @@ public: return bitmap; } - RegionLiveDesc* AllocateRegionLiveDesc() - { - return reinterpret_cast( - heapBitmap_[0].Allocate(Memory::Zone::ZoneType::LIVE_INFO, sizeof(RegionLiveDesc))); - } - private: size_t GetHeapBitmapSize(size_t heapSize) { @@ -174,8 +161,7 @@ private: constexpr uint8_t bitMarksSize = 64; // 3 bitmaps for each region: markBitmap,resurrectBitmap, enqueueBitmap. constexpr uint8_t nBitmapTypes = 3; - return unitCnt * sizeof(RegionLiveDesc) + - unitCnt * (sizeof(RegionBitmap) + (REGION_UNIT_SIZE / bitMarksSize)) * nBitmapTypes; + return unitCnt * (sizeof(RegionBitmap) + (REGION_UNIT_SIZE / bitMarksSize)) * nBitmapTypes; } Memory heapBitmap_[1]; diff --git a/common_components/heap/collector/region_bitmap.h b/common_components/heap/collector/region_bitmap.h index 9a97d1073e4828965f4a316fd71da08285a892e7..0282c27ae13baedac025c7e6704ed654a29860be 100755 --- a/common_components/heap/collector/region_bitmap.h +++ b/common_components/heap/collector/region_bitmap.h @@ -28,7 +28,6 @@ namespace common { static constexpr size_t kBitsPerByte = 8; static constexpr size_t kMarkedBytesPerBit = 8; static constexpr size_t kBitsPerWord = sizeof(uint64_t) * kBitsPerByte; -class RegionDesc; struct RegionBitmap { static constexpr uint8_t factor = 16; std::atomic partLiveBytes[factor]; @@ -41,6 +40,7 @@ struct RegionBitmap { static size_t GetRegionBitmapSize(size_t regionSize) { + CHECK_CC(regionSize % (kMarkedBytesPerBit * kBitsPerWord) == 0); return sizeof(RegionBitmap) + ((regionSize / (kMarkedBytesPerBit * kBitsPerWord)) * sizeof(uint64_t)); } @@ -71,13 +71,6 @@ struct RegionBitmap { return ret; } }; -struct RegionLiveDesc { - static constexpr HeapAddress TEMPORARY_PTR = 0x1234; - RegionDesc* relatedRegion = nullptr; - RegionBitmap* markBitmap = nullptr; - RegionBitmap* resurrectBitmap = nullptr; - RegionBitmap* enqueueBitmap = nullptr; -}; } // namespace common #endif // COMMON_COMPONENTS_HEAP_COLLECTOR_REGION_BITMAP_H diff --git a/common_components/heap/collector/region_rset.h b/common_components/heap/collector/region_rset.h index 2d31593573e6b3ae80c4924d3b5c325ed1fca4ba..c874e5b5e608a3536d0e231e89721324fa857de3 100644 --- a/common_components/heap/collector/region_rset.h +++ b/common_components/heap/collector/region_rset.h @@ -23,6 +23,7 @@ class RegionRSet { public: explicit RegionRSet(size_t regionSize) : cardCnt(regionSize / CARD_SIZE) { + CHECK_CC(regionSize % CARD_SIZE == 0); #ifdef _WIN64 void* startAddress = VirtualAlloc(NULL, cardCnt * sizeof(uint64_t), MEM_RESERVE, PAGE_READWRITE); if (startAddress == NULL) { @@ -63,7 +64,7 @@ public: size_t cardIdx = (offset / kMarkedBytesPerBit) / kBitsPerWord; size_t headMaskBitStart = (offset / kMarkedBytesPerBit) % kBitsPerWord; uint64_t headMaskBits = static_cast(1) << headMaskBitStart; - uint64_t card = cardTable[cardIdx].load(); + uint64_t card = cardTable[cardIdx].load(std::memory_order_relaxed); bool isMarked = ((card & headMaskBits) != 0); if (!isMarked) { card = cardTable[cardIdx].fetch_or(headMaskBits); diff --git a/common_components/heap/collector/trace_collector.h b/common_components/heap/collector/trace_collector.h index 62b12747f09d2222e48b84bc21f6973aabd0193f..86e59980da0f205feef64e859d697830c489f400 100755 --- a/common_components/heap/collector/trace_collector.h +++ b/common_components/heap/collector/trace_collector.h @@ -271,7 +271,7 @@ protected: void ResetBitmap(bool heapMarked) { - // if heap is marked and tracing result will be used during next gc, we should not reset liveInfo. + // if heap is marked and tracing result will be used during next gc, we should not reset liveInfo_. } uint32_t GetGCThreadCount(const bool isConcurrent) const @@ -334,6 +334,7 @@ public: DCHECK_CC(!stack.empty()); std::lock_guard guard(mtx_); stacks_.push_back(std::move(stack)); + cv_.notify_one(); } StackType PopWorkStack() diff --git a/common_components/heap/heap.cpp b/common_components/heap/heap.cpp index addd238634caad6932dce6f8b68ad57446ef4913..1c40d1bb85616232c7779bbcbd212e81a388daeb 100644 --- a/common_components/heap/heap.cpp +++ b/common_components/heap/heap.cpp @@ -36,6 +36,10 @@ #include #endif namespace common { +static_assert(Heap::NORMAL_UNIT_SIZE == RegionDesc::UNIT_SIZE); +static_assert(Heap::NORMAL_UNIT_HEADER_SIZE == RegionDesc::UNIT_HEADER_SIZE); +static_assert(Heap::NORMAL_UNIT_AVAILABLE_SIZE == RegionDesc::UNIT_AVAILABLE_SIZE); + std::atomic* Heap::currentBarrierPtr_ = nullptr; Barrier* Heap::stwBarrierPtr_ = nullptr; HeapAddress Heap::heapStartAddr_ = 0; diff --git a/common_components/heap/heap.h b/common_components/heap/heap.h index 8d444dcc5ae18bad74c7a2910dc0b185ca86cad3..73b39840488dcca792d43b6ceeebce16b159b4ef 100755 --- a/common_components/heap/heap.h +++ b/common_components/heap/heap.h @@ -34,6 +34,26 @@ class CollectorResources; class Heap { public: + // These need to keep same with that in `RegionDesc` + static constexpr size_t NORMAL_UNIT_SIZE = 256 * 1024; + static constexpr size_t NORMAL_UNIT_HEADER_SIZE = AlignUp(2 * sizeof(void *) + sizeof(uint8_t), 8); + static constexpr size_t NORMAL_UNIT_AVAILABLE_SIZE = NORMAL_UNIT_SIZE - NORMAL_UNIT_HEADER_SIZE; + + static constexpr size_t GetNormalRegionSize() + { + return NORMAL_UNIT_SIZE; + } + + static constexpr size_t GetNormalRegionHeaderSize() + { + return NORMAL_UNIT_HEADER_SIZE; + } + + static constexpr size_t GetNormalRegionAvailableSize() + { + return NORMAL_UNIT_AVAILABLE_SIZE; + } + static void throwOOM() { // Maybe we need to add heapdump logic here diff --git a/common_components/heap/verification.cpp b/common_components/heap/verification.cpp index 7ee2c041414d31e59851a1f437f884d498a80a30..440ec9715e8fee035e24ca6408754007755f9df0 100755 --- a/common_components/heap/verification.cpp +++ b/common_components/heap/verification.cpp @@ -101,6 +101,7 @@ std::string GetObjectInfo(const BaseObject* obj) } else { auto region = RegionDesc::GetRegionDescAt(reinterpret_cast(obj)); s << std::hex << "Type: 0x" << (int) region->GetRegionType() << ", " + << "Base: 0x" << region->GetRegionBase() << ", " << "Start: 0x" << region->GetRegionStart() << ", " << "End: 0x" << region->GetRegionEnd() << ", " << "AllocPtr: 0x" << region->GetRegionAllocPtr() << ", " diff --git a/common_components/heap/w_collector/idle_barrier.cpp b/common_components/heap/w_collector/idle_barrier.cpp index 4f23d406e8c354e1469c2481e297f02325192222..eaaa5fd028140d150163d61a0c2c4376c408327a 100755 --- a/common_components/heap/w_collector/idle_barrier.cpp +++ b/common_components/heap/w_collector/idle_barrier.cpp @@ -91,13 +91,14 @@ void IdleBarrier::UpdateRememberSet(BaseObject* object, BaseObject* ref) const { ASSERT(Heap::IsHeapAddress(ref)); ASSERT(object != nullptr); - RegionDesc* objRegion = RegionDesc::GetRegionDescAt(reinterpret_cast((void*)object)); - RegionDesc* refRegion = RegionDesc::GetRegionDescAt(reinterpret_cast((void*)ref)); - if ((!objRegion->IsInYoungSpace() && refRegion->IsInYoungSpace()) || - (objRegion->IsInFromSpace() && refRegion->IsInRecentSpace())) { - if (objRegion->MarkRSetCardTable(object)) { + RegionDesc::InlinedRegionMetaData *objMetaRegion = RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData( + reinterpret_cast(object)); + RegionDesc::InlinedRegionMetaData *refMetaRegion = RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData( + reinterpret_cast(ref)); + if (!objMetaRegion->IsInYoungSpaceForWB() && refMetaRegion->IsInYoungSpaceForWB()) { + if (objMetaRegion->MarkRSetCardTable(object)) { DLOG(BARRIER, "update point-out remember set of region %p, obj %p, ref: %p<%p>", - objRegion, object, ref, ref->GetTypeInfo()); + objMetaRegion->GetRegionDesc(), object, ref, ref->GetTypeInfo()); } } } diff --git a/common_components/heap/w_collector/w_collector.cpp b/common_components/heap/w_collector/w_collector.cpp index c638242d411a97a425541fa31a6ece5fa0ff9cd1..68df0f3656b41cce2ff14c1800d42a17c8018939 100755 --- a/common_components/heap/w_collector/w_collector.cpp +++ b/common_components/heap/w_collector/w_collector.cpp @@ -49,7 +49,7 @@ bool WCollector::MarkObject(BaseObject* obj, size_t cellCount) const } size_t size = cellCount == 0 ? obj->GetSize() : (cellCount + 1) * sizeof(uint64_t); region->AddLiveByteCount(size); - DLOG(TRACE, "mark obj %p<%p>(%zu) in region %p(%u)@%#zx, live %u", obj, obj->GetTypeInfo(), obj->GetSize(), + DLOG(TRACE, "mark obj %p<%p>(%zu) in region %p(%u)@%#zx, live %u", obj, obj->GetTypeInfo(), size, region, region->GetRegionType(), region->GetRegionStart(), region->GetLiveByteCount()); } return marked; diff --git a/common_components/heap/w_collector/w_collector.h b/common_components/heap/w_collector/w_collector.h index cb074ca433f282c6351c30692e377987c9564a8d..a22b2b1a50425ccaadb4651d773377b702da23d1 100755 --- a/common_components/heap/w_collector/w_collector.h +++ b/common_components/heap/w_collector/w_collector.h @@ -114,8 +114,9 @@ public: { // filter const string object. if (Heap::IsHeapAddress(obj)) { - auto regionInfo = RegionDesc::GetRegionDescAt(reinterpret_cast(obj)); - return regionInfo->IsFromRegion(); + RegionDesc::InlinedRegionMetaData *objMetaRegion = + RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData(reinterpret_cast(obj)); + return objMetaRegion->IsFromRegion(); } return false; diff --git a/common_components/serialize/serialize_utils.cpp b/common_components/serialize/serialize_utils.cpp index 68d4a400e28c334dc5a3630c18c3a4be767b9492..ae0902c833f6682bad1c56e9ba549693d035d4e1 100755 --- a/common_components/serialize/serialize_utils.cpp +++ b/common_components/serialize/serialize_utils.cpp @@ -47,9 +47,4 @@ SerializedBaseObjectSpace SerializeUtils::GetSerializeObjectSpace(uintptr_t obj) return SerializedBaseObjectSpace::OTHER; } } - -size_t SerializeUtils::GetRegionSize() -{ - return BaseRuntime::GetInstance()->GetHeapParam().regionSize * KB; -} } // namespace common diff --git a/common_components/serialize/serialize_utils.h b/common_components/serialize/serialize_utils.h index 7eec8691492f006ef5c8ba3594917e37b9a3ecf3..363d6a6531d7ee9e6669ccc3ea6fed17b12ace83 100644 --- a/common_components/serialize/serialize_utils.h +++ b/common_components/serialize/serialize_utils.h @@ -32,7 +32,6 @@ enum class SerializedBaseObjectSpace : uint8_t { class SerializeUtils { public: static SerializedBaseObjectSpace GetSerializeObjectSpace(uintptr_t obj); - static size_t GetRegionSize(); }; } // namespace common #endif // COMMON_COMPONENTS_SERIALIZE_UTILS_H diff --git a/ecmascript/compiler/call_signature.cpp b/ecmascript/compiler/call_signature.cpp index bcc4f64b3f31a4c16236c0c6b31e94aa69ef5cf0..6bf93385a06a69c56968f3d9ba1b177be5712744 100644 --- a/ecmascript/compiler/call_signature.cpp +++ b/ecmascript/compiler/call_signature.cpp @@ -948,6 +948,11 @@ DEF_CALL_SIGNATURE(VerifyBarrier) SETVALUEBARRIER_CALL_ARGS_SIGNATURE_COMMON(VerifyBarrier); } +DEF_CALL_SIGNATURE(CMCSetValueWithBarrier) +{ + SETVALUEBARRIER_CALL_ARGS_SIGNATURE_COMMON(CMCSetValueWithBarrier); +} + #undef SETVALUEBARRIER_CALL_ARGS_SIGNATURE_COMMON DEF_CALL_SIGNATURE(NewThisObjectChecked) @@ -3874,4 +3879,32 @@ DEF_CALL_SIGNATURE(LoadNativeModuleFailed) callSign->SetGCLeafFunction(true); callSign->SetTargetKind(CallSignature::TargetKind::RUNTIME_STUB_NO_GC); } + +DEF_CALL_SIGNATURE(MarkRSetCardTable) +{ + // 3 : 3 input parameters + CallSignature MarkRSetCardTable("MarkRSetCardTable", 0, 1, ArgumentsOrder::DEFAULT_ORDER, + VariableType::BOOL()); // 2 : 2 input parameters + *callSign = MarkRSetCardTable; + std::array params = { // 2 : 2 input parameters + VariableType::JS_POINTER(), + }; + callSign->SetParameters(params.data()); + callSign->SetGCLeafFunction(true); + callSign->SetTargetKind(CallSignature::TargetKind::RUNTIME_STUB_NO_GC); +} + +DEF_CALL_SIGNATURE(MarkInBuffer) +{ + // 3 : 3 input parameters + CallSignature MarkInBuffer("MarkInBuffer", 0, 1, ArgumentsOrder::DEFAULT_ORDER, + VariableType::BOOL()); + *callSign = MarkInBuffer; + std::array params = { // 1 : 1 input parameters + VariableType::JS_POINTER(), + }; + callSign->SetParameters(params.data()); + callSign->SetGCLeafFunction(true); + callSign->SetTargetKind(CallSignature::TargetKind::RUNTIME_STUB_NO_GC); +} } // namespace panda::ecmascript::kungfu diff --git a/ecmascript/compiler/call_signature.h b/ecmascript/compiler/call_signature.h index 2fdca48b989d51da687238512ac2a7a906ba76f0..59c932acec79d81708cc2af634f777a2eeb0ba87 100644 --- a/ecmascript/compiler/call_signature.h +++ b/ecmascript/compiler/call_signature.h @@ -706,7 +706,10 @@ private: V(ReadBarrier) \ V(ComputeStringHashcode) \ V(CopyCallTarget) \ - V(CopyArgvArray) + V(CopyArgvArray) \ + V(MarkRSetCardTable) \ + V(MarkInBuffer) \ + V(CMCSetValueWithBarrier) #define DECL_CALL_SIGNATURE(name) \ class name##CallSignature final { \ diff --git a/ecmascript/compiler/circuit_builder.h b/ecmascript/compiler/circuit_builder.h index da1fc944b99655de7acac7002a619b46ed8d100e..eb13face8e1ea43daf63fd53900bda9854d19710 100644 --- a/ecmascript/compiler/circuit_builder.h +++ b/ecmascript/compiler/circuit_builder.h @@ -986,6 +986,7 @@ public: void Store(VariableType type, GateRef glue, GateRef base, GateRef offset, GateRef value, MemoryAttribute mAttr = MemoryAttribute::Default()); + GateRef FetchOr(GateRef ptr, GateRef value); void StoreHClass(VariableType type, GateRef glue, GateRef base, GateRef offset, GateRef value, GateRef compValue, MemoryAttribute mAttr = MemoryAttribute::Default()); void StoreWithoutBarrier(VariableType type, GateRef addr, GateRef value, diff --git a/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp b/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp index 052adf1b7bbd61dd2abec7578fd5064353a3e41a..9d46f819626b5d6f8413ed3d898700e5226e3995 100644 --- a/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp +++ b/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp @@ -189,6 +189,7 @@ void LLVMIRBuilder::InitializeHandlers() {OpCode::UDIV, &LLVMIRBuilder::HandleUDiv}, {OpCode::AND, &LLVMIRBuilder::HandleIntAnd}, {OpCode::OR, &LLVMIRBuilder::HandleIntOr}, + {OpCode::FETCH_OR, &LLVMIRBuilder::HandleFetchOr}, {OpCode::XOR, &LLVMIRBuilder::HandleIntXor}, {OpCode::LSR, &LLVMIRBuilder::HandleIntLsr}, {OpCode::ASR, &LLVMIRBuilder::HandleIntAsr}, @@ -2046,6 +2047,13 @@ void LLVMIRBuilder::HandleIntOr(GateRef gate) VisitIntOr(gate, g0, g1); } +void LLVMIRBuilder::HandleFetchOr(GateRef gate) +{ + auto g0 = acc_.GetValueIn(gate, 0); + auto g1 = acc_.GetValueIn(gate, 1); + VisitFetchOr(gate, g0, g1); +} + void LLVMIRBuilder::HandleIntXor(GateRef gate) { auto g0 = acc_.GetIn(gate, 0); @@ -2560,6 +2568,19 @@ void LLVMIRBuilder::VisitIntOr(GateRef gate, GateRef e1, GateRef e2) } } +void LLVMIRBuilder::VisitFetchOr(GateRef gate, GateRef e1, GateRef e2) +{ + LLVMValueRef e1Value = GetLValue(e1); + LLVMValueRef e2Value = GetLValue(e2); + LLVMValueRef result = LLVMBuildAtomicRMW(builder_, LLVMAtomicRMWBinOpOr, e1Value, e2Value, + LLVMAtomicOrderingMonotonic, false); + Bind(gate, result); + + if (IsLogEnabled()) { + SetDebugInfo(gate, result); + } +} + void LLVMIRBuilder::HandleIntAnd(GateRef gate) { auto g0 = acc_.GetIn(gate, 0); diff --git a/ecmascript/compiler/common_stub_csigns.h b/ecmascript/compiler/common_stub_csigns.h index b853fac46215d915369cde7b6078e14687d7f7df..e7d40eb11ba922c1bae47b93cb9cb8f32a3c43bb 100644 --- a/ecmascript/compiler/common_stub_csigns.h +++ b/ecmascript/compiler/common_stub_csigns.h @@ -158,7 +158,8 @@ namespace panda::ecmascript::kungfu { V(ComputeStringHashcode) \ V(DefineNormalFuncForJit) \ V(DefineArrowFuncForJit) \ - V(DefineBaseConstructorForJit) + V(DefineBaseConstructorForJit) \ + V(CMCSetValueWithBarrier) #define COMMON_STUB_ID_LIST(V) \ COMMON_STUB_LIST(V) @@ -176,7 +177,7 @@ public: COMMON_STW_COPY_STUB_LIST(DEF_STUB_ID) #undef DEF_STUB_ID NUM_OF_STUBS, - NUM_OF_ALL_NORMAL_STUBS = DefineBaseConstructorForJit + 1, + NUM_OF_ALL_NORMAL_STUBS = CMCSetValueWithBarrier + 1, }; #define ASSERT_ID_EQUAL(name) \ static_assert((static_cast(ID::name##StwCopy)) == \ diff --git a/ecmascript/compiler/common_stubs.cpp b/ecmascript/compiler/common_stubs.cpp index 706a1181ba0daaa07cd3e01e45d0458c8c604a17..42374a64eb0eb63adff82cd7e470641d696b5325 100644 --- a/ecmascript/compiler/common_stubs.cpp +++ b/ecmascript/compiler/common_stubs.cpp @@ -1272,6 +1272,16 @@ void SetValueWithBarrierStubBuilder::GenerateCircuit() Return(); } +void CMCSetValueWithBarrierStubBuilder::GenerateCircuit() +{ + GateRef glue = PtrArgument(0); + GateRef obj = TaggedArgument(1); + GateRef offset = PtrArgument(2); // 2 : 3rd para + GateRef value = TaggedArgument(3); // 3 : 4th para + CMCSetValueWithBarrier(glue, obj, offset, value); + Return(); +} + void SetNonSValueWithBarrierStubBuilder::GenerateCircuit() { GateRef glue = PtrArgument(0); diff --git a/ecmascript/compiler/ir_builder.h b/ecmascript/compiler/ir_builder.h index e28da763dbfa0f233fb25ad35e02d1f53a87bb5d..4220ddd5a23326cd8f9d17ce6b250b85eb9c824f 100644 --- a/ecmascript/compiler/ir_builder.h +++ b/ecmascript/compiler/ir_builder.h @@ -121,6 +121,7 @@ enum class CallInfoKind : bool { V(HasPendingExceptionIntrinsic, (GateRef gate)) \ V(CheckObjectIsStringIntrinsic, (GateRef gate, GateRef e1, GateRef e2)) \ V(IsJsCOWArrayIntrinsic, (GateRef gate, GateRef e1, GateRef e2)) \ + V(FetchOr, (GateRef gate, GateRef e1, GateRef e2)) bool IsAddIntergerType(MachineType machineType); bool IsMulIntergerType(MachineType machineType); diff --git a/ecmascript/compiler/lcr_circuit_builder.cpp b/ecmascript/compiler/lcr_circuit_builder.cpp index dcb9d18940eddff3e36a13f3b5e144ea8193831e..cd1e3a6e0fb6379a7589df96317d160d4037626a 100644 --- a/ecmascript/compiler/lcr_circuit_builder.cpp +++ b/ecmascript/compiler/lcr_circuit_builder.cpp @@ -118,6 +118,16 @@ void CircuitBuilder::Store(VariableType type, GateRef glue, GateRef base, GateRe label->SetDepend(result); } +GateRef CircuitBuilder::FetchOr(GateRef ptr, GateRef value) +{ + auto label = GetCurrentLabel(); + auto depend = label->GetDepend(); + GateRef result = GetCircuit()->NewGate(circuit_->FetchOr(), + MachineType::I64, { depend, ptr, value }, acc_.GetGateType(value)); + label->SetDepend(result); + return result; +} + void CircuitBuilder::StoreHClass(VariableType type, GateRef glue, GateRef base, GateRef offset, GateRef value, GateRef compValue, MemoryAttribute mAttr) { diff --git a/ecmascript/compiler/lcr_opcodes.h b/ecmascript/compiler/lcr_opcodes.h index 7ae05ae4d703a3b2166584c0fcf6067777b38ea9..f418ef1040f63355a14d30c710d7aada4cab576c 100644 --- a/ecmascript/compiler/lcr_opcodes.h +++ b/ecmascript/compiler/lcr_opcodes.h @@ -32,6 +32,7 @@ namespace panda::ecmascript::kungfu { V(And, AND, GateFlags::NONE_FLAG, 0, 0, 2) \ V(Xor, XOR, GateFlags::NONE_FLAG, 0, 0, 2) \ V(Or, OR, GateFlags::NONE_FLAG, 0, 0, 2) \ + V(FetchOr, FETCH_OR, GateFlags::NONE_FLAG, 0, 1, 2) \ V(Lsl, LSL, GateFlags::NONE_FLAG, 0, 0, 2) \ V(Lsr, LSR, GateFlags::NONE_FLAG, 0, 0, 2) \ V(Asr, ASR, GateFlags::NONE_FLAG, 0, 0, 2) \ diff --git a/ecmascript/compiler/stub_builder-inl.h b/ecmascript/compiler/stub_builder-inl.h index 8b45278a708a3e8d25af9b757743363a46864dda..d35f120e11a5de1b975ec44d254c5d7a009809b6 100644 --- a/ecmascript/compiler/stub_builder-inl.h +++ b/ecmascript/compiler/stub_builder-inl.h @@ -580,6 +580,11 @@ inline GateRef StubBuilder::Int64Or(GateRef x, GateRef y) return env_->GetBuilder()->Int64Or(x, y); } +inline GateRef StubBuilder::Int64FetchOr(GateRef x, GateRef y) +{ + return env_->GetBuilder()->FetchOr(x, y); +} + inline GateRef StubBuilder::IntPtrOr(GateRef x, GateRef y) { return env_->GetBuilder()->IntPtrOr(x, y); diff --git a/ecmascript/compiler/stub_builder.cpp b/ecmascript/compiler/stub_builder.cpp index de1f495c142619ba00df099a6308ff008bf0ff49..7012c765aaf7428a8266f952a8e99850293f2177 100644 --- a/ecmascript/compiler/stub_builder.cpp +++ b/ecmascript/compiler/stub_builder.cpp @@ -24,6 +24,9 @@ #include "ecmascript/compiler/profiler_stub_builder.h" #include "ecmascript/elements.h" #include "ecmascript/compiler/stub_builder.h" + +#include "barrier_stub_builder.h" +#include "common_components/heap/allocator/region_desc.h" #include "ecmascript/ic/mega_ic_cache.h" #include "ecmascript/js_api/js_api_arraylist.h" #include "ecmascript/js_primitive_ref.h" @@ -1904,6 +1907,124 @@ void StubBuilder::VerifyBarrier(GateRef glue, GateRef obj, [[maybe_unused]] Gate env->SubCfgExit(); } +GateRef StubBuilder::GetCMCRegionDesc(GateRef obj) +{ + GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); + GateRef regionDescAddr = LoadPrimitive(VariableType::INT8(), metaDataAddr, + IntPtr(common::RegionDesc::REGION_DESC_IN_INLINED_METADATA_OFFSET)); + return regionDescAddr; +} + +GateRef StubBuilder::GetCMCRegionRSet(GateRef obj) +{ + GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~static_cast(common::RegionDesc::DEFAULT_REGION_UNIT_MASK))); + GateRef regionRSet = LoadPrimitive(VariableType::NATIVE_POINTER(), metaDataAddr, + IntPtr(common::RegionDesc::REGION_RSET_IN_INLINED_METADATA_OFFSET)); + return regionRSet; +} + +GateRef StubBuilder::GetCMCRegionType(GateRef obj) +{ + + GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), IntPtr(~common::RegionDesc::DEFAULT_REGION_UNIT_MASK)); + GateRef regionDescAddr = LoadPrimitive(VariableType::INT8(), metaDataAddr, + IntPtr(common::RegionDesc::REGION_TYPE_IN_INLINED_METADATA_OFFSET)); + return regionDescAddr; +} + +GateRef StubBuilder::IsInYoungSpace(GateRef regionType) +{ + auto env = GetEnvironment(); + GateRef ret = LogicOrBuilder(env). + Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::THREAD_LOCAL_REGION)))). + Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::RECENT_FULL_REGION)))). + Or(Int8Equal(regionType, Int8(static_cast(common::RegionDesc::RegionType::FROM_REGION)))). + Done(); + return ret; +} + +void StubBuilder::CMCSetValueWithBarrier(GateRef glue, GateRef obj, [[maybe_unused]]GateRef offset, GateRef value) +{ + auto env = GetEnvironment(); + Label entry(env); + env->SubCfgEntry(&entry); + Label exit(env); + + GateRef gcPhase = LoadPrimitive(VariableType::INT8(), glue, + Int64(JSThread::GlueData::GetSharedGCStateBitFieldOffset(false) + + JSThread::CMCGCPhaseBits::START_BIT / BITS_PER_BYTE)); + + Label checkOldToYoung(env); + Label markRSet(env); + Label notMarkRSet(env); + Label notIdlePhase(env); + BRANCH(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_IDLE)), &checkOldToYoung, ¬IdlePhase); + Bind(¬IdlePhase); + GateRef gcReason = LoadPrimitive(VariableType::INT32(), glue, + Int64(JSThread::GlueData::GetSharedGCStateBitFieldOffset(false) + + JSThread::CMCGCReasonBits::START_BIT / BITS_PER_BYTE)); + Label reasonNotYoung(env); + BRANCH(Int32Equal(gcReason, Int32(common::GCReason::GC_REASON_YOUNG)), &checkOldToYoung, &reasonNotYoung); + Bind(&reasonNotYoung); + GateRef needMarkPhase = LogicOrBuilder(env) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_COPY))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_FIX))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_PRECOPY))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_FINAL_MARK))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_REMARK_SATB))) + .Done(); + BRANCH(needMarkPhase, &checkOldToYoung, ¬MarkRSet); + + Bind(&checkOldToYoung); + { + GateRef objRegionType = GetCMCRegionType(obj); + GateRef valueRegionType = GetCMCRegionType(value); + GateRef isOldToYoung = LogicAndBuilder(env) + .And(BoolNot(IsInYoungSpace(objRegionType))) + .And(IsInYoungSpace(valueRegionType)).Done(); + BRANCH_UNLIKELY(isOldToYoung, &markRSet, ¬MarkRSet); + Bind(&markRSet); + { + CallNGCRuntime(glue, RTSTUB_ID(MarkRSetCardTable), {obj}); + Jump(¬MarkRSet); + // Label markBit(env); + // GateRef regionBase = IntPtrAnd(TaggedCastToIntPtr(obj), + // IntPtr(~static_cast(common::RegionDesc::DEFAULT_REGION_UNIT_MASK))); + // GateRef objOffset = TruncPtrToInt32(PtrSub(TaggedCastToIntPtr(obj), regionBase)); + // GateRef rset = GetCMCRegionRSet(obj); + // GateRef cardOffset = Int32Div(objOffset, Int32(common::kBitsPerWord)); + // GateRef headMaskBitStart = Int32Mod(Int32Div(objOffset, Int32(common::kMarkedBytesPerBit)), + // Int32(common::kBitsPerWord)); + // GateRef headMaskBits = Int64LSL(Int64(1) , ZExtInt32ToInt64(headMaskBitStart)); + // GateRef card = LoadPrimitive(VariableType::INT64(), rset, cardOffset); + // GateRef isMarked = Int64And(card, headMaskBits); + // BRANCH_NO_WEIGHT(Int64NotEqual(isMarked, Int64(0)), ¬MarkRSet, &markBit); + // Bind(&markBit); + // { + // Int64FetchOr(PtrAdd(rset, cardOffset), headMaskBits); + // Jump(¬MarkRSet); + // } + } + } + Bind(¬MarkRSet); + Label markInBuffer(env); + GateRef needMarkInBuffer = LogicOrBuilder(env) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_ENUM))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_MARK))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_FINAL_MARK))) + .Or(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_REMARK_SATB))) + .Done(); + BRANCH_UNLIKELY(needMarkInBuffer, &markInBuffer, &exit); + Bind(&markInBuffer); + { + CallNGCRuntime(glue, RTSTUB_ID(MarkInBuffer), {value}); + Jump(&exit); + } + Bind(&exit); + env->SubCfgExit(); +} + void StubBuilder::SetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value, MemoryAttribute::ShareFlag share) { @@ -1913,12 +2034,12 @@ void StubBuilder::SetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, Label exit(env); Label callRuntime(env); Label checkNext(env); - BRANCH_UNLIKELY(LoadPrimitive( + BRANCH_LIKELY(LoadPrimitive( VariableType::BOOL(), glue, IntPtr(JSThread::GlueData::GetIsEnableCMCGCOffset(env->Is32Bit()))), &callRuntime, &checkNext); Bind(&callRuntime); { - CallNGCRuntime(glue, RTSTUB_ID(CMCGCMarkingBarrier), {glue, obj, offset, value}); + CMCSetValueWithBarrier(glue, obj, offset, value); Jump(&exit); } Bind(&checkNext); diff --git a/ecmascript/compiler/stub_builder.h b/ecmascript/compiler/stub_builder.h index d02f1abfd7b97da8783aa3a4f07e88644856d7a5..138e15f3004f8306ba2cc3a2684f2663b848556b 100644 --- a/ecmascript/compiler/stub_builder.h +++ b/ecmascript/compiler/stub_builder.h @@ -253,6 +253,7 @@ public: GateRef Int32Xor(GateRef x, GateRef y); GateRef FixLoadType(GateRef x); GateRef Int64Or(GateRef x, GateRef y); + GateRef Int64FetchOr(GateRef x, GateRef y); GateRef IntPtrOr(GateRef x, GateRef y); GateRef Int64And(GateRef x, GateRef y); GateRef Int64Xor(GateRef x, GateRef y); @@ -811,6 +812,11 @@ public: void SetValueWithAttr(GateRef glue, GateRef obj, GateRef offset, GateRef key, GateRef value, GateRef attr); void SetValueWithRep(GateRef glue, GateRef obj, GateRef offset, GateRef value, GateRef rep, Label *repChange); void VerifyBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value); + GateRef GetCMCRegionDesc(GateRef obj); + GateRef GetCMCRegionRSet(GateRef obj); + GateRef GetCMCRegionType(GateRef obj); + GateRef IsInYoungSpace(GateRef regionType); + void CMCSetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value); void SetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value, MemoryAttribute::ShareFlag share = MemoryAttribute::UNKNOWN); GateRef GetValueWithBarrier(GateRef glue, GateRef addr); @@ -1306,6 +1312,11 @@ private: CallSignature *callSignature_ {nullptr}; Environment *env_; GateRef globalEnv_ {Gate::InvalidGateRef}; + + static constexpr uint64_t UNIT_SIZE = 4_KB; + static constexpr uint64_t OBJ_ALIGN_MASK = 0b111; + static constexpr uint64_t UNIT_ROLE_MASK = 0b11111; + static constexpr uint64_t REGION_TYPE_MASK = 0b11111; }; } // namespace panda::ecmascript::kungfu #endif // ECMASCRIPT_COMPILER_STUB_BUILDER_H diff --git a/ecmascript/js_thread.cpp b/ecmascript/js_thread.cpp index 2dd264ab2056faede6789cac17cc0d88653697e8..ccadaebb55c84377611233e8b697e6fc7a87d16a 100644 --- a/ecmascript/js_thread.cpp +++ b/ecmascript/js_thread.cpp @@ -38,6 +38,7 @@ #if defined(ECMASCRIPT_SUPPORT_CPUPROFILER) #include "ecmascript/dfx/cpu_profiler/cpu_profiler.h" #endif +#include "common_components/heap/allocator/region_desc.h" #include "ecmascript/dfx/vm_thread_control.h" #include "ecmascript/ecma_global_storage.h" #include "ecmascript/ic/properties_cache.h" @@ -217,6 +218,7 @@ JSThread::JSThread(EcmaVM *vm) : id_(os::thread::GetCurrentThreadId()), vm_(vm) glueData_.globalConst_ = new GlobalEnvConstants(); glueData_.baseAddress_ = TaggedStateWord::BASE_ADDRESS; glueData_.isEnableCMCGC_ = g_isEnableCMCGC; + glueData_.heapStartAddr = common::RegionDesc::GetHeapStartAddress(); } JSThread::JSThread(EcmaVM *vm, ThreadType threadType) : id_(os::thread::GetCurrentThreadId()), @@ -226,6 +228,7 @@ JSThread::JSThread(EcmaVM *vm, ThreadType threadType) : id_(os::thread::GetCurre // jit thread no need GCIterating readyForGCIterating_ = false; glueData_.isEnableCMCGC_ = g_isEnableCMCGC; + glueData_.heapStartAddr = common::RegionDesc::GetHeapStartAddress(); if (UNLIKELY(g_isEnableCMCGC)) { glueData_.threadHolder_ = ToUintPtr(ThreadHolder::CreateAndRegisterNewThreadHolder(nullptr)); } @@ -238,6 +241,7 @@ JSThread::JSThread(ThreadType threadType) : threadType_(threadType) // daemon thread no need GCIterating readyForGCIterating_ = false; glueData_.isEnableCMCGC_ = g_isEnableCMCGC; + glueData_.heapStartAddr = common::RegionDesc::GetHeapStartAddress(); } JSThread::~JSThread() diff --git a/ecmascript/js_thread.h b/ecmascript/js_thread.h index b919db11eb2a0e958885819a278d76d0f211b36c..e96f20f1c2550b71676915a3150253b37aaab1d9 100644 --- a/ecmascript/js_thread.h +++ b/ecmascript/js_thread.h @@ -37,6 +37,7 @@ #include "ecmascript/mem/visitor.h" #include "ecmascript/mutator_lock.h" #include "ecmascript/patch/patch_loader.h" +#include "common_components/heap/collector/gc_request.h" #include "common_interfaces/base_runtime.h" #include "common_interfaces/thread/base_thread.h" #include "common_interfaces/thread/thread_holder.h" @@ -165,6 +166,7 @@ public: static constexpr int CMC_GC_PHASE_BITFIELD_NUM = 8; static constexpr int CMC_GC_PHASE_BITFIELD_MASK = (((1 << CMC_GC_PHASE_BITFIELD_NUM) - 1) << CMC_GC_PHASE_BITFIELD_START); + static constexpr int CMC_GC_REASON_BITFIELD_NUM = 32; static constexpr int CHECK_SAFEPOINT_BITFIELD_NUM = 8; static constexpr int PGO_PROFILER_BITFIELD_START = 16; static constexpr int BOOL_BITFIELD_NUM = 1; @@ -172,9 +174,10 @@ public: static constexpr uint32_t RESERVE_STACK_SIZE = 128; static constexpr size_t DEFAULT_MAX_SYSTEM_STACK_SIZE = 8_MB; using MarkStatusBits = BitField; - using SharedMarkStatusBits = BitField; - using ReadBarrierStateBit = SharedMarkStatusBits::NextFlag; - using CMCGCPhaseBits = BitField; + using SharedMarkStatusBits = BitField; // 0 + using ReadBarrierStateBit = SharedMarkStatusBits::NextFlag; // 1 + using CMCGCPhaseBits = BitField; // 8-15 + using CMCGCReasonBits = CMCGCPhaseBits::NextField; using CheckSafePointBit = BitField; using VMNeedSuspensionBit = BitField; using VMHasSuspendedBit = VMNeedSuspensionBit::NextFlag; @@ -588,6 +591,16 @@ public: CMCGCPhaseBits::Set(gcPhase, &glueData_.sharedGCStateBitField_); } + common::GCReason GetCMCGCReason() const + { + return CMCGCReasonBits::Decode(glueData_.sharedGCStateBitField_); + } + + void SetCMCGCReason(common::GCReason gcReason) + { + CMCGCReasonBits::Set(gcReason, &glueData_.sharedGCStateBitField_); + } + void SetPGOProfilerEnable(bool enable) { PGOProfilerStatus status = @@ -1100,6 +1113,7 @@ public: base::AlignedPointer, base::AlignedPointer, base::AlignedPointer, + base::AlignedPointer, JSTaggedValue, base::AlignedPointer, base::AlignedPointer, @@ -1155,6 +1169,7 @@ public: ThreadHolderIndex, AllocBufferIndex, StateAndFlagsIndex, + HeapStartAddrIndex, ExceptionIndex, CurrentFrameIndex, LeaveFrameIndex, @@ -1208,6 +1223,11 @@ public: }; static_assert(static_cast(Index::NumOfMembers) == NumOfTypes); + static size_t GetHeapStartAddrOffset(bool isArch32) + { + return GetOffset(Index::HeapStartAddrIndex)>(isArch32); + } + static size_t GetThreadHolderOffset(bool isArch32) { return GetOffset(Index::ThreadHolderIndex)>(isArch32); @@ -1498,6 +1518,7 @@ public: alignas(EAS) uintptr_t threadHolder_ {0}; alignas(EAS) uintptr_t allocBuffer_ {0}; alignas(EAS) ThreadStateAndFlags stateAndFlags_ {}; + alignas(EAS) uintptr_t heapStartAddr {0}; alignas(EAS) JSTaggedValue exception_ {JSTaggedValue::Hole()}; alignas(EAS) JSTaggedType *currentFrame_ {nullptr}; alignas(EAS) JSTaggedType *leaveFrame_ {nullptr}; diff --git a/ecmascript/mem/cmc_gc/hooks.cpp b/ecmascript/mem/cmc_gc/hooks.cpp index fe902ae31db0320a9f2f3738ede178357eb3726a..18e629e9659e3171bb8494830f6c68249d1922d6 100644 --- a/ecmascript/mem/cmc_gc/hooks.cpp +++ b/ecmascript/mem/cmc_gc/hooks.cpp @@ -208,6 +208,7 @@ void VisitJSThread(void *jsThread, CommonRootVisitor visitor) void SynchronizeGCPhaseToJSThread(void *jsThread, GCPhase gcPhase) { reinterpret_cast(jsThread)->SetCMCGCPhase(gcPhase); + reinterpret_cast(jsThread)->SetCMCGCReason(Heap::GetHeap().GetGCReason()); if (panda::ecmascript::g_isEnableCMCGC) { // forcely enable read barrier for read barrier DFX #ifdef ENABLE_CMC_RB_DFX diff --git a/ecmascript/mem/space.cpp b/ecmascript/mem/space.cpp index 50c04f0967a31dcde343f907d3c8269b233f1304..fa4f072ebeee628320f6bfedcce72b552ccb2aae 100644 --- a/ecmascript/mem/space.cpp +++ b/ecmascript/mem/space.cpp @@ -13,12 +13,14 @@ * limitations under the License. */ +#include "ecmascript/mem/space.h" + +#include "common_components/heap/heap.h" +#include "common_interfaces/heap/heap_allocator.h" #include "ecmascript/js_tagged_value-inl.h" #include "ecmascript/mem/mem_controller.h" #include "ecmascript/mem/region-inl.h" -#include "ecmascript/mem/space.h" #include "ecmascript/platform/os.h" -#include "common_interfaces/heap/heap_allocator.h" namespace panda::ecmascript { Space::Space(BaseHeap* heap, HeapRegionAllocator *heapRegionAllocator, @@ -130,7 +132,8 @@ void* HugeMachineCodeSpace::AllocateFortForCMC(size_t objectSize, JSThread *thre ASSERT(pDesc != nullptr); MachineCodeDesc *desc = reinterpret_cast(pDesc); - size_t mutableSize = AlignUp(objectSize - desc->instructionsSize, PageSize()); + constexpr size_t REGION_HEADER_SIZE = common::Heap::GetNormalRegionHeaderSize(); + size_t mutableSize = AlignUp(objectSize + REGION_HEADER_SIZE - desc->instructionsSize, PageSize()); size_t fortSize = AlignUp(desc->instructionsSize, PageSize()); size_t allocSize = mutableSize + fortSize; uintptr_t machineCodeObj = static_cast( @@ -141,7 +144,7 @@ void* HugeMachineCodeSpace::AllocateFortForCMC(size_t objectSize, JSThread *thre return 0; } - desc->instructionsAddr = machineCodeObj + mutableSize; + desc->instructionsAddr = machineCodeObj - REGION_HEADER_SIZE + mutableSize; // Enable JitFort rights control [[maybe_unused]] void *addr = PageMapExecFortSpace((void *)desc->instructionsAddr, fortSize, diff --git a/ecmascript/serializer/base_deserializer.cpp b/ecmascript/serializer/base_deserializer.cpp index 09eb70d665a6a15a1a2f2843c8f0280ddf379949..e9f5b86e7e5a72e6073e56c59e5bdcab3b0e8f82 100644 --- a/ecmascript/serializer/base_deserializer.cpp +++ b/ecmascript/serializer/base_deserializer.cpp @@ -438,7 +438,7 @@ uintptr_t BaseDeserializer::RelocateObjectAddr(SerializedObjectSpace space, size switch (space) { case SerializedObjectSpace::REGULAR_SPACE: { if (currentRegularObjectAddr_ + objSize > - currentRegularRegionBeginAddr_ + common::SerializeUtils::GetRegionSize()) { + currentRegularRegionBeginAddr_ + common::Heap::GetNormalRegionAvailableSize()) { ASSERT(regularRegionIndex_ < regionVector_.size()); currentRegularObjectAddr_ = regionVector_[regularRegionIndex_++]; currentRegularRegionBeginAddr_ = currentRegularObjectAddr_; @@ -449,7 +449,7 @@ uintptr_t BaseDeserializer::RelocateObjectAddr(SerializedObjectSpace space, size } case SerializedObjectSpace::PIN_SPACE: { if (currentPinObjectAddr_ + objSize > - currentPinRegionBeginAddr_ + common::SerializeUtils::GetRegionSize()) { + currentPinRegionBeginAddr_ + common::Heap::GetNormalRegionAvailableSize()) { ASSERT(pinRegionIndex_ < regionVector_.size()); currentPinObjectAddr_ = regionVector_[pinRegionIndex_++]; currentPinRegionBeginAddr_ = currentPinObjectAddr_; @@ -690,7 +690,7 @@ void BaseDeserializer::AllocateToDifferentSpaces() void BaseDeserializer::AllocateToRegularSpace(size_t regularSpaceSize) { - if (regularSpaceSize <= common::SerializeUtils::GetRegionSize()) { + if (regularSpaceSize <= common::Heap::GetNormalRegionAvailableSize()) { currentRegularObjectAddr_ = common::HeapAllocator::AllocateNoGC(regularSpaceSize); } else { currentRegularObjectAddr_ = AllocateMultiCMCRegion(regularSpaceSize, regularRegionIndex_, @@ -703,7 +703,7 @@ void BaseDeserializer::AllocateToRegularSpace(size_t regularSpaceSize) } void BaseDeserializer::AllocateToPinSpace(size_t pinSpaceSize) { - if (pinSpaceSize <= common::SerializeUtils::GetRegionSize()) { + if (pinSpaceSize <= common::Heap::GetNormalRegionAvailableSize()) { currentPinObjectAddr_ = common::HeapAllocator::AllocatePinNoGC(pinSpaceSize); } else { currentPinObjectAddr_ = AllocateMultiCMCRegion(pinSpaceSize, pinRegionIndex_, RegionType::PinRegion); @@ -716,12 +716,13 @@ void BaseDeserializer::AllocateToPinSpace(size_t pinSpaceSize) uintptr_t BaseDeserializer::AllocateMultiCMCRegion(size_t spaceObjSize, size_t ®ionIndex, RegionType regionType) { - size_t regionSize = common::SerializeUtils::GetRegionSize(); - ASSERT(spaceObjSize > regionSize); + constexpr size_t REGION_SIZE = common::Heap::GetNormalRegionAvailableSize(); + ASSERT(REGION_SIZE != 0); + ASSERT(spaceObjSize > REGION_SIZE); regionIndex = regionVector_.size(); - size_t regionAlignedSize = AlignUp(spaceObjSize, regionSize); - ASSERT(regionSize != 0); - size_t regionNum = regionAlignedSize / regionSize; + size_t regionAlignedSize = SerializeData::AlignUpRegionAvailableSize(spaceObjSize); + ASSERT(regionAlignedSize % REGION_SIZE == 0); + size_t regionNum = regionAlignedSize / REGION_SIZE; uintptr_t firstRegionAddr = 0U; std::vector regionRemainSizeVector; size_t regionRemainSizeIndex = 0; @@ -748,11 +749,11 @@ uintptr_t BaseDeserializer::AllocateMultiCMCRegion(size_t spaceObjSize, size_t & // fill region not used size if (regionNum == 1) { // last region size_t lastRegionRemainSize = regionAlignedSize - spaceObjSize; - FreeObject::FillFreeObject(heap_, regionAddr + regionSize - lastRegionRemainSize, + FreeObject::FillFreeObject(heap_, regionAddr + REGION_SIZE - lastRegionRemainSize, lastRegionRemainSize); } else { - auto regionAliveObjSize = regionSize - regionRemainSizeVector[regionRemainSizeIndex++]; - FreeObject::FillFreeObject(heap_, regionAddr + regionAliveObjSize, regionSize - regionAliveObjSize); + auto regionAliveObjSize = REGION_SIZE - regionRemainSizeVector[regionRemainSizeIndex++]; + FreeObject::FillFreeObject(heap_, regionAddr + regionAliveObjSize, REGION_SIZE - regionAliveObjSize); } regionNum--; } diff --git a/ecmascript/serializer/serialize_data.h b/ecmascript/serializer/serialize_data.h index 0e7aa282489d2f25d63d4674e171b3d23fb17ec8..772812782c00e30809c68b01994d8f101d95a541 100644 --- a/ecmascript/serializer/serialize_data.h +++ b/ecmascript/serializer/serialize_data.h @@ -18,6 +18,7 @@ #include +#include "common_components/heap/heap.h" #include "common_components/serialize/serialize_utils.h" #include "ecmascript/js_tagged_value-inl.h" #include "ecmascript/mem/dyn_chunk.h" @@ -138,12 +139,12 @@ public: static size_t AlignUpRegionAvailableSize(size_t size) { if (g_isEnableCMCGC) { - size_t regionSize = common::SerializeUtils::GetRegionSize(); + constexpr size_t REGION_SIZE = common::Heap::GetNormalRegionAvailableSize(); if (size == 0) { - return regionSize; + return REGION_SIZE; } - ASSERT(regionSize != 0); - return ((size - 1) / regionSize + 1) * regionSize; // 1: align up + ASSERT(REGION_SIZE != 0); + return ((size - 1) / REGION_SIZE + 1) * REGION_SIZE; // 1: align up } else { if (size == 0) { return Region::GetRegionAvailableSize(); diff --git a/ecmascript/snapshot/mem/snapshot_processor.cpp b/ecmascript/snapshot/mem/snapshot_processor.cpp index e6e510a7c0ea404b6ac9d9450ba3b54dbd692b96..2d337f3003f804483c0c54374aff10571bf0c1ed 100644 --- a/ecmascript/snapshot/mem/snapshot_processor.cpp +++ b/ecmascript/snapshot/mem/snapshot_processor.cpp @@ -1004,7 +1004,7 @@ static uintptr_t g_nativeTable[] = { void SnapshotProcessor::Initialize() { if (g_isEnableCMCGC) { - commonRegionSize_ = common::SerializeUtils::GetRegionSize(); + commonRegionSize_ = common::Heap::GetNormalRegionAvailableSize(); regularObjAllocator_.Initialize(commonRegionSize_); pinnedObjAllocator_.Initialize(commonRegionSize_); largeObjAllocator_.Initialize(commonRegionSize_); @@ -1267,7 +1267,7 @@ void SnapshotProcessor::DeserializeSpaceObject(uintptr_t beginAddr, size_t objSi } if (spaceType != SerializedObjectSpace::LARGE_SPACE) { uintptr_t top = regionAddr + liveObjectSize; - uintptr_t end = regionAddr + common::SerializeUtils::GetRegionSize(); + uintptr_t end = regionAddr + common::Heap::GetNormalRegionAvailableSize(); FreeObject::FillFreeObject(sHeap_, top, end - top); } beginAddr += liveObjectSize; diff --git a/ecmascript/stubs/runtime_stub_list.h b/ecmascript/stubs/runtime_stub_list.h index 282953037edc822b58ebc8a68493c33a6a685a07..f00faf0b01ddabae23219f151d4a6882eece3ac7 100644 --- a/ecmascript/stubs/runtime_stub_list.h +++ b/ecmascript/stubs/runtime_stub_list.h @@ -226,7 +226,9 @@ namespace panda::ecmascript { V(GetExternalModuleVar) \ V(ReadBarrier) \ V(CopyCallTarget) \ - V(CopyArgvArray) + V(CopyArgvArray) \ + V(MarkRSetCardTable) \ + V(MarkInBuffer) // When ASM enters C++ via CallNGCRuntime, if the C++ process requires GetGlobalEnv(), // the current globalenv in ASM must be set to glue before CallNGCRuntime! diff --git a/ecmascript/stubs/runtime_stubs.cpp b/ecmascript/stubs/runtime_stubs.cpp index fb2b1d6d0e1d147664869bd8655f6bcdef139859..c9a930fba16c242751b6f74a9de9ded1f83ef619 100644 --- a/ecmascript/stubs/runtime_stubs.cpp +++ b/ecmascript/stubs/runtime_stubs.cpp @@ -48,6 +48,8 @@ #include "ecmascript/builtins/builtins_object.h" #include "ecmascript/module/module_value_accessor.h" #include "ecmascript/module/module_path_helper.h" +#include "common_components/heap/allocator/region_desc.h" +#include "common_components/mutator/mutator.h" #ifdef ARK_SUPPORT_INTL #include "ecmascript/js_collator.h" #include "ecmascript/js_locale.h" @@ -4994,6 +4996,19 @@ JSTaggedValue RuntimeStubs::GetExternalModuleVar(uintptr_t argGlue, JSFunction * return ModuleManager::GetExternalModuleVarFastPathForJIT(thread, index, jsFunc); } +bool RuntimeStubs::MarkRSetCardTable(BaseObject* obj) +{ + common::RegionDesc* region = common::RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); + return region->MarkRSetCardTable(obj); +} + +void RuntimeStubs::MarkInBuffer(BaseObject* ref) +{ + ref = reinterpret_cast(reinterpret_cast(ref) & ~(common::Barrier::TAG_WEAK)); + common::Mutator* mutator = common::Mutator::GetMutator(); + mutator->RememberObjectInSatbBuffer(ref); +} + void RuntimeStubs::Initialize(JSThread *thread) { #define DEF_RUNTIME_STUB(name) kungfu::RuntimeStubCSigns::ID_##name diff --git a/ecmascript/stubs/runtime_stubs.h b/ecmascript/stubs/runtime_stubs.h index 16f58c744d2c077a569b75bb0279e4277f322f79..36fd433439b36e2ae47c558c65f8d4d4ff1bbc39 100644 --- a/ecmascript/stubs/runtime_stubs.h +++ b/ecmascript/stubs/runtime_stubs.h @@ -16,6 +16,7 @@ #ifndef ECMASCRIPT_RUNTIME_STUBS_H #define ECMASCRIPT_RUNTIME_STUBS_H +#include "common_components/heap/allocator/region_desc.h" #include "ecmascript/frames.h" #include "ecmascript/ecma_macros.h" #include "ecmascript/js_tagged_value.h" @@ -177,6 +178,8 @@ public: static void LoadNativeModuleFailed(JSTaggedValue curModule); static void TraceLazyDeoptCommitSuccess(uintptr_t argGlue, JSHandle func); static JSTaggedValue GetExternalModuleVar(uintptr_t argGlue, JSFunction *jsFunc, int32_t index); + static bool MarkRSetCardTable(BaseObject* obj); + static void MarkInBuffer(BaseObject* ref); private: static void DumpToStreamWithHint(std::ostream &out, std::string_view prompt, JSTaggedValue value);