diff --git a/common_components/heap/allocator/region_desc.h b/common_components/heap/allocator/region_desc.h index 8fd719125431d84e18cae6514e6aeba73583be49..8cd4bfd00a619d177d3327d834a9e30ea42c6444 100755 --- a/common_components/heap/allocator/region_desc.h +++ b/common_components/heap/allocator/region_desc.h @@ -105,8 +105,6 @@ public: RegionDesc() { metadata.allocPtr = reinterpret_cast(nullptr); - metadata.markingLine = std::numeric_limits::max(); - metadata.forwardLine = std::numeric_limits::max(); metadata.freeSlot = nullptr; metadata.regionBase = reinterpret_cast(nullptr); metadata.regionStart = reinterpret_cast(nullptr); @@ -206,40 +204,6 @@ public: return nullptr; } - RegionBitmap* GetEnqueueBitmap() - { - RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); - if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { - return nullptr; - } - return bitmap; - } - - RegionBitmap* GetOrAllocEnqueueBitmap() - { - do { - RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); - if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { - continue; - } - if (LIKELY_CC(bitmap != nullptr)) { - return bitmap; - } - RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); - if (__atomic_compare_exchange_n(&metadata.liveInfo_.enqueueBitmap_, &bitmap, newValue, false, - std::memory_order_seq_cst, std::memory_order_relaxed)) { - RegionBitmap *allocated = - HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); - __atomic_store_n(&metadata.liveInfo_.enqueueBitmap_, allocated, std::memory_order_release); - DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc enqueuebitmap %p", - this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, metadata.liveInfo_.enqueueBitmap_); - return allocated; - } - } while (true); - - return nullptr; - } - void ResetMarkBit() { SetMarkedRegionFlag(0); @@ -281,21 +245,6 @@ public: return marked; } - bool EnqueueObject(const BaseObject* obj) - { - if (IsLargeRegion()) { - if (metadata.regionBits.AtomicGetValue(RegionBitOffset::BIT_OFFSET_ENQUEUED_REGION, 1) != 1) { - SetEnqueuedRegionFlag(1); - return false; - } - return true; - } - size_t offset = GetAddressOffset(reinterpret_cast(obj)); - bool marked = GetOrAllocEnqueueBitmap()->MarkBits(offset); - CHECK_CC(IsEnqueuedObject(obj)); - return marked; - } - bool IsResurrectedObject(const BaseObject* obj) { if (IsLargeRegion()) { @@ -322,19 +271,6 @@ public: return markBitmap->IsMarked(offset); } - bool IsEnqueuedObject(const BaseObject* obj) - { - if (IsLargeRegion()) { - return (metadata.regionBits.AtomicGetValue(RegionBitOffset::BIT_OFFSET_ENQUEUED_REGION, 1) == 1); - } - RegionBitmap* enqueBitmap = GetEnqueueBitmap(); - if (enqueBitmap == nullptr) { - return false; - } - size_t offset = GetAddressOffset(reinterpret_cast(obj)); - return enqueBitmap->IsMarked(offset); - } - RegionRSet* GetRSet() { return metadata.regionRSet; @@ -691,33 +627,28 @@ public: HeapAddress GetRegionAllocPtr() const { return metadata.allocPtr; } - HeapAddress GetMarkingLine() const { return metadata.markingLine; } - HeapAddress GetCopyLine() const { return metadata.forwardLine; } + HeapAddress GetMarkingLine() const + { + return InlinedRegionMetaData::GetInlinedRegionMetaData(this)->GetMarkingLine(); + } + HeapAddress GetCopyLine() const + { + return InlinedRegionMetaData::GetInlinedRegionMetaData(this)->GetCopyLine(); + } void SetMarkingLine() { - if (metadata.markingLine == std::numeric_limits::max()) { - uintptr_t line = GetRegionAllocPtr(); - metadata.markingLine = line; - DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu marking-line %#zx type %u", - this, GetRegionBase(), GetRegionStart(), GetRegionSize(), GetMarkingLine(), GetRegionType()); - } + InlinedRegionMetaData::GetInlinedRegionMetaData(this)->SetMarkingLine(); } void SetCopyLine() { - if (metadata.forwardLine == std::numeric_limits::max()) { - uintptr_t line = GetRegionAllocPtr(); - metadata.forwardLine = line; - DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu copy-line %#zx type %u", - this, GetRegionBase(), GetRegionStart(), GetRegionSize(), GetCopyLine(), GetRegionType()); - } + InlinedRegionMetaData::GetInlinedRegionMetaData(this)->SetCopyLine(); } void ClearMarkingCopyLine() { - metadata.markingLine = std::numeric_limits::max(); - metadata.forwardLine = std::numeric_limits::max(); + InlinedRegionMetaData::GetInlinedRegionMetaData(this)->ClearMarkingCopyLine(); } void ClearFreeSlot() @@ -996,17 +927,10 @@ public: return offsetof(UnitMetadata, regionEnd); } -private: - void VisitAllObjectsBefore(const std::function&& func, uintptr_t end); - - static constexpr int32_t MAX_RAW_POINTER_COUNT = std::numeric_limits::max(); - static constexpr int32_t BITS_4 = 4; - static constexpr int32_t BITS_5 = 5; - enum RegionBitOffset : uint8_t { BIT_OFFSET_REGION_TYPE = 0, // use mark-bitmap pointer instead - BIT_OFFSET_MARKED_REGION = BITS_5, + BIT_OFFSET_MARKED_REGION = 5, BIT_OFFSET_ENQUEUED_REGION = 6, BIT_OFFSET_RESURRECTED_REGION = 7, BIT_OFFSET_FIXED_REGION = 8, @@ -1014,6 +938,13 @@ private: BIT_OFFSET_IS_JITFORT_AWAIT_INSTALL = 16, }; +private: + void VisitAllObjectsBefore(const std::function&& func, uintptr_t end); + + static constexpr int32_t MAX_RAW_POINTER_COUNT = std::numeric_limits::max(); + static constexpr int32_t BITS_4 = 4; + static constexpr int32_t BITS_5 = 5; + struct ObjectSlot { HeapAddress next_ : 48; HeapAddress isFree_ : 1; @@ -1051,13 +982,11 @@ private: { markBitmap_ = nullptr; resurrectBitmap_ = nullptr; - enqueueBitmap_ = nullptr; } private: RegionDesc *relatedRegion_ {nullptr}; RegionBitmap *markBitmap_ {nullptr}; RegionBitmap *resurrectBitmap_ {nullptr}; - RegionBitmap *enqueueBitmap_ {nullptr}; friend class RegionDesc; }; @@ -1068,9 +997,6 @@ private: uintptr_t allocPtr; uintptr_t regionEnd; - // watermark set when gc phase transitions to pre-marking. - uintptr_t markingLine; - uintptr_t forwardLine; ObjectSlot* freeSlot; // `regionStart` is the header of the data, and `regionBase` is the header of the total region /** @@ -1092,7 +1018,7 @@ private: RegionLiveDesc liveInfo_ {}; - RegionRSet* regionRSet = nullptr;; + RegionRSet* regionRSet = nullptr; // the writing operation in C++ Bit-Field feature is not atomic, the we wants to // change the value, we must use specific interface implenmented by BitFields. @@ -1216,7 +1142,7 @@ public: // this could ONLY used in region that is ALIVE. class InlinedRegionMetaData { public: - static InlinedRegionMetaData *GetInlinedRegionMetaData(RegionDesc *region) + static InlinedRegionMetaData *GetInlinedRegionMetaData(const RegionDesc *region) { InlinedRegionMetaData *data = GetInlinedRegionMetaData(region->GetRegionStart()); DCHECK_CC(data->regionDesc_ == region); @@ -1228,7 +1154,7 @@ public: } explicit InlinedRegionMetaData(RegionDesc *regionDesc) - : regionDesc_(regionDesc), regionRSet_(regionDesc->GetRSet()), regionType_(regionDesc->GetRegionType()) + : regionDesc_(regionDesc), regionRSet_(regionDesc->GetRSet()), regionMarkMap_(regionDesc->GetMarkBitmap()), regionType_(regionDesc->GetRegionType()) { // Since this is a backup copy of `RegionDesc`, create rset at first to guarantee data consistency DCHECK_CC(regionRSet_ != nullptr); @@ -1260,7 +1186,7 @@ public: DCHECK_CC(RegionDesc::IsAliveRegionType(regionType_)); return regionType_; } - + bool IsInRecentSpace() const { RegionType type = GetRegionType(); @@ -1323,10 +1249,44 @@ public: size_t offset = GetAddressOffset(static_cast(reinterpret_cast(obj))); return GetRegionRSet()->MarkCardTable(offset); } + + HeapAddress GetMarkingLine() const { return markingLine_; } + HeapAddress GetCopyLine() const { return copyLine_; } + + void SetMarkingLine() + { + if (markingLine_ == std::numeric_limits::max()) { + uintptr_t line = regionDesc_->GetRegionAllocPtr(); + markingLine_ = line; + DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu marking-line %#zx type %u", + this, GetRegionBase(), GetRegionStart(), regionDesc_->GetRegionSize(), GetMarkingLine(), GetRegionType()); + } + } + + void SetCopyLine() + { + if (copyLine_ == std::numeric_limits::max()) { + uintptr_t line = regionDesc_->GetRegionAllocPtr(); + copyLine_ = line; + DLOG(REGION, "set region %p(base=%#zx)@%#zx+%zu copy-line %#zx type %u", + this, GetRegionBase(), GetRegionStart(), regionDesc_->GetRegionSize(), GetCopyLine(), GetRegionType()); + } + } + + void ClearMarkingCopyLine() + { + markingLine_ = std::numeric_limits::max(); + copyLine_ = std::numeric_limits::max(); + } + private: RegionDesc *regionDesc_ {nullptr}; RegionRSet *regionRSet_ {nullptr}; + RegionBitmap *regionMarkMap_ {nullptr}; RegionType regionType_ {}; + // watermark set when gc phase transitions to pre-marking. + uintptr_t markingLine_ { std::numeric_limits::max() }; + uintptr_t copyLine_ { std::numeric_limits::max() }; // fixme: inline more friend class RegionDesc; @@ -1356,7 +1316,6 @@ private: metadata.freeSlot = nullptr; SetRegionType(RegionType::FREE_REGION); SetUnitRole(uClass); - ClearMarkingCopyLine(); SetMarkedRegionFlag(0); SetEnqueuedRegionFlag(0); SetResurrectedRegionFlag(0); @@ -1394,6 +1353,9 @@ private: void InitMetaData(size_t nUnit, UnitRole uClass) { metadata.liveInfo_.Init(this); + GetOrAllocMarkBitmap(); + DCHECK_CC(GetMarkBitmap() != nullptr); + HeapAddress header = GetRegionBase(); void *ptr = reinterpret_cast(static_cast(header)); new (ptr) InlinedRegionMetaData(this); @@ -1410,8 +1372,14 @@ private: UnitMetadata metadata; public: friend constexpr size_t GetMetaDataInRegionOffset(); + static constexpr size_t REGION_DESC_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionDesc_); static constexpr size_t REGION_RSET_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionRSet_); static constexpr size_t REGION_TYPE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionType_); + static constexpr size_t MARK_LINE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, markingLine_); + static constexpr size_t COPY_LINE_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, copyLine_); + static constexpr size_t REGION_MARKMAP_IN_INLINED_METADATA_OFFSET = MEMBER_OFFSET(InlinedRegionMetaData, regionMarkMap_); + static constexpr size_t UNIT_BITS_IN_METADATA_OFFSET = MEMBER_OFFSET(UnitMetadata, unitBits); + static constexpr size_t REGION_BITS_IN_METADATA_OFFSET = MEMBER_OFFSET(UnitMetadata, regionBits); }; HeapAddress RegionDesc::InlinedRegionMetaData::GetRegionStart() const diff --git a/common_components/heap/allocator/regional_heap.h b/common_components/heap/allocator/regional_heap.h index 2bc0cc98d5c982ac558a5541bbd8e73f045aa1d8..60e41ee89781dd1f182ee4b54bfbb09b287d4aed 100755 --- a/common_components/heap/allocator/regional_heap.h +++ b/common_components/heap/allocator/regional_heap.h @@ -375,12 +375,6 @@ public: return regionInfo->ResurrentObject(obj); } - static bool EnqueueObject(const BaseObject* obj) - { - RegionDesc* regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); - return regionInfo->EnqueueObject(obj); - } - static bool IsMarkedObject(const BaseObject* obj) { RegionDesc* regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); @@ -393,12 +387,6 @@ public: return regionInfo->IsResurrectedObject(obj); } - static bool IsEnqueuedObject(const BaseObject* obj) - { - RegionDesc* regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); - return regionInfo->IsEnqueuedObject(obj); - } - static bool IsNewObjectSinceMarking(const BaseObject* object) { RegionDesc* region = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(object)); diff --git a/common_components/heap/ark_collector/ark_collector.cpp b/common_components/heap/ark_collector/ark_collector.cpp index 49985689be0ee104a6d9917c3539846510ad0b9a..1abfee918e3ac5ab49790dd532e606a4327ebfa3 100755 --- a/common_components/heap/ark_collector/ark_collector.cpp +++ b/common_components/heap/ark_collector/ark_collector.cpp @@ -322,8 +322,7 @@ public: } else { if (Heap::GetHeap().GetGCReason() != GC_REASON_YOUNG) { MarkObject(oldObj); - } else if (RegionalHeap::IsYoungSpaceObject(oldObj) && !RegionalHeap::IsNewObjectSinceMarking(oldObj) && - !RegionalHeap::IsMarkedObject(oldObj)) { + } else if (RegionalHeap::IsYoungSpaceObject(oldObj)) { // RSet don't protect exempted objects, we need to mark it MarkObject(oldObj); } @@ -341,9 +340,8 @@ private: void MarkToObject(BaseObject *oldVersion, BaseObject *toVersion) { // We've checked oldVersion is in fromSpace, no need to check markingLine - if (!collector_->MarkObject(oldVersion)) { - // No need to count oldVersion object size, as it has been copied. - collector_->MarkObject(toVersion); + collector_->MarkObject(oldVersion); + if (!collector_->MarkObject(toVersion)) { // oldVersion don't have valid type info, cannot push it collectStack_.Push(toVersion); } diff --git a/common_components/heap/collector/marking_collector.cpp b/common_components/heap/collector/marking_collector.cpp index 241f35a1681b566d433ae6ff8b52c66d429aa72d..7761798c66c5c52488b641b10d810a533bdd19d4 100755 --- a/common_components/heap/collector/marking_collector.cpp +++ b/common_components/heap/collector/marking_collector.cpp @@ -160,17 +160,13 @@ void MarkingCollector::ProcessMarkStack([[maybe_unused]] uint32_t threadIndex, P std::vector remarkStack; auto fetchFromSatbBuffer = [this, &markStack, &remarkStack]() { SatbBuffer::Instance().TryFetchOneRetiredNode(remarkStack); - bool needProcess = false; while (!remarkStack.empty()) { BaseObject *obj = remarkStack.back(); remarkStack.pop_back(); - if (Heap::IsHeapAddress(obj) && (!MarkObject(obj))) { - markStack.Push(obj); - needProcess = true; - DLOG(TRACE, "tracing take from satb buffer: obj %p", obj); - } + markStack.Push(obj); + DLOG(TRACE, "tracing take from satb buffer: obj %p", obj); } - return needProcess; + return true; }; size_t iterationCnt = 0; constexpr size_t maxIterationLoopNum = 1000; @@ -446,7 +442,12 @@ bool MarkingCollector::MarkSatbBuffer(GlobalMarkStack &globalMarkStack) while (!remarkStack.empty()) { // LCOV_EXCL_BR_LINE BaseObject* obj = remarkStack.back(); remarkStack.pop_back(); - if (Heap::IsHeapAddress(obj) && !this->MarkObject(obj)) { + + BaseObject* toObj = obj->GetForwardingPointer(); + if (toObj) { + collectStack.Push(toObj); + DLOG(TRACE, "satb buffer add obj %p", toObj); + } else { collectStack.Push(obj); DLOG(TRACE, "satb buffer add obj %p", obj); } diff --git a/common_components/heap/collector/region_bitmap.h b/common_components/heap/collector/region_bitmap.h index 0aeac4f6f4ed0a7e8a72ae17ac356b9ad4b2931b..82681ea025a124bfa8514e5aeba5ac500ef1989f 100755 --- a/common_components/heap/collector/region_bitmap.h +++ b/common_components/heap/collector/region_bitmap.h @@ -29,6 +29,7 @@ static constexpr size_t kBitsPerByte = 8; static constexpr size_t kMarkedBytesPerBit = 8; static constexpr size_t kBitsPerWord = sizeof(uint64_t) * kBitsPerByte; static constexpr size_t kBytesPerWord = sizeof(uint64_t) / sizeof(uint8_t); + struct RegionBitmap { static constexpr uint8_t factor = 16; std::atomic partLiveBytes[factor]; @@ -72,6 +73,8 @@ struct RegionBitmap { return ret; } }; +static constexpr size_t MARK_WORD_OFFSET_IN_REGION_BITMAP = offsetof(RegionBitmap, markWords); + } // namespace common #endif // COMMON_COMPONENTS_HEAP_COLLECTOR_REGION_BITMAP_H diff --git a/common_components/heap/heap.h b/common_components/heap/heap.h index e3b548e9df4723dc834a4519f3d9e051c7fed311..dc33d5c26699a9f08a31a012c2b35a056cb2ea1b 100755 --- a/common_components/heap/heap.h +++ b/common_components/heap/heap.h @@ -43,7 +43,7 @@ class Heap { public: // These need to keep same with that in `RegionDesc` static constexpr size_t NORMAL_UNIT_SIZE = 256 * 1024; - static constexpr size_t NORMAL_UNIT_HEADER_SIZE = AlignUp(2 * sizeof(void *) + sizeof(uint8_t), 8); + static constexpr size_t NORMAL_UNIT_HEADER_SIZE = AlignUp(3 * sizeof(void *) + sizeof(uint8_t) + 2 * sizeof(uintptr_t), 8); static constexpr size_t NORMAL_UNIT_AVAILABLE_SIZE = NORMAL_UNIT_SIZE - NORMAL_UNIT_HEADER_SIZE; static constexpr size_t GetNormalRegionSize() diff --git a/common_components/mutator/mutator.h b/common_components/mutator/mutator.h index b9ce1e978095e40b95ded034370dffec967346f2..c92e9fa904e248ed2efd728c6d233aa4c9d9c0f9 100755 --- a/common_components/mutator/mutator.h +++ b/common_components/mutator/mutator.h @@ -259,7 +259,16 @@ public: } #endif - NO_INLINE_CC void RememberObjectInSatbBuffer(const BaseObject* obj) { RememberObjectImpl(obj); } + NO_INLINE_CC void RememberObjectInSatbBuffer(const BaseObject* obj) + { + if (SatbBuffer::Instance().ShouldEnqueue(obj)) { + RememberObjectImpl(obj); + } + } + NO_INLINE_CC void RememberObjectInSatbBufferNoCheck(const BaseObject* obj) + { + RememberObjectImpl(obj); + } const SatbBuffer::TreapNode* GetSatbBufferNode() const { return satbNode_; } @@ -313,12 +322,8 @@ protected: private: void RememberObjectImpl(const BaseObject* obj) { - if (LIKELY_CC(Heap::IsHeapAddress(obj))) { - if (SatbBuffer::Instance().ShouldEnqueue(obj)) { - SatbBuffer::Instance().EnsureGoodNode(satbNode_); - satbNode_->Push(obj); - } - } + SatbBuffer::Instance().EnsureGoodNode(satbNode_); + satbNode_->Push(obj); } // Indicate the current mutator phase and use which barrier in concurrent gc // ATTENTION: THE LAYOUT FOR GCPHASE MUST NOT BE CHANGED! diff --git a/common_components/mutator/satb_buffer.cpp b/common_components/mutator/satb_buffer.cpp index 68c9d42ce45d0617764ee0988f1ecf28889d1633..1c910a028d7778ff91c381a03abd22781e515701 100755 --- a/common_components/mutator/satb_buffer.cpp +++ b/common_components/mutator/satb_buffer.cpp @@ -25,18 +25,23 @@ SatbBuffer& SatbBuffer::Instance() noexcept { return *g_instance; } bool SatbBuffer::ShouldEnqueue(const BaseObject* obj) { - if (UNLIKELY_CC(obj == nullptr)) { + if (UNLIKELY_CC(obj == nullptr || !Heap::IsHeapAddress(obj))) { return false; } + if (Heap::GetHeap().GetGCReason() == GC_REASON_YOUNG && !RegionalHeap::IsYoungSpaceObject(obj)) { return false; } + if (RegionalHeap::IsNewObjectSinceMarking(obj)) { return false; } - if (RegionalHeap::IsMarkedObject(obj)) { + + // already marked, no need remember + if (RegionalHeap::MarkObject(obj)) { return false; } - return !RegionalHeap::EnqueueObject(obj); + + return true; } } // namespace common diff --git a/ecmascript/compiler/barrier_stub_builder.cpp b/ecmascript/compiler/barrier_stub_builder.cpp index 3702fbf4c735e269e3dd9f481439c4fbb3fcee07..fb1b1ce5e0fe639fea6628a006f82148cd7006b6 100644 --- a/ecmascript/compiler/barrier_stub_builder.cpp +++ b/ecmascript/compiler/barrier_stub_builder.cpp @@ -810,11 +810,13 @@ void BarrierStubBuilder::DoReverseBarrier() GateRef ref = LoadPrimitive(VariableType::JS_ANY(), dstAddr_, offset); BRANCH(TaggedIsHeapObject(ref), &RefisTaggedObject, ¬MarkRSetLoopEnd); Bind(&RefisTaggedObject); - BRANCH_UNLIKELY(shouldProcessSATB, &markInBuffer, &exit); + + GateRef shouldEnqueue = ShouldEnqueueSATB(glue_, ref); + BRANCH_UNLIKELY(BitAnd(shouldProcessSATB, shouldEnqueue), &markInBuffer, &exit); Bind(&markInBuffer); { ASSERT(RuntimeStubCSigns::Get(RTSTUB_ID(MarkInBuffer))->IsNoTailCall()); - CallNGCRuntime(glue_, RTSTUB_ID(MarkInBuffer), {ref}); + CallNGCRuntime(glue_, RTSTUB_ID(MarkInBufferNoCheck), {ref}); Jump(¬MarkRSetLoopEnd); } diff --git a/ecmascript/compiler/call_signature.cpp b/ecmascript/compiler/call_signature.cpp index faff89b72dbcc7b2d74567d243a7a60e56b143d9..de2e28e4d157e44903db3b177b2f96b2ece082c8 100644 --- a/ecmascript/compiler/call_signature.cpp +++ b/ecmascript/compiler/call_signature.cpp @@ -3516,6 +3516,21 @@ DEF_CALL_SIGNATURE(MarkInBuffer) callSign->SetNoTailCall(true); } +DEF_CALL_SIGNATURE(MarkInBufferNoCheck) +{ + // 3 : 3 input parameters + CallSignature MarkInBufferNoCheck("MarkInBufferNoCheck", 0, 1, ArgumentsOrder::DEFAULT_ORDER, + VariableType::BOOL()); + *callSign = MarkInBufferNoCheck; + std::array params = { // 1 : 1 input parameters + VariableType::JS_POINTER(), + }; + callSign->SetParameters(params.data()); + callSign->SetGCLeafFunction(true); + callSign->SetTargetKind(CallSignature::TargetKind::RUNTIME_STUB_NO_GC); + callSign->SetNoTailCall(true); +} + DEF_CALL_SIGNATURE(BatchMarkInBuffer) { // 3 : 3 input parameters diff --git a/ecmascript/compiler/call_signature.h b/ecmascript/compiler/call_signature.h index a39c47be8a39ed7f25f24d0fcc1e284311acfb20..79d4242968b4818c57352e1d4e369a00ab811be8 100644 --- a/ecmascript/compiler/call_signature.h +++ b/ecmascript/compiler/call_signature.h @@ -718,6 +718,7 @@ private: V(CopyArgvArray) \ V(MarkRSetCardTable) \ V(MarkInBuffer) \ + V(MarkInBufferNoCheck) \ V(BatchMarkInBuffer) \ V(CMCSetValueWithBarrier) \ V(UpdateSharedModule) diff --git a/ecmascript/compiler/circuit_builder.h b/ecmascript/compiler/circuit_builder.h index 8a20ae7053ecf9bde2dd5e05ac6d0b5947a6d41c..a93e6946570ed09800311c75f3ea47461e2206a2 100644 --- a/ecmascript/compiler/circuit_builder.h +++ b/ecmascript/compiler/circuit_builder.h @@ -995,7 +995,7 @@ public: void Store(VariableType type, GateRef glue, GateRef base, GateRef offset, GateRef value, MemoryAttribute mAttr = MemoryAttribute::Default()); - GateRef FetchOr(GateRef ptr, GateRef value, MemoryAttribute mAttr); + GateRef FetchOr(VariableType type, GateRef ptr, GateRef value, MemoryAttribute mAttr); void StoreHClass(VariableType type, GateRef glue, GateRef base, GateRef offset, GateRef value, GateRef compValue, MemoryAttribute mAttr = MemoryAttribute::Default()); void StoreWithoutBarrier(VariableType type, GateRef addr, GateRef value, diff --git a/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp b/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp index 0d9475fbbbc78008a3c28917db98ffebc259385d..b7d256fcb432f53f1af430ae29fc0a928fdcf802 100644 --- a/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp +++ b/ecmascript/compiler/codegen/llvm/llvm_ir_builder.cpp @@ -1699,6 +1699,14 @@ void LLVMIRBuilder::VisitLoad(GateRef gate, GateRef base) LLVMValueRef result = LLVMBuildLoad(builder_, baseAddr, ""); auto order = acc_.GetMemoryAttribute(gate); switch (order.GetOrder()) { + case MemoryAttribute::ATOMIC: { + LLVMSetOrdering(result, LLVMAtomicOrderingMonotonic); + break; + } + case MemoryAttribute::MEMORY_ORDER_ACQUIRE: { + LLVMSetOrdering(result, LLVMAtomicOrderingAcquire); + break; + } case MemoryAttribute::MEMORY_ORDER_RELEASE: { LLVMSetOrdering(result, LLVMAtomicOrderingRelease); break; @@ -1730,6 +1738,14 @@ void LLVMIRBuilder::VisitStore(GateRef gate, GateRef base, GateRef value) LLVMValueRef result = LLVMBuildStore(builder_, data, baseAddr); auto order = acc_.GetMemoryAttribute(gate); switch (order.GetOrder()) { + case MemoryAttribute::ATOMIC: { + LLVMSetOrdering(result, LLVMAtomicOrderingMonotonic); + break; + } + case MemoryAttribute::MEMORY_ORDER_ACQUIRE: { + LLVMSetOrdering(result, LLVMAtomicOrderingAcquire); + break; + } case MemoryAttribute::MEMORY_ORDER_RELEASE: { LLVMSetOrdering(result, LLVMAtomicOrderingRelease); break; @@ -2580,6 +2596,7 @@ void LLVMIRBuilder::VisitFetchOr(GateRef gate, GateRef e1, GateRef e2) auto order = acc_.GetMemoryAttribute(gate); LLVMAtomicOrdering atomic_order = LLVMAtomicOrderingSequentiallyConsistent; switch (order.GetOrder()) { + case MemoryAttribute::ATOMIC: case MemoryAttribute::NOT_ATOMIC: { atomic_order = LLVMAtomicOrderingMonotonic; break; diff --git a/ecmascript/compiler/lcr_circuit_builder.cpp b/ecmascript/compiler/lcr_circuit_builder.cpp index 316369a1c2b13022be3450fd5afdbaa9d6cac1e0..4205412cf192c59758e5a2e6b167928b4a2c8ca0 100644 --- a/ecmascript/compiler/lcr_circuit_builder.cpp +++ b/ecmascript/compiler/lcr_circuit_builder.cpp @@ -118,13 +118,13 @@ void CircuitBuilder::Store(VariableType type, GateRef glue, GateRef base, GateRe label->SetDepend(result); } -GateRef CircuitBuilder::FetchOr(GateRef ptr, GateRef value, MemoryAttribute mAttr) +GateRef CircuitBuilder::FetchOr(VariableType type, GateRef ptr, GateRef value, MemoryAttribute mAttr) { auto label = GetCurrentLabel(); auto depend = label->GetDepend(); auto bits = LoadStoreAccessor::ToValue(mAttr); GateRef result = GetCircuit()->NewGate(circuit_->FetchOr(bits), - MachineType::I64, { depend, ptr, value }, acc_.GetGateType(value)); + type.GetMachineType(), { depend, ptr, value }, acc_.GetGateType(value)); label->SetDepend(result); return result; } diff --git a/ecmascript/compiler/mcr_gate_meta_data.h b/ecmascript/compiler/mcr_gate_meta_data.h index f9a8dd3e24b1d2b0e23e5a1edcb6182bc34f339c..40bc383c37f8b4f5fe00cbe53f1b75fa72130c25 100644 --- a/ecmascript/compiler/mcr_gate_meta_data.h +++ b/ecmascript/compiler/mcr_gate_meta_data.h @@ -431,7 +431,9 @@ public: enum Order { NOT_ATOMIC = 0, - MEMORY_ORDER_RELEASE + ATOMIC, + MEMORY_ORDER_RELEASE, + MEMORY_ORDER_ACQUIRE }; enum Barrier { @@ -451,6 +453,21 @@ public: return Create(NOT_ATOMIC); } + static MemoryAttribute Atomic() + { + return Create(ATOMIC); + } + + static MemoryAttribute Acquire() + { + return Create(MEMORY_ORDER_ACQUIRE); + } + + static MemoryAttribute Release() + { + return Create(MEMORY_ORDER_RELEASE); + } + static MemoryAttribute NeedBarrier() { return Create(NOT_ATOMIC, NEED_BARRIER); diff --git a/ecmascript/compiler/stub_builder-inl.h b/ecmascript/compiler/stub_builder-inl.h index 3881f4606e350e6bd024656e2f1fa859c5f7514c..68023721e6eff3d7c03fc20b1fbfe20e40c5028a 100644 --- a/ecmascript/compiler/stub_builder-inl.h +++ b/ecmascript/compiler/stub_builder-inl.h @@ -437,12 +437,12 @@ inline GateRef StubBuilder::LoadZeroOffset(VariableType type, GateRef glue, Gate return Load(type, glue, base, IntPtr(0), mAttr); } -inline GateRef StubBuilder::LoadPrimitive(VariableType type, GateRef base, GateRef offset) +inline GateRef StubBuilder::LoadPrimitive(VariableType type, GateRef base, GateRef offset, MemoryAttribute mAttr) { if (type == VariableType::NATIVE_POINTER()) { type = env_->IsArch64Bit() ? VariableType::INT64() : VariableType::INT32(); } - return env_->GetBuilder()->LoadWithoutBarrier(type, base, offset); + return env_->GetBuilder()->LoadWithoutBarrier(type, base, offset, mAttr); } inline GateRef StubBuilder::LoadZeroOffsetPrimitive(VariableType type, GateRef base) @@ -638,9 +638,19 @@ inline GateRef StubBuilder::Int64Or(GateRef x, GateRef y) return env_->GetBuilder()->Int64Or(x, y); } +inline GateRef StubBuilder::Int8FetchOr(GateRef x, GateRef y, MemoryAttribute mAttr) +{ + return env_->GetBuilder()->FetchOr(VariableType::INT8(), x, y, mAttr); +} + +inline GateRef StubBuilder::Int32FetchOr(GateRef x, GateRef y, MemoryAttribute mAttr) +{ + return env_->GetBuilder()->FetchOr(VariableType::INT32(), x, y, mAttr); +} + inline GateRef StubBuilder::Int64FetchOr(GateRef x, GateRef y, MemoryAttribute mAttr) { - return env_->GetBuilder()->FetchOr(x, y, mAttr); + return env_->GetBuilder()->FetchOr(VariableType::INT64(), x, y, mAttr); } inline GateRef StubBuilder::IntPtrOr(GateRef x, GateRef y) diff --git a/ecmascript/compiler/stub_builder.cpp b/ecmascript/compiler/stub_builder.cpp index d2559c6c0eb5640c6d9d4fc8c6a53c091dcdbdfd..a53f20b1de10b7072a001fce7049e91f8b83bebd 100644 --- a/ecmascript/compiler/stub_builder.cpp +++ b/ecmascript/compiler/stub_builder.cpp @@ -13,15 +13,21 @@ * limitations under the License. */ +#include "base_runtime.h" +#include "common_components/heap/collector/region_bitmap.h" #include "ecmascript/compiler/call_stub_builder.h" #include "ecmascript/compiler/access_object_stub_builder.h" #include "ecmascript/compiler/builtins/builtins_array_stub_builder.h" #include "ecmascript/compiler/builtins/builtins_proxy_stub_builder.h" #include "ecmascript/compiler/builtins/builtins_typedarray_stub_builder.h" #include "ecmascript/compiler/builtins/builtins_collection_stub_builder.h" +#include "ecmascript/compiler/circuit_builder.h" +#include "ecmascript/compiler/gate.h" +#include "ecmascript/compiler/mcr_gate_meta_data.h" #include "ecmascript/compiler/new_object_stub_builder.h" #include "ecmascript/compiler/object_operator_stub_builder.h" #include "ecmascript/compiler/profiler_stub_builder.h" +#include "ecmascript/compiler/variable_type.h" #include "ecmascript/elements.h" #include "ecmascript/compiler/stub_builder.h" #include "ecmascript/ic/mega_ic_cache.h" @@ -2008,6 +2014,167 @@ GateRef StubBuilder::ShouldProcessSATB(GateRef gcPhase) return shouldProcessSATB; } +// should be logically equal to RegionalHeap::IsNewObjectSinceMarking +GateRef StubBuilder::IsNewObjectSinceMarking(GateRef obj) +{ + GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~static_cast(common::RegionDesc::DEFAULT_REGION_UNIT_MASK))); + GateRef markLine = LoadPrimitive(VariableType::INT64(), metaDataAddr, + IntPtr(common::RegionDesc::MARK_LINE_IN_INLINED_METADATA_OFFSET)); + GateRef IsNewObjectSinceMarking = IntPtrLessThanOrEqual(ChangeInt64ToIntPtr(markLine), TaggedCastToIntPtr(obj)); + return IsNewObjectSinceMarking; +} + +// should be logically equal to RegionaDesc::MarkObject +GateRef StubBuilder::MarkObject(GateRef obj) +{ + auto env = GetEnvironment(); + Label entry(env); + Label exit(env); + Label isLargeRegion(env); + Label notLargeRegion(env); + Label bitmapMarked(env); + Label bitmapNotMarked(env); + Label largeRegionMarked(env); + Label largeRegionNotMarked(env); + env->SubCfgEntry(&entry); + DEFVARIABLE(isMarked, VariableType::BOOL(), False()); + + GateRef metaDataAddr = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~static_cast(common::RegionDesc::DEFAULT_REGION_UNIT_MASK))); + GateRef regionDescAddr = LoadPrimitive(VariableType::NATIVE_POINTER(), metaDataAddr, + IntPtr(common::RegionDesc::REGION_DESC_IN_INLINED_METADATA_OFFSET)); + GateRef unitBits = LoadPrimitive(VariableType::INT8(), regionDescAddr, + IntPtr(common::RegionDesc::UNIT_BITS_IN_METADATA_OFFSET)); + GateRef unitRole = Int8And(unitBits, Int8((1 << 5) - 1)); + + BRANCH(Int8Equal(unitBits, Int8(static_cast(common::RegionDesc::UnitRole::LARGE_SIZED_UNITS))), &isLargeRegion, ¬LargeRegion); + Bind(&isLargeRegion); + { + // Note: should be logically equal to RegionDesc::MarkObjectForLargeRegion + GateRef regionBits = LoadPrimitive(VariableType::INT32(), regionDescAddr, IntPtr(common::RegionDesc::REGION_BITS_IN_METADATA_OFFSET), MemoryAttribute::Atomic()); + GateRef mask = Int32LSL(Int32(1), Int32(common::RegionDesc::RegionBitOffset::BIT_OFFSET_MARKED_REGION)); + GateRef checkMarked = Int32NotEqual(Int32And(regionBits, mask), Int32(0)); + + Branch(checkMarked, &largeRegionMarked, &largeRegionNotMarked); + Bind(&largeRegionMarked); + { + isMarked = True(); + Jump(&exit); + } + + Bind(&largeRegionNotMarked); + GateRef old = Int32FetchOr(PtrAdd(regionDescAddr, IntPtr(common::RegionDesc::REGION_BITS_IN_METADATA_OFFSET)), mask, MemoryAttribute::Default()); + isMarked = False(); + Jump(&exit); + } + + Bind(¬LargeRegion); + GateRef regionBase = IntPtrAnd(TaggedCastToIntPtr(obj), + IntPtr(~static_cast(common::RegionDesc::DEFAULT_REGION_UNIT_MASK))); + GateRef offset = PtrSub(TaggedCastToIntPtr(obj), regionBase); + GateRef markBitMap = LoadPrimitive(VariableType::NATIVE_POINTER(), metaDataAddr, IntPtr(common::RegionDesc::REGION_MARKMAP_IN_INLINED_METADATA_OFFSET)); + GateRef headWordIdx = IntPtrDiv(IntPtrDiv(offset, IntPtr(common::kMarkedBytesPerBit)), + IntPtr(common::kBitsPerWord)); + GateRef headMaskBitStart = IntPtrMod(IntPtrDiv(offset, IntPtr(common::kMarkedBytesPerBit)), + IntPtr(common::kBitsPerWord)); + GateRef headMaskBits = Int64LSL(Int64(1), headMaskBitStart); + GateRef markWords = PtrAdd(markBitMap, IntPtr(common::MARK_WORD_OFFSET_IN_REGION_BITMAP)); + GateRef wordOffset = PtrMul(headWordIdx, IntPtr(common::kBytesPerWord)); + GateRef old = LoadPrimitive(VariableType::INT64(), markWords, wordOffset, MemoryAttribute::Atomic()); + GateRef checkMarked = Int64NotEqual(Int64And(old, headMaskBits), Int64(0)); + + Branch(checkMarked, &bitmapMarked, &bitmapNotMarked); + Bind(&bitmapMarked); + { + isMarked = True(); + Jump(&exit); + } + Bind(&bitmapNotMarked); + GateRef bitmapOld = Int64FetchOr(PtrAdd(markWords, wordOffset), headMaskBits, MemoryAttribute::Default()); + isMarked = Int64NotEqual(Int64And(bitmapOld, headMaskBits), Int64(0)); + Jump(&exit); + + Bind(&exit); + auto ret = *isMarked; + env->SubCfgExit(); + return ret; +} + +// should be logically equal to SatbBufer::ShouldEnqueue +GateRef StubBuilder::ShouldEnqueueSATB(GateRef glue, GateRef obj) +{ + auto env = GetEnvironment(); + Label entry(env); + env->SubCfgEntry(&entry); + Label exit(env); + Label isHeapAddress(env); + Label notHeapAddress(env); + Label isNullPtr(env); + Label notNullPtr(env); + Label isYoungObject(env); + Label notYoungObject(env); + Label isNewObject(env); + Label notNewObject(env); + Label isMarked(env); + Label notMarked(env); + + DEFVARIABLE(result, VariableType::BOOL(), False()); + + BRANCH_LIKELY(IsHeapAddress(glue, obj), &isHeapAddress, ¬HeapAddress); + Bind(¬HeapAddress); + { + result = False(); + Jump(&exit); + } + + Bind(&isHeapAddress); + BRANCH_UNLIKELY(IntPtrEuqal(TaggedCastToIntPtr(obj), IntPtr(0)), &isNullPtr, ¬NullPtr); + Bind(&isNullPtr); + { + result = False(); + Jump(&exit); + } + + Bind(¬NullPtr); + GateRef gcReason = GetGCReason(glue); + GateRef regionType = GetCMCRegionType(obj); + GateRef checkYoungObject = LogicAndBuilder(env) + .And(Int32Equal(gcReason, Int32(common::GC_REASON_YOUNG))) + .And(BoolNot(CMCIsInYoungSpace(regionType))) + .Done(); + Branch(checkYoungObject, ¬YoungObject, &isYoungObject); + Bind(¬YoungObject); + { + result = False(); + Jump(&exit); + } + + Bind(&isYoungObject); + Branch(IsNewObjectSinceMarking(obj), &isNewObject, ¬NewObject); + Bind(&isNewObject); + { + result = False(); + Jump(&exit); + } + + Bind(¬NewObject); + Branch(MarkObject(obj), &isMarked, ¬Marked); + Bind(&isMarked); + { + result = False(); + Jump(&exit); + } + + Bind(¬Marked); + result = True(); + Jump(&exit); + Bind(&exit); + auto ret = *result; + env->SubCfgExit(); + return ret; +} + GateRef StubBuilder::ShouldUpdateRememberSet(GateRef glue, GateRef gcPhase) { auto env = GetEnvironment(); @@ -2058,10 +2225,11 @@ void StubBuilder::CMCSetValueWithBarrier(GateRef glue, GateRef obj, [[maybe_unus Bind(¬MarkRSet); Label markInBuffer(env); GateRef shouldProcessSATB = ShouldProcessSATB(gcPhase); - BRANCH_UNLIKELY(shouldProcessSATB, &markInBuffer, &exit); + GateRef shouldEnqueue = ShouldEnqueueSATB(glue, obj); + BRANCH_UNLIKELY(BitAnd(shouldProcessSATB, shouldEnqueue), &markInBuffer, &exit); Bind(&markInBuffer); { - CallNGCRuntime(glue, RTSTUB_ID(MarkInBuffer), {value}); + CallNGCRuntime(glue, RTSTUB_ID(MarkInBufferNoCheck), {value}); Jump(&exit); } Bind(&exit); diff --git a/ecmascript/compiler/stub_builder.h b/ecmascript/compiler/stub_builder.h index 859a205a52c59609b9c39e5118afb315bc2522e7..45f4fe2c62d5a17128bf174bffbf572b4ceb907a 100644 --- a/ecmascript/compiler/stub_builder.h +++ b/ecmascript/compiler/stub_builder.h @@ -210,7 +210,8 @@ public: MemoryAttribute mAttr = MemoryAttribute::Default()); GateRef LoadZeroOffset(VariableType type, GateRef glue, GateRef base, MemoryAttribute mAttr = MemoryAttribute::Default()); - GateRef LoadPrimitive(VariableType type, GateRef base, GateRef offset); + GateRef LoadPrimitive(VariableType type, GateRef base, GateRef offset, + MemoryAttribute mAttr = MemoryAttribute::Default()); GateRef LoadZeroOffsetPrimitive(VariableType type, GateRef base); void Store(VariableType type, GateRef glue, @@ -260,6 +261,8 @@ public: GateRef Int32Xor(GateRef x, GateRef y); GateRef FixLoadType(GateRef x); GateRef Int64Or(GateRef x, GateRef y); + GateRef Int8FetchOr(GateRef x, GateRef y, MemoryAttribute mAttr); + GateRef Int32FetchOr(GateRef x, GateRef y, MemoryAttribute mAttr); GateRef Int64FetchOr(GateRef x, GateRef y, MemoryAttribute mAttr); GateRef IntPtrOr(GateRef x, GateRef y); GateRef Int64And(GateRef x, GateRef y); @@ -831,6 +834,9 @@ public: void MarkRSetCardTable(GateRef obj, Label *exit); GateRef ShouldGetGCReason(GateRef gcPhase); GateRef ShouldProcessSATB(GateRef gcPhase); + GateRef IsNewObjectSinceMarking(GateRef obj); + GateRef MarkObject(GateRef obj); + GateRef ShouldEnqueueSATB(GateRef glue, GateRef obj); GateRef ShouldUpdateRememberSet(GateRef glue, GateRef gcPhase); GateRef NeedSkipReadBarrier(GateRef glue); void CMCSetValueWithBarrier(GateRef glue, GateRef obj, GateRef offset, GateRef value); diff --git a/ecmascript/stubs/runtime_stub_list.h b/ecmascript/stubs/runtime_stub_list.h index f9dcd9c4f06b3c5bc5c9b67061685cdd4bbf862a..8d3dfe1b55606527bbd636365b22f167bc315cd6 100644 --- a/ecmascript/stubs/runtime_stub_list.h +++ b/ecmascript/stubs/runtime_stub_list.h @@ -225,6 +225,7 @@ namespace panda::ecmascript { V(CopyArgvArray) \ V(MarkRSetCardTable) \ V(MarkInBuffer) \ + V(MarkInBufferNoCheck) \ V(BatchMarkInBuffer) \ V(UpdateSharedModule) diff --git a/ecmascript/stubs/runtime_stubs.cpp b/ecmascript/stubs/runtime_stubs.cpp index 4cc4a0055051cf837cafcce7ad17a83c02f33961..a16c01b0d94a785eb9918528cabb5ddeb9587e82 100644 --- a/ecmascript/stubs/runtime_stubs.cpp +++ b/ecmascript/stubs/runtime_stubs.cpp @@ -5049,6 +5049,13 @@ void RuntimeStubs::MarkInBuffer(BaseObject* ref) mutator->RememberObjectInSatbBuffer(ref); } +void RuntimeStubs::MarkInBufferNoCheck(BaseObject* ref) +{ + ref = reinterpret_cast(reinterpret_cast(ref) & ~(common::Barrier::TAG_WEAK)); + common::Mutator* mutator = common::Mutator::GetMutator(); + mutator->RememberObjectInSatbBufferNoCheck(ref); +} + void RuntimeStubs::BatchMarkInBuffer(void* src, size_t count) { uintptr *srcPtr = reinterpret_cast(src); diff --git a/ecmascript/stubs/runtime_stubs.h b/ecmascript/stubs/runtime_stubs.h index 9305cb5491a424829fad7c8c4b657fed245ba0b5..2db59eae89297fd6c55326310075faa1c37075ed 100644 --- a/ecmascript/stubs/runtime_stubs.h +++ b/ecmascript/stubs/runtime_stubs.h @@ -184,6 +184,7 @@ public: static JSTaggedValue GetExternalModuleVar(uintptr_t argGlue, JSFunction *jsFunc, int32_t index); static bool MarkRSetCardTable(BaseObject* obj); static void MarkInBuffer(BaseObject* ref); + static void MarkInBufferNoCheck(BaseObject* ref); static void BatchMarkInBuffer(void* src, size_t count); private: