diff --git a/Apply-TBI-to-ZGC-of-JDK17.patch b/Apply-TBI-to-ZGC-of-JDK17.patch new file mode 100644 index 0000000000000000000000000000000000000000..28e1133f15369f81253657e98311cf3092b37850 --- /dev/null +++ b/Apply-TBI-to-ZGC-of-JDK17.patch @@ -0,0 +1,1728 @@ +From 4aea44c0438e3b635b6bea94869fd92ce96713a9 Mon Sep 17 00:00:00 2001 +Date: Wed, 29 Jun 2022 10:00:02 +0800 +Subject: Apply TBI to ZGC of JDK17 + +--- + src/hotspot/cpu/aarch64/aarch64.ad | 13 ++- + .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 5 + + .../cpu/aarch64/c1_LIRAssembler_aarch64.hpp | 9 +- + .../cpu/aarch64/compiledIC_aarch64.cpp | 9 +- + .../gc/shared/barrierSetNMethod_aarch64.cpp | 80 ++++++++++---- + .../gc/z/zBarrierSetAssembler_aarch64.cpp | 20 ++++ + .../gc/z/zBarrierSetAssembler_aarch64.hpp | 2 + + .../cpu/aarch64/gc/z/zGlobals_aarch64.cpp | 102 ++++++++++++++++++ + .../cpu/aarch64/gc/z/zGlobals_aarch64.hpp | 5 + + src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad | 8 +- + .../cpu/aarch64/globalDefinitions_aarch64.hpp | 3 + + src/hotspot/cpu/aarch64/globals_aarch64.hpp | 5 +- + .../aarch64/jvmciCodeInstaller_aarch64.cpp | 16 ++- + .../cpu/aarch64/macroAssembler_aarch64.cpp | 40 +++++-- + .../cpu/aarch64/nativeInst_aarch64.cpp | 31 +++++- + .../cpu/aarch64/nativeInst_aarch64.hpp | 16 ++- + .../cpu/aarch64/vm_version_aarch64.cpp | 28 +++++ + src/hotspot/os/linux/gc/z/zNUMA_linux.cpp | 4 + + .../gc/z/zPhysicalMemoryBacking_linux.cpp | 61 ++++++++++- + src/hotspot/share/asm/codeBuffer.cpp | 3 +- + src/hotspot/share/code/relocInfo.hpp | 29 ++++- + src/hotspot/share/gc/z/vmStructs_z.cpp | 1 + + src/hotspot/share/gc/z/vmStructs_z.hpp | 3 + + src/hotspot/share/gc/z/zAddress.cpp | 6 ++ + src/hotspot/share/gc/z/zAddress.hpp | 3 + + src/hotspot/share/gc/z/zAddress.inline.hpp | 30 ++++-- + src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 7 ++ + src/hotspot/share/gc/z/zGlobals.cpp | 1 + + src/hotspot/share/gc/z/zGlobals.hpp | 1 + + src/hotspot/share/gc/z/zMark.cpp | 7 ++ + src/hotspot/share/gc/z/zMarkStackEntry.hpp | 3 +- + src/hotspot/share/gc/z/zNMethod.cpp | 65 ++++++++++- + src/hotspot/share/gc/z/zNMethod.hpp | 6 +- + src/hotspot/share/gc/z/zNMethodData.cpp | 12 +++ + src/hotspot/share/gc/z/zNMethodData.hpp | 9 ++ + src/hotspot/share/gc/z/zPhysicalMemory.cpp | 54 ++++++++++ + src/hotspot/share/gc/z/zVirtualMemory.cpp | 15 +++ + .../writers/jfrStreamWriterHost.inline.hpp | 3 + + src/hotspot/share/prims/jni.cpp | 6 ++ + .../sun/jvm/hotspot/gc/z/ZAddress.java | 15 ++- + .../sun/jvm/hotspot/gc/z/ZCollectedHeap.java | 3 + + .../sun/jvm/hotspot/gc/z/ZGlobals.java | 4 + + .../hotspot/gc/z/ZGlobalsForVMStructs.java | 6 ++ + .../classes/sun/jvm/hotspot/gc/z/ZUtils.java | 5 + + 44 files changed, 685 insertions(+), 69 deletions(-) + +diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad +index a030555c9..7cc2e797a 100644 +--- a/src/hotspot/cpu/aarch64/aarch64.ad ++++ b/src/hotspot/cpu/aarch64/aarch64.ad +@@ -1750,7 +1750,11 @@ int MachCallStaticJavaNode::ret_addr_offset() + + int MachCallDynamicJavaNode::ret_addr_offset() + { +- return 16; // movz, movk, movk, bl ++ if (!UseTBI) { ++ return 16; // movz, movk, movk, bl ++ } else { ++ return 20; // movz, movk, movk, movk, bl ++ } + } + + int MachCallRuntimeNode::ret_addr_offset() { +@@ -1768,7 +1772,12 @@ int MachCallRuntimeNode::ret_addr_offset() { + if (cb) { + return 1 * NativeInstruction::instruction_size; + } else { +- return 6 * NativeInstruction::instruction_size; ++ // lea will emit 3 or an extra movk to make 4 instructions. ++ if (!UseTBI) { ++ return 6 * NativeInstruction::instruction_size; ++ } else { ++ return 7 * NativeInstruction::instruction_size; ++ } + } + } + +diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +index f488e863a..cfe8e85a1 100644 +--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +@@ -94,6 +94,11 @@ static void select_different_registers(Register preserve, + assert_different_registers(preserve, tmp1, tmp2, tmp3); + } + ++int LIR_Assembler::_call_stub_size = 13 * NativeInstruction::instruction_size; ++ ++void LIR_Assembler::init_for_tbi() { ++ _call_stub_size = 15 * NativeInstruction::instruction_size; ++} + + bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } + +diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp +index c31ad9b21..d39f57087 100644 +--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp +@@ -70,16 +70,19 @@ friend class ArrayCopyStub; + + void deoptimize_trap(CodeEmitInfo *info); + ++ // call stub: CompiledStaticCall::to_interp_stub_size() + ++ // CompiledStaticCall::to_trampoline_stub_size() ++ static int _call_stub_size; ++ + enum { +- // call stub: CompiledStaticCall::to_interp_stub_size() + +- // CompiledStaticCall::to_trampoline_stub_size() +- _call_stub_size = 13 * NativeInstruction::instruction_size, + _exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175), + _deopt_handler_size = 7 * NativeInstruction::instruction_size + }; + + public: + ++ static void init_for_tbi(); ++ + void store_parameter(Register r, int offset_from_esp_in_words); + void store_parameter(jint c, int offset_from_esp_in_words); + void store_parameter(jobject c, int offset_from_esp_in_words); +diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp +index e922fc1cd..c6a9d8387 100644 +--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp +@@ -71,8 +71,13 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) + #undef __ + + int CompiledStaticCall::to_interp_stub_size() { +- // isb; movk; movz; movz; movk; movz; movz; br +- return 8 * NativeInstruction::instruction_size; ++ if (!UseTBI) { ++ // isb; movk; movz; movz; movk; movz; movz; br ++ return 8 * NativeInstruction::instruction_size; ++ } else { ++ // emit_to_interp_stub will emit 2 extra movk instructions. ++ return 10 * NativeInstruction::instruction_size; ++ } + } + + int CompiledStaticCall::to_trampoline_stub_size() { +diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp +index 8598fb7e7..4942dca93 100644 +--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp +@@ -34,25 +34,6 @@ + #include "utilities/align.hpp" + #include "utilities/debug.hpp" + +-class NativeNMethodBarrier: public NativeInstruction { +- address instruction_address() const { return addr_at(0); } +- +- int *guard_addr() { +- return reinterpret_cast(instruction_address() + 10 * 4); +- } +- +-public: +- int get_value() { +- return Atomic::load_acquire(guard_addr()); +- } +- +- void set_value(int value) { +- Atomic::release_store(guard_addr(), value); +- } +- +- void verify() const; +-}; +- + // Store the instruction bitmask, bits and name for checking the barrier. + struct CheckInsn { + uint32_t mask; +@@ -60,7 +41,20 @@ struct CheckInsn { + const char *name; + }; + +-static const struct CheckInsn barrierInsn[] = { ++static const struct CheckInsn barrierInsn48[] = { ++ { 0xff000000, 0x18000000, "ldr (literal)" }, ++ { 0xfffff0ff, 0xd50330bf, "dmb" }, ++ { 0xffc00000, 0xb9400000, "ldr"}, ++ { 0x7f20001f, 0x6b00001f, "cmp"}, ++ { 0xff00001f, 0x54000000, "b.eq"}, ++ { 0xff800000, 0xd2800000, "mov"}, ++ { 0xff800000, 0xf2800000, "movk"}, ++ { 0xff800000, 0xf2800000, "movk"}, ++ { 0xfffffc1f, 0xd63f0000, "blr"}, ++ { 0xfc000000, 0x14000000, "b"} ++}; ++ ++static const struct CheckInsn barrierInsn64[] = { + { 0xff000000, 0x18000000, "ldr (literal)" }, + { 0xfffff0ff, 0xd50330bf, "dmb" }, + { 0xffc00000, 0xb9400000, "ldr"}, +@@ -69,16 +63,52 @@ static const struct CheckInsn barrierInsn[] = { + { 0xff800000, 0xd2800000, "mov"}, + { 0xff800000, 0xf2800000, "movk"}, + { 0xff800000, 0xf2800000, "movk"}, ++ { 0xff800000, 0xf2800000, "movk"}, + { 0xfffffc1f, 0xd63f0000, "blr"}, + { 0xfc000000, 0x14000000, "b"} + }; + ++static const unsigned int barrier_inst_len48 = sizeof(barrierInsn48) / sizeof(struct CheckInsn); ++static const unsigned int barrier_inst_len64 = sizeof(barrierInsn64) / sizeof(struct CheckInsn); ++ ++static int get_entry_barrier_size() { ++ return UseTBI ? (4 * (int)barrier_inst_len64) : (4 * (int)barrier_inst_len48); ++} ++ ++class NativeNMethodBarrier: public NativeInstruction { ++ address instruction_address() const { return addr_at(0); } ++ ++ int *guard_addr() { ++ return reinterpret_cast(instruction_address() + get_entry_barrier_size()); ++ } ++ ++public: ++ int get_value() { ++ return Atomic::load_acquire(guard_addr()); ++ } ++ ++ void set_value(int value) { ++ Atomic::release_store(guard_addr(), value); ++ } ++ ++ void verify() const; ++}; ++ + // The encodings must match the instructions emitted by + // BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific + // register numbers and immediate values in the encoding. + void NativeNMethodBarrier::verify() const { ++ const CheckInsn *barrierInsn; ++ unsigned int barrier_inst_len; ++ if (UseTBI) { ++ barrierInsn = barrierInsn64; ++ barrier_inst_len = barrier_inst_len64; ++ } else { ++ barrierInsn = barrierInsn48; ++ barrier_inst_len = barrier_inst_len48; ++ } + intptr_t addr = (intptr_t) instruction_address(); +- for(unsigned int i = 0; i < sizeof(barrierInsn)/sizeof(struct CheckInsn); i++ ) { ++ for(unsigned int i = 0; i < barrier_inst_len; i++) { + uint32_t inst = *((uint32_t*) addr); + if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) { + tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", addr, inst); +@@ -132,10 +162,14 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) { + // not find the expected native instruction at this offset, which needs updating. + // Note that this offset is invariant of PreserveFramePointer. + +-static const int entry_barrier_offset = -4 * 11; ++// offset = entry barrier insns (10 or 11, depending on UseTBI) + int32 (guard value) ++// @see BarrierSetAssembler::nmethod_entry_barrier ++static int entry_barrier_offset() { ++ return -(get_entry_barrier_size() + 4); ++} + + static NativeNMethodBarrier* native_nmethod_barrier(nmethod* nm) { +- address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset; ++ address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(); + NativeNMethodBarrier* barrier = reinterpret_cast(barrier_address); + debug_only(barrier->verify()); + return barrier; +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +index 6b42982ed..cafd4e58f 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +@@ -203,6 +203,26 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, + BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); + } + ++static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint32_t end) { ++ uint32_t imm_mask = ((1u << start) - 1u) ^ ((1u << (end + 1)) - 1u); ++ instr &= ~imm_mask; ++ instr |= imm << start; ++} ++ ++void ZBarrierSetAssembler::patch_barrier_relocation(address addr) { ++ uint32_t* const patch_addr = (uint32_t*)addr; ++ ++ // The next 3 insns should be movz, andr, cbnz. ++ assert(nativeInstruction_at(addr)->is_movz() && ++ Instruction_aarch64::extract(*(patch_addr + 1), 30, 24) == 0b0001010 && ++ Instruction_aarch64::extract(*(patch_addr + 2), 31, 24) == 0b10110101, ++ "wrong insns in barrier patch"); ++ ++ change_immediate(*patch_addr, (uint16_t) (ZAddressBadMask >> 48), 5, 20); ++ OrderAccess::fence(); ++ ICache::invalidate_word((address)patch_addr); ++} ++ + #ifdef COMPILER1 + + #undef __ +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +index cca873825..1594ffe64 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +@@ -77,6 +77,8 @@ public: + Register tmp, + Label& slowpath); + ++ void patch_barrier_relocation(address addr); ++ + #ifdef COMPILER1 + void generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const; +diff --git a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp +index 6aa6d41fe..031014511 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp +@@ -34,6 +34,7 @@ + #include + #endif // LINUX + ++// Address layout when using multi-mapping + // + // The heap can have three different layouts, depending on the max heap size. + // +@@ -142,6 +143,97 @@ + // * 63-48 Fixed (16-bits, always zero) + // + ++// Address layout when using TBI (Top Byte Ignore) ++// ++// The heap can have three different layouts, depending on the max heap size. ++// ++// Address Space & Pointer Layout 1 ++// -------------------------------- ++// ++// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) ++// . . ++// . . ++// . . ++// +--------------------------------+ 0x0000080000000000 (8TB) ++// | Heap | ++// +--------------------------------+ 0x0000040000000000 (4TB) ++// . . ++// +--------------------------------+ 0x0000000000000000 ++// ++// 6 5 5 4 4 ++// 2 9 8 2 1 0 ++// ++----+-------------------+-----------------------------------------------+ ++// 0|1111|000 00000000 000000|11 11111111 11111111 11111111 11111111 11111111| ++// ++----+-------------------+-----------------------------------------------+ ++// | | | ++// | | * 41-0 Object Offset (42-bits, 4TB address space) ++// | | ++// | * 58-42 Fixed (18-bits, always zero) ++// | ++// * 62-59 Metadata Bits (4-bits) 0001 = Marked0 ++// 0010 = Marked1 ++// 0100 = Remapped ++// 1000 = Finalizable ++// ++// ++// Address Space & Pointer Layout 2 ++// -------------------------------- ++// ++// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) ++// . . ++// . . ++// . . ++// +--------------------------------+ 0x0000100000000000 (16TB) ++// | Heap | ++// +--------------------------------+ 0x0000080000000000 (8TB) ++// . . ++// +--------------------------------+ 0x0000000000000000 ++// ++// 6 5 5 4 4 ++// 2 9 8 3 2 0 ++// ++----+------------------+------------------------------------------------+ ++// 0|1111|000 00000000 00000|111 11111111 11111111 11111111 11111111 11111111| ++// ++----+------------------+------------------------------------------------+ ++// | | | ++// | | * 42-0 Object Offset (43-bits, 8TB address space) ++// | | ++// | * 58-43 Fixed (17-bits, always zero) ++// | ++// * 62-59 Metadata Bits (4-bits) 0001 = Marked0 ++// 0010 = Marked1 ++// 0100 = Remapped ++// 1000 = Finalizable ++// ++// ++// Address Space & Pointer Layout 3 ++// -------------------------------- ++// ++// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) ++// . . ++// . . ++// . . ++// +--------------------------------+ 0x0000200000000000 (32TB) ++// | Heap | ++// +--------------------------------+ 0x0000100000000000 (16TB) ++// . . ++// +--------------------------------+ 0x0000000000000000 ++// ++// 6 5 5 4 4 ++// 2 9 8 4 3 0 ++// ++----+-----------------+-------------------------------------------------+ ++// 0|1111|000 00000000 0000|1111 11111111 11111111 11111111 11111111 11111111| ++// ++----+-----------------+-------------------------------------------------+ ++// | | | ++// | | * 43-0 Object Offset (44-bits, 16TB address space) ++// | | ++// | * 58-44 Fixed (17-bits, always zero) ++// | ++// * 62-59 Metadata Bits (4-bits) 0001 = Marked0 ++// 0010 = Marked1 ++// 0100 = Remapped ++// 1000 = Finalizable ++// ++ + // Default value if probing is not implemented for a certain platform: 128TB + static const size_t DEFAULT_MAX_ADDRESS_BIT = 47; + // Minimum value returned, if probing fails: 64GB +@@ -196,6 +288,13 @@ static size_t probe_valid_max_address_bit() { + #endif // LINUX + } + ++uintptr_t ZPlatformAddressBase() { ++ if (UseTBI) { ++ return (uintptr_t)1 << ZPlatformAddressOffsetBits(); ++ } ++ return 0; ++} ++ + size_t ZPlatformAddressOffsetBits() { + const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1; + const size_t max_address_offset_bits = valid_max_address_offset_bits - 3; +@@ -206,5 +305,8 @@ size_t ZPlatformAddressOffsetBits() { + } + + size_t ZPlatformAddressMetadataShift() { ++ if (UseTBI) { ++ return ZPlatformAddressMetadataShiftForTbi; ++ } + return ZPlatformAddressOffsetBits(); + } +diff --git a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp +index 3187808b6..264b51ec1 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp +@@ -28,6 +28,11 @@ const size_t ZPlatformGranuleSizeShift = 21; // 2MB + const size_t ZPlatformHeapViews = 3; + const size_t ZPlatformCacheLineSize = 64; + ++// The highest bit (bit 63) of the address is occupied by StackWatermarkState (see StackWatermarkState::create). ++// So here we use bits 59~62 as the metadata bits of ZGC. ++const size_t ZPlatformAddressMetadataShiftForTbi = 59; ++ ++uintptr_t ZPlatformAddressBase(); + size_t ZPlatformAddressOffsetBits(); + size_t ZPlatformAddressMetadataShift(); + +diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +index bd1c2cc9f..426a1cc2a 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad ++++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +@@ -26,6 +26,7 @@ source_hpp %{ + #include "gc/shared/gc_globals.hpp" + #include "gc/z/c2/zBarrierSetC2.hpp" + #include "gc/z/zThreadLocalData.hpp" ++#include "gc/z/zBarrierSetAssembler.hpp" + + %} + +@@ -36,7 +37,12 @@ static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address + return; + } + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data); +- __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ if (UseTBI) { ++ __ relocate(barrier_Relocation::spec()); ++ __ movz(tmp, barrier_Relocation::unpatched, 48); ++ } else { ++ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ } + __ andr(tmp, tmp, ref); + __ cbnz(tmp, *stub->entry()); + __ bind(*stub->continuation()); +diff --git a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp +index 3c779bb11..4a2af011f 100644 +--- a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp +@@ -67,4 +67,7 @@ const bool CCallingConventionRequiresIntsAsLongs = false; + #define NOT_R18_RESERVED(code) code + #endif + ++#define CLEAR_TOP_BYTE(addr) ((addr) & ((1UL << 56) - 1)) ++#define CLEAR_COLOR_BITS(addr) (UseTBI ? CLEAR_TOP_BYTE(addr) : (addr)) ++ + #endif // CPU_AARCH64_GLOBALDEFINITIONS_AARCH64_HPP +diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp +index 3ccf7630b..98c87b201 100644 +--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp +@@ -119,7 +119,10 @@ define_pd_global(intx, InlineSmallCode, 1000); + product(uint, OnSpinWaitInstCount, 1, DIAGNOSTIC, \ + "The number of OnSpinWaitInst instructions to generate." \ + "It cannot be used with OnSpinWaitInst=none.") \ +- range(1, 99) ++ range(1, 99) \ ++ product(bool, UseTBI, false, EXPERIMENTAL, \ ++ "Use the \"Top Byte Ignore\" feature for ZGC." \ ++ "And use 64 bit literal addresses instead of 48 bit.") + + // end of ARCH_FLAGS + +diff --git a/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp b/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp +index 17b978012..b52b3f2b3 100644 +--- a/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp +@@ -57,9 +57,19 @@ void CodeInstaller::pd_patch_OopConstant(int pc_offset, JVMCIObject constant, JV + assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 && + nativeInstruction_at(pc+4)->is_movk(), "wrong insn in patch"); + } else { +- // Move wide constant: movz n, movk, movk. +- assert(nativeInstruction_at(pc+4)->is_movk() +- && nativeInstruction_at(pc+8)->is_movk(), "wrong insn in patch"); ++ if (!UseTBI) { ++ // Move wide constant: movz n, movk, movk. ++ assert(nativeInstruction_at(pc+4)->is_movk() ++ && nativeInstruction_at(pc+8)->is_movk(), "wrong insn in patch"); ++ } else { ++ // Move wide constant: movz n, movk, movk, movk. ++ // JVMCI would require code generated for it to use 64 bit literals and ++ // it therefore disables UseTBI until support is enabled. ++ // @see src/hotspot/cpu/aarch64/vm_version_aarch64.cpp - VM_Version::initialize() ++ assert(nativeInstruction_at(pc+4)->is_movk() ++ && nativeInstruction_at(pc+8)->is_movk() ++ && nativeInstruction_at(pc+12)->is_movk(), "wrong insn in patch"); ++ } + } + } + #endif // ASSERT +diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +index 85ce4c44b..6294636c2 100644 +--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +@@ -174,9 +174,10 @@ int MacroAssembler::patch_oop(address insn_addr, address o) { + unsigned insn = *(unsigned*)insn_addr; + assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); + +- // OOPs are either narrow (32 bits) or wide (48 bits). We encode ++ // OOPs are either narrow (32 bits) or wide (48 or 64 bits). We encode + // narrow OOPs by setting the upper 16 bits in the first + // instruction. ++ // 64 bit addresses are only enabled with UseTBI set. + if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) { + // Move narrow OOP + uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); +@@ -191,6 +192,12 @@ int MacroAssembler::patch_oop(address insn_addr, address o) { + Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff); + Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff); + instructions = 3; ++ ++ if (UseTBI) { ++ assert(nativeInstruction_at(insn_addr+12)->is_movk(), "wrong insns in patch"); ++ Instruction_aarch64::patch(insn_addr+12, 20, 5, (dest >>= 16) & 0xffff); ++ instructions = 4; ++ } + } + return instructions * NativeInstruction::instruction_size; + } +@@ -277,12 +284,18 @@ address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) { + } + } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) { + uint32_t *insns = (uint32_t *)insn_addr; +- // Move wide constant: movz, movk, movk. See movptr(). +- assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch"); +- assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch"); +- return address(uint64_t(Instruction_aarch64::extract(insns[0], 20, 5)) ++ // Move wide constant: movz, movk, movk [, movk]. See movptr(). ++ assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch - 2nd movk missing"); ++ assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch - 3rd movk missing"); ++ uint64_t addr = uint64_t(Instruction_aarch64::extract(insns[0], 20, 5)) + + (uint64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16) +- + (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32)); ++ + (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32); ++ // Allow for getting the target address of a possible adddress. ++ if (UseTBI) { ++ assert(nativeInstruction_at(insns+3)->is_movk(), "wrong insns in patch - 4th movk missing."); ++ addr += uint64_t(Instruction_aarch64::extract(insns[3], 20, 5)) << 48; ++ } ++ return address(addr); + } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 && + Instruction_aarch64::extract(insn, 4, 0) == 0b11111) { + return 0; +@@ -1473,10 +1486,9 @@ void MacroAssembler::mov(Register r, Address dest) { + movptr(r, imm64); + } + +-// Move a constant pointer into r. In AArch64 mode the virtual +-// address space is 48 bits in size, so we only need three +-// instructions to create a patchable instruction sequence that can +-// reach anywhere. ++// Move a constant pointer into r. In AArch64 mode the virtual address space ++// is 48 bits in size or 52 bits. We need three or four instructions to create ++// a patchable instruction sequence that can reach anywhere. + void MacroAssembler::movptr(Register r, uintptr_t imm64) { + #ifndef PRODUCT + { +@@ -1485,12 +1497,18 @@ void MacroAssembler::movptr(Register r, uintptr_t imm64) { + block_comment(buffer); + } + #endif +- assert(imm64 < (1ull << 48), "48-bit overflow in address constant"); ++ if (!UseTBI) { ++ assert(imm64 < (1ull << 48), "48-bit overflow in address constant"); ++ } + movz(r, imm64 & 0xffff); + imm64 >>= 16; + movk(r, imm64 & 0xffff, 16); + imm64 >>= 16; + movk(r, imm64 & 0xffff, 32); ++ if (UseTBI) { ++ imm64 >>= 16; ++ movk(r, imm64 & 0xffff, 48); ++ } + } + + // Macro to mov replicated immediate to vector register. +diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp +index d808e4b5b..117686f8c 100644 +--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp +@@ -234,6 +234,14 @@ void NativeCall::insert(address code_pos, address entry) { Unimplemented(); } + + //------------------------------------------------------------------- + ++// movz, movk, movk. ++int NativeMovConstReg::instruction_size = 3 * NativeInstruction::instruction_size; ++ ++void NativeMovConstReg::init_for_tbi() { ++ // movz, movk, movk, movk. ++ instruction_size = 4 * NativeInstruction::instruction_size; ++} ++ + void NativeMovConstReg::verify() { + if (! (nativeInstruction_at(instruction_address())->is_movz() || + is_adrp_at(instruction_address()) || +@@ -439,8 +447,17 @@ bool NativeInstruction::is_general_jump() { + NativeInstruction* inst2 = nativeInstruction_at(addr_at(instruction_size * 2)); + if (inst2->is_movk()) { + NativeInstruction* inst3 = nativeInstruction_at(addr_at(instruction_size * 3)); +- if (inst3->is_blr()) { +- return true; ++ if (!UseTBI) { ++ if (inst3->is_blr()) { ++ return true; ++ } ++ } else { ++ if(inst3->is_movk()) { ++ NativeInstruction* inst4 = nativeInstruction_at(addr_at(instruction_size * 4)); ++ if (inst4->is_blr()) { ++ return true; ++ } ++ } + } + } + } +@@ -496,6 +513,16 @@ void NativeJump::patch_verified_entry(address entry, address verified_entry, add + ICache::invalidate_range(verified_entry, instruction_size); + } + ++// movz, movk, movk, br. ++int NativeGeneralJump::instruction_size = 4 * NativeInstruction::instruction_size; ++int NativeGeneralJump::next_instruction_offset = 4 * NativeInstruction::instruction_size; ++ ++void NativeGeneralJump::init_for_tbi() { ++ // movz, movk, movk, movk, br. ++ instruction_size = 5 * NativeInstruction::instruction_size; ++ next_instruction_offset = 5 * NativeInstruction::instruction_size; ++} ++ + void NativeGeneralJump::verify() { } + + void NativeGeneralJump::insert_unconditional(address code_pos, address entry) { +diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +index 75f2797c3..2460c02a1 100644 +--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +@@ -271,16 +271,19 @@ inline NativeCall* nativeCall_before(address return_address) { + class NativeMovConstReg: public NativeInstruction { + public: + enum Aarch64_specific_constants { +- instruction_size = 3 * 4, // movz, movk, movk. See movptr(). + instruction_offset = 0, + displacement_offset = 0, + }; + ++ static int instruction_size; ++ ++ static void init_for_tbi(); ++ + address instruction_address() const { return addr_at(instruction_offset); } + + address next_instruction_address() const { + if (nativeInstruction_at(instruction_address())->is_movz()) +- // Assume movz, movk, movk ++ // Assume movz, movk, movk [, movk]. + return addr_at(instruction_size); + else if (is_adrp_at(instruction_address())) + return addr_at(2*4); +@@ -469,12 +472,15 @@ inline NativeJump* nativeJump_at(address address) { + class NativeGeneralJump: public NativeJump { + public: + enum AArch64_specific_constants { +- instruction_size = 4 * 4, + instruction_offset = 0, +- data_offset = 0, +- next_instruction_offset = 4 * 4 ++ data_offset = 0 + }; + ++ static int instruction_size; ++ static int next_instruction_offset; ++ ++ static void init_for_tbi(); ++ + address jump_destination() const; + void set_jump_destination(address dest); + +diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +index f9de88613..0fa922ec9 100644 +--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +@@ -31,6 +31,10 @@ + #include "runtime/vm_version.hpp" + #include "utilities/formatBuffer.hpp" + #include "utilities/macros.hpp" ++#include "code/nativeInst.hpp" ++#ifdef COMPILER1 ++#include "c1/c1_LIRAssembler.hpp" ++#endif // COMPILER1 + + #include OS_HEADER_INLINE(os) + +@@ -66,6 +70,16 @@ static SpinWait get_spin_wait_desc() { + return SpinWait{}; + } + ++// Configure instruction sizes for nativeInst_aarch64 and c1_LIRAssembler_aarch64 ++// based on flag UseTBI. ++static void init_instruction_sizes_for_tbi() { ++ NativeMovConstReg::init_for_tbi(); ++ NativeGeneralJump::init_for_tbi(); ++#ifdef COMPILER1 ++ LIR_Assembler::init_for_tbi(); ++#endif // COMPILER1 ++} ++ + void VM_Version::initialize() { + _supports_cx8 = true; + _supports_atomic_getset4 = true; +@@ -406,6 +420,20 @@ void VM_Version::initialize() { + UsePopCountInstruction = true; + } + ++#if INCLUDE_JVMCI && defined(AARCH64) ++ if (UseTBI && EnableJVMCI) { ++ warning("64-bit Literal Addresses disabled due to EnableJVMCI."); ++ UseTBI = false; ++ } ++#endif ++ if (UseTBI && !UseZGC) { ++ warning("UseTBI only works when UseZGC is on."); ++ UseTBI = false; ++ } ++ if (UseTBI) { ++ init_instruction_sizes_for_tbi(); ++ } ++ + #ifdef COMPILER2 + if (FLAG_IS_DEFAULT(UseMultiplyToLenIntrinsic)) { + UseMultiplyToLenIntrinsic = true; +diff --git a/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp b/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp +index cfe25549f..cefe5d03d 100644 +--- a/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp ++++ b/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp +@@ -59,6 +59,10 @@ uint32_t ZNUMA::memory_id(uintptr_t addr) { + + uint32_t id = (uint32_t)-1; + ++#ifdef AARCH64 ++ addr = CLEAR_COLOR_BITS(addr); ++#endif ++ + if (ZSyscall::get_mempolicy((int*)&id, NULL, 0, (void*)addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) { + ZErrno err; + fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string()); +diff --git a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp +index 7a1130554..d0ffe93ce 100644 +--- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp ++++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp +@@ -120,6 +120,14 @@ ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) : + _available(0), + _initialized(false) { + ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Successfully initialized ++ _initialized = true; ++ return; ++ } ++#endif // AARCH64 ++ + // Create backing file + _fd = create_fd(ZFILENAME_HEAP); + if (_fd == -1) { +@@ -350,7 +358,12 @@ void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const { + // However, ZGC tends to create the most mappings and dominate the total count. + // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. + // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. +- const size_t required_max_map_count = (max_capacity / ZGranuleSize) * 3 * 1.2; ++#ifdef AARCH64 ++ const size_t required_max_map_multiple = UseTBI ? 1 : 3; ++#else ++ const size_t required_max_map_multiple = 3; ++#endif // AARCH64 ++ const size_t required_max_map_count = (max_capacity / ZGranuleSize) * required_max_map_multiple * 1.2; + if (actual_max_map_count < required_max_map_count) { + log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning_p(gc)("The system limit on number of memory mappings per process might be too low for the given"); +@@ -363,6 +376,13 @@ void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const { + } + + void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Warn if max map count is too low ++ warn_max_map_count(max_capacity); ++ return; ++ } ++#endif // AARCH64 + // Warn if available space is too low + warn_available_space(max_capacity); + +@@ -681,6 +701,13 @@ size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) cons + } + + size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // do nothing ++ return length; ++ } ++#endif // AARCH64 ++ + if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) { + // To get granule-level NUMA interleaving when using non-large pages, + // we must explicitly interleave the memory at commit/fallocate time. +@@ -693,6 +720,12 @@ size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const { + size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const { + log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", + offset / M, (offset + length) / M, length / M); ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Not yet supported. ++ return length; ++ } ++#endif // AARCH64 + + const ZErrno err = fallocate(true /* punch_hole */, offset, length); + if (err) { +@@ -704,6 +737,29 @@ size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const { + } + + void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const { ++#ifdef AARCH64 ++ if (UseTBI) { ++ int flags = MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE; ++ if (ZLargePages::is_explicit()) { ++ flags |= MAP_HUGETLB; ++ } ++ const void* const res = mmap((void*) CLEAR_TOP_BYTE(addr), size, PROT_READ | PROT_WRITE, flags, 0, 0); ++ if (res == MAP_FAILED) { ++ ZErrno err; ++ fatal("Failed to map memory (%s)", err.to_string()); ++ } ++ ++ // Advise on use of transparent huge pages before touching it ++ if (ZLargePages::is_transparent()) { ++ if (madvise((void*) CLEAR_TOP_BYTE(addr), size, MADV_HUGEPAGE) == -1) { ++ ZErrno err; ++ log_error(gc)("Failed to advise use of transparent huge pages (%s)", err.to_string()); ++ } ++ } ++ return; ++ } ++#endif // AARCH64 ++ + const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset); + if (res == MAP_FAILED) { + ZErrno err; +@@ -712,6 +768,9 @@ void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) + } + + void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const { ++#ifdef AARCH64 ++ addr = CLEAR_COLOR_BITS(addr); ++#endif // AARCH64 + // Note that we must keep the address space reservation intact and just detach + // the backing memory. For this reason we map a new anonymous, non-accessible + // and non-reserved page over the mapping instead of actually unmapping. +diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp +index 0012152d4..c89c8ee3b 100644 +--- a/src/hotspot/share/asm/codeBuffer.cpp ++++ b/src/hotspot/share/asm/codeBuffer.cpp +@@ -331,7 +331,8 @@ void CodeSection::relocate(address at, RelocationHolder const& spec, int format) + rtype == relocInfo::runtime_call_type || + rtype == relocInfo::internal_word_type|| + rtype == relocInfo::section_word_type || +- rtype == relocInfo::external_word_type, ++ rtype == relocInfo::external_word_type|| ++ rtype == relocInfo::barrier_type, + "code needs relocation information"); + // leave behind an indication that we attempted a relocation + DEBUG_ONLY(_locs_start = _locs_limit = (relocInfo*)badAddress); +diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp +index 55d4ac7c6..57d2126fb 100644 +--- a/src/hotspot/share/code/relocInfo.hpp ++++ b/src/hotspot/share/code/relocInfo.hpp +@@ -268,7 +268,8 @@ class relocInfo { + trampoline_stub_type = 13, // stub-entry for trampoline + runtime_call_w_cp_type = 14, // Runtime call which may load its target from the constant pool + data_prefix_tag = 15, // tag for a prefix (carries data arguments) +- type_mask = 15 // A mask which selects only the above values ++ barrier_type = 16, // GC barrier data ++ type_mask = 31 // A mask which selects only the above values + }; + + private: +@@ -307,12 +308,13 @@ class relocInfo { + visitor(poll_return) \ + visitor(section_word) \ + visitor(trampoline_stub) \ ++ visitor(barrier) \ + + + public: + enum { + value_width = sizeof(unsigned short) * BitsPerByte, +- type_width = 4, // == log2(type_mask+1) ++ type_width = 5, // == log2(type_mask+1) + nontype_width = value_width - type_width, + datalen_width = nontype_width-1, + datalen_tag = 1 << datalen_width, // or-ed into _value +@@ -994,6 +996,29 @@ class metadata_Relocation : public DataRelocation { + // Note: metadata_value transparently converts Universe::non_metadata_word to NULL. + }; + ++class barrier_Relocation : public DataRelocation { ++ ++ public: ++ // The uninitialized value used before the relocation has been patched. ++ // Code assumes that the unpatched value is zero. ++ static const int16_t unpatched = 0; ++ ++ static RelocationHolder spec() { ++ RelocationHolder rh = newHolder(); ++ new(rh) barrier_Relocation(); ++ return rh; ++ } ++ ++ private: ++ friend class RelocIterator; ++ barrier_Relocation() : DataRelocation(relocInfo::barrier_type) { } ++ ++ public: ++ virtual int offset() { ShouldNotReachHere(); return 0; } ++ virtual address value() { ShouldNotReachHere(); return NULL; } ++ virtual void set_value(address x) { ShouldNotReachHere(); } ++ virtual void set_value(address x, intptr_t o) { ShouldNotReachHere(); } ++}; + + class virtual_call_Relocation : public CallRelocation { + +diff --git a/src/hotspot/share/gc/z/vmStructs_z.cpp b/src/hotspot/share/gc/z/vmStructs_z.cpp +index c86d11c81..e68077ccb 100644 +--- a/src/hotspot/share/gc/z/vmStructs_z.cpp ++++ b/src/hotspot/share/gc/z/vmStructs_z.cpp +@@ -27,6 +27,7 @@ + ZGlobalsForVMStructs::ZGlobalsForVMStructs() : + _ZGlobalPhase(&ZGlobalPhase), + _ZGlobalSeqNum(&ZGlobalSeqNum), ++ _ZAddressBase(&ZAddressBase), + _ZAddressOffsetMask(&ZAddressOffsetMask), + _ZAddressMetadataMask(&ZAddressMetadataMask), + _ZAddressMetadataFinalizable(&ZAddressMetadataFinalizable), +diff --git a/src/hotspot/share/gc/z/vmStructs_z.hpp b/src/hotspot/share/gc/z/vmStructs_z.hpp +index 3c0eb9f74..f8754e0cb 100644 +--- a/src/hotspot/share/gc/z/vmStructs_z.hpp ++++ b/src/hotspot/share/gc/z/vmStructs_z.hpp +@@ -45,6 +45,7 @@ public: + + uint32_t* _ZGlobalSeqNum; + ++ uintptr_t* _ZAddressBase; + uintptr_t* _ZAddressOffsetMask; + uintptr_t* _ZAddressMetadataMask; + uintptr_t* _ZAddressMetadataFinalizable; +@@ -64,6 +65,7 @@ typedef ZAttachedArray ZAttachedArrayForForwardin + static_field(ZGlobalsForVMStructs, _instance_p, ZGlobalsForVMStructs*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZGlobalPhase, uint32_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZGlobalSeqNum, uint32_t*) \ ++ nonstatic_field(ZGlobalsForVMStructs, _ZAddressBase, uintptr_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressOffsetMask, uintptr_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressMetadataMask, uintptr_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressMetadataFinalizable, uintptr_t*) \ +@@ -120,6 +122,7 @@ typedef ZAttachedArray ZAttachedArrayForForwardin + declare_constant(ZPageSizeMediumShift) \ + declare_constant(ZAddressOffsetShift) \ + declare_constant(ZAddressOffsetBits) \ ++ declare_constant(ZAddressBase) \ + declare_constant(ZAddressOffsetMask) \ + declare_constant(ZAddressOffsetMax) + +diff --git a/src/hotspot/share/gc/z/zAddress.cpp b/src/hotspot/share/gc/z/zAddress.cpp +index cfa7c04d3..47935d4c1 100644 +--- a/src/hotspot/share/gc/z/zAddress.cpp ++++ b/src/hotspot/share/gc/z/zAddress.cpp +@@ -32,6 +32,12 @@ void ZAddress::set_good_mask(uintptr_t mask) { + } + + void ZAddress::initialize() { ++#ifdef AARCH64 ++ ZAddressBase = ZPlatformAddressBase(); ++#else ++ ZAddressBase = 0; ++#endif ++ + ZAddressOffsetBits = ZPlatformAddressOffsetBits(); + ZAddressOffsetMask = (((uintptr_t)1 << ZAddressOffsetBits) - 1) << ZAddressOffsetShift; + ZAddressOffsetMax = (uintptr_t)1 << ZAddressOffsetBits; +diff --git a/src/hotspot/share/gc/z/zAddress.hpp b/src/hotspot/share/gc/z/zAddress.hpp +index eddd10418..c9e7449dc 100644 +--- a/src/hotspot/share/gc/z/zAddress.hpp ++++ b/src/hotspot/share/gc/z/zAddress.hpp +@@ -52,6 +52,9 @@ public: + static bool is_remapped(uintptr_t value); + static bool is_in(uintptr_t value); + ++#ifdef AARCH64 ++ static uintptr_t base(uintptr_t value); ++#endif + static uintptr_t offset(uintptr_t value); + static uintptr_t good(uintptr_t value); + static uintptr_t good_or_null(uintptr_t value); +diff --git a/src/hotspot/share/gc/z/zAddress.inline.hpp b/src/hotspot/share/gc/z/zAddress.inline.hpp +index a151e7182..c8b560490 100644 +--- a/src/hotspot/share/gc/z/zAddress.inline.hpp ++++ b/src/hotspot/share/gc/z/zAddress.inline.hpp +@@ -31,6 +31,12 @@ + #include "utilities/macros.hpp" + #include "utilities/powerOfTwo.hpp" + ++#ifdef AARCH64 ++#define AARCH64_BASE(x) base(x) ++#else ++#define AARCH64_BASE(x) (x) ++#endif ++ + inline bool ZAddress::is_null(uintptr_t value) { + return value == 0; + } +@@ -90,7 +96,11 @@ inline bool ZAddress::is_remapped(uintptr_t value) { + + inline bool ZAddress::is_in(uintptr_t value) { + // Check that exactly one non-offset bit is set ++#ifdef AARCH64 ++ if (!is_power_of_2(value & ~ZAddressOffsetMask & ~ZAddressBase)) { ++#else + if (!is_power_of_2(value & ~ZAddressOffsetMask)) { ++#endif + return false; + } + +@@ -98,12 +108,18 @@ inline bool ZAddress::is_in(uintptr_t value) { + return value & (ZAddressMetadataMask & ~ZAddressMetadataFinalizable); + } + ++#ifdef AARCH64 ++inline uintptr_t ZAddress::base(uintptr_t value) { ++ return value | ZAddressBase; ++} ++#endif ++ + inline uintptr_t ZAddress::offset(uintptr_t value) { + return value & ZAddressOffsetMask; + } + + inline uintptr_t ZAddress::good(uintptr_t value) { +- return offset(value) | ZAddressGoodMask; ++ return AARCH64_BASE(offset(value) | ZAddressGoodMask); + } + + inline uintptr_t ZAddress::good_or_null(uintptr_t value) { +@@ -111,27 +127,29 @@ inline uintptr_t ZAddress::good_or_null(uintptr_t value) { + } + + inline uintptr_t ZAddress::finalizable_good(uintptr_t value) { +- return offset(value) | ZAddressMetadataFinalizable | ZAddressGoodMask; ++ return AARCH64_BASE(offset(value) | ZAddressMetadataFinalizable | ZAddressGoodMask); + } + + inline uintptr_t ZAddress::marked(uintptr_t value) { +- return offset(value) | ZAddressMetadataMarked; ++ return AARCH64_BASE(offset(value) | ZAddressMetadataMarked); + } + + inline uintptr_t ZAddress::marked0(uintptr_t value) { +- return offset(value) | ZAddressMetadataMarked0; ++ return AARCH64_BASE(offset(value) | ZAddressMetadataMarked0); + } + + inline uintptr_t ZAddress::marked1(uintptr_t value) { +- return offset(value) | ZAddressMetadataMarked1; ++ return AARCH64_BASE(offset(value) | ZAddressMetadataMarked1); + } + + inline uintptr_t ZAddress::remapped(uintptr_t value) { +- return offset(value) | ZAddressMetadataRemapped; ++ return AARCH64_BASE(offset(value) | ZAddressMetadataRemapped); + } + + inline uintptr_t ZAddress::remapped_or_null(uintptr_t value) { + return is_null(value) ? 0 : remapped(value); + } + ++#undef AARCH64_BASE ++ + #endif // SHARE_GC_Z_ZADDRESS_INLINE_HPP +diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +index 9916178cc..e5a2ab09a 100644 +--- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp ++++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +@@ -55,6 +55,13 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { + return false; + } + ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Heal barriers ++ ZNMethod::nmethod_patch_barriers(nm); ++ } ++#endif ++ + // Heal oops + ZNMethod::nmethod_oops_barrier(nm); + +diff --git a/src/hotspot/share/gc/z/zGlobals.cpp b/src/hotspot/share/gc/z/zGlobals.cpp +index 28200e23b..cc82e9335 100644 +--- a/src/hotspot/share/gc/z/zGlobals.cpp ++++ b/src/hotspot/share/gc/z/zGlobals.cpp +@@ -49,6 +49,7 @@ static uint32_t* ZAddressCalculateBadMaskHighOrderBitsAddr() { + + uint32_t* ZAddressBadMaskHighOrderBitsAddr = ZAddressCalculateBadMaskHighOrderBitsAddr(); + ++uintptr_t ZAddressBase; + size_t ZAddressOffsetBits; + uintptr_t ZAddressOffsetMask; + size_t ZAddressOffsetMax; +diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp +index 300f6489a..783411a82 100644 +--- a/src/hotspot/share/gc/z/zGlobals.hpp ++++ b/src/hotspot/share/gc/z/zGlobals.hpp +@@ -102,6 +102,7 @@ extern uint32_t* ZAddressBadMaskHighOrderBitsAddr; + const int ZAddressBadMaskHighOrderBitsOffset = LITTLE_ENDIAN_ONLY(4) BIG_ENDIAN_ONLY(0); + + // Pointer part of address ++extern uintptr_t ZAddressBase; + extern size_t ZAddressOffsetBits; + const size_t ZAddressOffsetShift = 0; + extern uintptr_t ZAddressOffsetMask; +diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp +index edc80925d..8a527107a 100644 +--- a/src/hotspot/share/gc/z/zMark.cpp ++++ b/src/hotspot/share/gc/z/zMark.cpp +@@ -643,6 +643,13 @@ public: + } + + if (ZNMethod::is_armed(nm)) { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Heal barriers ++ ZNMethod::nmethod_patch_barriers(nm); ++ } ++#endif ++ + ZNMethod::nmethod_oops_do_inner(nm, _cl); + ZNMethod::disarm(nm); + } +diff --git a/src/hotspot/share/gc/z/zMarkStackEntry.hpp b/src/hotspot/share/gc/z/zMarkStackEntry.hpp +index fbb798cc8..ba66717c1 100644 +--- a/src/hotspot/share/gc/z/zMarkStackEntry.hpp ++++ b/src/hotspot/share/gc/z/zMarkStackEntry.hpp +@@ -78,7 +78,8 @@ private: + typedef ZBitField field_follow; + typedef ZBitField field_inc_live; + typedef ZBitField field_mark; +- typedef ZBitField field_object_address; ++ // Set ValueShift to 2 to avoid the top bits being cleared. ++ typedef ZBitField field_object_address; + typedef ZBitField field_partial_array_length; + typedef ZBitField field_partial_array_offset; + +diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp +index 71f510c2e..777ade668 100644 +--- a/src/hotspot/share/gc/z/zNMethod.cpp ++++ b/src/hotspot/share/gc/z/zNMethod.cpp +@@ -28,7 +28,10 @@ + #include "gc/shared/barrierSet.hpp" + #include "gc/shared/barrierSetNMethod.hpp" + #include "gc/shared/suspendibleThreadSet.hpp" ++#include "gc/z/zArray.inline.hpp" + #include "gc/z/zBarrier.inline.hpp" ++#include "gc/z/zBarrierSet.hpp" ++#include "gc/z/zBarrierSetAssembler.hpp" + #include "gc/z/zGlobals.hpp" + #include "gc/z/zLock.inline.hpp" + #include "gc/z/zNMethod.hpp" +@@ -54,12 +57,23 @@ static void set_gc_data(nmethod* nm, ZNMethodData* data) { + } + + void ZNMethod::attach_gc_data(nmethod* nm) { ++#ifdef AARCH64 ++ ZArray
barriers; ++#endif + GrowableArray immediate_oops; + bool non_immediate_oops = false; + +- // Find all oop relocations ++ // Find all barrier and oop relocations + RelocIterator iter(nm); + while (iter.next()) { ++#ifdef AARCH64 ++ if (UseTBI && iter.type() == relocInfo::barrier_type) { ++ // Barrier relocation ++ barrier_Relocation* const reloc = iter.barrier_reloc(); ++ barriers.push(reloc->addr()); ++ continue; ++ } ++#endif + if (iter.type() != relocInfo::oop_type) { + // Not an oop + continue; +@@ -92,9 +106,16 @@ void ZNMethod::attach_gc_data(nmethod* nm) { + ZNMethodDataOops* const new_oops = ZNMethodDataOops::create(immediate_oops, non_immediate_oops); + ZNMethodDataOops* const old_oops = data->swap_oops(new_oops); + ZNMethodDataOops::destroy(old_oops); ++ ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Attach barriers in GC data ++ data->swap_barriers(&barriers); ++ } ++#endif + } + +-ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) { ++ZReentrantLock* ZNMethod::lock_for_nmethod(const nmethod* nm) { + return gc_data(nm)->lock(); + } + +@@ -116,6 +137,20 @@ void ZNMethod::log_register(const nmethod* nm) { + oops->immediates_count(), + oops->has_non_immediates() ? "Yes" : "No"); + ++#ifdef AARCH64 ++ if (UseTBI) { ++ LogTarget(Trace, gc, nmethod, barrier) log_barriers; ++ if (log_barriers.is_enabled()) { ++ // Print nmethod barriers ++ ZLocker locker(lock_for_nmethod(nm)); ++ ZArrayIterator
iter(gc_data(nm)->barriers()); ++ for (address b; iter.next(&b);) { ++ log_barriers.print(" Barrier: " PTR_FORMAT, p2i(b)); ++ } ++ } ++ } ++#endif ++ + LogTarget(Trace, gc, nmethod, oops) log_oops; + if (!log_oops.is_enabled()) { + return; +@@ -164,6 +199,15 @@ void ZNMethod::register_nmethod(nmethod* nm) { + + log_register(nm); + ++#ifdef AARCH64 ++ if (UseTBI) { ++ ZLocker locker(lock_for_nmethod(nm)); ++ // Patch nmathod barriers ++ nmethod_patch_barriers(nm); ++ } ++#endif ++ ++ // Register nmethod + ZNMethodTable::register_nmethod(nm); + + // Disarm nmethod entry barrier +@@ -206,6 +250,16 @@ void ZNMethod::disarm(nmethod* nm) { + bs->disarm(nm); + } + ++#ifdef AARCH64 ++void ZNMethod::nmethod_patch_barriers(nmethod* nm) { ++ ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler(); ++ ZArrayIterator
iter(gc_data(nm)->barriers()); ++ for (address barrier; iter.next(&barrier);) { ++ bs_asm->patch_barrier_relocation(barrier); ++ } ++} ++#endif ++ + void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) { + ZLocker locker(ZNMethod::lock_for_nmethod(nm)); + if (!nm->is_alive()) { +@@ -329,6 +383,13 @@ public: + ZLocker locker(ZNMethod::lock_for_nmethod(nm)); + + if (ZNMethod::is_armed(nm)) { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Heal barriers ++ ZNMethod::nmethod_patch_barriers(nm); ++ } ++#endif ++ + // Heal oops and disarm + ZNMethod::nmethod_oops_barrier(nm); + ZNMethod::disarm(nm); +diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp +index 40ac93adb..bfe6af377 100644 +--- a/src/hotspot/share/gc/z/zNMethod.hpp ++++ b/src/hotspot/share/gc/z/zNMethod.hpp +@@ -48,6 +48,10 @@ public: + static bool is_armed(nmethod* nm); + static void disarm(nmethod* nm); + ++#ifdef AARCH64 ++ static void nmethod_patch_barriers(nmethod* nm); ++#endif ++ + static void nmethod_oops_do(nmethod* nm, OopClosure* cl); + static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl); + +@@ -57,7 +61,7 @@ public: + static void nmethods_do_end(); + static void nmethods_do(NMethodClosure* cl); + +- static ZReentrantLock* lock_for_nmethod(nmethod* nm); ++ static ZReentrantLock* lock_for_nmethod(const nmethod* nm); + + static void unlink(ZWorkers* workers, bool unloading_occurred); + static void purge(ZWorkers* workers); +diff --git a/src/hotspot/share/gc/z/zNMethodData.cpp b/src/hotspot/share/gc/z/zNMethodData.cpp +index c6efbfe66..70d06557f 100644 +--- a/src/hotspot/share/gc/z/zNMethodData.cpp ++++ b/src/hotspot/share/gc/z/zNMethodData.cpp +@@ -86,3 +86,15 @@ ZNMethodDataOops* ZNMethodData::swap_oops(ZNMethodDataOops* new_oops) { + _oops = new_oops; + return old_oops; + } ++ ++#ifdef AARCH64 ++const ZArray
* ZNMethodData::barriers() const { ++ assert(_lock.is_owned(), "Should be owned"); ++ return &_barriers; ++} ++ ++void ZNMethodData::swap_barriers(ZArray
* new_barriers) { ++ ZLocker locker(&_lock); ++ _barriers.swap(new_barriers); ++} ++#endif +\ No newline at end of file +diff --git a/src/hotspot/share/gc/z/zNMethodData.hpp b/src/hotspot/share/gc/z/zNMethodData.hpp +index 7afd60105..1013855c9 100644 +--- a/src/hotspot/share/gc/z/zNMethodData.hpp ++++ b/src/hotspot/share/gc/z/zNMethodData.hpp +@@ -24,6 +24,7 @@ + #ifndef SHARE_GC_Z_ZNMETHODDATA_HPP + #define SHARE_GC_Z_ZNMETHODDATA_HPP + ++#include "gc/z/zArray.hpp" + #include "gc/z/zAttachedArray.hpp" + #include "gc/z/zLock.hpp" + #include "memory/allocation.hpp" +@@ -57,6 +58,9 @@ class ZNMethodData : public CHeapObj { + private: + ZReentrantLock _lock; + ZNMethodDataOops* volatile _oops; ++#ifdef AARCH64 ++ ZArray
_barriers; ++#endif + + public: + ZNMethodData(); +@@ -66,6 +70,11 @@ public: + + ZNMethodDataOops* oops() const; + ZNMethodDataOops* swap_oops(ZNMethodDataOops* oops); ++ ++#ifdef AARCH64 ++ const ZArray
* barriers() const; ++ void swap_barriers(ZArray
* barriers); ++#endif + }; + + #endif // SHARE_GC_Z_ZNMETHODDATA_HPP +diff --git a/src/hotspot/share/gc/z/zPhysicalMemory.cpp b/src/hotspot/share/gc/z/zPhysicalMemory.cpp +index f2b80b02e..9df44677a 100644 +--- a/src/hotspot/share/gc/z/zPhysicalMemory.cpp ++++ b/src/hotspot/share/gc/z/zPhysicalMemory.cpp +@@ -277,13 +277,21 @@ void ZPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max + + void ZPhysicalMemoryManager::nmt_commit(uintptr_t offset, size_t size) const { + // From an NMT point of view we treat the first heap view (marked0) as committed ++#ifdef AARCH64 ++ const uintptr_t addr = UseTBI ? ZAddress::base(offset) : ZAddress::marked0(offset); ++#else // AARCH64 + const uintptr_t addr = ZAddress::marked0(offset); ++#endif // AARCH64 + MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC); + } + + void ZPhysicalMemoryManager::nmt_uncommit(uintptr_t offset, size_t size) const { + if (MemTracker::tracking_level() > NMT_minimal) { ++#ifdef AARCH64 ++ const uintptr_t addr = UseTBI ? ZAddress::base(offset) : ZAddress::marked0(offset); ++#else // AARCH64 + const uintptr_t addr = ZAddress::marked0(offset); ++#endif // AARCH64 + Tracker tracker(Tracker::uncommit); + tracker.record((address)addr, size); + } +@@ -291,6 +299,13 @@ void ZPhysicalMemoryManager::nmt_uncommit(uintptr_t offset, size_t size) const { + + void ZPhysicalMemoryManager::alloc(ZPhysicalMemory& pmem, size_t size) { + assert(is_aligned(size, ZGranuleSize), "Invalid size"); ++#ifdef AARCH64 ++ if (UseTBI) { ++ // We don't use _manager to alloc addresses. ++ pmem.add_segment(ZPhysicalMemorySegment(0, size, false /* committed */)); ++ return; ++ } ++#endif // AARCH64 + + // Allocate segments + while (size > 0) { +@@ -303,6 +318,13 @@ void ZPhysicalMemoryManager::alloc(ZPhysicalMemory& pmem, size_t size) { + } + + void ZPhysicalMemoryManager::free(const ZPhysicalMemory& pmem) { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // We don't use _manager to alloc addresses. ++ return; ++ } ++#endif // AARCH64 ++ + // Free segments + for (int i = 0; i < pmem.nsegments(); i++) { + const ZPhysicalMemorySegment& segment = pmem.segment(i); +@@ -381,6 +403,13 @@ void ZPhysicalMemoryManager::unmap_view(uintptr_t addr, size_t size) const { + } + + void ZPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const { ++#ifdef AARCH64 ++ if (UseTBI) { ++ pretouch_view(ZAddress::base(offset), size); ++ return; ++ } ++#endif // AARCH64 ++ + if (ZVerifyViews) { + // Pre-touch good view + pretouch_view(ZAddress::good(offset), size); +@@ -394,6 +423,13 @@ void ZPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const { + + void ZPhysicalMemoryManager::map(uintptr_t offset, const ZPhysicalMemory& pmem) const { + const size_t size = pmem.size(); ++#ifdef AARCH64 ++ if (UseTBI) { ++ map_view(ZAddress::base(offset), pmem); ++ nmt_commit(offset, size); ++ return; ++ } ++#endif // AARCH64 + + if (ZVerifyViews) { + // Map good view +@@ -410,6 +446,12 @@ void ZPhysicalMemoryManager::map(uintptr_t offset, const ZPhysicalMemory& pmem) + + void ZPhysicalMemoryManager::unmap(uintptr_t offset, size_t size) const { + nmt_uncommit(offset, size); ++#ifdef AARCH64 ++ if (UseTBI) { ++ unmap_view(ZAddress::base(offset), size); ++ return; ++ } ++#endif // AARCH64 + + if (ZVerifyViews) { + // Unmap good view +@@ -423,12 +465,24 @@ void ZPhysicalMemoryManager::unmap(uintptr_t offset, size_t size) const { + } + + void ZPhysicalMemoryManager::debug_map(uintptr_t offset, const ZPhysicalMemory& pmem) const { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Does nothing when using VA-masking ++ return; ++ } ++#endif // AARCH64 + // Map good view + assert(ZVerifyViews, "Should be enabled"); + map_view(ZAddress::good(offset), pmem); + } + + void ZPhysicalMemoryManager::debug_unmap(uintptr_t offset, size_t size) const { ++#ifdef AARCH64 ++ if (UseTBI) { ++ // Does nothing when using VA-masking ++ return; ++ } ++#endif // AARCH64 + // Unmap good view + assert(ZVerifyViews, "Should be enabled"); + unmap_view(ZAddress::good(offset), size); +diff --git a/src/hotspot/share/gc/z/zVirtualMemory.cpp b/src/hotspot/share/gc/z/zVirtualMemory.cpp +index 4f9c9bd4b..cb50ae803 100644 +--- a/src/hotspot/share/gc/z/zVirtualMemory.cpp ++++ b/src/hotspot/share/gc/z/zVirtualMemory.cpp +@@ -104,6 +104,21 @@ size_t ZVirtualMemoryManager::reserve_discontiguous(size_t size) { + bool ZVirtualMemoryManager::reserve_contiguous(uintptr_t start, size_t size) { + assert(is_aligned(size, ZGranuleSize), "Must be granule aligned"); + ++#ifdef AARCH64 ++ if (UseTBI) { ++ const uintptr_t addr = ZAddress::base(start); ++ if (!pd_reserve(addr, size)) { ++ return false; ++ } ++ nmt_reserve(addr, size); ++ ++ // Make the address range free ++ _manager.free(start, size); ++ ++ return true; ++ } ++#endif // AARCH64 ++ + // Reserve address views + const uintptr_t marked0 = ZAddress::marked0(start); + const uintptr_t marked1 = ZAddress::marked1(start); +diff --git a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp +index f8900a13b..356c6ca8b 100644 +--- a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp ++++ b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp +@@ -28,6 +28,7 @@ + #include "jfr/jni/jfrJavaSupport.hpp" + #include "jfr/writers/jfrStreamWriterHost.hpp" + #include "runtime/os.hpp" ++#include "runtime/globals.hpp" + + template + StreamWriterHost::StreamWriterHost(typename Adapter::StorageType* storage, Thread* thread) : +@@ -74,6 +75,9 @@ inline void StreamWriterHost::write_bytes(void* dest, const void* b + template + inline void StreamWriterHost::write_bytes(const u1* buf, intptr_t len) { + assert(len >= 0, "invariant"); ++#ifdef AARCH64 ++ buf = (const u1*) CLEAR_COLOR_BITS((uintptr_t) buf); ++#endif + while (len > 0) { + const unsigned int nBytes = len > INT_MAX ? INT_MAX : (unsigned int)len; + const ssize_t num_written = (ssize_t)os::write(_fd, buf, nBytes); +diff --git a/src/hotspot/share/prims/jni.cpp b/src/hotspot/share/prims/jni.cpp +index cd0115248..41e946563 100644 +--- a/src/hotspot/share/prims/jni.cpp ++++ b/src/hotspot/share/prims/jni.cpp +@@ -2831,6 +2831,9 @@ JNI_ENTRY(void*, jni_GetPrimitiveArrayCritical(JNIEnv *env, jarray array, jboole + type = TypeArrayKlass::cast(a->klass())->element_type(); + } + void* ret = arrayOop(a)->base(type); ++#ifdef AARCH64 ++ ret = (void*) CLEAR_COLOR_BITS((uintptr_t) ret); ++#endif // AARCH64 + HOTSPOT_JNI_GETPRIMITIVEARRAYCRITICAL_RETURN(ret); + return ret; + JNI_END +@@ -2893,6 +2896,9 @@ JNI_ENTRY(const jchar*, jni_GetStringCritical(JNIEnv *env, jstring string, jbool + } + if (isCopy != NULL) *isCopy = JNI_TRUE; + } ++#ifdef AARCH64 ++ ret = (jchar*) CLEAR_COLOR_BITS((uintptr_t) ret); ++#endif // AARCH64 + HOTSPOT_JNI_GETSTRINGCRITICAL_RETURN((uint16_t *) ret); + return ret; + JNI_END +diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java +index 52d9555e4..4a449d9d4 100644 +--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java ++++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java +@@ -51,12 +51,18 @@ class ZAddress { + return !is_weak_bad(value); + } + ++ static long base(long address) { ++ // ZAddressBase is a non-zero value only when flag UseTBI is on. ++ // So nothing changes when the arch is not Aarch64 or UseTBI is off. ++ return address | ZGlobals.ZAddressBase(); ++ } ++ + static long offset(Address address) { + return as_long(address) & ZGlobals.ZAddressOffsetMask(); + } + + static Address good(Address value) { +- return VM.getVM().getDebugger().newAddress(offset(value) | ZGlobals.ZAddressGoodMask()); ++ return VM.getVM().getDebugger().newAddress(base(offset(value) | ZGlobals.ZAddressGoodMask())); + } + + static Address good_or_null(Address value) { +@@ -69,9 +75,14 @@ class ZAddress { + + static boolean isIn(Address addr) { + long value = as_long(addr); +- if (!isPowerOf2(value & ~ZGlobals.ZAddressOffsetMask())) { ++ if (!isPowerOf2(value & ~ZGlobals.ZAddressOffsetMask() & ~ZGlobals.ZAddressBase())) { + return false; + } + return (value & (ZGlobals.ZAddressMetadataMask() & ~ZGlobals.ZAddressMetadataFinalizable())) != 0L; + } ++ ++ static Address clearTopByte(Address value) { ++ // (1L << 56) - 1 = 0x 00ff ffff ffff ffff ++ return VM.getVM().getDebugger().newAddress(as_long(value) & ((1L << 56) - 1)); ++ } + } +diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java +index c55370ef7..dd74913dd 100644 +--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java ++++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java +@@ -92,6 +92,9 @@ public class ZCollectedHeap extends CollectedHeap { + if (oopAddress == null) { + return null; + } ++ if (ZUtils.getUseTBI()) { ++ oopAddress = ZAddress.clearTopByte(oopAddress); ++ } + + return oopAddress.addOffsetToAsOopHandle(0); + } +diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java +index e01f7b832..41c9bb37b 100644 +--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java ++++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java +@@ -98,6 +98,10 @@ public class ZGlobals { + return instance().ZGlobalSeqNum(); + } + ++ public static long ZAddressBase() { ++ return instance().ZAddressBase(); ++ } ++ + public static long ZAddressOffsetMask() { + return instance().ZAddressOffsetMask(); + } +diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java +index 28d33f4d9..12054c92a 100644 +--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java ++++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java +@@ -34,6 +34,7 @@ import sun.jvm.hotspot.types.TypeDataBase; + class ZGlobalsForVMStructs extends VMObject { + private static AddressField ZGlobalPhaseField; + private static AddressField ZGlobalSeqNumField; ++ private static AddressField ZAddressBaseField; + private static AddressField ZAddressOffsetMaskField; + private static AddressField ZAddressMetadataMaskField; + private static AddressField ZAddressMetadataFinalizableField; +@@ -52,6 +53,7 @@ class ZGlobalsForVMStructs extends VMObject { + + ZGlobalPhaseField = type.getAddressField("_ZGlobalPhase"); + ZGlobalSeqNumField = type.getAddressField("_ZGlobalSeqNum"); ++ ZAddressBaseField = type.getAddressField("_ZAddressBase"); + ZAddressOffsetMaskField = type.getAddressField("_ZAddressOffsetMask"); + ZAddressMetadataMaskField = type.getAddressField("_ZAddressMetadataMask"); + ZAddressMetadataFinalizableField = type.getAddressField("_ZAddressMetadataFinalizable"); +@@ -74,6 +76,10 @@ class ZGlobalsForVMStructs extends VMObject { + return ZGlobalSeqNumField.getValue(addr).getJIntAt(0); + } + ++ long ZAddressBase() { ++ return ZAddressBaseField.getValue(addr).getJLongAt(0); ++ } ++ + long ZAddressOffsetMask() { + return ZAddressOffsetMaskField.getValue(addr).getJLongAt(0); + } +diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java +index 2029a71da..65930a006 100644 +--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java ++++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java +@@ -37,4 +37,9 @@ class ZUtils { + long adjusted = size + mask; + return adjusted & ~mask; + } ++ ++ static boolean getUseTBI() { ++ VM.Flag flag = VM.getVM().getCommandLineFlag("UseTBI"); ++ return flag == null ? false : flag.getBool(); ++ } + } +-- +2.17.1 + diff --git a/openjdk-17.spec b/openjdk-17.spec index 150e5265b8bf5ef1d64161d6f8eee0cebb6c7d53..80785feba22e80a4dad565649fe49c3053d339b5 100644 --- a/openjdk-17.spec +++ b/openjdk-17.spec @@ -885,7 +885,7 @@ Provides: java-src%{?1} = %{epoch}:%{version}-%{release} Name: java-%{javaver}-%{origin} Version: %{newjavaver}.%{buildver} -Release: 2 +Release: 3 # java-1.5.0-ibm from jpackage.org set Epoch to 1 for unknown reasons # and this change was brought into RHEL-4. java-1.5.0-ibm packages @@ -971,6 +971,7 @@ Patch16: Clean-up-JDK17-codeDEX-fix-Non-static-numa_node_dist.patch # 17.0.4 Patch17: 8290705_fix_StringConcat_validate_mem_flow_asserts_with_unexpected_userStoreI.patch +Patch18: Apply-TBI-to-ZGC-of-JDK17.patch BuildRequires: autoconf BuildRequires: automake @@ -980,6 +981,7 @@ BuildRequires: cups-devel BuildRequires: desktop-file-utils # elfutils only are OK for build without AOT BuildRequires: elfutils-devel +BuildRequires: elfutils-extra BuildRequires: fontconfig-devel BuildRequires: freetype-devel BuildRequires: giflib-devel @@ -1204,6 +1206,7 @@ pushd %{top_level_dir_name} %patch15 -p1 %patch16 -p1 %patch17 -p1 +%patch18 -p1 popd # openjdk # Extract systemtap tapsets @@ -1755,6 +1758,9 @@ cjc.mainProgram(arg) %changelog +* Thu Sep 15 2022 kuenking111 - 1:17.0.4.8-0.3 +- add Apply-TBI-to-ZGC-of-JDK17.patch + * Fri Aug 5 2022 kuenking111 - 1:17.0.4.8-0.2 - del pr3183-rh1340845-support_system_crypto_policy.patch - del rh1648249-add_commented_out_nss_cfg_provider_to_java_security.patch