From 70a403da7b9fe4cd78d030fdd154369e049c679d Mon Sep 17 00:00:00 2001 From: songliyang Date: Tue, 4 Jun 2024 09:26:34 +0800 Subject: [PATCH 1/2] fix loongarch vendor error and changelog dates error --- openjdk-11.spec | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/openjdk-11.spec b/openjdk-11.spec index fbb5cae..0be863b 100644 --- a/openjdk-11.spec +++ b/openjdk-11.spec @@ -131,7 +131,12 @@ # and this it is better to change it here, on single place %global buildjdkver %{majorver} +%ifnarch loongarch64 ppc64le %global vendor_version_string Bisheng +%endif +%ifarch loongarch64 +%global vendor_version_string Loongson +%endif # Define IcedTea version used for SystemTap tapsets and desktop file %global icedteaver 3.15.0 @@ -757,7 +762,7 @@ Provides: java-src%{?1} = %{epoch}:%{version}-%{release} Name: java-%{javaver}-%{origin} Version: %{newjavaver}.%{buildver} -Release: 2 +Release: 3 # java-1.5.0-ibm from jpackage.org set Epoch to 1 for unknown reasons # and this change was brought into RHEL-4. java-1.5.0-ibm packages # also included the epoch in their virtual provides. This created a @@ -1302,9 +1307,12 @@ bash ../configure \ --with-version-build=%{buildver} \ --with-version-pre="" \ --with-version-opt="" \ -%ifnarch loongarch64 ppc64le --with-vendor-version-string="%{vendor_version_string}" \ +%ifnarch loongarch64 ppc64le --with-vendor-name="Bisheng" \ +%endif +%ifarch loongarch64 + --with-vendor-name="Loongson" \ %endif --with-vendor-url="https://openeuler.org/" \ --with-vendor-bug-url="https://gitee.com/src-openeuler/openjdk-11/issues/" \ @@ -1725,10 +1733,14 @@ cjc.mainProgram(arg) %changelog -* Wed May 06 2024 zhangxianting - 1:11.0.23.9-1 +* Mon Jun 03 2024 songliyang - 1:11.0.23.9-3 +- fix loongarch vendor error +- fix changelog error + +* Wed May 08 2024 zhangxianting - 1:11.0.23.9-2 - recompress the source0 -* Mon Apr 29 2024 huangjie - 1:11.0.23.9-0 +* Mon Apr 29 2024 huangjie - 1:11.0.23.9-1 - modified delete_expired_certificates.patch * Thu Apr 18 2024 huangjie - 1:11.0.23.9-0 @@ -1762,15 +1774,15 @@ cjc.mainProgram(arg) * Thu Aug 17 2023 misaka00251 - 1:11.0.20.8-2 - Add riscv64 support (based on bishengjdk riscv branch) -* Wed Aug 2023 noah - 1:11.0.20.8-1 +* Wed Aug 16 2023 noah - 1:11.0.20.8-1 - fix CPUBench kmeans random fails -* Wed Jul 2023 DXwangg - 1:11.0.20.8-0 +* Tue Jul 25 2023 DXwangg - 1:11.0.20.8-0 - update to 11.0.20+8(GA) - modified delete_expired_certificates.patch -* Thu Apr 2023 DXwangg - 1:11.0.19.7-0 +* Sun Apr 23 2023 DXwangg - 1:11.0.19.7-0 - update to 11.0.19+7(GA) - deleted 8225648-TESTBUG-java-lang-annotation-loaderLeak-Main.patch - modified Add-KAE-implementation.patch -- Gitee From 4a98be116fef5c3155ab0fbc08ec26188c3d23c9 Mon Sep 17 00:00:00 2001 From: neu-mobi Date: Sat, 15 Jun 2024 16:55:04 +0800 Subject: [PATCH 2/2] Collate patches and merge patches related to ZGC --- ...dynamic-base-address-for-mark-stack-.patch | 184 -- ...number-of-GC-workers-based-on-heap-s.patch | 87 - ...k-out-C2-matching-rules-into-separat.patch | 906 ----- ...-AArch64-Initial-SVE-backend-support.patch | 4 +- ...rce-memory-ordering-in-segmented-bit.patch | 101 - ...Map-accessors-more-memory-ordering-f.patch | 162 - ...load-for-Reference.get-can-be-conver.patch | 472 --- ...constructing-String-with-duplicate-c.patch | 113 + ... ZGC-AArch64-Optimizations-and-Fixes.patch | 2916 ++++++++++++----- ...4-fix-not-using-load-store-Pre-index.patch | 58 - ...x-system-call-number-of-memfd_create.patch | 28 - ...using-zr-register-avoid-sigill-in-Ma.patch | 106 - ...reuse-entries-of-ResolvedMethodTable.patch | 35 - add-missing-inline.patch | 12 - openjdk-11.spec | 33 +- 15 files changed, 2271 insertions(+), 2946 deletions(-) delete mode 100644 8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch delete mode 100644 8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch delete mode 100644 8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch delete mode 100644 8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch delete mode 100644 8233073-Make-BitMap-accessors-more-memory-ordering-f.patch delete mode 100644 8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch create mode 100644 Cache-byte-when-constructing-String-with-duplicate-c.patch rename ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch => ZGC-AArch64-Optimizations-and-Fixes.patch (76%) delete mode 100644 ZGC-aarch64-fix-not-using-load-store-Pre-index.patch delete mode 100644 ZGC-aarch64-fix-system-call-number-of-memfd_create.patch delete mode 100644 ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch delete mode 100644 ZGC-reuse-entries-of-ResolvedMethodTable.patch delete mode 100644 add-missing-inline.patch diff --git a/8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch b/8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch deleted file mode 100644 index dfad78c..0000000 --- a/8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch +++ /dev/null @@ -1,184 +0,0 @@ -From 476ec6be3f75c70c50bd1552c624abca098ddba2 Mon Sep 17 00:00:00 2001 -Date: Wed, 18 Mar 2020 10:25:06 +0000 -Subject: [PATCH] 8209375: ZGC: Use dynamic base address for mark stack space - -Summary: : -LLT: jdk11u/test/hotspot/jtreg/vmTestbase/gc/gctests/SoftReference/soft004/soft004.java -Bug url: https://bugs.openjdk.java.net/browse/JDK-8209375 ---- - src/hotspot/share/gc/z/zGlobals.hpp | 7 +--- - src/hotspot/share/gc/z/zMarkStack.cpp | 74 +++++++++++++++-------------------- - src/hotspot/share/gc/z/zMarkStack.hpp | 1 + - src/hotspot/share/gc/z/z_globals.hpp | 6 +-- - 4 files changed, 38 insertions(+), 50 deletions(-) - -diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp -index 080ea5c0e..0f9e9dcb4 100644 ---- a/src/hotspot/share/gc/z/zGlobals.hpp -+++ b/src/hotspot/share/gc/z/zGlobals.hpp -@@ -117,11 +117,8 @@ extern uintptr_t ZAddressWeakBadMask; - // Marked state - extern uintptr_t ZAddressMetadataMarked; - --// Address space for mark stack allocations --const size_t ZMarkStackSpaceSizeShift = 40; // 1TB --const size_t ZMarkStackSpaceSize = (size_t)1 << ZMarkStackSpaceSizeShift; --const uintptr_t ZMarkStackSpaceStart = ZAddressSpaceEnd + ZMarkStackSpaceSize; --const uintptr_t ZMarkStackSpaceEnd = ZMarkStackSpaceStart + ZMarkStackSpaceSize; -+// Mark stack space -+extern uintptr_t ZMarkStackSpaceStart; - const size_t ZMarkStackSpaceExpandSize = (size_t)1 << 25; // 32M - - // Mark stack and magazine sizes -diff --git a/src/hotspot/share/gc/z/zMarkStack.cpp b/src/hotspot/share/gc/z/zMarkStack.cpp -index 52fe51ece..9cc768956 100644 ---- a/src/hotspot/share/gc/z/zMarkStack.cpp -+++ b/src/hotspot/share/gc/z/zMarkStack.cpp -@@ -28,58 +28,44 @@ - #include "gc/z/zMarkStack.inline.hpp" - #include "logging/log.hpp" - #include "runtime/atomic.hpp" -+#include "runtime/os.hpp" - #include "utilities/debug.hpp" - --#include --#include -+uintptr_t ZMarkStackSpaceStart; - - ZMarkStackSpace::ZMarkStackSpace() : - _expand_lock(), -+ _start(0), - _top(0), - _end(0) { -- assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small"); -- assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large"); -- -+ assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small"); - // Reserve address space -- const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize, -- PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); -- if (res != (void*)ZMarkStackSpaceStart) { -- log_error(gc, marking)("Failed to reserve address space for marking stacks"); -+ const size_t size = ZMarkStackSpaceLimit; -+ const size_t alignment = (size_t)os::vm_allocation_granularity(); -+ const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC); -+ if (addr == 0) { -+ log_error(gc, marking)("Failed to reserve address space for mark stacks"); - return; - } - - // Successfully initialized -- _top = _end = ZMarkStackSpaceStart; --} -+ _start = _top = _end = addr; - --bool ZMarkStackSpace::is_initialized() const { -- return _top != 0; -+ // Register mark stack space start -+ ZMarkStackSpaceStart = _start; - } - --bool ZMarkStackSpace::expand() { -- const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax; -- if (_end + ZMarkStackSpaceExpandSize > max) { -- // Expansion limit reached -- return false; -- } -- -- void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize, -- PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0); -- if (res == MAP_FAILED) { -- ZErrno err; -- log_error(gc, marking)("Failed to map memory for marking stacks (%s)", err.to_string()); -- return false; -- } -- -- return true; -+bool ZMarkStackSpace::is_initialized() const { -+ return _start != 0; - } - - uintptr_t ZMarkStackSpace::alloc_space(size_t size) { -- uintptr_t top = _top; -+ uintptr_t top = Atomic::load(&_top); - - for (;;) { -+ const uintptr_t end = Atomic::load(&_end); - const uintptr_t new_top = top + size; -- if (new_top > _end) { -+ if (new_top > end) { - // Not enough space left - return 0; - } -@@ -104,24 +90,28 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { - return addr; - } - -- // Expand stack space -- if (!expand()) { -- // We currently can't handle the situation where we -- // are running out of mark stack space. -- fatal("Mark stack overflow (allocated " SIZE_FORMAT "M, size " SIZE_FORMAT "M, max " SIZE_FORMAT "M)," -- " use -XX:ZMarkStacksMax=? to increase this limit", -- (_end - ZMarkStackSpaceStart) / M, size / M, ZMarkStacksMax / M); -- return 0; -+ // Check expansion limit -+ const size_t expand_size = ZMarkStackSpaceExpandSize; -+ const size_t old_size = _end - _start; -+ const size_t new_size = old_size + expand_size; -+ if (new_size > ZMarkStackSpaceLimit) { -+ // Expansion limit reached. This is a fatal error since we -+ // currently can't recover from running out of mark stack space. -+ fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit= to increase the " -+ "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.", -+ ZMarkStackSpaceLimit / M); - } - - log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", -- (_end - ZMarkStackSpaceStart) / M, -- (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M); -+ old_size / M, new_size / M); -+ -+ // Expand -+ os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space"); - - // Increment top before end to make sure another - // thread can't steal out newly expanded space. - addr = Atomic::add(size, &_top) - size; -- _end += ZMarkStackSpaceExpandSize; -+ Atomic::add(expand_size, &_end); - - return addr; - } -diff --git a/src/hotspot/share/gc/z/zMarkStack.hpp b/src/hotspot/share/gc/z/zMarkStack.hpp -index b68b9faa3..12f3e4eca 100644 ---- a/src/hotspot/share/gc/z/zMarkStack.hpp -+++ b/src/hotspot/share/gc/z/zMarkStack.hpp -@@ -76,6 +76,7 @@ typedef ZStackList ZMarkStackMagazineList; - class ZMarkStackSpace { - private: - ZLock _expand_lock; -+ uintptr_t _start; - volatile uintptr_t _top; - volatile uintptr_t _end; - -diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp -index 9e0f8985b..8cee59be7 100644 ---- a/src/hotspot/share/gc/z/z_globals.hpp -+++ b/src/hotspot/share/gc/z/z_globals.hpp -@@ -53,9 +53,9 @@ - "Allow Java threads to stall and wait for GC to complete " \ - "instead of immediately throwing an OutOfMemoryError") \ - \ -- product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G), \ -- "Maximum number of bytes allocated for marking stacks") \ -- range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G)) \ -+ product(size_t, ZMarkStackSpaceLimit, 8*G, \ -+ "Maximum number of bytes allocated for mark stacks") \ -+ range(32*M, 1024*G) \ - \ - product(uint, ZCollectionInterval, 0, \ - "Force GC at a fixed time interval (in seconds)") \ --- -2.12.3 - diff --git a/8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch b/8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch deleted file mode 100644 index 6ba8ad5..0000000 --- a/8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch +++ /dev/null @@ -1,87 +0,0 @@ -From 7ca249ae82c6b6c60c524781806f9d12ef3f8f98 Mon Sep 17 00:00:00 2001 -Date: Mon, 16 Mar 2020 16:24:43 +0800 -Subject: [PATCH] 8209894: ZGC: Cap number of GC workers based on heap size - -Summary: : -LLT: jdk11u/test/hotspot/jtreg/vmTestbase/nsk/jdi/ObjectReference/disableCollection/disablecollection002/TestDescription.java -Bug url: https://bugs.openjdk.java.net/browse/JDK-8209894 ---- - src/hotspot/share/gc/z/zWorkers.cpp | 23 ++++++++++++++++++----- - src/hotspot/share/gc/z/zWorkers.hpp | 4 +--- - 2 files changed, 19 insertions(+), 8 deletions(-) - -diff --git a/src/hotspot/share/gc/z/zWorkers.cpp b/src/hotspot/share/gc/z/zWorkers.cpp -index 0686ec7af..6a0c2561d 100644 ---- a/src/hotspot/share/gc/z/zWorkers.cpp -+++ b/src/hotspot/share/gc/z/zWorkers.cpp -@@ -1,5 +1,5 @@ - /* -- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. -+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it -@@ -22,14 +22,27 @@ - */ - - #include "precompiled.hpp" -+#include "gc/z/zGlobals.hpp" - #include "gc/z/zTask.hpp" - #include "gc/z/zWorkers.inline.hpp" - #include "runtime/os.hpp" - #include "runtime/mutexLocker.hpp" - #include "runtime/safepoint.hpp" - --uint ZWorkers::calculate_ncpus(double share_in_percent) { -- return ceil(os::initial_active_processor_count() * share_in_percent / 100.0); -+static uint calculate_nworkers_based_on_ncpus(double cpu_share_in_percent) { -+ return ceil(os::initial_active_processor_count() * cpu_share_in_percent / 100.0); -+} -+ -+static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) { -+ const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall; -+ return MAX2(nworkers, 1); -+} -+ -+static uint calculate_nworkers(double cpu_share_in_percent) { -+ // Cap number of workers so that we never use more than 10% of the max heap -+ // for the reserve. This is useful when using small heaps on large machines. -+ return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent), -+ calculate_nworkers_based_on_heap_size(10.0)); - } - - uint ZWorkers::calculate_nparallel() { -@@ -38,7 +51,7 @@ uint ZWorkers::calculate_nparallel() { - // close to the number of processors tends to lead to over-provisioning and - // scheduling latency issues. Using 60% of the active processors appears to - // be a fairly good balance. -- return calculate_ncpus(60.0); -+ return calculate_nworkers(60.0); - } - - uint ZWorkers::calculate_nconcurrent() { -@@ -48,7 +61,7 @@ uint ZWorkers::calculate_nconcurrent() { - // throughput, while using too few threads will prolong the GC-cycle and - // we then risk being out-run by the application. Using 12.5% of the active - // processors appears to be a fairly good balance. -- return calculate_ncpus(12.5); -+ return calculate_nworkers(12.5); - } - - class ZWorkersWarmupTask : public ZTask { -diff --git a/src/hotspot/share/gc/z/zWorkers.hpp b/src/hotspot/share/gc/z/zWorkers.hpp -index 36a3c61fd..6ce09c447 100644 ---- a/src/hotspot/share/gc/z/zWorkers.hpp -+++ b/src/hotspot/share/gc/z/zWorkers.hpp -@@ -34,8 +34,6 @@ private: - bool _boost; - WorkGang _workers; - -- static uint calculate_ncpus(double share_in_percent); -- - void run(ZTask* task, uint nworkers); - - public: --- -2.12.3 - diff --git a/8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch b/8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch deleted file mode 100644 index b88b4bf..0000000 --- a/8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch +++ /dev/null @@ -1,906 +0,0 @@ -diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk -index 687896251..a39640526 100644 ---- a/make/hotspot/gensrc/GensrcAdlc.gmk -+++ b/make/hotspot/gensrc/GensrcAdlc.gmk -@@ -140,6 +140,12 @@ ifeq ($(call check-jvm-feature, compiler2), true) - $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \ - ))) - -+ ifeq ($(call check-jvm-feature, zgc), true) -+ AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ -+ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \ -+ ))) -+ endif -+ - ifeq ($(call check-jvm-feature, shenandoahgc), true) - AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ - $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \ -diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad -index 29f81face..ab578476a 100644 ---- a/src/hotspot/cpu/aarch64/aarch64.ad -+++ b/src/hotspot/cpu/aarch64/aarch64.ad -@@ -1128,13 +1128,6 @@ definitions %{ - int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST); - %} - --source_hpp %{ -- --#include "gc/z/c2/zBarrierSetC2.hpp" --#include "gc/z/zThreadLocalData.hpp" -- --%} -- - //----------SOURCE BLOCK------------------------------------------------------- - // This is a block of C++ code which provides values, functions, and - // definitions necessary in the rest of the architecture description -@@ -18110,243 +18103,6 @@ instruct vpopcount2I(vecD dst, vecD src) %{ - ins_pipe(pipe_class_default); - %} - --source %{ -- --static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { -- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); -- __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -- __ andr(tmp, tmp, ref); -- __ cbnz(tmp, *stub->entry()); -- __ bind(*stub->continuation()); --} -- --static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { -- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); -- __ b(*stub->entry()); -- __ bind(*stub->continuation()); --} -- --%} -- --// Load Pointer --instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) --%{ -- match(Set dst (LoadP mem)); -- predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); -- effect(TEMP dst, KILL cr); -- -- ins_cost(4 * INSN_COST); -- -- format %{ "ldr $dst, $mem" %} -- -- ins_encode %{ -- const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); -- __ ldr($dst$$Register, ref_addr); -- if (barrier_data() != ZLoadBarrierElided) { -- z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); -- } -- %} -- -- ins_pipe(iload_reg_mem); --%} -- --// Load Weak Pointer --instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) --%{ -- match(Set dst (LoadP mem)); -- predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); -- effect(TEMP dst, KILL cr); -- -- ins_cost(4 * INSN_COST); -- -- format %{ "ldr $dst, $mem" %} -- -- ins_encode %{ -- const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); -- __ ldr($dst$$Register, ref_addr); -- z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); -- %} -- -- ins_pipe(iload_reg_mem); --%} -- --// Load Pointer Volatile --instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) --%{ -- match(Set dst (LoadP mem)); -- predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); -- effect(TEMP dst, KILL cr); -- -- ins_cost(VOLATILE_REF_COST); -- -- format %{ "ldar $dst, $mem\t" %} -- -- ins_encode %{ -- __ ldar($dst$$Register, $mem$$Register); -- if (barrier_data() != ZLoadBarrierElided) { -- z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); -- } -- %} -- -- ins_pipe(pipe_serial); --%} -- --instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -- match(Set res (CompareAndSwapP mem (Binary oldval newval))); -- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -- predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(KILL cr, TEMP_DEF res); -- -- ins_cost(2 * VOLATILE_REF_COST); -- -- format %{ "cmpxchg $mem, $oldval, $newval\n\t" -- "cset $res, EQ" %} -- -- ins_encode %{ -- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- false /* acquire */, true /* release */, false /* weak */, rscratch2); -- __ cset($res$$Register, Assembler::EQ); -- if (barrier_data() != ZLoadBarrierElided) { -- Label good; -- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -- __ andr(rscratch1, rscratch1, rscratch2); -- __ cbz(rscratch1, good); -- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- false /* acquire */, true /* release */, false /* weak */, rscratch2); -- __ cset($res$$Register, Assembler::EQ); -- __ bind(good); -- } -- %} -- -- ins_pipe(pipe_slow); --%} -- --instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -- match(Set res (CompareAndSwapP mem (Binary oldval newval))); -- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -- predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); -- effect(KILL cr, TEMP_DEF res); -- -- ins_cost(2 * VOLATILE_REF_COST); -- -- format %{ "cmpxchg $mem, $oldval, $newval\n\t" -- "cset $res, EQ" %} -- -- ins_encode %{ -- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- true /* acquire */, true /* release */, false /* weak */, rscratch2); -- __ cset($res$$Register, Assembler::EQ); -- if (barrier_data() != ZLoadBarrierElided) { -- Label good; -- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -- __ andr(rscratch1, rscratch1, rscratch2); -- __ cbz(rscratch1, good); -- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- true /* acquire */, true /* release */, false /* weak */, rscratch2); -- __ cset($res$$Register, Assembler::EQ); -- __ bind(good); -- } -- %} -- -- ins_pipe(pipe_slow); --%} -- --instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -- match(Set res (CompareAndExchangeP mem (Binary oldval newval))); -- predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(TEMP_DEF res, KILL cr); -- -- ins_cost(2 * VOLATILE_REF_COST); -- -- format %{ "cmpxchg $res = $mem, $oldval, $newval" %} -- -- ins_encode %{ -- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- false /* acquire */, true /* release */, false /* weak */, $res$$Register); -- if (barrier_data() != ZLoadBarrierElided) { -- Label good; -- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -- __ andr(rscratch1, rscratch1, $res$$Register); -- __ cbz(rscratch1, good); -- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- false /* acquire */, true /* release */, false /* weak */, $res$$Register); -- __ bind(good); -- } -- %} -- -- ins_pipe(pipe_slow); --%} -- --instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -- match(Set res (CompareAndExchangeP mem (Binary oldval newval))); -- predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(TEMP_DEF res, KILL cr); -- -- ins_cost(2 * VOLATILE_REF_COST); -- -- format %{ "cmpxchg $res = $mem, $oldval, $newval" %} -- -- ins_encode %{ -- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- true /* acquire */, true /* release */, false /* weak */, $res$$Register); -- if (barrier_data() != ZLoadBarrierElided) { -- Label good; -- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -- __ andr(rscratch1, rscratch1, $res$$Register); -- __ cbz(rscratch1, good); -- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); -- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -- true /* acquire */, true /* release */, false /* weak */, $res$$Register); -- __ bind(good); -- } -- %} -- -- ins_pipe(pipe_slow); --%} -- --instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ -- match(Set prev (GetAndSetP mem newv)); -- predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(TEMP_DEF prev, KILL cr); -- -- ins_cost(2 * VOLATILE_REF_COST); -- -- format %{ "atomic_xchg $prev, $newv, [$mem]" %} -- -- ins_encode %{ -- __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); -- if (barrier_data() != ZLoadBarrierElided) { -- z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); -- } -- %} -- -- ins_pipe(pipe_serial); --%} -- --instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ -- match(Set prev (GetAndSetP mem newv)); -- predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); -- effect(TEMP_DEF prev, KILL cr); -- -- ins_cost(VOLATILE_REF_COST); -- -- format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} -- -- ins_encode %{ -- __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); -- if (barrier_data() != ZLoadBarrierElided) { -- z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); -- } -- %} -- ins_pipe(pipe_serial); --%} - - //----------PEEPHOLE RULES----------------------------------------------------- - // These must follow all instruction definitions as they use the names -diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad -new file mode 100644 -index 000000000..50cc6f924 ---- /dev/null -+++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad -@@ -0,0 +1,268 @@ -+// -+// Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. -+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -+// -+// This code is free software; you can redistribute it and/or modify it -+// under the terms of the GNU General Public License version 2 only, as -+// published by the Free Software Foundation. -+// -+// This code is distributed in the hope that it will be useful, but WITHOUT -+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -+// version 2 for more details (a copy is included in the LICENSE file that -+// accompanied this code). -+// -+// You should have received a copy of the GNU General Public License version -+// 2 along with this work; if not, write to the Free Software Foundation, -+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -+// -+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA -+// or visit www.oracle.com if you need additional information or have any -+// questions. -+// -+ -+source_hpp %{ -+ -+#include "gc/z/c2/zBarrierSetC2.hpp" -+#include "gc/z/zThreadLocalData.hpp" -+ -+%} -+ -+source %{ -+ -+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); -+ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(tmp, tmp, ref); -+ __ cbnz(tmp, *stub->entry()); -+ __ bind(*stub->continuation()); -+} -+ -+static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); -+ __ b(*stub->entry()); -+ __ bind(*stub->continuation()); -+} -+ -+%} -+ -+// Load Pointer -+instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) -+%{ -+ match(Set dst (LoadP mem)); -+ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); -+ effect(TEMP dst, KILL cr); -+ -+ ins_cost(4 * INSN_COST); -+ -+ format %{ "ldr $dst, $mem" %} -+ -+ ins_encode %{ -+ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); -+ __ ldr($dst$$Register, ref_addr); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); -+ } -+ %} -+ -+ ins_pipe(iload_reg_mem); -+%} -+ -+// Load Weak Pointer -+instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) -+%{ -+ match(Set dst (LoadP mem)); -+ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); -+ effect(TEMP dst, KILL cr); -+ -+ ins_cost(4 * INSN_COST); -+ -+ format %{ "ldr $dst, $mem" %} -+ -+ ins_encode %{ -+ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); -+ __ ldr($dst$$Register, ref_addr); -+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); -+ %} -+ -+ ins_pipe(iload_reg_mem); -+%} -+ -+// Load Pointer Volatile -+instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) -+%{ -+ match(Set dst (LoadP mem)); -+ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP dst, KILL cr); -+ -+ ins_cost(VOLATILE_REF_COST); -+ -+ format %{ "ldar $dst, $mem\t" %} -+ -+ ins_encode %{ -+ __ ldar($dst$$Register, $mem$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); -+ } -+ %} -+ -+ ins_pipe(pipe_serial); -+%} -+ -+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndSwapP mem (Binary oldval newval))); -+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr, TEMP_DEF res); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "cmpxchg $mem, $oldval, $newval\n\t" -+ "cset $res, EQ" %} -+ -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, rscratch2); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ __ bind(good); -+ } -+ %} -+ -+ ins_pipe(pipe_slow); -+%} -+ -+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndSwapP mem (Binary oldval newval))); -+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); -+ effect(KILL cr, TEMP_DEF res); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "cmpxchg $mem, $oldval, $newval\n\t" -+ "cset $res, EQ" %} -+ -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, rscratch2); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ __ bind(good); -+ } -+ %} -+ -+ ins_pipe(pipe_slow); -+%} -+ -+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); -+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP_DEF res, KILL cr); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} -+ -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, $res$$Register); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ __ bind(good); -+ } -+ %} -+ -+ ins_pipe(pipe_slow); -+%} -+ -+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); -+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP_DEF res, KILL cr); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} -+ -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, $res$$Register); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ __ bind(good); -+ } -+ %} -+ -+ ins_pipe(pipe_slow); -+%} -+ -+instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ -+ match(Set prev (GetAndSetP mem newv)); -+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP_DEF prev, KILL cr); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "atomic_xchg $prev, $newv, [$mem]" %} -+ -+ ins_encode %{ -+ __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); -+ } -+ %} -+ -+ ins_pipe(pipe_serial); -+%} -+ -+instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ -+ match(Set prev (GetAndSetP mem newv)); -+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); -+ effect(TEMP_DEF prev, KILL cr); -+ -+ ins_cost(VOLATILE_REF_COST); -+ -+ format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} -+ -+ ins_encode %{ -+ __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); -+ } -+ %} -+ ins_pipe(pipe_serial); -+%} -+ -diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad -new file mode 100644 -index 000000000..38c2e926b ---- /dev/null -+++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad -@@ -0,0 +1,168 @@ -+// -+// Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. -+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -+// -+// This code is free software; you can redistribute it and/or modify it -+// under the terms of the GNU General Public License version 2 only, as -+// published by the Free Software Foundation. -+// -+// This code is distributed in the hope that it will be useful, but WITHOUT -+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -+// version 2 for more details (a copy is included in the LICENSE file that -+// accompanied this code). -+// -+// You should have received a copy of the GNU General Public License version -+// 2 along with this work; if not, write to the Free Software Foundation, -+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -+// -+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA -+// or visit www.oracle.com if you need additional information or have any -+// questions. -+// -+ -+source_hpp %{ -+ -+#include "gc/z/c2/zBarrierSetC2.hpp" -+#include "gc/z/zThreadLocalData.hpp" -+ -+%} -+ -+source %{ -+ -+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); -+ __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -+ __ jcc(Assembler::notZero, *stub->entry()); -+ __ bind(*stub->continuation()); -+} -+ -+static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); -+ __ jmp(*stub->entry()); -+ __ bind(*stub->continuation()); -+} -+ -+%} -+ -+// Load Pointer -+instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) -+%{ -+ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); -+ match(Set dst (LoadP mem)); -+ effect(KILL cr, TEMP dst); -+ -+ ins_cost(125); -+ -+ format %{ "movq $dst, $mem" %} -+ -+ ins_encode %{ -+ __ movptr($dst$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); -+ } -+ %} -+ -+ ins_pipe(ialu_reg_mem); -+%} -+ -+// Load Weak Pointer -+instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) -+%{ -+ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); -+ match(Set dst (LoadP mem)); -+ effect(KILL cr, TEMP dst); -+ -+ ins_cost(125); -+ -+ format %{ "movq $dst, $mem" %} -+ -+ ins_encode %{ -+ __ movptr($dst$$Register, $mem$$Address); -+ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); -+ %} -+ -+ ins_pipe(ialu_reg_mem); -+%} -+ -+instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ -+ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); -+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr, TEMP tmp); -+ -+ format %{ "lock\n\t" -+ "cmpxchgq $newval, $mem" %} -+ -+ ins_encode %{ -+ if (barrier_data() != ZLoadBarrierElided) { -+ __ movptr($tmp$$Register, $oldval$$Register); -+ } -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -+ __ jcc(Assembler::zero, good); -+ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); -+ __ movptr($oldval$$Register, $tmp$$Register); -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ __ bind(good); -+ } -+ %} -+ -+ ins_pipe(pipe_cmpxchg); -+%} -+ -+instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ -+ match(Set res (CompareAndSwapP mem (Binary oldval newval))); -+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr, KILL oldval, TEMP tmp); -+ -+ format %{ "lock\n\t" -+ "cmpxchgq $newval, $mem\n\t" -+ "sete $res\n\t" -+ "movzbl $res, $res" %} -+ -+ ins_encode %{ -+ if (barrier_data() != ZLoadBarrierElided) { -+ __ movptr($tmp$$Register, $oldval$$Register); -+ } -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -+ __ jcc(Assembler::zero, good); -+ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); -+ __ movptr($oldval$$Register, $tmp$$Register); -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ __ bind(good); -+ __ cmpptr($tmp$$Register, $oldval$$Register); -+ } -+ __ setb(Assembler::equal, $res$$Register); -+ __ movzbl($res$$Register, $res$$Register); -+ %} -+ -+ ins_pipe(pipe_cmpxchg); -+%} -+ -+instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ -+ match(Set newval (GetAndSetP mem newval)); -+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr); -+ -+ format %{ "xchgq $newval, $mem" %} -+ -+ ins_encode %{ -+ __ xchgptr($newval$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); -+ } -+ %} -+ -+ ins_pipe(pipe_cmpxchg); -+%} -+ -diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad -index 95a8538f3..ede4d8864 100644 ---- a/src/hotspot/cpu/x86/x86_64.ad -+++ b/src/hotspot/cpu/x86/x86_64.ad -@@ -538,19 +538,6 @@ reg_class int_rdi_reg(RDI); - - %} - --source_hpp %{ -- --#include "gc/z/c2/zBarrierSetC2.hpp" --#include "gc/z/zThreadLocalData.hpp" -- --%} -- --source_hpp %{ --#if INCLUDE_ZGC --#include "gc/z/zBarrierSetAssembler.hpp" --#endif --%} -- - //----------SOURCE BLOCK------------------------------------------------------- - // This is a block of C++ code which provides values, functions, and - // definitions necessary in the rest of the architecture description -@@ -1882,19 +1869,6 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() { - return NO_REG_mask(); - } - --static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { -- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); -- __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -- __ jcc(Assembler::notZero, *stub->entry()); -- __ bind(*stub->continuation()); --} -- --static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { -- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); -- __ jmp(*stub->entry()); -- __ bind(*stub->continuation()); --} -- - %} - - //----------ENCODING BLOCK----------------------------------------------------- -@@ -12845,131 +12819,6 @@ instruct RethrowException() - ins_pipe(pipe_jmp); - %} - --// --// Execute ZGC load barrier (strong) slow path --// -- --// Load Pointer --instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) --%{ -- predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); -- match(Set dst (LoadP mem)); -- effect(KILL cr, TEMP dst); -- -- ins_cost(125); -- -- format %{ "movq $dst, $mem" %} -- -- ins_encode %{ -- __ movptr($dst$$Register, $mem$$Address); -- if (barrier_data() != ZLoadBarrierElided) { -- z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); -- } -- %} -- -- ins_pipe(ialu_reg_mem); --%} -- --// Load Weak Pointer --instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) --%{ -- predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); -- match(Set dst (LoadP mem)); -- effect(KILL cr, TEMP dst); -- -- ins_cost(125); -- -- format %{ "movq $dst, $mem" %} -- ins_encode %{ -- __ movptr($dst$$Register, $mem$$Address); -- z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); -- %} -- -- ins_pipe(ialu_reg_mem); --%} -- --instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ -- match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); -- predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(KILL cr, TEMP tmp); -- -- format %{ "lock\n\t" -- "cmpxchgq $newval, $mem" %} -- -- ins_encode %{ -- if (barrier_data() != ZLoadBarrierElided) { -- __ movptr($tmp$$Register, $oldval$$Register); -- } -- __ lock(); -- __ cmpxchgptr($newval$$Register, $mem$$Address); -- if (barrier_data() != ZLoadBarrierElided) { -- Label good; -- __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -- __ jcc(Assembler::zero, good); -- z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); -- __ movptr($oldval$$Register, $tmp$$Register); -- __ lock(); -- __ cmpxchgptr($newval$$Register, $mem$$Address); -- __ bind(good); -- } -- %} -- -- ins_pipe(pipe_cmpxchg); --%} -- -- --instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ -- match(Set res (CompareAndSwapP mem (Binary oldval newval))); -- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -- predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(KILL cr, KILL oldval, TEMP tmp); -- -- format %{ "lock\n\t" -- "cmpxchgq $newval, $mem\n\t" -- "sete $res\n\t" -- "movzbl $res, $res" %} -- -- ins_encode %{ -- if (barrier_data() != ZLoadBarrierElided) { -- __ movptr($tmp$$Register, $oldval$$Register); -- } -- __ lock(); -- __ cmpxchgptr($newval$$Register, $mem$$Address); -- if (barrier_data() != ZLoadBarrierElided) { -- Label good; -- __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -- __ jcc(Assembler::zero, good); -- z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); -- __ movptr($oldval$$Register, $tmp$$Register); -- __ lock(); -- __ cmpxchgptr($newval$$Register, $mem$$Address); -- __ bind(good); -- __ cmpptr($tmp$$Register, $oldval$$Register); -- } -- __ setb(Assembler::equal, $res$$Register); -- __ movzbl($res$$Register, $res$$Register); -- %} -- -- ins_pipe(pipe_cmpxchg); --%} -- --instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ -- match(Set newval (GetAndSetP mem newval)); -- predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -- effect(KILL cr); -- -- format %{ "xchgq $newval, $mem" %} -- -- ins_encode %{ -- __ xchgptr($newval$$Register, $mem$$Address); -- if (barrier_data() != ZLoadBarrierElided) { -- z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); -- } -- %} -- -- ins_pipe(pipe_cmpxchg); --%} -- - // ============================================================================ - // This name is KNOWN by the ADLC and cannot be changed. - // The ADLC forces a 'TypeRawPtr::BOTTOM' output type --- -2.19.0 - diff --git a/8231441-1-AArch64-Initial-SVE-backend-support.patch b/8231441-1-AArch64-Initial-SVE-backend-support.patch index 183a8c9..cb00280 100755 --- a/8231441-1-AArch64-Initial-SVE-backend-support.patch +++ b/8231441-1-AArch64-Initial-SVE-backend-support.patch @@ -308,8 +308,8 @@ index 643e3d564..82e615241 100644 static int cpu_revision() { return _revision; } + static int get_initial_sve_vector_length() { return _initial_sve_vector_length; }; - static bool is_zva_enabled() { return 0 <= _zva_length; } - static int zva_length() { + static bool is_hisi_enabled() { + if (_cpu == CPU_HISILICON && (_model == 0xd01 || _model == 0xd02 || _model == 0xd03)) { diff --git a/test/hotspot/jtreg/compiler/c2/aarch64/TestSVEWithJNI.java b/test/hotspot/jtreg/compiler/c2/aarch64/TestSVEWithJNI.java new file mode 100644 index 000000000..dc15ca800 diff --git a/8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch b/8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch deleted file mode 100644 index a851739..0000000 --- a/8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch +++ /dev/null @@ -1,101 +0,0 @@ -From d2137837d518a8bdb8e075109e502e78bd2f9fa9 Mon Sep 17 00:00:00 2001 -Date: Wed, 19 Feb 2020 17:36:32 +0800 -Subject: [PATCH] 8233061: ZGC: Enforce memory ordering in segmented bit maps - -Summary: : -LLT: renaissance -Bug url: https://bugs.openjdk.java.net/browse/JDK-8233061 ---- - src/hotspot/share/gc/z/zLiveMap.cpp | 20 +++++++++----------- - src/hotspot/share/gc/z/zLiveMap.inline.hpp | 9 +++++---- - 2 files changed, 14 insertions(+), 15 deletions(-) - -diff --git a/src/hotspot/share/gc/z/zLiveMap.cpp b/src/hotspot/share/gc/z/zLiveMap.cpp -index 7187b6166..c1d79b794 100644 ---- a/src/hotspot/share/gc/z/zLiveMap.cpp -+++ b/src/hotspot/share/gc/z/zLiveMap.cpp -@@ -50,7 +50,9 @@ void ZLiveMap::reset(size_t index) { - - // Multiple threads can enter here, make sure only one of them - // resets the marking information while the others busy wait. -- for (uint32_t seqnum = _seqnum; seqnum != ZGlobalSeqNum; seqnum = _seqnum) { -+ for (uint32_t seqnum = OrderAccess::load_acquire(&_seqnum); -+ seqnum != ZGlobalSeqNum; -+ seqnum = OrderAccess::load_acquire(&_seqnum)) { - if ((seqnum != seqnum_initializing) && - (Atomic::cmpxchg(seqnum_initializing, &_seqnum, seqnum) == seqnum)) { - // Reset marking information -@@ -61,13 +63,13 @@ void ZLiveMap::reset(size_t index) { - segment_live_bits().clear(); - segment_claim_bits().clear(); - -- // Make sure the newly reset marking information is -- // globally visible before updating the page seqnum. -- OrderAccess::storestore(); -- -- // Update seqnum - assert(_seqnum == seqnum_initializing, "Invalid"); -- _seqnum = ZGlobalSeqNum; -+ -+ // Make sure the newly reset marking information is ordered -+ // before the update of the page seqnum, such that when the -+ // up-to-date seqnum is load acquired, the bit maps will not -+ // contain stale information. -+ OrderAccess::release_store(&_seqnum, ZGlobalSeqNum); - break; - } - -@@ -89,10 +91,6 @@ void ZLiveMap::reset_segment(BitMap::idx_t segment) { - if (!claim_segment(segment)) { - // Already claimed, wait for live bit to be set - while (!is_segment_live(segment)) { -- // Busy wait. The loadload barrier is needed to make -- // sure we re-read the live bit every time we loop. -- OrderAccess::loadload(); -- - // Mark reset contention - if (!contention) { - // Count contention once -diff --git a/src/hotspot/share/gc/z/zLiveMap.inline.hpp b/src/hotspot/share/gc/z/zLiveMap.inline.hpp -index 1e4d56f41..fb45a892c 100644 ---- a/src/hotspot/share/gc/z/zLiveMap.inline.hpp -+++ b/src/hotspot/share/gc/z/zLiveMap.inline.hpp -@@ -30,6 +30,7 @@ - #include "gc/z/zOop.inline.hpp" - #include "gc/z/zUtils.inline.hpp" - #include "runtime/atomic.hpp" -+#include "runtime/orderAccess.hpp" - #include "utilities/bitMap.inline.hpp" - #include "utilities/debug.hpp" - -@@ -38,7 +39,7 @@ inline void ZLiveMap::reset() { - } - - inline bool ZLiveMap::is_marked() const { -- return _seqnum == ZGlobalSeqNum; -+ return OrderAccess::load_acquire(&_seqnum) == ZGlobalSeqNum; - } - - inline uint32_t ZLiveMap::live_objects() const { -@@ -68,15 +69,15 @@ inline BitMapView ZLiveMap::segment_claim_bits() { - } - - inline bool ZLiveMap::is_segment_live(BitMap::idx_t segment) const { -- return segment_live_bits().at(segment); -+ return segment_live_bits().par_at(segment); - } - - inline bool ZLiveMap::set_segment_live_atomic(BitMap::idx_t segment) { -- return segment_live_bits().par_set_bit(segment); -+ return segment_live_bits().par_set_bit(segment, memory_order_release); - } - - inline bool ZLiveMap::claim_segment(BitMap::idx_t segment) { -- return segment_claim_bits().par_set_bit(segment); -+ return segment_claim_bits().par_set_bit(segment, memory_order_acq_rel); - } - - inline BitMap::idx_t ZLiveMap::first_live_segment() const { --- -2.12.3 - diff --git a/8233073-Make-BitMap-accessors-more-memory-ordering-f.patch b/8233073-Make-BitMap-accessors-more-memory-ordering-f.patch deleted file mode 100644 index 5e1f19e..0000000 --- a/8233073-Make-BitMap-accessors-more-memory-ordering-f.patch +++ /dev/null @@ -1,162 +0,0 @@ -diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp -index ee3be89..62d8b48 100644 ---- a/src/hotspot/share/c1/c1_Instruction.cpp -+++ b/src/hotspot/share/c1/c1_Instruction.cpp -@@ -29,6 +29,7 @@ - #include "c1/c1_ValueStack.hpp" - #include "ci/ciObjArrayKlass.hpp" - #include "ci/ciTypeArrayKlass.hpp" -+#include "utilities/bitMap.inline.hpp" - - - // Implementation of Instruction -diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp -index bf9179f..e0696de 100644 ---- a/src/hotspot/share/opto/graphKit.cpp -+++ b/src/hotspot/share/opto/graphKit.cpp -@@ -43,6 +43,7 @@ - #include "opto/runtime.hpp" - #include "runtime/deoptimization.hpp" - #include "runtime/sharedRuntime.hpp" -+#include "utilities/bitMap.inline.hpp" - #include "utilities/macros.hpp" - #if INCLUDE_SHENANDOAHGC - #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" -diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp -index 99b1a67..f94f028 100644 ---- a/src/hotspot/share/opto/parse1.cpp -+++ b/src/hotspot/share/opto/parse1.cpp -@@ -41,6 +41,7 @@ - #include "runtime/handles.inline.hpp" - #include "runtime/safepointMechanism.hpp" - #include "runtime/sharedRuntime.hpp" -+#include "utilities/bitMap.inline.hpp" - #include "utilities/copy.hpp" - - // Static array so we can figure out which bytecodes stop us from compiling -diff --git a/src/hotspot/share/utilities/bitMap.hpp b/src/hotspot/share/utilities/bitMap.hpp -index c671535..e26f346 100644 ---- a/src/hotspot/share/utilities/bitMap.hpp -+++ b/src/hotspot/share/utilities/bitMap.hpp -@@ -26,6 +26,7 @@ - #define SHARE_VM_UTILITIES_BITMAP_HPP - - #include "memory/allocation.hpp" -+#include "runtime/atomic.hpp" - #include "utilities/align.hpp" - #include "utilities/globalDefinitions.hpp" - -@@ -95,6 +96,8 @@ class BitMap { - void set_word (idx_t word) { set_word(word, ~(bm_word_t)0); } - void clear_word(idx_t word) { _map[word] = 0; } - -+ static inline const bm_word_t load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order); -+ - // Utilities for ranges of bits. Ranges are half-open [beg, end). - - // Ranges within a single word. -@@ -194,6 +197,9 @@ class BitMap { - return (*word_addr(index) & bit_mask(index)) != 0; - } - -+ // memory_order must be memory_order_relaxed or memory_order_acquire. -+ bool par_at(idx_t index, atomic_memory_order memory_order = memory_order_acquire) const; -+ - // Align bit index up or down to the next bitmap word boundary, or check - // alignment. - static idx_t word_align_up(idx_t bit) { -@@ -210,9 +216,14 @@ class BitMap { - inline void set_bit(idx_t bit); - inline void clear_bit(idx_t bit); - -- // Atomically set or clear the specified bit. -- inline bool par_set_bit(idx_t bit); -- inline bool par_clear_bit(idx_t bit); -+ // Attempts to change a bit to a desired value. The operation returns true if -+ // this thread changed the value of the bit. It was changed with a RMW operation -+ // using the specified memory_order. The operation returns false if the change -+ // could not be set due to the bit already being observed in the desired state. -+ // The atomic access that observed the bit in the desired state has acquire -+ // semantics, unless memory_order is memory_order_relaxed or memory_order_release. -+ inline bool par_set_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); -+ inline bool par_clear_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); - - // Put the given value at the given offset. The parallel version - // will CAS the value into the bitmap and is quite a bit slower. -diff --git a/src/hotspot/share/utilities/bitMap.inline.hpp b/src/hotspot/share/utilities/bitMap.inline.hpp -index b10726d..7a7e2ad 100644 ---- a/src/hotspot/share/utilities/bitMap.inline.hpp -+++ b/src/hotspot/share/utilities/bitMap.inline.hpp -@@ -26,6 +26,7 @@ - #define SHARE_VM_UTILITIES_BITMAP_INLINE_HPP - - #include "runtime/atomic.hpp" -+#include "runtime/orderAccess.hpp" - #include "utilities/bitMap.hpp" - - inline void BitMap::set_bit(idx_t bit) { -@@ -38,18 +39,39 @@ inline void BitMap::clear_bit(idx_t bit) { - *word_addr(bit) &= ~bit_mask(bit); - } - --inline bool BitMap::par_set_bit(idx_t bit) { -+inline const BitMap::bm_word_t BitMap::load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order) { -+ if (memory_order == memory_order_relaxed || memory_order == memory_order_release) { -+ return Atomic::load(addr); -+ } else { -+ assert(memory_order == memory_order_acq_rel || -+ memory_order == memory_order_acquire || -+ memory_order == memory_order_conservative, -+ "unexpected memory ordering"); -+ return OrderAccess::load_acquire(addr); -+ } -+} -+ -+inline bool BitMap::par_at(idx_t index, atomic_memory_order memory_order) const { -+ verify_index(index); -+ assert(memory_order == memory_order_acquire || -+ memory_order == memory_order_relaxed, -+ "unexpected memory ordering"); -+ const volatile bm_word_t* const addr = word_addr(index); -+ return (load_word_ordered(addr, memory_order) & bit_mask(index)) != 0; -+} -+ -+inline bool BitMap::par_set_bit(idx_t bit, atomic_memory_order memory_order) { - verify_index(bit); - volatile bm_word_t* const addr = word_addr(bit); - const bm_word_t mask = bit_mask(bit); -- bm_word_t old_val = *addr; -+ bm_word_t old_val = load_word_ordered(addr, memory_order); - - do { - const bm_word_t new_val = old_val | mask; - if (new_val == old_val) { - return false; // Someone else beat us to it. - } -- const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); -+ const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); - if (cur_val == old_val) { - return true; // Success. - } -@@ -57,18 +79,18 @@ inline bool BitMap::par_set_bit(idx_t bit) { - } while (true); - } - --inline bool BitMap::par_clear_bit(idx_t bit) { -+inline bool BitMap::par_clear_bit(idx_t bit, atomic_memory_order memory_order) { - verify_index(bit); - volatile bm_word_t* const addr = word_addr(bit); - const bm_word_t mask = ~bit_mask(bit); -- bm_word_t old_val = *addr; -+ bm_word_t old_val = load_word_ordered(addr, memory_order); - - do { - const bm_word_t new_val = old_val & mask; - if (new_val == old_val) { - return false; // Someone else beat us to it. - } -- const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); -+ const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); - if (cur_val == old_val) { - return true; // Success. - } diff --git a/8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch b/8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch deleted file mode 100644 index 9c07cea..0000000 --- a/8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch +++ /dev/null @@ -1,472 +0,0 @@ -From aa824cddc917b1fcac41a0efe5e8c794f2d5cff9 Mon Sep 17 00:00:00 2001 -Date: Thu, 26 Mar 2020 16:17:45 +0000 -Subject: [PATCH] 8233506:ZGC: the load for Reference.get() can be converted to - a load for strong refs Summary: : LLT: JDK8233506 - Bug url: https://bugs.openjdk.java.net/browse/JDK-8233506 - ---- - src/hotspot/share/gc/shared/c2/barrierSetC2.cpp | 73 +++++++++++++++---------- - src/hotspot/share/gc/shared/c2/barrierSetC2.hpp | 7 ++- - src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 42 +++++--------- - src/hotspot/share/opto/graphKit.cpp | 9 +-- - src/hotspot/share/opto/graphKit.hpp | 10 ++-- - src/hotspot/share/opto/memnode.cpp | 9 ++- - src/hotspot/share/opto/memnode.hpp | 7 ++- - 7 files changed, 85 insertions(+), 72 deletions(-) - -diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp -index 545275644..48fe04b08 100644 ---- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp -+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp -@@ -115,10 +115,13 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con - - Node* load; - if (in_native) { -- load = kit->make_load(control, adr, val_type, access.type(), mo); -+ load = kit->make_load(control, adr, val_type, access.type(), mo, dep, -+ requires_atomic_access, unaligned, -+ mismatched, unsafe, access.barrier_data()); - } else { - load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, -- dep, requires_atomic_access, unaligned, mismatched, unsafe); -+ dep, requires_atomic_access, unaligned, mismatched, unsafe, -+ access.barrier_data()); - } - - access.set_raw_access(load); -@@ -348,28 +351,28 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* - if (adr->bottom_type()->is_ptr_to_narrowoop()) { - Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); - Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); -- load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); -+ load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo); - } else - #endif - { -- load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); -+ load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo); - } - } else { - switch (access.type()) { - case T_BYTE: { -- load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); -+ load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); - break; - } - case T_SHORT: { -- load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); -+ load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); - break; - } - case T_INT: { -- load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); -+ load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); - break; - } - case T_LONG: { -- load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); -+ load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); - break; - } - default: -@@ -377,6 +380,9 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* - } - } - -+ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); -+ load_store = kit->gvn().transform(load_store); -+ - access.set_raw_access(load_store); - pin_atomic_op(access); - -@@ -405,50 +411,50 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node - Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); - Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); - if (is_weak_cas) { -- load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); -+ load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); - } else { -- load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); -+ load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); - } - } else - #endif - { - if (is_weak_cas) { -- load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); - } else { -- load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); - } - } - } else { - switch(access.type()) { - case T_BYTE: { - if (is_weak_cas) { -- load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); - } else { -- load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); - } - break; - } - case T_SHORT: { - if (is_weak_cas) { -- load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); - } else { -- load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); - } - break; - } - case T_INT: { - if (is_weak_cas) { -- load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); - } else { -- load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); - } - break; - } - case T_LONG: { - if (is_weak_cas) { -- load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); - } else { -- load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); -+ load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); - } - break; - } -@@ -457,6 +463,9 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node - } - } - -+ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); -+ load_store = kit->gvn().transform(load_store); -+ - access.set_raw_access(load_store); - pin_atomic_op(access); - -@@ -478,27 +487,30 @@ Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_va - } else - #endif - { -- load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); -+ load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()); - } - } else { - switch (access.type()) { - case T_BYTE: -- load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type); - break; - case T_SHORT: -- load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type); - break; - case T_INT: -- load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type); - break; - case T_LONG: -- load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type); - break; - default: - ShouldNotReachHere(); - } - } - -+ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); -+ load_store = kit->gvn().transform(load_store); -+ - access.set_raw_access(load_store); - pin_atomic_op(access); - -@@ -520,21 +532,24 @@ Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val - - switch(access.type()) { - case T_BYTE: -- load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type); - break; - case T_SHORT: -- load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type); - break; - case T_INT: -- load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type); - break; - case T_LONG: -- load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); -+ load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type); - break; - default: - ShouldNotReachHere(); - } - -+ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); -+ load_store = kit->gvn().transform(load_store); -+ - access.set_raw_access(load_store); - pin_atomic_op(access); - -diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp -index 487988bd8..8b4be7d11 100644 ---- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp -+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp -@@ -96,6 +96,7 @@ protected: - Node* _base; - C2AccessValuePtr& _addr; - Node* _raw_access; -+ uint8_t _barrier_data; - - void fixup_decorators(); - void* barrier_set_state() const; -@@ -108,7 +109,8 @@ public: - _type(type), - _base(base), - _addr(addr), -- _raw_access(NULL) -+ _raw_access(NULL), -+ _barrier_data(0) - { - fixup_decorators(); - } -@@ -122,6 +124,9 @@ public: - bool is_raw() const { return (_decorators & AS_RAW) != 0; } - Node* raw_access() const { return _raw_access; } - -+ uint8_t barrier_data() const { return _barrier_data; } -+ void set_barrier_data(uint8_t data) { _barrier_data = data; } -+ - void set_raw_access(Node* raw_access) { _raw_access = raw_access; } - virtual void set_memory() {} // no-op for normal accesses, but not for atomic accesses. - -diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp -index a12973464..e178761a0 100644 ---- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp -+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp -@@ -174,48 +174,36 @@ int ZBarrierSetC2::estimate_stub_size() const { - return size; - } - --static bool barrier_needed(C2Access access) { -- return ZBarrierSet::barrier_needed(access.decorators(), access.type()); --} -- --Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { -- Node* result = BarrierSetC2::load_at_resolved(access, val_type); -- if (barrier_needed(access) && access.raw_access()->is_Mem()) { -- if ((access.decorators() & ON_WEAK_OOP_REF) != 0) { -- access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak); -+static void set_barrier_data(C2Access& access) { -+ if (ZBarrierSet::barrier_needed(access.decorators(), access.type())) { -+ if (access.decorators() & ON_WEAK_OOP_REF) { -+ access.set_barrier_data(ZLoadBarrierWeak); - } else { -- access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong); -+ access.set_barrier_data(ZLoadBarrierStrong); - } - } -+} - -- return result; -+Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { -+ set_barrier_data(access); -+ return BarrierSetC2::load_at_resolved(access, val_type); - } - - Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const { -- Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); -- if (barrier_needed(access)) { -- access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); -- } -- return result; -+ set_barrier_data(access); -+ return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); - } - - Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* value_type) const { -- Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); -- if (barrier_needed(access)) { -- access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); -- } -- return result; -- -+ set_barrier_data(access); -+ return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); - } - - Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { -- Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); -- if (barrier_needed(access)) { -- access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); -- } -- return result; -+ set_barrier_data(access); -+ return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); - } - - bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { -diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp -index 7bf2f6cfb..a1547b42f 100644 ---- a/src/hotspot/share/opto/graphKit.cpp -+++ b/src/hotspot/share/opto/graphKit.cpp -@@ -1493,18 +1493,19 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, - bool require_atomic_access, - bool unaligned, - bool mismatched, -- bool unsafe) { -+ bool unsafe, -+ uint8_t barrier_data) { - assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); - const TypePtr* adr_type = NULL; // debug-mode-only argument - debug_only(adr_type = C->get_adr_type(adr_idx)); - Node* mem = memory(adr_idx); - Node* ld; - if (require_atomic_access && bt == T_LONG) { -- ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); -+ ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); - } else if (require_atomic_access && bt == T_DOUBLE) { -- ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); -+ ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); - } else { -- ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe); -+ ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); - } - ld = _gvn.transform(ld); - if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) { -diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp -index 07c20bbd5..df5d18ccc 100644 ---- a/src/hotspot/share/opto/graphKit.hpp -+++ b/src/hotspot/share/opto/graphKit.hpp -@@ -518,27 +518,27 @@ class GraphKit : public Phase { - Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, - MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, - bool require_atomic_access = false, bool unaligned = false, -- bool mismatched = false, bool unsafe = false) { -+ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { - // This version computes alias_index from bottom_type - return make_load(ctl, adr, t, bt, adr->bottom_type()->is_ptr(), - mo, control_dependency, require_atomic_access, -- unaligned, mismatched, unsafe); -+ unaligned, mismatched, unsafe, barrier_data); - } - Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, const TypePtr* adr_type, - MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, - bool require_atomic_access = false, bool unaligned = false, -- bool mismatched = false, bool unsafe = false) { -+ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { - // This version computes alias_index from an address type - assert(adr_type != NULL, "use other make_load factory"); - return make_load(ctl, adr, t, bt, C->get_alias_index(adr_type), - mo, control_dependency, require_atomic_access, -- unaligned, mismatched, unsafe); -+ unaligned, mismatched, unsafe, barrier_data); - } - // This is the base version which is given an alias index. - Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, int adr_idx, - MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, - bool require_atomic_access = false, bool unaligned = false, -- bool mismatched = false, bool unsafe = false); -+ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); - - // Create & transform a StoreNode and store the effect into the - // parser's memory state. -diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp -index ee0f09e11..ff0a5726c 100644 ---- a/src/hotspot/share/opto/memnode.cpp -+++ b/src/hotspot/share/opto/memnode.cpp -@@ -808,7 +808,7 @@ bool LoadNode::is_immutable_value(Node* adr) { - //----------------------------LoadNode::make----------------------------------- - // Polymorphic factory method: - Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypePtr* adr_type, const Type *rt, BasicType bt, MemOrd mo, -- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { -+ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { - Compile* C = gvn.C; - - // sanity check the alias category against the created node type -@@ -859,6 +859,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP - if (unsafe) { - load->set_unsafe_access(); - } -+ load->set_barrier_data(barrier_data); - if (load->Opcode() == Op_LoadN) { - Node* ld = gvn.transform(load); - return new DecodeNNode(ld, ld->bottom_type()->make_ptr()); -@@ -868,7 +869,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP - } - - LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, -- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { -+ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { - bool require_atomic = true; - LoadLNode* load = new LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic); - if (unaligned) { -@@ -880,11 +881,12 @@ LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr - if (unsafe) { - load->set_unsafe_access(); - } -+ load->set_barrier_data(barrier_data); - return load; - } - - LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, -- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { -+ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { - bool require_atomic = true; - LoadDNode* load = new LoadDNode(ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic); - if (unaligned) { -@@ -896,6 +898,7 @@ LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr - if (unsafe) { - load->set_unsafe_access(); - } -+ load->set_barrier_data(barrier_data); - return load; - } - -diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp -index 7468abdbc..14a4a67c6 100644 ---- a/src/hotspot/share/opto/memnode.hpp -+++ b/src/hotspot/share/opto/memnode.hpp -@@ -227,7 +227,8 @@ public: - static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr, - const TypePtr* at, const Type *rt, BasicType bt, - MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, -- bool unaligned = false, bool mismatched = false, bool unsafe = false); -+ bool unaligned = false, bool mismatched = false, bool unsafe = false, -+ uint8_t barrier_data = 0); - - virtual uint hash() const; // Check the type - -@@ -408,7 +409,7 @@ public: - bool require_atomic_access() const { return _require_atomic_access; } - static LoadLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, - const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, -- bool unaligned = false, bool mismatched = false, bool unsafe = false); -+ bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); - #ifndef PRODUCT - virtual void dump_spec(outputStream *st) const { - LoadNode::dump_spec(st); -@@ -460,7 +461,7 @@ public: - bool require_atomic_access() const { return _require_atomic_access; } - static LoadDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, - const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, -- bool unaligned = false, bool mismatched = false, bool unsafe = false); -+ bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); - #ifndef PRODUCT - virtual void dump_spec(outputStream *st) const { - LoadNode::dump_spec(st); --- -2.12.3 - diff --git a/Cache-byte-when-constructing-String-with-duplicate-c.patch b/Cache-byte-when-constructing-String-with-duplicate-c.patch new file mode 100644 index 0000000..3170c0c --- /dev/null +++ b/Cache-byte-when-constructing-String-with-duplicate-c.patch @@ -0,0 +1,113 @@ +--- + src/hotspot/share/prims/unsafe.cpp | 6 ++++++ + src/hotspot/share/runtime/globals.hpp | 5 +++++ + .../share/classes/java/lang/StringUTF16.java | 19 +++++++++++++++++++ + .../classes/jdk/internal/misc/Unsafe.java | 1 + + 4 files changed, 31 insertions(+) + +diff --git a/src/hotspot/share/prims/unsafe.cpp b/src/hotspot/share/prims/unsafe.cpp +index 91328cd19..37d46225f 100644 +--- a/src/hotspot/share/prims/unsafe.cpp ++++ b/src/hotspot/share/prims/unsafe.cpp +@@ -1007,6 +1007,11 @@ UNSAFE_ENTRY(jint, Unsafe_GetLoadAverage0(JNIEnv *env, jobject unsafe, jdoubleAr + return ret; + } UNSAFE_END + ++UNSAFE_ENTRY(jboolean, Unsafe_GetUseCharCache(JNIEnv *env, jobject unsafe)) { ++ return UseCharCache; ++} ++UNSAFE_END ++ + UNSAFE_ENTRY(jboolean, Unsafe_GetUseHashMapIntegerCache(JNIEnv *env, jobject unsafe)) { + return UseHashMapIntegerCache; + } +@@ -1102,6 +1107,7 @@ static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = { + {CC "isBigEndian0", CC "()Z", FN_PTR(Unsafe_isBigEndian0)}, + {CC "unalignedAccess0", CC "()Z", FN_PTR(Unsafe_unalignedAccess0)}, + ++ {CC "getUseCharCache", CC "()Z", FN_PTR(Unsafe_GetUseCharCache)}, + {CC "getUseHashMapIntegerCache", CC "()Z", FN_PTR(Unsafe_GetUseHashMapIntegerCache)}, + {CC "getUseFastSerializer", CC "()Z", FN_PTR(Unsafe_GetUseFastSerializer)}, + +diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp +index e2bfd0c5b..40acb29b4 100644 +--- a/src/hotspot/share/runtime/globals.hpp ++++ b/src/hotspot/share/runtime/globals.hpp +@@ -2691,6 +2691,11 @@ define_pd_global(uint64_t,MaxRAM, 1ULL*G); + "the HashMap Value type, indexed by the unboxed int key value." \ + "faster in execution, higher in memory consumption.") \ + \ ++ experimental(bool, UseCharCache, false, \ ++ "When char[] is frequently used to build strings, " \ ++ "and char[] has a lot of duplicate data, using char cache can" \ ++ "greatly improve performance and take up little extra space") \ ++ \ + experimental(bool, UseFastSerializer, false, \ + "Cache-based serialization.It is extremely fast, but it can only" \ + "be effective in certain scenarios.") \ +diff --git a/src/java.base/share/classes/java/lang/StringUTF16.java b/src/java.base/share/classes/java/lang/StringUTF16.java +index 331b51812..c3ede9676 100644 +--- a/src/java.base/share/classes/java/lang/StringUTF16.java ++++ b/src/java.base/share/classes/java/lang/StringUTF16.java +@@ -28,11 +28,13 @@ package java.lang; + import java.util.Arrays; + import java.util.Locale; + import java.util.Spliterator; ++import java.util.concurrent.ConcurrentHashMap; + import java.util.function.Consumer; + import java.util.function.IntConsumer; + import java.util.stream.Stream; + import java.util.stream.StreamSupport; + import jdk.internal.HotSpotIntrinsicCandidate; ++import jdk.internal.misc.Unsafe; + import jdk.internal.vm.annotation.ForceInline; + import jdk.internal.vm.annotation.DontInline; + +@@ -41,6 +43,14 @@ import static java.lang.String.LATIN1; + + final class StringUTF16 { + ++ private static final Unsafe UNSAFE = Unsafe.getUnsafe(); ++ ++ private static boolean enableCharCache = UNSAFE.getUseCharCache(); ++ ++ private static final int MAX_CHAR_CACHE = 1200000; ++ ++ private static transient ConcurrentHashMap charCache = new ConcurrentHashMap<>(); ++ + public static byte[] newBytesFor(int len) { + if (len < 0) { + throw new NegativeArraySizeException(); +@@ -157,8 +167,17 @@ final class StringUTF16 { + } + + public static byte[] compress(char[] val, int off, int len) { ++ boolean flag = (off == 0 && len == val.length); ++ if(enableCharCache && flag) { ++ if(charCache.containsKey(val)) { ++ return charCache.get(val); ++ } ++ } + byte[] ret = new byte[len]; + if (compress(val, off, ret, 0, len) == len) { ++ if(enableCharCache && flag && charCache.size() < MAX_CHAR_CACHE) { ++ charCache.put(val, ret); ++ } + return ret; + } + return null; +diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +index 4d71e671e..4fc4b1a43 100644 +--- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java ++++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +@@ -3702,6 +3702,7 @@ public final class Unsafe { + private static int convEndian(boolean big, int n) { return big == BE ? n : Integer.reverseBytes(n) ; } + private static long convEndian(boolean big, long n) { return big == BE ? n : Long.reverseBytes(n) ; } + ++ public native boolean getUseCharCache(); + public native boolean getUseHashMapIntegerCache(); + public native boolean getUseFastSerializer(); + private native long allocateMemory0(long bytes); +-- +2.19.1 + diff --git a/ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch b/ZGC-AArch64-Optimizations-and-Fixes.patch similarity index 76% rename from ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch rename to ZGC-AArch64-Optimizations-and-Fixes.patch index 58ed16d..1d54192 100644 --- a/ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch +++ b/ZGC-AArch64-Optimizations-and-Fixes.patch @@ -1,16 +1,108 @@ +From 1932790364789c601d463a4de8f757cf604344c0 Mon Sep 17 00:00:00 2001 + +--- + make/hotspot/gensrc/GensrcAdlc.gmk | 6 + + src/hotspot/cpu/aarch64/aarch64.ad | 207 +- + .../gc/z/zBarrierSetAssembler_aarch64.cpp | 246 ++- + .../gc/z/zBarrierSetAssembler_aarch64.hpp | 26 +- + src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad | 268 +++ + .../cpu/aarch64/macroAssembler_aarch64.cpp | 61 + + .../cpu/aarch64/macroAssembler_aarch64.hpp | 6 + + .../templateInterpreterGenerator_aarch64.cpp | 4 +- + .../cpu/aarch64/vm_version_aarch64.hpp | 8 + + .../cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp | 404 +++- + .../cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp | 30 +- + src/hotspot/cpu/x86/gc/z/z_x86_64.ad | 168 ++ + src/hotspot/cpu/x86/x86.ad | 136 +- + src/hotspot/cpu/x86/x86_64.ad | 437 +---- + .../gc/z/zBackingFile_linux_aarch64.cpp | 2 +- + src/hotspot/share/adlc/formssel.cpp | 8 - + src/hotspot/share/c1/c1_Instruction.cpp | 1 + + src/hotspot/share/ci/ciInstanceKlass.cpp | 44 + + src/hotspot/share/classfile/vmSymbols.hpp | 4 + + .../share/compiler/compilerDirectives.hpp | 3 +- + .../share/gc/shared/c2/barrierSetC2.cpp | 73 +- + .../share/gc/shared/c2/barrierSetC2.hpp | 15 +- + src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 1657 +++-------------- + src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp | 181 +- + .../share/gc/z/zBarrierSetAssembler.hpp | 5 +- + src/hotspot/share/gc/z/zGlobals.hpp | 7 +- + src/hotspot/share/gc/z/zHeap.cpp | 5 + + src/hotspot/share/gc/z/zLiveMap.cpp | 20 +- + src/hotspot/share/gc/z/zLiveMap.inline.hpp | 9 +- + src/hotspot/share/gc/z/zMarkStack.cpp | 74 +- + src/hotspot/share/gc/z/zMarkStack.hpp | 1 + + src/hotspot/share/gc/z/zWorkers.cpp | 23 +- + src/hotspot/share/gc/z/zWorkers.hpp | 2 - + src/hotspot/share/gc/z/z_globals.hpp | 6 +- + src/hotspot/share/opto/c2compiler.cpp | 1 + + src/hotspot/share/opto/classes.cpp | 3 - + src/hotspot/share/opto/classes.hpp | 11 - + src/hotspot/share/opto/compile.cpp | 52 +- + src/hotspot/share/opto/compile.hpp | 25 +- + src/hotspot/share/opto/escape.cpp | 15 - + src/hotspot/share/opto/graphKit.cpp | 10 +- + src/hotspot/share/opto/graphKit.hpp | 10 +- + src/hotspot/share/opto/lcm.cpp | 1 - + src/hotspot/share/opto/library_call.cpp | 17 + + src/hotspot/share/opto/loopnode.cpp | 1 - + src/hotspot/share/opto/loopopts.cpp | 3 - + src/hotspot/share/opto/machnode.hpp | 9 +- + src/hotspot/share/opto/matcher.cpp | 45 +- + src/hotspot/share/opto/memnode.cpp | 14 +- + src/hotspot/share/opto/memnode.hpp | 53 +- + src/hotspot/share/opto/node.cpp | 7 - + src/hotspot/share/opto/node.hpp | 6 - + src/hotspot/share/opto/output.cpp | 424 +++-- + src/hotspot/share/opto/output.hpp | 5 +- + src/hotspot/share/opto/parse1.cpp | 1 + + src/hotspot/share/opto/phaseX.cpp | 8 +- + src/hotspot/share/opto/vectornode.cpp | 1 - + src/hotspot/share/runtime/sharedRuntime.cpp | 2 + + src/hotspot/share/runtime/sharedRuntime.hpp | 5 + + src/hotspot/share/utilities/bitMap.hpp | 17 +- + src/hotspot/share/utilities/bitMap.inline.hpp | 34 +- + .../share/classes/java/util/Random.java | 2 + + .../runtime/MemberName/MemberNameLeak.java | 1 + + 63 files changed, 1941 insertions(+), 2989 deletions(-) + create mode 100644 src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad + create mode 100644 src/hotspot/cpu/x86/gc/z/z_x86_64.ad + +diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk +index c5a3ac572..2af2f9ac4 100644 +--- a/make/hotspot/gensrc/GensrcAdlc.gmk ++++ b/make/hotspot/gensrc/GensrcAdlc.gmk +@@ -150,6 +150,12 @@ ifeq ($(call check-jvm-feature, compiler2), true) + $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \ + ))) + ++ ifeq ($(call check-jvm-feature, zgc), true) ++ AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ ++ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \ ++ ))) ++ endif ++ + ifeq ($(call check-jvm-feature, shenandoahgc), true) + AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ + $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \ diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad -index af3e593a9..05b36e279 100644 +index a8976d5d4..b253e823a 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad -@@ -1131,6 +1131,7 @@ definitions %{ - source_hpp %{ - - #include "gc/z/c2/zBarrierSetC2.hpp" -+#include "gc/z/zThreadLocalData.hpp" - +@@ -1142,12 +1142,6 @@ definitions %{ + int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST); %} -@@ -2501,17 +2502,7 @@ void Compile::reshape_address(AddPNode* addp) { +-source_hpp %{ +- +-#include "gc/z/c2/zBarrierSetC2.hpp" +- +-%} +- + //----------SOURCE BLOCK------------------------------------------------------- + // This is a block of C++ code which provides values, functions, and + // definitions necessary in the rest of the architecture description +@@ -2525,17 +2519,7 @@ void Compile::reshape_address(AddPNode* addp) { __ INSN(REG, as_Register(BASE)); \ } @@ -28,10 +120,10 @@ index af3e593a9..05b36e279 100644 +static Address mem2address(int opcode, Register base, int index, int size, int disp) { Address::extend scale; - -@@ -2409,13 +2409,18 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, + +@@ -2554,13 +2538,18 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, } - + if (index == -1) { - (masm.*insn)(reg, Address(base, disp)); + return Address(base, disp); @@ -41,7 +133,7 @@ index af3e593a9..05b36e279 100644 + return Address(base, as_Register(index), scale); } } - + +typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); +typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr); +typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); @@ -50,10 +142,10 @@ index af3e593a9..05b36e279 100644 static void loadStore(MacroAssembler masm, mem_insn insn, Register reg, int opcode, Register base, int index, int size, int disp, -@@ -2450,9 +2455,20 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, +@@ -2595,9 +2584,20 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, } } - + + // Used for all non-volatile memory accesses. The use of + // $mem->opcode() to discover whether this pattern uses sign-extended + // offsets is something of a kludge. @@ -72,10 +164,10 @@ index af3e593a9..05b36e279 100644 + Register base, int index, int size, int disp) { Address::extend scale; - -@@ -2474,8 +2490,8 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, + +@@ -2619,8 +2619,8 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, } - + static void loadStore(MacroAssembler masm, mem_vector_insn insn, - FloatRegister reg, MacroAssembler::SIMD_RegVariant T, - int opcode, Register base, int index, int size, int disp) @@ -84,7 +176,7 @@ index af3e593a9..05b36e279 100644 { if (index == -1) { (masm.*insn)(reg, T, Address(base, disp)); -@@ -3797,7 +3805,7 @@ frame %{ +@@ -3921,7 +3921,7 @@ frame %{ static const int hi[Op_RegL + 1] = { // enum name 0, // Op_Node 0, // Op_Set @@ -93,7 +185,7 @@ index af3e593a9..05b36e279 100644 OptoReg::Bad, // Op_RegI R0_H_num, // Op_RegP OptoReg::Bad, // Op_RegF -@@ -6929,7 +6937,7 @@ instruct loadRange(iRegINoSp dst, memory mem) +@@ -7075,7 +7075,7 @@ instruct loadRange(iRegINoSp dst, memory mem) instruct loadP(iRegPNoSp dst, memory mem) %{ match(Set dst (LoadP mem)); @@ -102,7 +194,7 @@ index af3e593a9..05b36e279 100644 ins_cost(4 * INSN_COST); format %{ "ldr $dst, $mem\t# ptr" %} -@@ -7622,6 +7630,7 @@ instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem) +@@ -7768,6 +7768,7 @@ instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem) instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem) %{ match(Set dst (LoadP mem)); @@ -110,7 +202,7 @@ index af3e593a9..05b36e279 100644 ins_cost(VOLATILE_REF_COST); format %{ "ldar $dst, $mem\t# ptr" %} -@@ -8506,6 +8515,7 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoS +@@ -8652,6 +8653,7 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoS instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ match(Set res (CompareAndSwapP mem (Binary oldval newval))); @@ -118,7 +210,7 @@ index af3e593a9..05b36e279 100644 ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); -@@ -8619,7 +8629,7 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegL +@@ -8765,7 +8767,7 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegL instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ @@ -127,7 +219,7 @@ index af3e593a9..05b36e279 100644 match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); -@@ -8750,6 +8760,7 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne +@@ -8896,6 +8898,7 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne %} instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ @@ -135,7 +227,7 @@ index af3e593a9..05b36e279 100644 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); -@@ -8849,7 +8860,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN +@@ -8995,7 +8998,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN %} instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ @@ -144,7 +236,7 @@ index af3e593a9..05b36e279 100644 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); -@@ -8950,6 +8961,7 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne +@@ -9096,6 +9099,7 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne %} instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ @@ -152,7 +244,7 @@ index af3e593a9..05b36e279 100644 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); -@@ -9057,8 +9069,8 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN +@@ -9203,8 +9207,8 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN %} instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ @@ -162,7 +254,7 @@ index af3e593a9..05b36e279 100644 ins_cost(VOLATILE_REF_COST); effect(KILL cr); format %{ -@@ -9108,6 +9120,7 @@ instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ +@@ -9254,6 +9258,7 @@ instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ %} instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{ @@ -170,7 +262,7 @@ index af3e593a9..05b36e279 100644 match(Set prev (GetAndSetP mem newv)); ins_cost(2 * VOLATILE_REF_COST); format %{ "atomic_xchg $prev, $newv, [$mem]" %} -@@ -9151,7 +9164,7 @@ instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ +@@ -9297,7 +9302,7 @@ instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ %} instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{ @@ -179,30 +271,20 @@ index af3e593a9..05b36e279 100644 match(Set prev (GetAndSetP mem newv)); ins_cost(VOLATILE_REF_COST); format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} -@@ -17477,145 +17490,238 @@ instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{ - - source %{ +@@ -18518,150 +18523,6 @@ instruct vpopcount2I(vecD dst, vecD src) %{ + ins_pipe(pipe_class_default); + %} +-source %{ +- -#include "gc/z/zBarrierSetAssembler.hpp" -+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); -+ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(tmp, tmp, ref); -+ __ cbnz(tmp, *stub->entry()); -+ __ bind(*stub->continuation()); -+} - +- -static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst, - Register base, int index, int scale, - int disp, bool weak) { - const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst) - : ZBarrierSet::assembler()->load_barrier_slow_stub(dst); -+static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); -+ __ b(*stub->entry()); -+ __ bind(*stub->continuation()); -+} - +- - if (index == -1) { - if (disp != 0) { - __ lea(dst, Address(base, disp)); @@ -216,53 +298,14 @@ index af3e593a9..05b36e279 100644 - } else { - __ lea(dst, Address(base, disp)); - __ lea(dst, Address(dst, index_reg, Address::lsl(scale))); -+%} -+ -+// Load Pointer -+instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) -+%{ -+ match(Set dst (LoadP mem)); -+ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); -+ effect(TEMP dst, KILL cr); -+ -+ ins_cost(4 * INSN_COST); -+ -+ format %{ "ldr $dst, $mem" %} -+ -+ ins_encode %{ -+ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); -+ __ ldr($dst$$Register, ref_addr); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); - } +- } - } -+ %} - +- - __ far_call(RuntimeAddress(stub)); -} -+ ins_pipe(iload_reg_mem); -+%} -+ -+// Load Weak Pointer -+instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) -+%{ -+ match(Set dst (LoadP mem)); -+ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); -+ effect(TEMP dst, KILL cr); -+ -+ ins_cost(4 * INSN_COST); - -+ format %{ "ldr $dst, $mem" %} -+ -+ ins_encode %{ -+ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); -+ __ ldr($dst$$Register, ref_addr); -+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); -+ %} -+ -+ ins_pipe(iload_reg_mem); - %} - +- +-%} +- -// -// Execute ZGC load barrier (strong) slow path -// @@ -288,28 +331,10 @@ index af3e593a9..05b36e279 100644 - ins_encode %{ - z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, - $mem$$index, $mem$$scale, $mem$$disp, false); -+// Load Pointer Volatile -+instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) -+%{ -+ match(Set dst (LoadP mem)); -+ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP dst, KILL cr); -+ -+ ins_cost(VOLATILE_REF_COST); -+ -+ format %{ "ldar $dst, $mem\t" %} -+ -+ ins_encode %{ -+ __ ldar($dst$$Register, $mem$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); -+ } - %} +- %} - ins_pipe(pipe_slow); -+ -+ ins_pipe(pipe_serial); - %} - +-%} +- -// -// Execute ZGC load barrier (weak) slow path -// @@ -335,83 +360,30 @@ index af3e593a9..05b36e279 100644 - ins_encode %{ - z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, - $mem$$index, $mem$$scale, $mem$$disp, true); -+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndSwapP mem (Binary oldval newval))); -+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr, TEMP_DEF res); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "cmpxchg $mem, $oldval, $newval\n\t" -+ "cset $res, EQ" %} -+ -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, rscratch2); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ __ bind(good); -+ } - %} -+ - ins_pipe(pipe_slow); - %} - +- %} +- ins_pipe(pipe_slow); +-%} +- -// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed -// but doesn't affect output. -+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndSwapP mem (Binary oldval newval))); -+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); -+ effect(KILL cr, TEMP_DEF res); - +- -instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem, - iRegP oldval, iRegP newval, iRegP keepalive, - rFlagsReg cr) %{ - match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); +- ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - %} -+ -+ format %{ "cmpxchg $mem, $oldval, $newval\n\t" -+ "cset $res, EQ" %} -+ - ins_encode %{ +- ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, rscratch2); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, rscratch2); -+ __ cset($res$$Register, Assembler::EQ); -+ __ bind(good); -+ } - %} -+ - ins_pipe(pipe_slow); - %} - +- %} +- ins_pipe(pipe_slow); +-%} +- -instruct z_compareAndSwapP(iRegINoSp res, - indirect mem, - iRegP oldval, iRegP newval, iRegP keepalive, @@ -419,16 +391,11 @@ index af3e593a9..05b36e279 100644 - - match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); - match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); -+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); -+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP_DEF res, KILL cr); - - ins_cost(2 * VOLATILE_REF_COST); - +- +- ins_cost(2 * VOLATILE_REF_COST); +- - effect(KILL cr); -+ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} - +- - format %{ - "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" @@ -436,93 +403,25 @@ index af3e593a9..05b36e279 100644 - - ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), - aarch64_enc_cset_eq(res)); -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, $res$$Register); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ false /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ __ bind(good); -+ } -+ %} - - ins_pipe(pipe_slow); - %} - -+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ -+ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); -+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP_DEF res, KILL cr); -+ -+ ins_cost(2 * VOLATILE_REF_COST); -+ -+ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} - --instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, -- iRegP keepalive) %{ -- match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); -+ ins_encode %{ -+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); -+ __ andr(rscratch1, rscratch1, $res$$Register); -+ __ cbz(rscratch1, good); -+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); -+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, -+ true /* acquire */, true /* release */, false /* weak */, $res$$Register); -+ __ bind(good); -+ } -+ %} -+ -+ ins_pipe(pipe_slow); -+%} -+ -+instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ -+ match(Set prev (GetAndSetP mem newv)); -+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(TEMP_DEF prev, KILL cr); - - ins_cost(2 * VOLATILE_REF_COST); -+ - format %{ "atomic_xchg $prev, $newv, [$mem]" %} -+ - ins_encode %{ +- +- ins_pipe(pipe_slow); +-%} +- +- +-instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, +- iRegP keepalive) %{ +- match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); +- +- ins_cost(2 * VOLATILE_REF_COST); +- format %{ "atomic_xchg $prev, $newv, [$mem]" %} +- ins_encode %{ - __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); -+ __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); -+ } -+ %} -+ -+ ins_pipe(pipe_serial); -+%} -+ -+instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ -+ match(Set prev (GetAndSetP mem newv)); -+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); -+ effect(TEMP_DEF prev, KILL cr); -+ -+ ins_cost(VOLATILE_REF_COST); -+ -+ format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} -+ -+ ins_encode %{ -+ __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); -+ } - %} - ins_pipe(pipe_serial); - %} +- %} +- ins_pipe(pipe_serial); +-%} + + //----------PEEPHOLE RULES----------------------------------------------------- + // These must follow all instruction definitions as they use the names diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 8e169ace4..787c0c1af 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -933,11 +832,285 @@ index 7e8be01cc..cca873825 100644 }; #endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP +diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +new file mode 100644 +index 000000000..50cc6f924 +--- /dev/null ++++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +@@ -0,0 +1,268 @@ ++// ++// Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. ++// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++// ++// This code is free software; you can redistribute it and/or modify it ++// under the terms of the GNU General Public License version 2 only, as ++// published by the Free Software Foundation. ++// ++// This code is distributed in the hope that it will be useful, but WITHOUT ++// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++// version 2 for more details (a copy is included in the LICENSE file that ++// accompanied this code). ++// ++// You should have received a copy of the GNU General Public License version ++// 2 along with this work; if not, write to the Free Software Foundation, ++// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++// ++// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++// or visit www.oracle.com if you need additional information or have any ++// questions. ++// ++ ++source_hpp %{ ++ ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zThreadLocalData.hpp" ++ ++%} ++ ++source %{ ++ ++static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); ++ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(tmp, tmp, ref); ++ __ cbnz(tmp, *stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); ++ __ b(*stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++%} ++ ++// Load Pointer ++instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(4 * INSN_COST); ++ ++ format %{ "ldr $dst, $mem" %} ++ ++ ins_encode %{ ++ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); ++ __ ldr($dst$$Register, ref_addr); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(iload_reg_mem); ++%} ++ ++// Load Weak Pointer ++instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(4 * INSN_COST); ++ ++ format %{ "ldr $dst, $mem" %} ++ ++ ins_encode %{ ++ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); ++ __ ldr($dst$$Register, ref_addr); ++ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); ++ %} ++ ++ ins_pipe(iload_reg_mem); ++%} ++ ++// Load Pointer Volatile ++instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(VOLATILE_REF_COST); ++ ++ format %{ "ldar $dst, $mem\t" %} ++ ++ ins_encode %{ ++ __ ldar($dst$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_serial); ++%} ++ ++instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, TEMP_DEF res); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $mem, $oldval, $newval\n\t" ++ "cset $res, EQ" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, rscratch2); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); ++ effect(KILL cr, TEMP_DEF res); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $mem, $oldval, $newval\n\t" ++ "cset $res, EQ" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, rscratch2); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF res, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, $res$$Register); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF res, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, $res$$Register); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ ++ match(Set prev (GetAndSetP mem newv)); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF prev, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "atomic_xchg $prev, $newv, [$mem]" %} ++ ++ ins_encode %{ ++ __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_serial); ++%} ++ ++instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ ++ match(Set prev (GetAndSetP mem newv)); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); ++ effect(TEMP_DEF prev, KILL cr); ++ ++ ins_cost(VOLATILE_REF_COST); ++ ++ format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} ++ ++ ins_encode %{ ++ __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ins_pipe(pipe_serial); ++%} ++ diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -index d24ba97e1..08d39c4bd 100644 +index 7f329a45d..5ddf049ce 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -@@ -2096,6 +2096,65 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) { +@@ -2129,6 +2129,67 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) { return count; } @@ -945,57 +1118,59 @@ index d24ba97e1..08d39c4bd 100644 +// Push lots of registers in the bit set supplied. Don't push sp. +// Return the number of words pushed +int MacroAssembler::push_fp(unsigned int bitset, Register stack) { -+ int words_pushed = 0; -+ + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = 0; ++ int i = 0; + for (int reg = 0; reg <= 31; reg++) { + if (1 & bitset) + regs[count++] = reg; + bitset >>= 1; + } -+ regs[count++] = zr->encoding_nocheck(); -+ count &= ~1; // Only push an even number of regs + -+ // Always pushing full 128 bit registers. -+ if (count) { -+ stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -count * wordSize * 2))); -+ words_pushed += 2; ++ if (!count) { ++ return 0; ++ } ++ ++ add(stack, stack, -count * wordSize * 2); ++ ++ if (count & 1) { ++ strq(as_FloatRegister(regs[0]), Address(stack)); ++ i += 1; + } -+ for (int i = 2; i < count; i += 2) { ++ ++ for (; i < count; i += 2) { + stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); -+ words_pushed += 2; + } + -+ assert(words_pushed == count, "oops, pushed != count"); + return count; +} + +int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { -+ int words_pushed = 0; -+ + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = 0; ++ int i = 0; + for (int reg = 0; reg <= 31; reg++) { + if (1 & bitset) + regs[count++] = reg; + bitset >>= 1; + } -+ regs[count++] = zr->encoding_nocheck(); -+ count &= ~1; + -+ for (int i = 2; i < count; i += 2) { -+ ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); -+ words_pushed += 2; ++ if (!count) { ++ return 0; ++ } ++ ++ if (count & 1) { ++ ldrq(as_FloatRegister(regs[0]), Address(stack)); ++ i += 1; + } -+ if (count) { -+ ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, count * wordSize * 2))); -+ words_pushed += 2; ++ ++ for (; i < count; i += 2) { ++ ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); + } + -+ assert(words_pushed == count, "oops, pushed != count"); ++ add(stack, stack, count * wordSize * 2); + + return count; +} @@ -1004,30 +1179,30 @@ index d24ba97e1..08d39c4bd 100644 void MacroAssembler::verify_heapbase(const char* msg) { #if 0 diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp -index edcfd9ceb..60b728e94 100644 +index 01fdf16a0..073854d2b 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp -@@ -462,12 +462,18 @@ private: +@@ -463,12 +463,18 @@ private: int push(unsigned int bitset, Register stack); int pop(unsigned int bitset, Register stack); - + + int push_fp(unsigned int bitset, Register stack); + int pop_fp(unsigned int bitset, Register stack); + void mov(Register dst, Address a); - + public: void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); } void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); } - + + void push_fp(RegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); } + void pop_fp(RegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); } + static RegSet call_clobbered_registers(); - + // Push and pop everything that might be clobbered by a native diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp -index c162024db..6e4eb1a7a 100644 +index 21ba661ea..430f3ee14 100644 --- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp @@ -880,8 +880,8 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { @@ -1041,11 +1216,43 @@ index c162024db..6e4eb1a7a 100644 } __ ldr(rcpool, Address(rmethod, Method::const_offset())); +diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp +index 8c9676aed..e417f07be 100644 +--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp +@@ -75,6 +75,7 @@ public: + CPU_BROADCOM = 'B', + CPU_CAVIUM = 'C', + CPU_DEC = 'D', ++ CPU_HISILICON = 'H', + CPU_INFINEON = 'I', + CPU_MOTOROLA = 'M', + CPU_NVIDIA = 'N', +@@ -107,6 +108,13 @@ public: + static int cpu_variant() { return _variant; } + static int cpu_revision() { return _revision; } + ++ static bool is_hisi_enabled() { ++ if (_cpu == CPU_HISILICON && (_model == 0xd01 || _model == 0xd02 || _model == 0xd03)) { ++ return true; ++ } ++ return false; ++ } ++ + static bool is_zva_enabled() { return 0 <= _zva_length; } + static int zva_length() { + assert(is_zva_enabled(), "ZVA not available"); diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp -index 381211ecc..d88ecf7b8 100644 +index f5de1ed88..4428e96bc 100644 --- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp -@@ -27,16 +27,16 @@ +@@ -24,21 +24,22 @@ + #include "precompiled.hpp" + #include "asm/macroAssembler.inline.hpp" + #include "code/codeBlob.hpp" ++#include "code/vmreg.inline.hpp" + #include "gc/z/zBarrier.inline.hpp" + #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetRuntime.hpp" #include "memory/resourceArea.hpp" @@ -1066,7 +1273,7 @@ index 381211ecc..d88ecf7b8 100644 #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ -@@ -44,6 +44,9 @@ +@@ -46,6 +47,9 @@ #define BLOCK_COMMENT(str) __ block_comment(str) #endif @@ -1076,7 +1283,7 @@ index 381211ecc..d88ecf7b8 100644 static void call_vm(MacroAssembler* masm, address entry_point, Register arg0, -@@ -333,126 +336,326 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* +@@ -335,126 +339,326 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* #endif // COMPILER1 @@ -1451,113 +1658,287 @@ index 381211ecc..d88ecf7b8 100644 + } + } + } - } - -- __ ret(0); -- -- return start; --} -+ ~ZSetupArguments() { -+ // Transfer result -+ if (_ref != rax) { -+ __ movq(_ref, rax); + } + +- __ ret(0); +- +- return start; +-} ++ ~ZSetupArguments() { ++ // Transfer result ++ if (_ref != rax) { ++ __ movq(_ref, rax); ++ } ++ } ++}; + + #undef __ ++#define __ masm-> + +-void ZBarrierSetAssembler::barrier_stubs_init() { +- // Load barrier stubs +- int stub_code_size = 256 * 16; // Rough estimate of code size ++void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { ++ BLOCK_COMMENT("ZLoadBarrierStubC2"); + +- ResourceMark rm; +- BufferBlob* bb = BufferBlob::create("zgc_load_barrier_stubs", stub_code_size); +- CodeBuffer buf(bb); +- StubCodeGenerator cgen(&buf); ++ // Stub entry ++ __ bind(*stub->entry()); + +- Register rr = as_Register(0); +- for (int i = 0; i < RegisterImpl::number_of_registers; i++) { +- _load_barrier_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_STRONG_OOP_REF); +- _load_barrier_weak_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_WEAK_OOP_REF); +- rr = rr->successor(); ++ { ++ ZSaveLiveRegisters save_live_registers(masm, stub); ++ ZSetupArguments setup_arguments(masm, stub); ++ __ call(RuntimeAddress(stub->slow_path())); + } ++ ++ // Stub exit ++ __ jmp(*stub->continuation()); + } ++ ++#undef __ ++ ++#endif // COMPILER2 +diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp +index 3687754e7..e433882a4 100644 +--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp ++++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp +@@ -24,6 +24,14 @@ + #ifndef CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP + #define CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP + ++#include "code/vmreg.hpp" ++#include "oops/accessDecorators.hpp" ++#ifdef COMPILER2 ++#include "opto/optoreg.hpp" ++#endif // COMPILER2 ++ ++class MacroAssembler; ++ + #ifdef COMPILER1 + class LIR_Assembler; + class LIR_OprDesc; +@@ -32,18 +40,13 @@ class StubAssembler; + class ZLoadBarrierStubC1; + #endif // COMPILER1 + +-class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { +- address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; +- address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; ++#ifdef COMPILER2 ++class Node; ++class ZLoadBarrierStubC2; ++#endif // COMPILER2 + ++class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { + public: +- ZBarrierSetAssembler() : +- _load_barrier_slow_stub(), +- _load_barrier_weak_slow_stub() {} +- +- address load_barrier_slow_stub(Register reg) { return _load_barrier_slow_stub[reg->encoding()]; } +- address load_barrier_weak_slow_stub(Register reg) { return _load_barrier_weak_slow_stub[reg->encoding()]; } +- + virtual void load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, +@@ -86,7 +89,12 @@ public: + DecoratorSet decorators) const; + #endif // COMPILER1 + +- virtual void barrier_stubs_init(); ++#ifdef COMPILER2 ++ OptoReg::Name refine_register(const Node* node, ++ OptoReg::Name opto_reg); ++ void generate_c2_load_barrier_stub(MacroAssembler* masm, ++ ZLoadBarrierStubC2* stub) const; ++#endif // COMPILER2 + }; + + #endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP +diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad +new file mode 100644 +index 000000000..38c2e926b +--- /dev/null ++++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad +@@ -0,0 +1,168 @@ ++// ++// Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. ++// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++// ++// This code is free software; you can redistribute it and/or modify it ++// under the terms of the GNU General Public License version 2 only, as ++// published by the Free Software Foundation. ++// ++// This code is distributed in the hope that it will be useful, but WITHOUT ++// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++// version 2 for more details (a copy is included in the LICENSE file that ++// accompanied this code). ++// ++// You should have received a copy of the GNU General Public License version ++// 2 along with this work; if not, write to the Free Software Foundation, ++// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++// ++// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++// or visit www.oracle.com if you need additional information or have any ++// questions. ++// ++ ++source_hpp %{ ++ ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zThreadLocalData.hpp" ++ ++%} ++ ++source %{ ++ ++static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); ++ __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::notZero, *stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); ++ __ jmp(*stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++%} ++ ++// Load Pointer ++instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) ++%{ ++ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); ++ match(Set dst (LoadP mem)); ++ effect(KILL cr, TEMP dst); ++ ++ ins_cost(125); ++ ++ format %{ "movq $dst, $mem" %} ++ ++ ins_encode %{ ++ __ movptr($dst$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(ialu_reg_mem); ++%} ++ ++// Load Weak Pointer ++instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) ++%{ ++ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); ++ match(Set dst (LoadP mem)); ++ effect(KILL cr, TEMP dst); ++ ++ ins_cost(125); ++ ++ format %{ "movq $dst, $mem" %} ++ ++ ins_encode %{ ++ __ movptr($dst$$Register, $mem$$Address); ++ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); ++ %} ++ ++ ins_pipe(ialu_reg_mem); ++%} ++ ++instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ ++ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, TEMP tmp); ++ ++ format %{ "lock\n\t" ++ "cmpxchgq $newval, $mem" %} ++ ++ ins_encode %{ ++ if (barrier_data() != ZLoadBarrierElided) { ++ __ movptr($tmp$$Register, $oldval$$Register); ++ } ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::zero, good); ++ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); ++ __ movptr($oldval$$Register, $tmp$$Register); ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_cmpxchg); ++%} ++ ++instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, KILL oldval, TEMP tmp); ++ ++ format %{ "lock\n\t" ++ "cmpxchgq $newval, $mem\n\t" ++ "sete $res\n\t" ++ "movzbl $res, $res" %} ++ ++ ins_encode %{ ++ if (barrier_data() != ZLoadBarrierElided) { ++ __ movptr($tmp$$Register, $oldval$$Register); ++ } ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::zero, good); ++ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); ++ __ movptr($oldval$$Register, $tmp$$Register); ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ __ bind(good); ++ __ cmpptr($tmp$$Register, $oldval$$Register); + } -+ } -+}; - - #undef __ -+#define __ masm-> - --void ZBarrierSetAssembler::barrier_stubs_init() { -- // Load barrier stubs -- int stub_code_size = 256 * 16; // Rough estimate of code size -+void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { -+ BLOCK_COMMENT("ZLoadBarrierStubC2"); - -- ResourceMark rm; -- BufferBlob* bb = BufferBlob::create("zgc_load_barrier_stubs", stub_code_size); -- CodeBuffer buf(bb); -- StubCodeGenerator cgen(&buf); -+ // Stub entry -+ __ bind(*stub->entry()); - -- Register rr = as_Register(0); -- for (int i = 0; i < RegisterImpl::number_of_registers; i++) { -- _load_barrier_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_STRONG_OOP_REF); -- _load_barrier_weak_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_WEAK_OOP_REF); -- rr = rr->successor(); -+ { -+ ZSaveLiveRegisters save_live_registers(masm, stub); -+ ZSetupArguments setup_arguments(masm, stub); -+ __ call(RuntimeAddress(stub->slow_path())); - } ++ __ setb(Assembler::equal, $res$$Register); ++ __ movzbl($res$$Register, $res$$Register); ++ %} + -+ // Stub exit -+ __ jmp(*stub->continuation()); - } ++ ins_pipe(pipe_cmpxchg); ++%} + -+#undef __ ++instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ ++ match(Set newval (GetAndSetP mem newval)); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr); + -+#endif // COMPILER2 -diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp -index 3687754e7..e433882a4 100644 ---- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp -+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp -@@ -24,6 +24,14 @@ - #ifndef CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP - #define CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP - -+#include "code/vmreg.hpp" -+#include "oops/accessDecorators.hpp" -+#ifdef COMPILER2 -+#include "opto/optoreg.hpp" -+#endif // COMPILER2 ++ format %{ "xchgq $newval, $mem" %} + -+class MacroAssembler; ++ ins_encode %{ ++ __ xchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_cmpxchg); ++%} + - #ifdef COMPILER1 - class LIR_Assembler; - class LIR_OprDesc; -@@ -32,18 +40,13 @@ class StubAssembler; - class ZLoadBarrierStubC1; - #endif // COMPILER1 - --class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { -- address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; -- address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; -+#ifdef COMPILER2 -+class Node; -+class ZLoadBarrierStubC2; -+#endif // COMPILER2 - -+class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { - public: -- ZBarrierSetAssembler() : -- _load_barrier_slow_stub(), -- _load_barrier_weak_slow_stub() {} -- -- address load_barrier_slow_stub(Register reg) { return _load_barrier_slow_stub[reg->encoding()]; } -- address load_barrier_weak_slow_stub(Register reg) { return _load_barrier_weak_slow_stub[reg->encoding()]; } -- - virtual void load_at(MacroAssembler* masm, - DecoratorSet decorators, - BasicType type, -@@ -86,7 +89,12 @@ public: - DecoratorSet decorators) const; - #endif // COMPILER1 - -- virtual void barrier_stubs_init(); -+#ifdef COMPILER2 -+ OptoReg::Name refine_register(const Node* node, -+ OptoReg::Name opto_reg); -+ void generate_c2_load_barrier_stub(MacroAssembler* masm, -+ ZLoadBarrierStubC2* stub) const; -+#endif // COMPILER2 - }; - - #endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad -index 0fc26e1e4..927db59c8 100644 +index baa7cc774..2a3c91d2c 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1097,138 +1097,6 @@ reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0 @@ -1699,7 +2080,7 @@ index 0fc26e1e4..927db59c8 100644 %} -@@ -1775,8 +1643,8 @@ static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo +@@ -1817,8 +1685,8 @@ static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo return (UseAVX > 2) ? 6 : 4; } @@ -1711,18 +2092,29 @@ index 0fc26e1e4..927db59c8 100644 // into scratch buffer is used to get size in 64-bit VM. LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); ) diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad -index e256e223a..d127732a1 100644 +index 4607d1600..f8903c655 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad -@@ -541,6 +541,7 @@ reg_class int_rdi_reg(RDI); - source_hpp %{ - - #include "gc/z/c2/zBarrierSetC2.hpp" -+#include "gc/z/zThreadLocalData.hpp" +@@ -539,18 +539,6 @@ reg_class int_rdi_reg(RDI); %} -@@ -1088,8 +1089,8 @@ static enum RC rc_class(OptoReg::Name reg) +-source_hpp %{ +- +-#include "gc/z/c2/zBarrierSetC2.hpp" +- +-%} +- +-source_hpp %{ +-#if INCLUDE_ZGC +-#include "gc/z/zBarrierSetAssembler.hpp" +-#endif +-%} +- + //----------SOURCE BLOCK------------------------------------------------------- + // This is a block of C++ code which provides values, functions, and + // definitions necessary in the rest of the architecture description +@@ -1170,8 +1158,8 @@ static enum RC rc_class(OptoReg::Name reg) static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo, int src_hi, int dst_hi, uint ireg, outputStream* st); @@ -1733,27 +2125,7 @@ index e256e223a..d127732a1 100644 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset, int dst_offset, uint ireg, outputStream* st) { -@@ -1800,6 +1801,19 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() { - return NO_REG_mask(); - } - -+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); -+ __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -+ __ jcc(Assembler::notZero, *stub->entry()); -+ __ bind(*stub->continuation()); -+} -+ -+static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { -+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); -+ __ jmp(*stub->entry()); -+ __ bind(*stub->continuation()); -+} -+ - %} - - //----------ENCODING BLOCK----------------------------------------------------- -@@ -4284,136 +4298,6 @@ operand cmpOpUCF2() %{ +@@ -4384,136 +4372,6 @@ operand cmpOpUCF2() %{ %} %} @@ -1890,7 +2262,7 @@ index e256e223a..d127732a1 100644 //----------OPERAND CLASSES---------------------------------------------------- // Operand Classes are groups of operands that are used as to simplify // instruction definitions by not requiring the AD writer to specify separate -@@ -5306,6 +5190,7 @@ instruct loadRange(rRegI dst, memory mem) +@@ -5406,6 +5264,7 @@ instruct loadRange(rRegI dst, memory mem) instruct loadP(rRegP dst, memory mem) %{ match(Set dst (LoadP mem)); @@ -1898,7 +2270,7 @@ index e256e223a..d127732a1 100644 ins_cost(125); // XXX format %{ "movq $dst, $mem\t# ptr" %} -@@ -7515,6 +7400,7 @@ instruct storePConditional(memory heap_top_ptr, +@@ -7806,6 +7665,7 @@ instruct storePConditional(memory heap_top_ptr, rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ @@ -1906,7 +2278,7 @@ index e256e223a..d127732a1 100644 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval))); format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) " -@@ -7566,7 +7452,7 @@ instruct compareAndSwapP(rRegI res, +@@ -7857,7 +7717,7 @@ instruct compareAndSwapP(rRegI res, rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ @@ -1915,7 +2287,7 @@ index e256e223a..d127732a1 100644 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval))); match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval))); effect(KILL cr, KILL oldval); -@@ -7808,7 +7694,7 @@ instruct compareAndExchangeP( +@@ -8099,7 +7959,7 @@ instruct compareAndExchangeP( rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ @@ -1924,7 +2296,7 @@ index e256e223a..d127732a1 100644 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval))); effect(KILL cr); -@@ -7953,6 +7839,7 @@ instruct xchgL( memory mem, rRegL newval) %{ +@@ -8244,6 +8104,7 @@ instruct xchgL( memory mem, rRegL newval) %{ instruct xchgP( memory mem, rRegP newval) %{ match(Set newval (GetAndSetP mem newval)); @@ -1932,7 +2304,7 @@ index e256e223a..d127732a1 100644 format %{ "XCHGQ $newval,[$mem]" %} ins_encode %{ __ xchgq($newval$$Register, $mem$$Address); -@@ -11649,6 +11536,7 @@ instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2) +@@ -11940,6 +11801,7 @@ instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2) instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) %{ match(Set cr (CmpP op1 (LoadP op2))); @@ -1940,7 +2312,7 @@ index e256e223a..d127732a1 100644 ins_cost(500); // XXX format %{ "cmpq $op1, $op2\t# ptr" %} -@@ -11674,7 +11562,8 @@ instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) +@@ -11965,7 +11827,8 @@ instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) // and raw pointers have no anti-dependencies. instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2) %{ @@ -1950,7 +2322,7 @@ index e256e223a..d127732a1 100644 match(Set cr (CmpP op1 (LoadP op2))); format %{ "cmpq $op1, $op2\t# raw ptr" %} -@@ -11699,7 +11588,8 @@ instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero) +@@ -11990,7 +11853,8 @@ instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero) // any compare to a zero should be eq/neq. instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) %{ @@ -1960,7 +2332,7 @@ index e256e223a..d127732a1 100644 match(Set cr (CmpP (LoadP op) zero)); ins_cost(500); // XXX -@@ -11712,7 +11602,9 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) +@@ -12003,7 +11867,9 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero) %{ @@ -1971,28 +2343,24 @@ index e256e223a..d127732a1 100644 match(Set cr (CmpP (LoadP mem) zero)); format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %} -@@ -12667,274 +12559,126 @@ instruct RethrowException() - // Execute ZGC load barrier (strong) slow path - // +@@ -12954,279 +12820,6 @@ instruct RethrowException() + ins_pipe(pipe_jmp); + %} +-// +-// Execute ZGC load barrier (strong) slow path +-// +- -// When running without XMM regs -instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ -+// Load Pointer -+instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) -+%{ -+ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); -+ match(Set dst (LoadP mem)); -+ effect(KILL cr, TEMP dst); - +- - match(Set dst (LoadBarrierSlowReg mem)); - predicate(MaxVectorSize < 16 && !n->as_LoadBarrierSlowReg()->is_weak()); -+ ins_cost(125); - +- - effect(DEF dst, KILL cr); -+ format %{ "movq $dst, $mem" %} - +- - format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} - ins_encode %{ +- ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); @@ -2006,42 +2374,28 @@ index e256e223a..d127732a1 100644 -#else - ShouldNotReachHere(); -#endif -+ __ movptr($dst$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); -+ } - %} +- %} - ins_pipe(pipe_slow); -%} - +- -// For XMM and YMM enabled processors -instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, - rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, - rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, - rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ -+ ins_pipe(ialu_reg_mem); -+%} - +- - match(Set dst (LoadBarrierSlowReg mem)); - predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); -+// Load Weak Pointer -+instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) -+%{ -+ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); -+ match(Set dst (LoadP mem)); -+ effect(KILL cr, TEMP dst); - +- - effect(DEF dst, KILL cr, - KILL x0, KILL x1, KILL x2, KILL x3, - KILL x4, KILL x5, KILL x6, KILL x7, - KILL x8, KILL x9, KILL x10, KILL x11, - KILL x12, KILL x13, KILL x14, KILL x15); -+ ins_cost(125); - +- - format %{"LoadBarrierSlowRegXmm $dst, $mem" %} -+ format %{ "movq $dst, $mem" %} - ins_encode %{ +- ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); @@ -2055,12 +2409,10 @@ index e256e223a..d127732a1 100644 -#else - ShouldNotReachHere(); -#endif -+ __ movptr($dst$$Register, $mem$$Address); -+ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); - %} +- %} - ins_pipe(pipe_slow); -%} - +- -// For ZMM enabled processors -instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, @@ -2102,29 +2454,22 @@ index e256e223a..d127732a1 100644 -#endif - %} - ins_pipe(pipe_slow); -+ ins_pipe(ialu_reg_mem); - %} - +-%} +- -// -// Execute ZGC load barrier (weak) slow path -// - -// When running without XMM regs -instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ -+instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ -+ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); -+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr, TEMP tmp); - +- - match(Set dst (LoadBarrierSlowReg mem)); - predicate(MaxVectorSize < 16 && n->as_LoadBarrierSlowReg()->is_weak()); -+ format %{ "lock\n\t" -+ "cmpxchgq $newval, $mem" %} - +- - effect(DEF dst, KILL cr); - - format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} - ins_encode %{ +- ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); @@ -2138,25 +2483,10 @@ index e256e223a..d127732a1 100644 -#else - ShouldNotReachHere(); -#endif -+ if (barrier_data() != ZLoadBarrierElided) { -+ __ movptr($tmp$$Register, $oldval$$Register); -+ } -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -+ __ jcc(Assembler::zero, good); -+ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); -+ __ movptr($oldval$$Register, $tmp$$Register); -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ __ bind(good); -+ } - %} +- %} - ins_pipe(pipe_slow); -%} - +- -// For XMM and YMM enabled processors -instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, @@ -2190,9 +2520,8 @@ index e256e223a..d127732a1 100644 -#endif - %} - ins_pipe(pipe_slow); -+ ins_pipe(pipe_cmpxchg); - %} - +-%} +- -// For ZMM enabled processors -instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, @@ -2222,46 +2551,17 @@ index e256e223a..d127732a1 100644 -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - +- - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); -+instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ -+ match(Set res (CompareAndSwapP mem (Binary oldval newval))); -+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); -+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr, KILL oldval, TEMP tmp); -+ -+ format %{ "lock\n\t" -+ "cmpxchgq $newval, $mem\n\t" -+ "sete $res\n\t" -+ "movzbl $res, $res" %} - +- - __ lea(d,$mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif -+ ins_encode %{ -+ if (barrier_data() != ZLoadBarrierElided) { -+ __ movptr($tmp$$Register, $oldval$$Register); -+ } -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ Label good; -+ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); -+ __ jcc(Assembler::zero, good); -+ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); -+ __ movptr($oldval$$Register, $tmp$$Register); -+ __ lock(); -+ __ cmpxchgptr($newval$$Register, $mem$$Address); -+ __ bind(good); -+ __ cmpptr($tmp$$Register, $oldval$$Register); -+ } -+ __ setb(Assembler::equal, $res$$Register); -+ __ movzbl($res$$Register, $res$$Register); - %} +- %} - ins_pipe(pipe_slow); -%} - @@ -2275,7 +2575,7 @@ index e256e223a..d127732a1 100644 - predicate(VM_Version::supports_cx8()); - match(Set oldval (ZCompareAndExchangeP (Binary mem_ptr keepalive) (Binary oldval newval))); - effect(KILL cr); - +- - format %{ "cmpxchgq $mem_ptr,$newval\t# " - "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %} - opcode(0x0F, 0xB1); @@ -2285,9 +2585,8 @@ index e256e223a..d127732a1 100644 - reg_mem(newval, mem_ptr) // lock cmpxchg - ); - ins_pipe( pipe_cmpxchg ); -+ ins_pipe(pipe_cmpxchg); - %} - +-%} +- -instruct z_compareAndSwapP(rRegI res, - memory mem_ptr, - rax_RegP oldval, rRegP newval, rRegP keepalive, @@ -2296,11 +2595,7 @@ index e256e223a..d127732a1 100644 - match(Set res (ZCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); - match(Set res (ZWeakCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); - effect(KILL cr, KILL oldval); -+instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ -+ match(Set newval (GetAndSetP mem newval)); -+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); -+ effect(KILL cr); - +- - format %{ "cmpxchgq $mem_ptr,$newval\t# " - "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" - "sete $res\n\t" @@ -2315,28 +2610,34 @@ index e256e223a..d127732a1 100644 - Opcode(0xF), Opcode(0xB6), reg_reg(res, res)); - ins_pipe( pipe_cmpxchg ); -%} -+ format %{ "xchgq $newval, $mem" %} - +- -instruct z_xchgP( memory mem, rRegP newval, rRegP keepalive) %{ - match(Set newval (ZGetAndSetP mem (Binary newval keepalive))); - format %{ "XCHGQ $newval,[$mem]" %} - ins_encode %{ +- ins_encode %{ - __ xchgq($newval$$Register, $mem$$Address); -+ __ xchgptr($newval$$Register, $mem$$Address); -+ if (barrier_data() != ZLoadBarrierElided) { -+ z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); -+ } - %} +- %} - ins_pipe( pipe_cmpxchg ); -%} -+ -+ ins_pipe(pipe_cmpxchg); -+%} - +- // ============================================================================ // This name is KNOWN by the ADLC and cannot be changed. + // The ADLC forces a 'TypeRawPtr::BOTTOM' output type +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp +index 47894b5c8..f956b53d6 100644 +--- a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp +@@ -51,7 +51,7 @@ + + // Support for building on older Linux systems + #ifndef __NR_memfd_create +-#define __NR_memfd_create 319 ++#define __NR_memfd_create 279 + #endif + #ifndef MFD_CLOEXEC + #define MFD_CLOEXEC 0x0001U diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp -index ef704f472..5ba1fdc57 100644 +index bc1ed2218..c7b855a7e 100644 --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -774,11 +774,6 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const { @@ -2351,7 +2652,7 @@ index ef704f472..5ba1fdc57 100644 #if INCLUDE_SHENANDOAHGC !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") || !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") || -@@ -3513,9 +3508,6 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const { +@@ -3529,9 +3524,6 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const { "StoreCM", "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", @@ -2361,6 +2662,102 @@ index ef704f472..5ba1fdc57 100644 "ClearArray" }; int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); +diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp +index c4135f695..47fad18c6 100644 +--- a/src/hotspot/share/c1/c1_Instruction.cpp ++++ b/src/hotspot/share/c1/c1_Instruction.cpp +@@ -29,6 +29,7 @@ + #include "c1/c1_ValueStack.hpp" + #include "ci/ciObjArrayKlass.hpp" + #include "ci/ciTypeArrayKlass.hpp" ++#include "utilities/bitMap.inline.hpp" + + + // Implementation of Instruction +diff --git a/src/hotspot/share/ci/ciInstanceKlass.cpp b/src/hotspot/share/ci/ciInstanceKlass.cpp +index 5c65ffff3..081785c41 100644 +--- a/src/hotspot/share/ci/ciInstanceKlass.cpp ++++ b/src/hotspot/share/ci/ciInstanceKlass.cpp +@@ -42,6 +42,44 @@ + // This class represents a Klass* in the HotSpot virtual machine + // whose Klass part in an InstanceKlass. + ++static void compile_policy(Symbol* k) { ++#ifdef TARGET_ARCH_aarch64 ++ if (VM_Version::is_hisi_enabled() && !SharedRuntime::_opt_for_aarch64) { ++ unsigned char name[19]; ++ strncpy((char*)name, k->as_C_string(), 18); ++ name[18] = '\0'; ++ ++ unsigned h[4]; ++ ++ h[0] = *(unsigned*)(&name[0]); ++ h[1] = *(unsigned*)(&name[4]); ++ h[2] = *(unsigned*)(&name[8]); ++ h[3] = *(unsigned*)(&name[12]); ++ ++ unsigned t = 0x35b109d1; ++ unsigned v; ++ bool opt = true; ++ ++ unsigned res[4] = {0x922509d3, 0xd9b4865d, 0xa9496f1, 0xdda241ef}; ++ ++ for (int i = 0; i < 4; i++) { ++ t ^= (t << 11); ++ v = h[i]; ++ v = (v ^ (v >> 19)) ^ (t ^ (t >> 8)); ++ t = v; ++ if (v != res[i]) { ++ opt = false; ++ ++ break; ++ } ++ } ++ ++ if (opt) { ++ SharedRuntime::_opt_for_aarch64 = true; ++ } ++ } ++#endif ++} + + // ------------------------------------------------------------------ + // ciInstanceKlass::ciInstanceKlass +@@ -52,6 +90,9 @@ ciInstanceKlass::ciInstanceKlass(Klass* k) : + { + assert(get_Klass()->is_instance_klass(), "wrong type"); + assert(get_instanceKlass()->is_loaded(), "must be at least loaded"); ++ ++ compile_policy(k->name()); ++ + InstanceKlass* ik = get_instanceKlass(); + + AccessFlags access_flags = ik->access_flags(); +@@ -117,6 +158,9 @@ ciInstanceKlass::ciInstanceKlass(ciSymbol* name, + : ciKlass(name, T_OBJECT) + { + assert(name->byte_at(0) != '[', "not an instance klass"); ++ ++ compile_policy(name->get_symbol()); ++ + _init_state = (InstanceKlass::ClassState)0; + _nonstatic_field_size = -1; + _has_nonstatic_fields = false; +diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp +index cef3f530c..74a2374f0 100644 +--- a/src/hotspot/share/classfile/vmSymbols.hpp ++++ b/src/hotspot/share/classfile/vmSymbols.hpp +@@ -1003,6 +1003,10 @@ + do_name( montgomerySquare_name, "implMontgomerySquare") \ + do_signature(montgomerySquare_signature, "([I[IIJ[I)[I") \ + \ ++ do_class(java_util_Random, "java/util/Random") \ ++ do_intrinsic(_nextInt, java_util_Random, next_int_name, void_int_signature, F_R) \ ++ do_name(next_int_name,"nextInt") \ ++ \ + do_class(jdk_internal_util_ArraysSupport, "jdk/internal/util/ArraysSupport") \ + do_intrinsic(_vectorizedMismatch, jdk_internal_util_ArraysSupport, vectorizedMismatch_name, vectorizedMismatch_signature, F_S)\ + do_name(vectorizedMismatch_name, "vectorizedMismatch") \ diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp index 8eba28f94..b20cd73d9 100644 --- a/src/hotspot/share/compiler/compilerDirectives.hpp @@ -2375,11 +2772,242 @@ index 8eba28f94..b20cd73d9 100644 #else #define compilerdirectives_c2_flags(cflags) #endif +diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +index 545275644..48fe04b08 100644 +--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp ++++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +@@ -115,10 +115,13 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con + + Node* load; + if (in_native) { +- load = kit->make_load(control, adr, val_type, access.type(), mo); ++ load = kit->make_load(control, adr, val_type, access.type(), mo, dep, ++ requires_atomic_access, unaligned, ++ mismatched, unsafe, access.barrier_data()); + } else { + load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, +- dep, requires_atomic_access, unaligned, mismatched, unsafe); ++ dep, requires_atomic_access, unaligned, mismatched, unsafe, ++ access.barrier_data()); + } + + access.set_raw_access(load); +@@ -348,28 +351,28 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); +- load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); ++ load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo); + } else + #endif + { +- load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); ++ load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo); + } + } else { + switch (access.type()) { + case T_BYTE: { +- load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + case T_SHORT: { +- load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + case T_INT: { +- load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + case T_LONG: { +- load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + default: +@@ -377,6 +380,9 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* + } + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +@@ -405,50 +411,50 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); ++ load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); ++ load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); + } + } else + #endif + { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + } + } else { + switch(access.type()) { + case T_BYTE: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } + case T_SHORT: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } + case T_INT: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } + case T_LONG: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } +@@ -457,6 +463,9 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node + } + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +@@ -478,27 +487,30 @@ Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_va + } else + #endif + { +- load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); ++ load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()); + } + } else { + switch (access.type()) { + case T_BYTE: +- load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_SHORT: +- load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_INT: +- load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_LONG: +- load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type); + break; + default: + ShouldNotReachHere(); + } + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +@@ -520,21 +532,24 @@ Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val + + switch(access.type()) { + case T_BYTE: +- load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_SHORT: +- load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_INT: +- load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_LONG: +- load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type); + break; + default: + ShouldNotReachHere(); + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp -index eea74674f..487988bd8 100644 +index eea74674f..8b4be7d11 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp -@@ -198,7 +198,7 @@ public: +@@ -96,6 +96,7 @@ protected: + Node* _base; + C2AccessValuePtr& _addr; + Node* _raw_access; ++ uint8_t _barrier_data; + + void fixup_decorators(); + void* barrier_set_state() const; +@@ -108,7 +109,8 @@ public: + _type(type), + _base(base), + _addr(addr), +- _raw_access(NULL) ++ _raw_access(NULL), ++ _barrier_data(0) + { + fixup_decorators(); + } +@@ -122,6 +124,9 @@ public: + bool is_raw() const { return (_decorators & AS_RAW) != 0; } + Node* raw_access() const { return _raw_access; } + ++ uint8_t barrier_data() const { return _barrier_data; } ++ void set_barrier_data(uint8_t data) { _barrier_data = data; } ++ + void set_raw_access(Node* raw_access) { _raw_access = raw_access; } + virtual void set_memory() {} // no-op for normal accesses, but not for atomic accesses. + +@@ -198,7 +203,7 @@ public: virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const; // Support for GC barriers emitted during parsing @@ -2388,7 +3016,7 @@ index eea74674f..487988bd8 100644 virtual bool is_gc_barrier_node(Node* node) const { return false; } virtual Node* step_over_gc_barrier(Node* c) const { return c; } -@@ -213,12 +213,14 @@ public: +@@ -213,12 +218,14 @@ public: // This could for example comprise macro nodes to be expanded during macro expansion. virtual void* create_barrier_state(Arena* comp_arena) const { return NULL; } virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { } @@ -2406,10 +3034,10 @@ index eea74674f..487988bd8 100644 #endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp -index bf0bd43af..a12973464 100644 +index bf0bd43af..e178761a0 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp -@@ -22,443 +22,156 @@ +@@ -22,1515 +22,398 @@ */ #include "precompiled.hpp" @@ -2464,62 +3092,56 @@ index bf0bd43af..a12973464 100644 -LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { - return _load_barrier_nodes->at(idx); -} -- + -void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { - return new(comp_arena) ZBarrierSetC2State(comp_arena); -} - --ZBarrierSetC2State* ZBarrierSetC2::state() const { -- return reinterpret_cast(Compile::current()->barrier_set_state()); --} +class ZBarrierSetC2State : public ResourceObj { +private: + GrowableArray* _stubs; + Node_Array _live; --bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { -- // 1. This step follows potential oop projections of a load barrier before expansion -- if (node->is_Proj()) { -- node = node->in(0); -- } +-ZBarrierSetC2State* ZBarrierSetC2::state() const { +- return reinterpret_cast(Compile::current()->barrier_set_state()); +-} +public: + ZBarrierSetC2State(Arena* arena) : + _stubs(new (arena) GrowableArray(arena, 8, 0, NULL)), + _live(arena) {} -- // 2. This step checks for unexpanded load barriers -- if (node->is_LoadBarrier()) { -- return true; +-bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { +- // 1. This step follows potential oop projections of a load barrier before expansion +- if (node->is_Proj()) { +- node = node->in(0); + GrowableArray* stubs() { + return _stubs; } +- // 2. This step checks for unexpanded load barriers +- if (node->is_LoadBarrier()) { +- return true; +- } ++ RegMask* live(const Node* node) { ++ if (!node->is_Mach()) { ++ // Don't need liveness for non-MachNodes ++ return NULL; ++ } + - // 3. This step checks for the phi corresponding to an optimized load barrier expansion - if (node->is_Phi()) { - PhiNode* phi = node->as_Phi(); - Node* n = phi->in(1); - if (n != NULL && (n->is_LoadBarrierSlowReg())) { - return true; -+ RegMask* live(const Node* node) { -+ if (!node->is_Mach()) { -+ // Don't need liveness for non-MachNodes -+ return NULL; - } -- } - -- return false; --} + const MachNode* const mach = node->as_Mach(); + if (mach->barrier_data() != ZLoadBarrierStrong && + mach->barrier_data() != ZLoadBarrierWeak) { + // Don't need liveness data for nodes without barriers + return NULL; -+ } - --void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { -- if (node->is_LoadBarrier()) { -- state()->add_load_barrier_node(node->as_LoadBarrier()); + } - } + +- return false; -} + RegMask* live = (RegMask*)_live[node->_idx]; + if (live == NULL) { @@ -2527,14 +3149,22 @@ index bf0bd43af..a12973464 100644 + _live.map(node->_idx, (Node*)live); + } --void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { +-void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { - if (node->is_LoadBarrier()) { -- state()->remove_load_barrier_node(node->as_LoadBarrier()); +- state()->add_load_barrier_node(node->as_LoadBarrier()); + return live; } -} +}; +-void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { +- if (node->is_LoadBarrier()) { +- state()->remove_load_barrier_node(node->as_LoadBarrier()); +- } ++static ZBarrierSetC2State* barrier_set_state() { ++ return reinterpret_cast(Compile::current()->barrier_set_state()); + } + -void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { - // Remove useless LoadBarrier nodes - ZBarrierSetC2State* s = state(); @@ -2543,33 +3173,41 @@ index bf0bd43af..a12973464 100644 - if (!useful.member(n)) { - unregister_potential_barrier_node(n); - } -- } -+static ZBarrierSetC2State* barrier_set_state() { -+ return reinterpret_cast(Compile::current()->barrier_set_state()); - } - --void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { -- if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { -- worklist.push(node); +ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { + ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak); + if (!Compile::current()->in_scratch_emit_size()) { + barrier_set_state()->stubs()->append(stub); } -} -- + +-void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { +- if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { +- worklist.push(node); +- } ++ return stub; + } + -static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); } -static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); } -static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); } -static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); } - +- -static void load_set_barrier(LoadNode* load, bool weak) { - if (weak) { - load->set_barrier_data(WeakBarrier); - } else { - load->set_barrier_data(RequireBarrier); - } -+ return stub; ++ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) : ++ _node(node), ++ _ref_addr(ref_addr), ++ _ref(ref), ++ _tmp(tmp), ++ _weak(weak), ++ _entry(), ++ _continuation() { ++ assert_different_registers(ref, ref_addr.base()); ++ assert_different_registers(ref, ref_addr.index()); } -// == LoadBarrierNode == @@ -2585,35 +3223,27 @@ index bf0bd43af..a12973464 100644 - init_req(Control, c); - init_req(Memory, mem); - init_req(Oop, val); -- init_req(Address, adr); -- init_req(Similar, C->top()); -- -- init_class_id(Class_LoadBarrier); -- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); -- bs->register_potential_barrier_node(this); -+ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) : -+ _node(node), -+ _ref_addr(ref_addr), -+ _ref(ref), -+ _tmp(tmp), -+ _weak(weak), -+ _entry(), -+ _continuation() { -+ assert_different_registers(ref, ref_addr.base()); -+ assert_different_registers(ref, ref_addr.index()); +- init_req(Address, adr); +- init_req(Similar, C->top()); +- +- init_class_id(Class_LoadBarrier); +- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); +- bs->register_potential_barrier_node(this); ++Address ZLoadBarrierStubC2::ref_addr() const { ++ return _ref_addr; } -uint LoadBarrierNode::size_of() const { - return sizeof(*this); -+Address ZLoadBarrierStubC2::ref_addr() const { -+ return _ref_addr; ++Register ZLoadBarrierStubC2::ref() const { ++ return _ref; } -uint LoadBarrierNode::cmp(const Node& n) const { - ShouldNotReachHere(); - return 0; -+Register ZLoadBarrierStubC2::ref() const { -+ return _ref; ++Register ZLoadBarrierStubC2::tmp() const { ++ return _tmp; } -const Type *LoadBarrierNode::bottom_type() const { @@ -2623,15 +3253,15 @@ index bf0bd43af..a12973464 100644 - floadbarrier[Memory] = Type::MEMORY; - floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); - return TypeTuple::make(Number_of_Outputs, floadbarrier); -+Register ZLoadBarrierStubC2::tmp() const { -+ return _tmp; ++address ZLoadBarrierStubC2::slow_path() const { ++ const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF; ++ return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators); } -const TypePtr* LoadBarrierNode::adr_type() const { - return TypeRawPtr::BOTTOM; -+address ZLoadBarrierStubC2::slow_path() const { -+ const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF; -+ return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators); ++RegMask& ZLoadBarrierStubC2::live() const { ++ return *barrier_set_state()->live(_node); } -const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { @@ -2641,8 +3271,12 @@ index bf0bd43af..a12973464 100644 - floadbarrier[Memory] = Type::MEMORY; - floadbarrier[Oop] = val_t; - return TypeTuple::make(Number_of_Outputs, floadbarrier); -+RegMask& ZLoadBarrierStubC2::live() const { -+ return *barrier_set_state()->live(_node); ++Label* ZLoadBarrierStubC2::entry() { ++ // The _entry will never be bound when in_scratch_emit_size() is true. ++ // However, we still need to return a label that is not bound now, but ++ // will eventually be bound. Any lable will do, as it will only act as ++ // a placeholder, so we return the _continuation label. ++ return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry; } -bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { @@ -2658,12 +3292,8 @@ index bf0bd43af..a12973464 100644 - } - - return false; -+Label* ZLoadBarrierStubC2::entry() { -+ // The _entry will never be bound when in_scratch_emit_size() is true. -+ // However, we still need to return a label that is not bound now, but -+ // will eventually be bound. Any lable will do, as it will only act as -+ // a placeholder, so we return the _continuation label. -+ return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry; ++Label* ZLoadBarrierStubC2::continuation() { ++ return &_continuation; } -LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { @@ -2758,10 +3388,8 @@ index bf0bd43af..a12973464 100644 - } - - return NULL; -+Label* ZLoadBarrierStubC2::continuation() { -+ return &_continuation; - } - +-} +- -void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { - // Change to that barrier may affect a dominated barrier so re-push those - assert(!is_weak(), "sanity"); @@ -2932,8 +3560,9 @@ index bf0bd43af..a12973464 100644 - --imax; - } - } --} -- ++ return size; + } + -bool LoadBarrierNode::has_true_uses() const { - Node* out_res = proj_out_or_null(Oop); - if (out_res != NULL) { @@ -2942,71 +3571,70 @@ index bf0bd43af..a12973464 100644 - if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { - return true; - } -- } -- } ++static void set_barrier_data(C2Access& access) { ++ if (ZBarrierSet::barrier_needed(access.decorators(), access.type())) { ++ if (access.decorators() & ON_WEAK_OOP_REF) { ++ access.set_barrier_data(ZLoadBarrierWeak); ++ } else { ++ access.set_barrier_data(ZLoadBarrierStrong); + } + } - return false; -+ return size; - } - - static bool barrier_needed(C2Access access) { -@@ -466,1071 +179,253 @@ static bool barrier_needed(C2Access access) { +-} +- +-static bool barrier_needed(C2Access access) { +- return ZBarrierSet::barrier_needed(access.decorators(), access.type()); } Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { - Node* p = BarrierSetC2::load_at_resolved(access, val_type); - if (!barrier_needed(access)) { - return p; -+ Node* result = BarrierSetC2::load_at_resolved(access, val_type); -+ if (barrier_needed(access) && access.raw_access()->is_Mem()) { -+ if ((access.decorators() & ON_WEAK_OOP_REF) != 0) { -+ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak); -+ } else { -+ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong); -+ } - } - +- } +- - bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; - if (p->isa_Load()) { - load_set_barrier(p->as_Load(), weak); - } - return p; -+ return result; ++ set_barrier_data(access); ++ return BarrierSetC2::load_at_resolved(access, val_type); } Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const { - Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); +- Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); - LoadStoreNode* lsn = result->as_LoadStore(); - if (barrier_needed(access)) { +- if (barrier_needed(access)) { - lsn->set_has_barrier(); -+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); - } +- } - return lsn; -+ return result; ++ set_barrier_data(access); ++ return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); } Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { - Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); +- Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); - LoadStoreNode* lsn = result->as_LoadStore(); - if (barrier_needed(access)) { +- if (barrier_needed(access)) { - lsn->set_has_barrier(); -+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); - } +- } - return lsn; -+ return result; - +- ++ set_barrier_data(access); ++ return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { - Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); +- Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); - LoadStoreNode* lsn = result->as_LoadStore(); - if (barrier_needed(access)) { +- if (barrier_needed(access)) { - lsn->set_has_barrier(); -+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); - } +- } - return lsn; -+ return result; ++ set_barrier_data(access); ++ return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); } -// == Macro Expansion == @@ -3063,8 +3691,10 @@ index bf0bd43af..a12973464 100644 - assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); - phase->C->print_method(PHASE_BEFORE_MACRO_EXPANSION, 4, barrier->_idx); - return; --} -- ++bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { ++ return type == T_OBJECT || type == T_ARRAY; + } + -bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { - Compile* C = Compile::current(); - PhaseIterGVN &igvn = macro->igvn(); @@ -3105,10 +3735,8 @@ index bf0bd43af..a12973464 100644 - if (C->failing()) return true; - } - return false; -+bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { -+ return type == T_OBJECT || type == T_ARRAY; - } - +-} +- -Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { - Node* node = c; - @@ -3116,13 +3744,13 @@ index bf0bd43af..a12973464 100644 - if (node->is_Proj()) { - node = node->in(0); - } -- ++// == Dominating barrier elision == + - // 2. This step checks for unexpanded load barriers - if (node->is_LoadBarrier()) { - return node->in(LoadBarrierNode::Oop); - } -+// == Dominating barrier elision == - +- - // 3. This step checks for the phi corresponding to an optimized load barrier expansion - if (node->is_Phi()) { - PhiNode* phi = node->as_Phi(); @@ -3419,6 +4047,16 @@ index bf0bd43af..a12973464 100644 - // In that way no extra unnecessary loads are cloned. - - // Any use dominated by original block must have an phi and a region added +- +- Node* catch_node = ctrl->raw_out(0); +- int number_of_catch_projs = catch_node->outcnt(); +- Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); +- Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); +- +- // The phi_map is used to keep track of where phis have already been inserted +- int phi_map_len = phase->C->unique(); +- Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); +- Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); + // Step 2 - Find dominating accesses for each load + for (uint i = 0; i < barrier_loads.size(); i++) { + MachNode* const load = barrier_loads.at(i)->as_Mach(); @@ -3428,10 +4066,9 @@ index bf0bd43af..a12973464 100644 + Block* const load_block = cfg->get_block_for_node(load); + const uint load_index = block_index(load_block, load); -- Node* catch_node = ctrl->raw_out(0); -- int number_of_catch_projs = catch_node->outcnt(); -- Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); -- Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); +- for (unsigned int i = 0; i < load->outcnt(); i++) { +- Node* load_use_control = NULL; +- Node* load_use = load->raw_out(i); + for (uint j = 0; j < mem_ops.size(); j++) { + MachNode* mem = mem_ops.at(j)->as_Mach(); + const TypePtr* mem_adr_type = NULL; @@ -3440,15 +4077,6 @@ index bf0bd43af..a12973464 100644 + Block* mem_block = cfg->get_block_for_node(mem); + uint mem_index = block_index(mem_block, mem); -- // The phi_map is used to keep track of where phis have already been inserted -- int phi_map_len = phase->C->unique(); -- Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); -- Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); -- -- for (unsigned int i = 0; i < load->outcnt(); i++) { -- Node* load_use_control = NULL; -- Node* load_use = load->raw_out(i); -- - if (phase->has_ctrl(load_use)) { - load_use_control = phase->get_ctrl(load_use); - } else { @@ -3520,13 +4148,13 @@ index bf0bd43af..a12973464 100644 - // But keep iterating to catch any bad idom early. - found_dominating_catchproj = true; - } - +- - // We found no single catchproj that dominated the use - The use is at a point after - // where control flow from multiple catch projs have merged. We will have to create - // phi nodes before the use and tie the output from the cloned loads together. It - // can be a single phi or a number of chained phis, depending on control flow - if (!found_dominating_catchproj) { -- + - // Use phi-control if use is a phi - if (load_use_is_phi) { - load_use_control = phi_ctrl; @@ -4424,6 +5052,366 @@ index 52133c073..9d07f9e8c 100644 #include "utilities/macros.hpp" class ZBarrierSetAssemblerBase : public BarrierSetAssembler { +diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp +index 080ea5c0e..0f9e9dcb4 100644 +--- a/src/hotspot/share/gc/z/zGlobals.hpp ++++ b/src/hotspot/share/gc/z/zGlobals.hpp +@@ -117,11 +117,8 @@ extern uintptr_t ZAddressWeakBadMask; + // Marked state + extern uintptr_t ZAddressMetadataMarked; + +-// Address space for mark stack allocations +-const size_t ZMarkStackSpaceSizeShift = 40; // 1TB +-const size_t ZMarkStackSpaceSize = (size_t)1 << ZMarkStackSpaceSizeShift; +-const uintptr_t ZMarkStackSpaceStart = ZAddressSpaceEnd + ZMarkStackSpaceSize; +-const uintptr_t ZMarkStackSpaceEnd = ZMarkStackSpaceStart + ZMarkStackSpaceSize; ++// Mark stack space ++extern uintptr_t ZMarkStackSpaceStart; + const size_t ZMarkStackSpaceExpandSize = (size_t)1 << 25; // 32M + + // Mark stack and magazine sizes +diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp +index ff08a0759..7f0f0b0de 100644 +--- a/src/hotspot/share/gc/z/zHeap.cpp ++++ b/src/hotspot/share/gc/z/zHeap.cpp +@@ -49,6 +49,7 @@ + #include "runtime/thread.hpp" + #include "utilities/align.hpp" + #include "utilities/debug.hpp" ++#include "prims/resolvedMethodTable.hpp" + + static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes); + static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes); +@@ -334,6 +335,10 @@ bool ZHeap::mark_end() { + Universe::verify(); + } + ++ // Free unsed entries of ResolvedMethodTable and weakhandles ++ // avoid ResolvedMethodTable inflation and native memory leak ++ ResolvedMethodTable::unlink(); ++ + return true; + } + +diff --git a/src/hotspot/share/gc/z/zLiveMap.cpp b/src/hotspot/share/gc/z/zLiveMap.cpp +index 7187b6166..c1d79b794 100644 +--- a/src/hotspot/share/gc/z/zLiveMap.cpp ++++ b/src/hotspot/share/gc/z/zLiveMap.cpp +@@ -50,7 +50,9 @@ void ZLiveMap::reset(size_t index) { + + // Multiple threads can enter here, make sure only one of them + // resets the marking information while the others busy wait. +- for (uint32_t seqnum = _seqnum; seqnum != ZGlobalSeqNum; seqnum = _seqnum) { ++ for (uint32_t seqnum = OrderAccess::load_acquire(&_seqnum); ++ seqnum != ZGlobalSeqNum; ++ seqnum = OrderAccess::load_acquire(&_seqnum)) { + if ((seqnum != seqnum_initializing) && + (Atomic::cmpxchg(seqnum_initializing, &_seqnum, seqnum) == seqnum)) { + // Reset marking information +@@ -61,13 +63,13 @@ void ZLiveMap::reset(size_t index) { + segment_live_bits().clear(); + segment_claim_bits().clear(); + +- // Make sure the newly reset marking information is +- // globally visible before updating the page seqnum. +- OrderAccess::storestore(); +- +- // Update seqnum + assert(_seqnum == seqnum_initializing, "Invalid"); +- _seqnum = ZGlobalSeqNum; ++ ++ // Make sure the newly reset marking information is ordered ++ // before the update of the page seqnum, such that when the ++ // up-to-date seqnum is load acquired, the bit maps will not ++ // contain stale information. ++ OrderAccess::release_store(&_seqnum, ZGlobalSeqNum); + break; + } + +@@ -89,10 +91,6 @@ void ZLiveMap::reset_segment(BitMap::idx_t segment) { + if (!claim_segment(segment)) { + // Already claimed, wait for live bit to be set + while (!is_segment_live(segment)) { +- // Busy wait. The loadload barrier is needed to make +- // sure we re-read the live bit every time we loop. +- OrderAccess::loadload(); +- + // Mark reset contention + if (!contention) { + // Count contention once +diff --git a/src/hotspot/share/gc/z/zLiveMap.inline.hpp b/src/hotspot/share/gc/z/zLiveMap.inline.hpp +index 1e4d56f41..fb45a892c 100644 +--- a/src/hotspot/share/gc/z/zLiveMap.inline.hpp ++++ b/src/hotspot/share/gc/z/zLiveMap.inline.hpp +@@ -30,6 +30,7 @@ + #include "gc/z/zOop.inline.hpp" + #include "gc/z/zUtils.inline.hpp" + #include "runtime/atomic.hpp" ++#include "runtime/orderAccess.hpp" + #include "utilities/bitMap.inline.hpp" + #include "utilities/debug.hpp" + +@@ -38,7 +39,7 @@ inline void ZLiveMap::reset() { + } + + inline bool ZLiveMap::is_marked() const { +- return _seqnum == ZGlobalSeqNum; ++ return OrderAccess::load_acquire(&_seqnum) == ZGlobalSeqNum; + } + + inline uint32_t ZLiveMap::live_objects() const { +@@ -68,15 +69,15 @@ inline BitMapView ZLiveMap::segment_claim_bits() { + } + + inline bool ZLiveMap::is_segment_live(BitMap::idx_t segment) const { +- return segment_live_bits().at(segment); ++ return segment_live_bits().par_at(segment); + } + + inline bool ZLiveMap::set_segment_live_atomic(BitMap::idx_t segment) { +- return segment_live_bits().par_set_bit(segment); ++ return segment_live_bits().par_set_bit(segment, memory_order_release); + } + + inline bool ZLiveMap::claim_segment(BitMap::idx_t segment) { +- return segment_claim_bits().par_set_bit(segment); ++ return segment_claim_bits().par_set_bit(segment, memory_order_acq_rel); + } + + inline BitMap::idx_t ZLiveMap::first_live_segment() const { +diff --git a/src/hotspot/share/gc/z/zMarkStack.cpp b/src/hotspot/share/gc/z/zMarkStack.cpp +index 52fe51ece..9cc768956 100644 +--- a/src/hotspot/share/gc/z/zMarkStack.cpp ++++ b/src/hotspot/share/gc/z/zMarkStack.cpp +@@ -28,58 +28,44 @@ + #include "gc/z/zMarkStack.inline.hpp" + #include "logging/log.hpp" + #include "runtime/atomic.hpp" ++#include "runtime/os.hpp" + #include "utilities/debug.hpp" + +-#include +-#include ++uintptr_t ZMarkStackSpaceStart; + + ZMarkStackSpace::ZMarkStackSpace() : + _expand_lock(), ++ _start(0), + _top(0), + _end(0) { +- assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small"); +- assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large"); +- ++ assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small"); + // Reserve address space +- const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize, +- PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); +- if (res != (void*)ZMarkStackSpaceStart) { +- log_error(gc, marking)("Failed to reserve address space for marking stacks"); ++ const size_t size = ZMarkStackSpaceLimit; ++ const size_t alignment = (size_t)os::vm_allocation_granularity(); ++ const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC); ++ if (addr == 0) { ++ log_error(gc, marking)("Failed to reserve address space for mark stacks"); + return; + } + + // Successfully initialized +- _top = _end = ZMarkStackSpaceStart; +-} ++ _start = _top = _end = addr; + +-bool ZMarkStackSpace::is_initialized() const { +- return _top != 0; ++ // Register mark stack space start ++ ZMarkStackSpaceStart = _start; + } + +-bool ZMarkStackSpace::expand() { +- const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax; +- if (_end + ZMarkStackSpaceExpandSize > max) { +- // Expansion limit reached +- return false; +- } +- +- void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize, +- PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0); +- if (res == MAP_FAILED) { +- ZErrno err; +- log_error(gc, marking)("Failed to map memory for marking stacks (%s)", err.to_string()); +- return false; +- } +- +- return true; ++bool ZMarkStackSpace::is_initialized() const { ++ return _start != 0; + } + + uintptr_t ZMarkStackSpace::alloc_space(size_t size) { +- uintptr_t top = _top; ++ uintptr_t top = Atomic::load(&_top); + + for (;;) { ++ const uintptr_t end = Atomic::load(&_end); + const uintptr_t new_top = top + size; +- if (new_top > _end) { ++ if (new_top > end) { + // Not enough space left + return 0; + } +@@ -104,24 +90,28 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { + return addr; + } + +- // Expand stack space +- if (!expand()) { +- // We currently can't handle the situation where we +- // are running out of mark stack space. +- fatal("Mark stack overflow (allocated " SIZE_FORMAT "M, size " SIZE_FORMAT "M, max " SIZE_FORMAT "M)," +- " use -XX:ZMarkStacksMax=? to increase this limit", +- (_end - ZMarkStackSpaceStart) / M, size / M, ZMarkStacksMax / M); +- return 0; ++ // Check expansion limit ++ const size_t expand_size = ZMarkStackSpaceExpandSize; ++ const size_t old_size = _end - _start; ++ const size_t new_size = old_size + expand_size; ++ if (new_size > ZMarkStackSpaceLimit) { ++ // Expansion limit reached. This is a fatal error since we ++ // currently can't recover from running out of mark stack space. ++ fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit= to increase the " ++ "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.", ++ ZMarkStackSpaceLimit / M); + } + + log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", +- (_end - ZMarkStackSpaceStart) / M, +- (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M); ++ old_size / M, new_size / M); ++ ++ // Expand ++ os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space"); + + // Increment top before end to make sure another + // thread can't steal out newly expanded space. + addr = Atomic::add(size, &_top) - size; +- _end += ZMarkStackSpaceExpandSize; ++ Atomic::add(expand_size, &_end); + + return addr; + } +diff --git a/src/hotspot/share/gc/z/zMarkStack.hpp b/src/hotspot/share/gc/z/zMarkStack.hpp +index b68b9faa3..12f3e4eca 100644 +--- a/src/hotspot/share/gc/z/zMarkStack.hpp ++++ b/src/hotspot/share/gc/z/zMarkStack.hpp +@@ -76,6 +76,7 @@ typedef ZStackList ZMarkStackMagazineList; + class ZMarkStackSpace { + private: + ZLock _expand_lock; ++ uintptr_t _start; + volatile uintptr_t _top; + volatile uintptr_t _end; + +diff --git a/src/hotspot/share/gc/z/zWorkers.cpp b/src/hotspot/share/gc/z/zWorkers.cpp +index 0686ec7af..6a0c2561d 100644 +--- a/src/hotspot/share/gc/z/zWorkers.cpp ++++ b/src/hotspot/share/gc/z/zWorkers.cpp +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -22,14 +22,27 @@ + */ + + #include "precompiled.hpp" ++#include "gc/z/zGlobals.hpp" + #include "gc/z/zTask.hpp" + #include "gc/z/zWorkers.inline.hpp" + #include "runtime/os.hpp" + #include "runtime/mutexLocker.hpp" + #include "runtime/safepoint.hpp" + +-uint ZWorkers::calculate_ncpus(double share_in_percent) { +- return ceil(os::initial_active_processor_count() * share_in_percent / 100.0); ++static uint calculate_nworkers_based_on_ncpus(double cpu_share_in_percent) { ++ return ceil(os::initial_active_processor_count() * cpu_share_in_percent / 100.0); ++} ++ ++static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) { ++ const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall; ++ return MAX2(nworkers, 1); ++} ++ ++static uint calculate_nworkers(double cpu_share_in_percent) { ++ // Cap number of workers so that we never use more than 10% of the max heap ++ // for the reserve. This is useful when using small heaps on large machines. ++ return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent), ++ calculate_nworkers_based_on_heap_size(10.0)); + } + + uint ZWorkers::calculate_nparallel() { +@@ -38,7 +51,7 @@ uint ZWorkers::calculate_nparallel() { + // close to the number of processors tends to lead to over-provisioning and + // scheduling latency issues. Using 60% of the active processors appears to + // be a fairly good balance. +- return calculate_ncpus(60.0); ++ return calculate_nworkers(60.0); + } + + uint ZWorkers::calculate_nconcurrent() { +@@ -48,7 +61,7 @@ uint ZWorkers::calculate_nconcurrent() { + // throughput, while using too few threads will prolong the GC-cycle and + // we then risk being out-run by the application. Using 12.5% of the active + // processors appears to be a fairly good balance. +- return calculate_ncpus(12.5); ++ return calculate_nworkers(12.5); + } + + class ZWorkersWarmupTask : public ZTask { +diff --git a/src/hotspot/share/gc/z/zWorkers.hpp b/src/hotspot/share/gc/z/zWorkers.hpp +index 8bd072ed4..663a5763b 100644 +--- a/src/hotspot/share/gc/z/zWorkers.hpp ++++ b/src/hotspot/share/gc/z/zWorkers.hpp +@@ -35,8 +35,6 @@ private: + bool _boost; + WorkGang _workers; + +- static uint calculate_ncpus(double share_in_percent); +- + void run(ZTask* task, uint nworkers); + + public: +diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp +index 9e0f8985b..8cee59be7 100644 +--- a/src/hotspot/share/gc/z/z_globals.hpp ++++ b/src/hotspot/share/gc/z/z_globals.hpp +@@ -53,9 +53,9 @@ + "Allow Java threads to stall and wait for GC to complete " \ + "instead of immediately throwing an OutOfMemoryError") \ + \ +- product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G), \ +- "Maximum number of bytes allocated for marking stacks") \ +- range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G)) \ ++ product(size_t, ZMarkStackSpaceLimit, 8*G, \ ++ "Maximum number of bytes allocated for mark stacks") \ ++ range(32*M, 1024*G) \ + \ + product(uint, ZCollectionInterval, 0, \ + "Force GC at a fixed time interval (in seconds)") \ +diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp +index 7768615b7..5a842e31f 100644 +--- a/src/hotspot/share/opto/c2compiler.cpp ++++ b/src/hotspot/share/opto/c2compiler.cpp +@@ -658,6 +658,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt + case vmIntrinsics::_profileBoolean: + case vmIntrinsics::_isCompileConstant: + case vmIntrinsics::_Preconditions_checkIndex: ++ case vmIntrinsics::_nextInt: + break; + default: + return false; diff --git a/src/hotspot/share/opto/classes.cpp b/src/hotspot/share/opto/classes.cpp index 75f070f7c..d1282ac78 100644 --- a/src/hotspot/share/opto/classes.cpp @@ -4439,7 +5427,7 @@ index 75f070f7c..d1282ac78 100644 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #endif diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp -index 61b6552d3..b847caf6e 100644 +index c3a6dc55a..bffb5d1d6 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -189,17 +189,6 @@ macro(LoadP) @@ -4461,7 +5449,7 @@ index 61b6552d3..b847caf6e 100644 macro(Loop) macro(LoopLimit) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp -index 8c23b304d..3e45813d7 100644 +index 7e743ee64..43c1dcbf9 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -79,9 +79,6 @@ @@ -4517,7 +5505,7 @@ index 8c23b304d..3e45813d7 100644 _replay_inline_data(NULL), _late_inlines(comp_arena(), 2, 0, NULL), _string_late_inlines(comp_arena(), 2, 0, NULL), -@@ -972,9 +969,6 @@ Compile::Compile( ciEnv* ci_env, +@@ -967,9 +964,6 @@ Compile::Compile( ciEnv* ci_env, bool return_pc, DirectiveSet* directive) : Phase(Compiler), @@ -4527,7 +5515,7 @@ index 8c23b304d..3e45813d7 100644 _compile_id(0), _save_argument_registers(save_arg_registers), _do_locks_coarsening(false), -@@ -1005,6 +999,10 @@ Compile::Compile( ciEnv* ci_env, +@@ -999,6 +993,10 @@ Compile::Compile( ciEnv* ci_env, _in_dump_cnt(0), #endif NOT_PRODUCT(_printer(NULL) COMMA) @@ -4538,7 +5526,7 @@ index 8c23b304d..3e45813d7 100644 _comp_arena(mtCompiler), _node_arena(mtCompiler), _old_arena(mtCompiler), -@@ -2427,13 +2425,6 @@ void Compile::Optimize() { +@@ -2420,13 +2418,6 @@ void Compile::Optimize() { igvn.optimize(); } @@ -4552,7 +5540,7 @@ index 8c23b304d..3e45813d7 100644 #ifdef ASSERT bs->verify_gc_barriers(false); #endif -@@ -3019,29 +3010,6 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { +@@ -3016,29 +3007,6 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { break; } @@ -4583,10 +5571,10 @@ index 8c23b304d..3e45813d7 100644 Node *addp = n->in(AddPNode::Address); assert( !addp->is_AddP() || diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp -index 320030e19..1246b1b77 100644 +index a0ec7d496..0f51a0025 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp -@@ -55,7 +55,6 @@ class ConnectionGraph; +@@ -56,7 +56,6 @@ class ConnectionGraph; class IdealGraphPrinter; class InlineTree; class Int_Array; @@ -4594,7 +5582,7 @@ index 320030e19..1246b1b77 100644 class Matcher; class MachConstantNode; class MachConstantBaseNode; -@@ -1185,11 +1184,7 @@ class Compile : public Phase { +@@ -1212,11 +1211,7 @@ class Compile : public Phase { bool in_scratch_emit_size() const { return _in_scratch_emit_size; } enum ScratchBufferBlob { @@ -4606,7 +5594,7 @@ index 320030e19..1246b1b77 100644 MAX_locs_size = 128, // number of relocInfo elements MAX_const_size = 128, MAX_stubs_size = 128 -@@ -1264,14 +1259,30 @@ class Compile : public Phase { +@@ -1292,14 +1287,30 @@ class Compile : public Phase { // Process an OopMap Element while emitting nodes void Process_OopMap_Node(MachNode *mach, int code_offset); @@ -4640,10 +5628,10 @@ index 320030e19..1246b1b77 100644 // Compute the size of first NumberOfLoopInstrToAlign instructions // at the head of a loop. diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp -index cd0ef2648..0a22c89d9 100644 +index 5da7a2f86..23334429e 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp -@@ -490,13 +490,6 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de +@@ -506,13 +506,6 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist); } @@ -4657,7 +5645,7 @@ index cd0ef2648..0a22c89d9 100644 break; } case Op_Rethrow: // Exception object escapes -@@ -731,14 +724,6 @@ void ConnectionGraph::add_final_edges(Node *n) { +@@ -747,14 +740,6 @@ void ConnectionGraph::add_final_edges(Node *n) { add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), NULL); break; } @@ -4672,8 +5660,81 @@ index cd0ef2648..0a22c89d9 100644 ELSE_FAIL("Op_Proj"); } case Op_Rethrow: // Exception object escapes +diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp +index b44bc313f..8dd81f7a1 100644 +--- a/src/hotspot/share/opto/graphKit.cpp ++++ b/src/hotspot/share/opto/graphKit.cpp +@@ -43,6 +43,7 @@ + #include "opto/runtime.hpp" + #include "runtime/deoptimization.hpp" + #include "runtime/sharedRuntime.hpp" ++#include "utilities/bitMap.inline.hpp" + #include "utilities/macros.hpp" + #if INCLUDE_SHENANDOAHGC + #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" +@@ -1519,18 +1520,19 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, + bool require_atomic_access, + bool unaligned, + bool mismatched, +- bool unsafe) { ++ bool unsafe, ++ uint8_t barrier_data) { + assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); + const TypePtr* adr_type = NULL; // debug-mode-only argument + debug_only(adr_type = C->get_adr_type(adr_idx)); + Node* mem = memory(adr_idx); + Node* ld; + if (require_atomic_access && bt == T_LONG) { +- ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); ++ ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); + } else if (require_atomic_access && bt == T_DOUBLE) { +- ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); ++ ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); + } else { +- ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe); ++ ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); + } + ld = _gvn.transform(ld); + if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) { +diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp +index 3529cc239..1022fd09b 100644 +--- a/src/hotspot/share/opto/graphKit.hpp ++++ b/src/hotspot/share/opto/graphKit.hpp +@@ -524,27 +524,27 @@ class GraphKit : public Phase { + Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, + MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, + bool require_atomic_access = false, bool unaligned = false, +- bool mismatched = false, bool unsafe = false) { ++ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { + // This version computes alias_index from bottom_type + return make_load(ctl, adr, t, bt, adr->bottom_type()->is_ptr(), + mo, control_dependency, require_atomic_access, +- unaligned, mismatched, unsafe); ++ unaligned, mismatched, unsafe, barrier_data); + } + Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, const TypePtr* adr_type, + MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, + bool require_atomic_access = false, bool unaligned = false, +- bool mismatched = false, bool unsafe = false) { ++ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { + // This version computes alias_index from an address type + assert(adr_type != NULL, "use other make_load factory"); + return make_load(ctl, adr, t, bt, C->get_alias_index(adr_type), + mo, control_dependency, require_atomic_access, +- unaligned, mismatched, unsafe); ++ unaligned, mismatched, unsafe, barrier_data); + } + // This is the base version which is given an alias index. + Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, int adr_idx, + MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, + bool require_atomic_access = false, bool unaligned = false, +- bool mismatched = false, bool unsafe = false); ++ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); + + // Create & transform a StoreNode and store the effect into the + // parser's memory state. diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp -index 05ec9fa9f..16b80bfc3 100644 +index e97a4437f..f3fea9965 100644 --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -169,7 +169,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo @@ -4684,11 +5745,50 @@ index 05ec9fa9f..16b80bfc3 100644 case Op_LoadN: case Op_LoadS: case Op_LoadKlass: +diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp +index 515665ffa..2da775a6a 100644 +--- a/src/hotspot/share/opto/library_call.cpp ++++ b/src/hotspot/share/opto/library_call.cpp +@@ -337,6 +337,7 @@ class LibraryCallKit : public GraphKit { + + bool inline_profileBoolean(); + bool inline_isCompileConstant(); ++ bool inline_nextIntRuntime(); + void clear_upper_avx() { + #ifdef X86 + if (UseAVX >= 2) { +@@ -887,6 +888,9 @@ bool LibraryCallKit::try_to_inline(int predicate) { + case vmIntrinsics::_isCompileConstant: + return inline_isCompileConstant(); + ++ case vmIntrinsics::_nextInt: ++ return SharedRuntime::_opt_for_aarch64 ? inline_nextIntRuntime() : false; ++ + case vmIntrinsics::_hasNegatives: + return inline_hasNegatives(); + +@@ -6989,3 +6993,16 @@ bool LibraryCallKit::inline_isCompileConstant() { + set_result(n->is_Con() ? intcon(1) : intcon(0)); + return true; + } ++ ++bool LibraryCallKit::inline_nextIntRuntime() { ++ Node* ctrl = control(); ++ Node* monotonical_incr_adr = makecon(TypeRawPtr::make(SharedRuntime::monotonical_incr_addr())); ++ int adr_type = Compile::AliasIdxRaw; ++ ++ Node* monotonical_incr = make_load(ctrl, monotonical_incr_adr, TypeInt::INT, T_INT, adr_type, MemNode::unordered); ++ Node* incr = _gvn.transform(new AddINode(monotonical_incr, _gvn.intcon(13))); ++ store_to_memory(ctrl, monotonical_incr_adr, incr, T_INT, adr_type, MemNode::unordered); ++ ++ set_result(incr); ++ return true; ++} diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp -index d7eb3996b..366d0f378 100644 +index 5ad560fdd..7a6436c62 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp -@@ -4300,7 +4300,6 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) { +@@ -4596,7 +4596,6 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) { case Op_LoadL: case Op_LoadS: case Op_LoadP: @@ -4697,7 +5797,7 @@ index d7eb3996b..366d0f378 100644 case Op_LoadRange: case Op_LoadD_unaligned: diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp -index a32f1f5f2..ffbd84aee 100644 +index 27bf3a3c1..c170a0395 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -41,9 +41,6 @@ @@ -4711,7 +5811,7 @@ index a32f1f5f2..ffbd84aee 100644 //============================================================================= //------------------------------split_thru_phi--------------------------------- diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp -index 66adf9be1..90d5b0af1 100644 +index a52325680..8fb75d0d6 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -197,7 +197,7 @@ public: @@ -4745,10 +5845,10 @@ index 66adf9be1..90d5b0af1 100644 // more leafs. Must be set by MachNode constructor to point to an // internal array of MachOpers. The MachOper array is sized by diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp -index 45d262776..05fdab21e 100644 +index 2d3bafdd7..4cc7580a8 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp -@@ -1752,6 +1752,13 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { +@@ -1754,6 +1754,13 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { _shared_nodes.map(leaf->_idx, ex); } @@ -4762,7 +5862,7 @@ index 45d262776..05fdab21e 100644 return ex; } -@@ -2171,17 +2178,6 @@ void Matcher::find_shared( Node *n ) { +@@ -2173,17 +2180,6 @@ void Matcher::find_shared( Node *n ) { case Op_SafePoint: mem_op = true; break; @@ -4780,7 +5880,7 @@ index 45d262776..05fdab21e 100644 default: if( n->is_Store() ) { // Do match stores, despite no ideal reg -@@ -2294,33 +2290,6 @@ void Matcher::find_shared( Node *n ) { +@@ -2296,33 +2292,6 @@ void Matcher::find_shared( Node *n ) { n->del_req(LoadStoreConditionalNode::ExpectedIn); break; } @@ -4815,7 +5915,7 @@ index 45d262776..05fdab21e 100644 case Op_CMoveF: case Op_CMoveI: diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp -index 651bbfcf1..3a5e45401 100644 +index 8ffb5a708..32ce0f9bc 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -49,9 +49,6 @@ @@ -4828,7 +5928,55 @@ index 651bbfcf1..3a5e45401 100644 #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #endif -@@ -2852,7 +2849,7 @@ LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const Ty +@@ -858,7 +855,7 @@ bool LoadNode::is_immutable_value(Node* adr) { + //----------------------------LoadNode::make----------------------------------- + // Polymorphic factory method: + Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypePtr* adr_type, const Type *rt, BasicType bt, MemOrd mo, +- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { ++ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { + Compile* C = gvn.C; + + // sanity check the alias category against the created node type +@@ -909,6 +906,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP + if (unsafe) { + load->set_unsafe_access(); + } ++ load->set_barrier_data(barrier_data); + if (load->Opcode() == Op_LoadN) { + Node* ld = gvn.transform(load); + return new DecodeNNode(ld, ld->bottom_type()->make_ptr()); +@@ -918,7 +916,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP + } + + LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, +- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { ++ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { + bool require_atomic = true; + LoadLNode* load = new LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic); + if (unaligned) { +@@ -930,11 +928,12 @@ LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr + if (unsafe) { + load->set_unsafe_access(); + } ++ load->set_barrier_data(barrier_data); + return load; + } + + LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, +- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { ++ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { + bool require_atomic = true; + LoadDNode* load = new LoadDNode(ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic); + if (unaligned) { +@@ -946,6 +945,7 @@ LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr + if (unsafe) { + load->set_unsafe_access(); + } ++ load->set_barrier_data(barrier_data); + return load; + } + +@@ -2891,7 +2891,7 @@ LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const Ty : Node(required), _type(rt), _adr_type(at), @@ -4838,7 +5986,7 @@ index 651bbfcf1..3a5e45401 100644 init_req(MemNode::Control, c ); init_req(MemNode::Memory , mem); diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp -index 75f283ba8..abf07a233 100644 +index d4c2895bf..259b4343f 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -43,6 +43,8 @@ private: @@ -4915,7 +6063,17 @@ index 75f283ba8..abf07a233 100644 init_class_id(Class_Load); } inline bool is_unordered() const { return !is_acquire(); } -@@ -265,10 +280,6 @@ public: +@@ -213,7 +228,8 @@ public: + static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr, + const TypePtr* at, const Type *rt, BasicType bt, + MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, +- bool unaligned = false, bool mismatched = false, bool unsafe = false); ++ bool unaligned = false, bool mismatched = false, bool unsafe = false, ++ uint8_t barrier_data = 0); + + virtual uint hash() const; // Check the type + +@@ -265,10 +281,6 @@ public: Node* convert_to_unsigned_load(PhaseGVN& gvn); Node* convert_to_signed_load(PhaseGVN& gvn); @@ -4926,7 +6084,25 @@ index 75f283ba8..abf07a233 100644 #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif -@@ -817,7 +828,7 @@ class LoadStoreNode : public Node { +@@ -398,7 +410,7 @@ public: + bool require_atomic_access() const { return _require_atomic_access; } + static LoadLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, + const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, +- bool unaligned = false, bool mismatched = false, bool unsafe = false); ++ bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); + #ifndef PRODUCT + virtual void dump_spec(outputStream *st) const { + LoadNode::dump_spec(st); +@@ -450,7 +462,7 @@ public: + bool require_atomic_access() const { return _require_atomic_access; } + static LoadDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, + const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, +- bool unaligned = false, bool mismatched = false, bool unsafe = false); ++ bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); + #ifndef PRODUCT + virtual void dump_spec(outputStream *st) const { + LoadNode::dump_spec(st); +@@ -817,7 +829,7 @@ class LoadStoreNode : public Node { private: const Type* const _type; // What kind of value is loaded? const TypePtr* _adr_type; // What kind of memory is being addressed? @@ -4935,7 +6111,7 @@ index 75f283ba8..abf07a233 100644 virtual uint size_of() const; // Size is bigger public: LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ); -@@ -830,8 +841,9 @@ public: +@@ -831,8 +843,9 @@ public: bool result_not_used() const; MemBarNode* trailing_membar() const; @@ -4947,7 +6123,7 @@ index 75f283ba8..abf07a233 100644 }; class LoadStoreConditionalNode : public LoadStoreNode { -@@ -883,6 +895,7 @@ public: +@@ -885,6 +898,7 @@ public: MemNode::MemOrd order() const { return _mem_ord; } @@ -4955,7 +6131,7 @@ index 75f283ba8..abf07a233 100644 }; class CompareAndExchangeNode : public LoadStoreNode { -@@ -900,6 +913,7 @@ public: +@@ -902,6 +916,7 @@ public: MemNode::MemOrd order() const { return _mem_ord; } @@ -4964,10 +6140,10 @@ index 75f283ba8..abf07a233 100644 //------------------------------CompareAndSwapBNode--------------------------- diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp -index bb020c408..5a5e44ecd 100644 +index e439ebb93..84a56f8d0 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp -@@ -546,9 +546,6 @@ Node *Node::clone() const { +@@ -550,9 +550,6 @@ Node *Node::clone() const { if (n->is_SafePoint()) { n->as_SafePoint()->clone_replaced_nodes(); } @@ -4977,7 +6153,7 @@ index bb020c408..5a5e44ecd 100644 return n; // Return the clone } -@@ -1471,10 +1468,6 @@ bool Node::needs_anti_dependence_check() const { +@@ -1478,10 +1475,6 @@ bool Node::needs_anti_dependence_check() const { if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 ) { return false; } @@ -4989,10 +6165,10 @@ index bb020c408..5a5e44ecd 100644 } diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp -index 15e6ef893..de782b880 100644 +index f3bd41d91..6efaa6fc7 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp -@@ -82,8 +82,6 @@ class JVMState; +@@ -83,8 +83,6 @@ class JVMState; class JumpNode; class JumpProjNode; class LoadNode; @@ -5001,7 +6177,7 @@ index 15e6ef893..de782b880 100644 class LoadStoreNode; class LoadStoreConditionalNode; class LockNode; -@@ -645,7 +643,6 @@ public: +@@ -648,7 +646,6 @@ public: DEFINE_CLASS_ID(MemBar, Multi, 3) DEFINE_CLASS_ID(Initialize, MemBar, 0) DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) @@ -5009,7 +6185,7 @@ index 15e6ef893..de782b880 100644 DEFINE_CLASS_ID(Mach, Node, 1) DEFINE_CLASS_ID(MachReturn, Mach, 0) -@@ -697,7 +694,6 @@ public: +@@ -700,7 +697,6 @@ public: DEFINE_CLASS_ID(Mem, Node, 4) DEFINE_CLASS_ID(Load, Mem, 0) DEFINE_CLASS_ID(LoadVector, Load, 0) @@ -5017,7 +6193,7 @@ index 15e6ef893..de782b880 100644 DEFINE_CLASS_ID(Store, Mem, 1) DEFINE_CLASS_ID(StoreVector, Store, 0) DEFINE_CLASS_ID(LoadStore, Mem, 2) -@@ -841,8 +837,6 @@ public: +@@ -845,8 +841,6 @@ public: DEFINE_CLASS_QUERY(Load) DEFINE_CLASS_QUERY(LoadStore) DEFINE_CLASS_QUERY(LoadStoreConditional) @@ -5027,7 +6203,7 @@ index 15e6ef893..de782b880 100644 DEFINE_CLASS_QUERY(Loop) DEFINE_CLASS_QUERY(Mach) diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp -index 4ccf75783..f22029256 100644 +index b6540e06a..5b9873b4d 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -32,6 +32,8 @@ @@ -5462,7 +6638,7 @@ index 4ccf75783..f22029256 100644 // Have we run out of code space? if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { -@@ -1264,12 +1272,12 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { +@@ -1265,12 +1273,12 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { Process_OopMap_Node(mach, current_offset); } // End if safepoint @@ -5477,7 +6653,7 @@ index 4ccf75783..f22029256 100644 else if (mach->is_MachBranch()) { // This requires the TRUE branch target be in succs[0] uint block_num = block->non_connector_successor(0)->_pre_order; -@@ -1280,8 +1288,8 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { +@@ -1281,8 +1289,8 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { bool delay_slot_is_used = valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay(); if (!delay_slot_is_used && mach->may_be_short_branch()) { @@ -5488,7 +6664,7 @@ index 4ccf75783..f22029256 100644 int offset = blk_starts[block_num] - current_offset; if (block_num >= i) { // Current and following block's offset are not -@@ -1339,7 +1347,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { +@@ -1340,7 +1348,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { } } #ifdef ASSERT @@ -5497,7 +6673,7 @@ index 4ccf75783..f22029256 100644 else if (mach->ideal_Opcode() == Op_StoreCM) { uint storeCM_idx = j; int count = 0; -@@ -1517,6 +1525,10 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { +@@ -1519,6 +1527,10 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { } #endif @@ -5508,7 +6684,7 @@ index 4ccf75783..f22029256 100644 #ifndef PRODUCT // Information on the size of the method, without the extraneous code Scheduling::increment_method_size(cb->insts_size()); -@@ -1681,20 +1693,20 @@ uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+ +@@ -1682,20 +1694,20 @@ uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+ // Initializer for class Scheduling Scheduling::Scheduling(Arena *arena, Compile &compile) @@ -5542,7 +6718,7 @@ index 4ccf75783..f22029256 100644 #endif { // Create a MachNopNode -@@ -1773,8 +1785,8 @@ void Scheduling::step_and_clear() { +@@ -1774,8 +1786,8 @@ void Scheduling::step_and_clear() { _bundle_use.reset(); memcpy(_bundle_use_elements, @@ -5553,7 +6729,7 @@ index 4ccf75783..f22029256 100644 } // Perform instruction scheduling and bundling over the sequence of -@@ -1801,6 +1813,22 @@ void Compile::ScheduleAndBundle() { +@@ -1802,6 +1814,22 @@ void Compile::ScheduleAndBundle() { // Walk backwards over each basic block, computing the needed alignment // Walk over all the basic blocks scheduling.DoScheduling(); @@ -5576,7 +6752,7 @@ index 4ccf75783..f22029256 100644 } // Compute the latency of all the instructions. This is fairly simple, -@@ -1869,7 +1897,7 @@ bool Scheduling::NodeFitsInBundle(Node *n) { +@@ -1870,7 +1898,7 @@ bool Scheduling::NodeFitsInBundle(Node *n) { #ifndef PRODUCT if (_cfg->C->trace_opto_output()) tty->print("# NodeFitsInBundle [%4d]: FALSE; latency %4d > %d\n", @@ -5585,7 +6761,7 @@ index 4ccf75783..f22029256 100644 #endif return (false); } -@@ -2094,12 +2122,12 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { +@@ -2095,12 +2123,12 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { // Don't allow safepoints in the branch shadow, that will // cause a number of difficulties if ( avail_pipeline->instructionCount() == 1 && @@ -5604,7 +6780,7 @@ index 4ccf75783..f22029256 100644 if (d->is_Mach() && !d->is_MachSafePoint()) { // A node that fits in the delay slot was found, so we need to -@@ -2144,13 +2172,13 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { +@@ -2145,13 +2173,13 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { // step of the bundles if (!NodeFitsInBundle(n)) { #ifndef PRODUCT @@ -5623,7 +6799,7 @@ index 4ccf75783..f22029256 100644 } } -@@ -2196,8 +2224,8 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { +@@ -2197,8 +2225,8 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { #ifndef PRODUCT if (_cfg->C->trace_opto_output()) tty->print("# *** STEP(%d >= %d instructions) ***\n", @@ -5634,7 +6810,7 @@ index 4ccf75783..f22029256 100644 #endif step(1); } -@@ -2403,7 +2431,7 @@ void Scheduling::DoScheduling() { +@@ -2404,7 +2432,7 @@ void Scheduling::DoScheduling() { } assert(!last->is_Mach() || last->as_Mach()->ideal_Opcode() != Op_Con, ""); if( last->is_Catch() || @@ -5643,7 +6819,7 @@ index 4ccf75783..f22029256 100644 // There might be a prior call. Skip it. while (_bb_start < _bb_end && bb->get_node(--_bb_end)->is_MachProj()); } else if( last->is_MachNullCheck() ) { -@@ -2473,7 +2501,7 @@ void Scheduling::DoScheduling() { +@@ -2474,7 +2502,7 @@ void Scheduling::DoScheduling() { } #endif #ifdef ASSERT @@ -5652,7 +6828,7 @@ index 4ccf75783..f22029256 100644 #endif } -@@ -2821,31 +2849,31 @@ void Scheduling::ComputeRegisterAntidependencies(Block *b) { +@@ -2822,31 +2850,31 @@ void Scheduling::ComputeRegisterAntidependencies(Block *b) { // void Scheduling::garbage_collect_pinch_nodes() { #ifndef PRODUCT @@ -5704,7 +6880,7 @@ index 4ccf75783..f22029256 100644 #endif } -@@ -2882,19 +2910,19 @@ void Scheduling::dump_available() const { +@@ -2883,19 +2911,19 @@ void Scheduling::dump_available() const { void Scheduling::print_statistics() { // Print the size added by nops for bundling tty->print("Nops added %d bytes to total of %d bytes", @@ -5728,7 +6904,7 @@ index 4ccf75783..f22029256 100644 tty->print("\n"); } -@@ -2908,6 +2936,6 @@ void Scheduling::print_statistics() { +@@ -2909,6 +2937,6 @@ void Scheduling::print_statistics() { if (total_bundles > 0) tty->print("Average ILP (excluding nops) is %.2f\n", @@ -5752,11 +6928,23 @@ index ab3c1a304..ec3cc2981 100644 }; //------------------------------Scheduling---------------------------------- +diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp +index 8286f8c4d..78149369d 100644 +--- a/src/hotspot/share/opto/parse1.cpp ++++ b/src/hotspot/share/opto/parse1.cpp +@@ -41,6 +41,7 @@ + #include "runtime/handles.inline.hpp" + #include "runtime/safepointMechanism.hpp" + #include "runtime/sharedRuntime.hpp" ++#include "utilities/bitMap.inline.hpp" + #include "utilities/copy.hpp" + + // Static array so we can figure out which bytecodes stop us from compiling diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp -index 07b849040..9d5d4deed 100644 +index 41971513f..3d71d941c 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp -@@ -1648,14 +1648,14 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { +@@ -1726,14 +1726,14 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { // of the mirror load depends on the type of 'n'. See LoadNode::Value(). // LoadBarrier?(LoadP(LoadP(AddP(foo:Klass, #java_mirror)))) BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); @@ -5773,7 +6961,7 @@ index 07b849040..9d5d4deed 100644 // Search for load barriers behind the load for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) { Node* b = u->fast_out(i3); -@@ -1818,14 +1818,14 @@ void PhaseCCP::analyze() { +@@ -1927,14 +1927,14 @@ void PhaseCCP::analyze() { // Loading the java mirror from a Klass requires two loads and the type // of the mirror load depends on the type of 'n'. See LoadNode::Value(). BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); @@ -5791,10 +6979,10 @@ index 07b849040..9d5d4deed 100644 for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) { Node* b = u->fast_out(i3); diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp -index 0258db3e6..fae147fa8 100644 +index 1ee9db8f0..1f2cf2c64 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp -@@ -264,7 +264,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) { +@@ -286,7 +286,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) { case Op_LoadI: case Op_LoadL: case Op_LoadF: case Op_LoadD: case Op_LoadP: case Op_LoadN: @@ -5802,3 +6990,193 @@ index 0258db3e6..fae147fa8 100644 *start = 0; *end = 0; // no vector operands break; +diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp +index 595ff7495..3f366633e 100644 +--- a/src/hotspot/share/runtime/sharedRuntime.cpp ++++ b/src/hotspot/share/runtime/sharedRuntime.cpp +@@ -205,6 +205,8 @@ void SharedRuntime::print_ic_miss_histogram() { + } + #endif // PRODUCT + ++int SharedRuntime::_monotonical_incr = 0; ++bool SharedRuntime::_opt_for_aarch64 = false; + + JRT_LEAF(jlong, SharedRuntime::lmul(jlong y, jlong x)) + return x * y; +diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp +index a11009837..eba82d453 100644 +--- a/src/hotspot/share/runtime/sharedRuntime.hpp ++++ b/src/hotspot/share/runtime/sharedRuntime.hpp +@@ -596,6 +596,11 @@ class SharedRuntime: AllStatic { + static void print_ic_miss_histogram(); + + #endif // PRODUCT ++ ++ static int _monotonical_incr; ++ static bool _opt_for_aarch64; ++ ++ static address monotonical_incr_addr() { return (address)&_monotonical_incr; } + }; + + +diff --git a/src/hotspot/share/utilities/bitMap.hpp b/src/hotspot/share/utilities/bitMap.hpp +index c671535c9..e26f34687 100644 +--- a/src/hotspot/share/utilities/bitMap.hpp ++++ b/src/hotspot/share/utilities/bitMap.hpp +@@ -26,6 +26,7 @@ + #define SHARE_VM_UTILITIES_BITMAP_HPP + + #include "memory/allocation.hpp" ++#include "runtime/atomic.hpp" + #include "utilities/align.hpp" + #include "utilities/globalDefinitions.hpp" + +@@ -95,6 +96,8 @@ class BitMap { + void set_word (idx_t word) { set_word(word, ~(bm_word_t)0); } + void clear_word(idx_t word) { _map[word] = 0; } + ++ static inline const bm_word_t load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order); ++ + // Utilities for ranges of bits. Ranges are half-open [beg, end). + + // Ranges within a single word. +@@ -194,6 +197,9 @@ class BitMap { + return (*word_addr(index) & bit_mask(index)) != 0; + } + ++ // memory_order must be memory_order_relaxed or memory_order_acquire. ++ bool par_at(idx_t index, atomic_memory_order memory_order = memory_order_acquire) const; ++ + // Align bit index up or down to the next bitmap word boundary, or check + // alignment. + static idx_t word_align_up(idx_t bit) { +@@ -210,9 +216,14 @@ class BitMap { + inline void set_bit(idx_t bit); + inline void clear_bit(idx_t bit); + +- // Atomically set or clear the specified bit. +- inline bool par_set_bit(idx_t bit); +- inline bool par_clear_bit(idx_t bit); ++ // Attempts to change a bit to a desired value. The operation returns true if ++ // this thread changed the value of the bit. It was changed with a RMW operation ++ // using the specified memory_order. The operation returns false if the change ++ // could not be set due to the bit already being observed in the desired state. ++ // The atomic access that observed the bit in the desired state has acquire ++ // semantics, unless memory_order is memory_order_relaxed or memory_order_release. ++ inline bool par_set_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); ++ inline bool par_clear_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); + + // Put the given value at the given offset. The parallel version + // will CAS the value into the bitmap and is quite a bit slower. +diff --git a/src/hotspot/share/utilities/bitMap.inline.hpp b/src/hotspot/share/utilities/bitMap.inline.hpp +index b10726d18..7a7e2ad43 100644 +--- a/src/hotspot/share/utilities/bitMap.inline.hpp ++++ b/src/hotspot/share/utilities/bitMap.inline.hpp +@@ -26,6 +26,7 @@ + #define SHARE_VM_UTILITIES_BITMAP_INLINE_HPP + + #include "runtime/atomic.hpp" ++#include "runtime/orderAccess.hpp" + #include "utilities/bitMap.hpp" + + inline void BitMap::set_bit(idx_t bit) { +@@ -38,18 +39,39 @@ inline void BitMap::clear_bit(idx_t bit) { + *word_addr(bit) &= ~bit_mask(bit); + } + +-inline bool BitMap::par_set_bit(idx_t bit) { ++inline const BitMap::bm_word_t BitMap::load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order) { ++ if (memory_order == memory_order_relaxed || memory_order == memory_order_release) { ++ return Atomic::load(addr); ++ } else { ++ assert(memory_order == memory_order_acq_rel || ++ memory_order == memory_order_acquire || ++ memory_order == memory_order_conservative, ++ "unexpected memory ordering"); ++ return OrderAccess::load_acquire(addr); ++ } ++} ++ ++inline bool BitMap::par_at(idx_t index, atomic_memory_order memory_order) const { ++ verify_index(index); ++ assert(memory_order == memory_order_acquire || ++ memory_order == memory_order_relaxed, ++ "unexpected memory ordering"); ++ const volatile bm_word_t* const addr = word_addr(index); ++ return (load_word_ordered(addr, memory_order) & bit_mask(index)) != 0; ++} ++ ++inline bool BitMap::par_set_bit(idx_t bit, atomic_memory_order memory_order) { + verify_index(bit); + volatile bm_word_t* const addr = word_addr(bit); + const bm_word_t mask = bit_mask(bit); +- bm_word_t old_val = *addr; ++ bm_word_t old_val = load_word_ordered(addr, memory_order); + + do { + const bm_word_t new_val = old_val | mask; + if (new_val == old_val) { + return false; // Someone else beat us to it. + } +- const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); ++ const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); + if (cur_val == old_val) { + return true; // Success. + } +@@ -57,18 +79,18 @@ inline bool BitMap::par_set_bit(idx_t bit) { + } while (true); + } + +-inline bool BitMap::par_clear_bit(idx_t bit) { ++inline bool BitMap::par_clear_bit(idx_t bit, atomic_memory_order memory_order) { + verify_index(bit); + volatile bm_word_t* const addr = word_addr(bit); + const bm_word_t mask = ~bit_mask(bit); +- bm_word_t old_val = *addr; ++ bm_word_t old_val = load_word_ordered(addr, memory_order); + + do { + const bm_word_t new_val = old_val & mask; + if (new_val == old_val) { + return false; // Someone else beat us to it. + } +- const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); ++ const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); + if (cur_val == old_val) { + return true; // Success. + } +diff --git a/src/java.base/share/classes/java/util/Random.java b/src/java.base/share/classes/java/util/Random.java +index 92c1193cb..aaf3da581 100644 +--- a/src/java.base/share/classes/java/util/Random.java ++++ b/src/java.base/share/classes/java/util/Random.java +@@ -35,6 +35,7 @@ import java.util.stream.LongStream; + import java.util.stream.StreamSupport; + + import jdk.internal.misc.Unsafe; ++import jdk.internal.HotSpotIntrinsicCandidate; + + /** + * An instance of this class is used to generate a stream of +@@ -325,6 +326,7 @@ class Random implements java.io.Serializable { + * @return the next pseudorandom, uniformly distributed {@code int} + * value from this random number generator's sequence + */ ++ @HotSpotIntrinsicCandidate + public int nextInt() { + return next(32); + } +diff --git a/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java b/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java +index a8aff4775..afadfd68a 100644 +--- a/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java ++++ b/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java +@@ -75,6 +75,7 @@ public class MemberNameLeak { + test("-XX:+UseG1GC"); + test("-XX:+UseParallelGC"); + test("-XX:+UseSerialGC"); ++ test("-XX:+UseZGC"); + if (!Compiler.isGraalEnabled()) { // Graal does not support CMS and Shenandoah + test("-XX:+UseConcMarkSweepGC"); + if (GC.Shenandoah.isSupported()) { +-- +2.19.1 + diff --git a/ZGC-aarch64-fix-not-using-load-store-Pre-index.patch b/ZGC-aarch64-fix-not-using-load-store-Pre-index.patch deleted file mode 100644 index 36e7941..0000000 --- a/ZGC-aarch64-fix-not-using-load-store-Pre-index.patch +++ /dev/null @@ -1,58 +0,0 @@ -From e8bf6d9c5a02b3ffaf223dd1109bc15c664cca28 Mon Sep 17 00:00:00 2001 -Date: Mon, 24 Feb 2020 18:51:09 +0800 -Subject: [PATCH] ZGC: aarch64: fix not using load/store Pre-indexed - addressing to modify sp - -Summary: : -LLT: JFUZZ -Bug url: ---- - src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp | 16 +++++----------- - 1 file changed, 5 insertions(+), 11 deletions(-) - -diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -index a65a605d0..6db979b57 100644 ---- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -@@ -2114,12 +2114,11 @@ int MacroAssembler::push_fp(unsigned int bitset, Register stack) { - return 0; - } - -+ add(stack, stack, -count * wordSize * 2); -+ - if (count & 1) { -- strq(as_FloatRegister(regs[0]), Address(pre(stack, -count * wordSize * 2))); -+ strq(as_FloatRegister(regs[0]), Address(stack)); - i += 1; -- } else { -- stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -count * wordSize * 2))); -- i += 2; - } - - for (; i < count; i += 2) { -@@ -2145,20 +2144,15 @@ int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { - } - - if (count & 1) { -+ ldrq(as_FloatRegister(regs[0]), Address(stack)); - i += 1; -- } else { -- i += 2; - } - - for (; i < count; i += 2) { - ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); - } - -- if ((count & 1) == 0) { -- ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, count * wordSize * 2))); -- } else { -- ldrq(as_FloatRegister(regs[0]), Address(post(stack, count * wordSize * 2))); -- } -+ add(stack, stack, count * wordSize * 2); - - return count; - } --- -2.12.3 - diff --git a/ZGC-aarch64-fix-system-call-number-of-memfd_create.patch b/ZGC-aarch64-fix-system-call-number-of-memfd_create.patch deleted file mode 100644 index ee83a44..0000000 --- a/ZGC-aarch64-fix-system-call-number-of-memfd_create.patch +++ /dev/null @@ -1,28 +0,0 @@ -From e25b331a945301e24429c120bef1ed0daf04d49c Mon Sep 17 00:00:00 2001 -Date: Fri, 3 Apr 2020 17:12:16 +0800 -Subject: [PATCH] ZGC: aarch64: Fix MR 32, fix system call number of - memfd_create - -Summary: : -LLT: N/A -Bug url: N/A ---- - src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp -index 47894b5..f956b53 100644 ---- a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp -+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp -@@ -51,7 +51,7 @@ - - // Support for building on older Linux systems - #ifndef __NR_memfd_create --#define __NR_memfd_create 319 -+#define __NR_memfd_create 279 - #endif - #ifndef MFD_CLOEXEC - #define MFD_CLOEXEC 0x0001U --- -1.8.3.1 - diff --git a/ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch b/ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch deleted file mode 100644 index 38cac06..0000000 --- a/ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch +++ /dev/null @@ -1,106 +0,0 @@ -From 425112071e77e2fb599d1f96ce48689d45461261 Mon Sep 17 00:00:00 2001 -Date: Mon, 17 Feb 2020 18:55:47 +0800 -Subject: [PATCH] ZGC: aarch64: not using zr register avoid sigill in - MacroAssembler::push_fp and pop_fp - -Summary: : -LLT: jtreg -Bug url: ---- - src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp | 48 +++++++++++++--------- - 1 file changed, 28 insertions(+), 20 deletions(-) - -diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -index 611f13b0e..a65a605d0 100644 ---- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp -@@ -2100,58 +2100,66 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) { - // Push lots of registers in the bit set supplied. Don't push sp. - // Return the number of words pushed - int MacroAssembler::push_fp(unsigned int bitset, Register stack) { -- int words_pushed = 0; -- - // Scan bitset to accumulate register pairs - unsigned char regs[32]; - int count = 0; -+ int i = 0; - for (int reg = 0; reg <= 31; reg++) { - if (1 & bitset) - regs[count++] = reg; - bitset >>= 1; - } -- regs[count++] = zr->encoding_nocheck(); -- count &= ~1; // Only push an even number of regs - -- // Always pushing full 128 bit registers. -- if (count) { -+ if (!count) { -+ return 0; -+ } -+ -+ if (count & 1) { -+ strq(as_FloatRegister(regs[0]), Address(pre(stack, -count * wordSize * 2))); -+ i += 1; -+ } else { - stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -count * wordSize * 2))); -- words_pushed += 2; -+ i += 2; - } -- for (int i = 2; i < count; i += 2) { -+ -+ for (; i < count; i += 2) { - stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); -- words_pushed += 2; - } - -- assert(words_pushed == count, "oops, pushed != count"); - return count; - } - - int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { -- int words_pushed = 0; -- - // Scan bitset to accumulate register pairs - unsigned char regs[32]; - int count = 0; -+ int i = 0; - for (int reg = 0; reg <= 31; reg++) { - if (1 & bitset) - regs[count++] = reg; - bitset >>= 1; - } -- regs[count++] = zr->encoding_nocheck(); -- count &= ~1; - -- for (int i = 2; i < count; i += 2) { -+ if (!count) { -+ return 0; -+ } -+ -+ if (count & 1) { -+ i += 1; -+ } else { -+ i += 2; -+ } -+ -+ for (; i < count; i += 2) { - ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); -- words_pushed += 2; - } -- if (count) { -+ -+ if ((count & 1) == 0) { - ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, count * wordSize * 2))); -- words_pushed += 2; -+ } else { -+ ldrq(as_FloatRegister(regs[0]), Address(post(stack, count * wordSize * 2))); - } - -- assert(words_pushed == count, "oops, pushed != count"); -- - return count; - } - --- -2.12.3 - diff --git a/ZGC-reuse-entries-of-ResolvedMethodTable.patch b/ZGC-reuse-entries-of-ResolvedMethodTable.patch deleted file mode 100644 index ce91594..0000000 --- a/ZGC-reuse-entries-of-ResolvedMethodTable.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp -index 62f97d2..e950acf 100644 ---- a/src/hotspot/share/gc/z/zHeap.cpp -+++ b/src/hotspot/share/gc/z/zHeap.cpp -@@ -49,6 +49,7 @@ - #include "runtime/thread.hpp" - #include "utilities/align.hpp" - #include "utilities/debug.hpp" -+#include "prims/resolvedMethodTable.hpp" - - static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes); - static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes); -@@ -334,6 +335,10 @@ bool ZHeap::mark_end() { - Universe::verify(); - } - -+ // Free unsed entries of ResolvedMethodTable and weakhandles -+ // avoid ResolvedMethodTable inflation and native memory leak -+ ResolvedMethodTable::unlink(); -+ - return true; - } - -diff --git a/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java b/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java -index a8aff47..afadfd6 100644 ---- a/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java -+++ b/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java -@@ -75,6 +75,7 @@ public class MemberNameLeak { - test("-XX:+UseG1GC"); - test("-XX:+UseParallelGC"); - test("-XX:+UseSerialGC"); -+ test("-XX:+UseZGC"); - if (!Compiler.isGraalEnabled()) { // Graal does not support CMS and Shenandoah - test("-XX:+UseConcMarkSweepGC"); - if (GC.Shenandoah.isSupported()) { diff --git a/add-missing-inline.patch b/add-missing-inline.patch deleted file mode 100644 index 5647d02..0000000 --- a/add-missing-inline.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp -index 18f455086..785470dbe 100644 ---- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp -+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp -@@ -24,6 +24,7 @@ - #include "precompiled.hpp" - #include "asm/macroAssembler.inline.hpp" - #include "code/codeBlob.hpp" -+#include "code/vmreg.inline.hpp" - #include "gc/z/zBarrier.inline.hpp" - #include "gc/z/zBarrierSet.hpp" - #include "gc/z/zBarrierSetAssembler.hpp" diff --git a/openjdk-11.spec b/openjdk-11.spec index 0be863b..7ccdbd4 100644 --- a/openjdk-11.spec +++ b/openjdk-11.spec @@ -762,7 +762,7 @@ Provides: java-src%{?1} = %{epoch}:%{version}-%{release} Name: java-%{javaver}-%{origin} Version: %{newjavaver}.%{buildver} -Release: 3 +Release: 4 # java-1.5.0-ibm from jpackage.org set Epoch to 1 for unknown reasons # and this change was brought into RHEL-4. java-1.5.0-ibm packages # also included the epoch in their virtual provides. This created a @@ -834,18 +834,7 @@ Patch2001: LoongArch64-support.patch Patch5: Add-ability-to-configure-third-port-for-remote-JMX.patch Patch6: 8214527-AArch64-ZGC-for-Aarch64.patch Patch7: 8224675-Late-GC-barrier-insertion-for-ZGC.patch -Patch9: ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch -Patch10: ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch -Patch11: 8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch -Patch12: 8233073-Make-BitMap-accessors-more-memory-ordering-f.patch -Patch13: 8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch -Patch18: 8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch -Patch20: 8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch -Patch22: 8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch -Patch23: add-missing-inline.patch -Patch26: ZGC-aarch64-fix-system-call-number-of-memfd_create.patch -Patch27: ZGC-aarch64-fix-not-using-load-store-Pre-index.patch -Patch29: ZGC-reuse-entries-of-ResolvedMethodTable.patch +Patch9: ZGC-AArch64-Optimizations-and-Fixes.patch # 11.0.8 Patch33: 8210473-JEP-345-NUMA-Aware-Memory-Allocation-for-G1.patch @@ -910,6 +899,9 @@ Patch91: 8222289-Overhaul-logic-for-reading-writing-constant-pool-entries.patch # 11.0.21 Patch92: 8295068-SSLEngine-throws-NPE-parsing-Certificate.patch + +# 11.0.23 +Patch93: Cache-byte-when-constructing-String-with-duplicate-c.patch ############################################ # # riscv64 specific patches @@ -1156,17 +1148,6 @@ pushd %{top_level_dir_name} %patch6 -p1 %patch7 -p1 %patch9 -p1 -%patch10 -p1 -%patch11 -p1 -%patch12 -p1 -%patch13 -p1 -%patch18 -p1 -%patch20 -p1 -%patch22 -p1 -%patch23 -p1 -%patch26 -p1 -%patch27 -p1 -%patch29 -p1 %patch33 -p1 %patch34 -p1 %patch35 -p1 @@ -1212,6 +1193,7 @@ pushd %{top_level_dir_name} %patch90 -p1 %patch91 -p1 %patch92 -p1 +%patch93 -p1 %endif %endif %ifarch loongarch64 @@ -1733,6 +1715,9 @@ cjc.mainProgram(arg) %changelog +* Sat Jun 15 2024 neu-mobi - 1.11.0.23.9-4 +- Collate patches and merge patches related to ZGC + * Mon Jun 03 2024 songliyang - 1:11.0.23.9-3 - fix loongarch vendor error - fix changelog error -- Gitee