infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-04-09 14:38:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

8381003: [REDO] Mitigate Neoverse-N1 erratum 1542419 negative impact on GCs and JIT performance

Browse Source 
Reviewed-by: aph
This commit is contained in:
Evgeny Astigeevich 2026-04-07 10:35:31 +00:00
parent 0b803bd34e
commit 9cf2b686bd
24 changed files with 1371 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
View File

@ -879,7 +879,9 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
ShouldNotReachHere();
}
ICache::invalidate_word((address)patch_addr);
if (!UseSingleICacheInvalidation) {
ICache::invalidate_word((address)patch_addr);
}
}
#ifdef COMPILER1

4
src/hotspot/cpu/aarch64/globals_aarch64.hpp
View File

@ -131,6 +131,10 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Branch Protection to use: none, standard, pac-ret") \
product(bool, AlwaysMergeDMB, true, DIAGNOSTIC, \
"Always merge DMB instructions in code emission") \
product(bool, NeoverseN1ICacheErratumMitigation, false, DIAGNOSTIC, \
"Enable workaround for Neoverse N1 erratum 1542419") \
product(bool, UseSingleICacheInvalidation, false, DIAGNOSTIC, \
"Defer multiple ICache invalidation to single invalidation") \
// end of ARCH_FLAGS

7
src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
View File

@ -54,7 +54,12 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
bytes = MacroAssembler::pd_patch_instruction_size(addr(), x);
break;
}
ICache::invalidate_range(addr(), bytes);
if (UseSingleICacheInvalidation) {
assert(_binding != nullptr, "expect to be called with RelocIterator in use");
} else {
ICache::invalidate_range(addr(), bytes);
}
}
address Relocation::pd_call_destination(address orig_addr) {

59
src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
View File

@ -24,6 +24,7 @@
*
*/
#include "logging/log.hpp"
#include "pauth_aarch64.hpp"
#include "register_aarch64.hpp"
#include "runtime/arguments.hpp"
@ -52,6 +53,10 @@ uintptr_t VM_Version::_pac_mask;
SpinWait VM_Version::_spin_wait;
bool VM_Version::_cache_dic_enabled;
bool VM_Version::_cache_idc_enabled;
bool VM_Version::_ic_ivau_trapped;
const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr };
static SpinWait get_spin_wait_desc() {
@ -85,6 +90,19 @@ static SpinWait get_spin_wait_desc() {
return spin_wait;
}
static bool has_neoverse_n1_errata_1542419() {
const int major_rev_num = VM_Version::cpu_variant();
const int minor_rev_num = VM_Version::cpu_revision();
// Neoverse N1: 0xd0c
// Erratum 1542419 affects r3p0, r3p1 and r4p0.
// It is fixed in r4p1 and later revisions, which are not affected.
return (VM_Version::cpu_family() == VM_Version::CPU_ARM &&
VM_Version::model_is(0xd0c) &&
((major_rev_num == 3 && minor_rev_num == 0) ||
(major_rev_num == 3 && minor_rev_num == 1) ||
(major_rev_num == 4 && minor_rev_num == 0)));
}
void VM_Version::initialize() {
#define SET_CPU_FEATURE_NAME(id, name, bit) \
_features_names[bit] = XSTR(name);
@ -96,6 +114,10 @@ void VM_Version::initialize() {
_supports_atomic_getset8 = true;
_supports_atomic_getadd8 = true;
_cache_dic_enabled = false;
_cache_idc_enabled = false;
_ic_ivau_trapped = false;
get_os_cpu_info();
_cpu_features = _features;
@ -676,6 +698,43 @@ void VM_Version::initialize() {
clear_feature(CPU_SVE);
}
if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation) && is_cache_idc_enabled() && is_cache_dic_enabled()) {
FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
}
if (FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && has_neoverse_n1_errata_1542419()
&& is_cache_idc_enabled() && !is_cache_dic_enabled()) {
if (_ic_ivau_trapped) {
FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, true);
} else {
log_info(os)("IC IVAU is not trapped; disabling NeoverseN1ICacheErratumMitigation");
FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, false);
}
}
if (NeoverseN1ICacheErratumMitigation) {
if (!has_neoverse_n1_errata_1542419()) {
vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set for the CPU not having Neoverse N1 errata 1542419");
}
// If the user explicitly set the flag, verify the trap is active.
if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !_ic_ivau_trapped) {
vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but IC IVAU is not trapped. "
"The optimization is not safe on this system.");
}
if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation)) {
FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
}
if (!UseSingleICacheInvalidation) {
vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but UseSingleICacheInvalidation is not enabled");
}
}
if (UseSingleICacheInvalidation
&& (!is_cache_idc_enabled() || (!is_cache_dic_enabled() && !NeoverseN1ICacheErratumMitigation))) {
vm_exit_during_initialization("UseSingleICacheInvalidation is set but neither IDC nor DIC nor NeoverseN1ICacheErratumMitigation is enabled");
}
// Construct the "features" string
stringStream ss(512);
ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision);

10
src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
View File

@ -58,6 +58,12 @@ protected:
// When _prefer_sve_merging_mode_cpy is true, `cpy (imm, zeroing)` is
// implemented as `movi; cpy(imm, merging)`.
static constexpr bool _prefer_sve_merging_mode_cpy = true;
static bool _cache_dic_enabled;
static bool _cache_idc_enabled;
// IC IVAU trap probe for Neoverse N1 erratum 1542419.
// Set by get_os_cpu_info() on Linux via ic_ivau_probe_linux_aarch64.S.
static bool _ic_ivau_trapped;
static SpinWait _spin_wait;
@ -257,6 +263,10 @@ public:
return vector_length_in_bytes <= 16;
}
static bool is_cache_dic_enabled() { return _cache_dic_enabled; }
static bool is_cache_idc_enabled() { return _cache_idc_enabled; }
static bool is_ic_ivau_trapped() { return _ic_ivau_trapped; }
static void get_cpu_features_name(void* features_buffer, stringStream& ss);
// Returns names of features present in features_set1 but not in features_set2

69
src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "defs.S.inc"
# Probe whether IC IVAU is trapped.
#
# Returns 1 if IC IVAU is trapped (did not fault), 0 if not trapped
# (faulted on VA 0x0, signal handler redirected to continuation).
#
# int ic_ivau_probe(void);
DECLARE_FUNC(ic_ivau_probe):
DECLARE_FUNC(_ic_ivau_probe_fault):
ic ivau, xzr
mov x0, #1
ret
DECLARE_FUNC(_ic_ivau_probe_continuation):
mov x0, #0
ret
/* Emit .note.gnu.property section in case of PAC or BTI being enabled. */
#ifdef __ARM_FEATURE_BTI_DEFAULT
#ifdef __ARM_FEATURE_PAC_DEFAULT
#define GNU_PROPERTY_AARCH64_FEATURE 3
#else
#define GNU_PROPERTY_AARCH64_FEATURE 1
#endif
#else
#ifdef __ARM_FEATURE_PAC_DEFAULT
#define GNU_PROPERTY_AARCH64_FEATURE 2
#else
#define GNU_PROPERTY_AARCH64_FEATURE 0
#endif
#endif
#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
.pushsection .note.gnu.property, "a"
.align 3
.long 4 /* name length */
.long 0x10 /* data length */
.long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
.string "GNU" /* vendor name */
.long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
.long 4 /* pr_datasze */
.long GNU_PROPERTY_AARCH64_FEATURE
.long 0
.popsection
#endif

28
src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "runtime/icache.hpp"
#include "utilities/globalDefinitions.hpp"
NOT_PRODUCT(THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr;)

104
src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
View File

@ -26,6 +26,11 @@
#ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
#define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
#include "memory/allocation.hpp"
#include "runtime/vm_version.hpp"
#include "utilities/globalDefinitions.hpp"
#include "vm_version_aarch64.hpp"
// Interface for updating the instruction cache. Whenever the VM
// modifies code, part of the processor instruction cache potentially
// has to be flushed.
@ -37,8 +42,105 @@ class ICache : public AbstractICache {
__builtin___clear_cache((char *)addr, (char *)(addr + 4));
}
static void invalidate_range(address start, int nbytes) {
__builtin___clear_cache((char *)start, (char *)(start + nbytes));
if (NeoverseN1ICacheErratumMitigation) {
assert(VM_Version::is_cache_idc_enabled(),
"Expect CTR_EL0.IDC to be enabled for Neoverse N1 with erratum "
"1542419");
assert(!VM_Version::is_cache_dic_enabled(),
"Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
"1542419");
assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
asm volatile("dsb ish \n"
"ic ivau, xzr \n"
"dsb ish \n"
"isb \n"
: : : "memory");
} else {
__builtin___clear_cache((char *)start, (char *)(start + nbytes));
}
}
};
class AArch64ICacheInvalidationContext : StackObj {
private:
#ifdef ASSERT
static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context;
#endif
bool _has_modified_code;
public:
NONCOPYABLE(AArch64ICacheInvalidationContext);
AArch64ICacheInvalidationContext()
: _has_modified_code(false) {
assert(_current_context == nullptr, "nested ICacheInvalidationContext not supported");
#ifdef ASSERT
_current_context = this;
#endif
}
~AArch64ICacheInvalidationContext() {
NOT_PRODUCT(_current_context = nullptr);
if (!_has_modified_code || !UseSingleICacheInvalidation) {
return;
}
assert(VM_Version::is_cache_idc_enabled(), "Expect CTR_EL0.IDC to be enabled");
asm volatile("dsb ish" : : : "memory");
if (NeoverseN1ICacheErratumMitigation) {
assert(!VM_Version::is_cache_dic_enabled(),
"Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
"1542419");
assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
// Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature
// may fetch stale instructions when software depends on
// prefetch-speculation-protection instead of explicit synchronization.
//
// Neoverse-N1 implementation mitigates the errata 1542419 with a
// workaround:
// - Disable coherent icache.
// - Trap IC IVAU instructions.
// - Execute:
// - tlbi vae3is, xzr
// - dsb sy
// - Ignore trapped IC IVAU instructions.
//
// `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all
// possible levels). It waits for all memory accesses using in-scope old
// translation information to complete before it is considered complete.
//
// As this workaround has significant overhead, Arm Neoverse N1 (MP050)
// Software Developer Errata Notice version 29.0 suggests:
//
// "Since one TLB inner-shareable invalidation is enough to avoid this
// erratum, the number of injected TLB invalidations should be minimized
// in the trap handler to mitigate the performance impact due to this
// workaround."
// As the address for icache invalidation is not relevant and
// IC IVAU instruction is ignored, we use XZR in it.
asm volatile(
"ic ivau, xzr \n"
"dsb ish \n"
:
:
: "memory");
} else {
assert(VM_Version::is_cache_dic_enabled(), "Expect CTR_EL0.DIC to be enabled");
}
asm volatile("isb" : : : "memory");
}
void set_has_modified_code() {
_has_modified_code = true;
}
};
#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext
#endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP

11
src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
View File

@ -77,6 +77,11 @@
#define REG_LR 30
#define REG_BCP 22
// IC IVAU trap probe.
// Defined in ic_ivau_probe_linux_aarch64.S.
extern "C" char _ic_ivau_probe_fault[] __attribute__ ((visibility ("hidden")));
extern "C" char _ic_ivau_probe_continuation[] __attribute__ ((visibility ("hidden")));
NOINLINE address os::current_stack_pointer() {
return (address)__builtin_frame_address(0);
}
@ -228,6 +233,12 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
}
}
// IC IVAU trap probe during VM_Version initialization.
// If IC IVAU is not trapped, it faults on unmapped VA 0x0.
if (sig == SIGSEGV && pc == (address)_ic_ivau_probe_fault) {
stub = (address)_ic_ivau_probe_continuation;
}
if (thread->thread_state() == _thread_in_Java) {
// Java thread running in Java code => find exception handler if any
// a fault inside compiled code, the interpreter, or a stub

10
src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
View File

@ -31,6 +31,10 @@
#include <sys/auxv.h>
#include <sys/prctl.h>
// IC IVAU trap probe.
// Defined in ic_ivau_probe_linux_aarch64.S.
extern "C" int ic_ivau_probe(void);
#ifndef HWCAP_AES
#define HWCAP_AES (1<<3)
#endif
@ -182,6 +186,12 @@ void VM_Version::get_os_cpu_info() {
_icache_line_size = (1 << (ctr_el0 & 0x0f)) * 4;
_dcache_line_size = (1 << ((ctr_el0 >> 16) & 0x0f)) * 4;
_cache_idc_enabled = ((ctr_el0 >> 28) & 0x1) != 0;
_cache_dic_enabled = ((ctr_el0 >> 29) & 0x1) != 0;
// Probe whether IC IVAU is trapped.
// Must run before VM_Version::initialize() sets NeoverseN1ICacheErratumMitigation.
_ic_ivau_trapped = (ic_ivau_probe() == 1);
if (!(dczid_el0 & 0x10)) {
_zva_length = 4 << (dczid_el0 & 0xf);

5
src/hotspot/share/code/codeBlob.cpp
View File

@ -326,6 +326,11 @@ RuntimeBlob::RuntimeBlob(
: CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments,
align_up(cb->total_relocation_size(), oopSize))
{
if (code_size() == 0) {
// Nothing to copy
return;
}
cb->copy_code_and_locs_to(this);
// Flush generated code

32
src/hotspot/share/code/nmethod.cpp
View File

@ -1332,6 +1332,7 @@ nmethod::nmethod(
code_buffer->copy_values_to(this);
post_init();
ICache::invalidate_range(code_begin(), code_size());
}
if (PrintNativeNMethods || PrintDebugInfo || PrintRelocations || PrintDependencies) {
@ -1811,6 +1812,7 @@ nmethod::nmethod(
init_immutable_data_ref_count();
post_init();
ICache::invalidate_range(code_begin(), code_size());
// we use the information of entry points to find out if a method is
// static or non static
@ -2038,7 +2040,7 @@ void nmethod::copy_values(GrowableArray<jobject>* array) {
// The code and relocations have already been initialized by the
// CodeBlob constructor, so it is valid even at this early point to
// iterate over relocations and patch the code.
fix_oop_relocations(nullptr, nullptr, /*initialize_immediates=*/ true);
fix_oop_relocations(/*initialize_immediates=*/ true);
}
void nmethod::copy_values(GrowableArray<Metadata*>* array) {
@ -2050,24 +2052,42 @@ void nmethod::copy_values(GrowableArray<Metadata*>* array) {
}
}
void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) {
bool nmethod::fix_oop_relocations(bool initialize_immediates) {
// re-patch all oop-bearing instructions, just in case some oops moved
RelocIterator iter(this, begin, end);
RelocIterator iter(this);
bool modified_code = false;
while (iter.next()) {
if (iter.type() == relocInfo::oop_type) {
oop_Relocation* reloc = iter.oop_reloc();
if (initialize_immediates && reloc->oop_is_immediate()) {
if (!reloc->oop_is_immediate()) {
// Refresh the oop-related bits of this instruction.
reloc->set_value(reloc->value());
modified_code = true;
} else if (initialize_immediates) {
oop* dest = reloc->oop_addr();
jobject obj = *reinterpret_cast<jobject*>(dest);
initialize_immediate_oop(dest, obj);
}
// Refresh the oop-related bits of this instruction.
reloc->fix_oop_relocation();
} else if (iter.type() == relocInfo::metadata_type) {
metadata_Relocation* reloc = iter.metadata_reloc();
reloc->fix_metadata_relocation();
modified_code |= !reloc->metadata_is_immediate();
}
}
return modified_code;
}
void nmethod::fix_oop_relocations() {
ICacheInvalidationContext icic;
fix_oop_relocations(&icic);
}
void nmethod::fix_oop_relocations(ICacheInvalidationContext* icic) {
assert(icic != nullptr, "must provide context to track if code was modified");
bool modified_code = fix_oop_relocations(/*initialize_immediates=*/ false);
if (modified_code) {
icic->set_has_modified_code();
}
}
static void install_post_call_nop_displacement(nmethod* nm, address pc) {

7
src/hotspot/share/code/nmethod.hpp
View File

@ -41,6 +41,7 @@ class Dependencies;
class DirectiveSet;
class DebugInformationRecorder;
class ExceptionHandlerTable;
class ICacheInvalidationContext;
class ImplicitExceptionTable;
class JvmtiThreadState;
class MetadataClosure;
@ -801,15 +802,15 @@ public:
// Relocation support
private:
void fix_oop_relocations(address begin, address end, bool initialize_immediates);
bool fix_oop_relocations(bool initialize_immediates);
inline void initialize_immediate_oop(oop* dest, jobject handle);
protected:
address oops_reloc_begin() const;
public:
void fix_oop_relocations(address begin, address end) { fix_oop_relocations(begin, end, false); }
void fix_oop_relocations() { fix_oop_relocations(nullptr, nullptr, false); }
void fix_oop_relocations(ICacheInvalidationContext* icic);
void fix_oop_relocations();
bool is_at_poll_return(address pc);
bool is_at_poll_or_poll_return(address pc);

9
src/hotspot/share/code/relocInfo.cpp
View File

@ -590,15 +590,6 @@ oop oop_Relocation::oop_value() {
return *oop_addr();
}
void oop_Relocation::fix_oop_relocation() {
if (!oop_is_immediate()) {
// get the oop from the pool, and re-insert it into the instruction:
set_value(value());
}
}
void oop_Relocation::verify_oop_relocation() {
if (!oop_is_immediate()) {
// get the oop from the pool, and re-insert it into the instruction:

2
src/hotspot/share/code/relocInfo.hpp
View File

@ -988,8 +988,6 @@ class oop_Relocation : public DataRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
void fix_oop_relocation(); // reasserts oop value
void verify_oop_relocation();
address value() override { return *reinterpret_cast<address*>(oop_addr()); }

14
src/hotspot/share/gc/z/zBarrierSetNMethod.cpp
View File

@ -33,6 +33,7 @@
#include "gc/z/zThreadLocalData.hpp"
#include "gc/z/zUncoloredRoot.inline.hpp"
#include "logging/log.hpp"
#include "runtime/icache.hpp"
#include "runtime/threadWXSetters.inline.hpp"
bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
@ -70,12 +71,15 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
return false;
}
// Heal barriers
ZNMethod::nmethod_patch_barriers(nm);
{
ICacheInvalidationContext icic;
// Heal barriers
ZNMethod::nmethod_patch_barriers(nm, &icic);
// Heal oops
ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm));
ZNMethod::nmethod_oops_do_inner(nm, &cl);
// Heal oops
ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm));
ZNMethod::nmethod_oops_do_inner(nm, &cl, &icic);
}
const uintptr_t prev_color = ZNMethod::color(nm);
const uintptr_t new_color = *ZPointerStoreGoodMaskLowOrderBitsAddr;

14
src/hotspot/share/gc/z/zGeneration.cpp
View File

@ -58,6 +58,7 @@
#include "prims/jvmtiTagMap.hpp"
#include "runtime/continuation.hpp"
#include "runtime/handshake.hpp"
#include "runtime/icache.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/threads.hpp"
#include "runtime/vmOperations.hpp"
@ -1434,12 +1435,15 @@ public:
virtual void do_nmethod(nmethod* nm) {
ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
if (_bs_nm->is_armed(nm)) {
// Heal barriers
ZNMethod::nmethod_patch_barriers(nm);
{
ICacheInvalidationContext icic;
// Heal barriers
ZNMethod::nmethod_patch_barriers(nm, &icic);
// Heal oops
ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm));
ZNMethod::nmethod_oops_do_inner(nm, &cl);
// Heal oops
ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm));
ZNMethod::nmethod_oops_do_inner(nm, &cl, &icic);
}
log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old remapping", p2i(nm));

31
src/hotspot/share/gc/z/zMark.cpp
View File

@ -59,6 +59,7 @@
#include "oops/oop.inline.hpp"
#include "runtime/continuation.hpp"
#include "runtime/handshake.hpp"
#include "runtime/icache.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/safepointMechanism.hpp"
@ -718,12 +719,15 @@ public:
virtual void do_nmethod(nmethod* nm) {
ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
if (_bs_nm->is_armed(nm)) {
// Heal barriers
ZNMethod::nmethod_patch_barriers(nm);
{
ICacheInvalidationContext icic;
// Heal barriers
ZNMethod::nmethod_patch_barriers(nm, &icic);
// Heal oops
ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm));
ZNMethod::nmethod_oops_do_inner(nm, &cl);
// Heal oops
ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm));
ZNMethod::nmethod_oops_do_inner(nm, &cl, &icic);
}
// CodeCache unloading support
nm->mark_as_maybe_on_stack();
@ -753,10 +757,6 @@ public:
if (_bs_nm->is_armed(nm)) {
const uintptr_t prev_color = ZNMethod::color(nm);
// Heal oops
ZUncoloredRootMarkYoungOopClosure cl(prev_color);
ZNMethod::nmethod_oops_do_inner(nm, &cl);
// Disarm only the young marking, not any potential old marking cycle
const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
@ -767,9 +767,16 @@ public:
// Check if disarming for young mark, completely disarms the nmethod entry barrier
const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr);
if (complete_disarm) {
// We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming
ZNMethod::nmethod_patch_barriers(nm);
{
ICacheInvalidationContext icic;
if (complete_disarm) {
// We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming
ZNMethod::nmethod_patch_barriers(nm, &icic);
}
// Heal oops
ZUncoloredRootMarkYoungOopClosure cl(prev_color);
ZNMethod::nmethod_oops_do_inner(nm, &cl, &icic);
}
_bs_nm->guard_with(nm, (int)untype(new_disarm_value_ptr));

16
src/hotspot/share/gc/z/zNMethod.cpp
View File

@ -50,6 +50,7 @@
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/continuation.hpp"
#include "runtime/icache.hpp"
#include "utilities/debug.hpp"
static ZNMethodData* gc_data(const nmethod* nm) {
@ -245,8 +246,16 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) {
}
void ZNMethod::nmethod_patch_barriers(nmethod* nm) {
ICacheInvalidationContext icic;
nmethod_patch_barriers(nm, &icic);
}
void ZNMethod::nmethod_patch_barriers(nmethod* nm, ICacheInvalidationContext* icic) {
ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler();
ZArrayIterator<ZNMethodDataBarrier> iter(gc_data(nm)->barriers());
if (gc_data(nm)->barriers()->is_nonempty()) {
icic->set_has_modified_code();
}
for (ZNMethodDataBarrier barrier; iter.next(&barrier);) {
bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format);
}
@ -258,6 +267,11 @@ void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) {
}
void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
ICacheInvalidationContext icic;
nmethod_oops_do_inner(nm, cl, &icic);
}
void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, ICacheInvalidationContext* icic) {
// Process oops table
{
oop* const begin = nm->oops_begin();
@ -283,7 +297,7 @@ void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
// Process non-immediate oops
if (data->has_non_immediate_oops()) {
nm->fix_oop_relocations();
nm->fix_oop_relocations(icic);
}
}

2
src/hotspot/share/gc/z/zNMethod.hpp
View File

@ -56,9 +56,11 @@ public:
static void set_guard_value(nmethod* nm, int value);
static void nmethod_patch_barriers(nmethod* nm);
static void nmethod_patch_barriers(nmethod* nm, ICacheInvalidationContext* icic);
static void nmethod_oops_do(nmethod* nm, OopClosure* cl);
static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl);
static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, ICacheInvalidationContext* icic);
static void nmethods_do_begin(bool secondary);
static void nmethods_do_end(bool secondary);

23
src/hotspot/share/runtime/icache.hpp
View File

@ -129,4 +129,27 @@ class ICacheStubGenerator : public StubCodeGenerator {
void generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub);
};
class DefaultICacheInvalidationContext : StackObj {
public:
NONCOPYABLE(DefaultICacheInvalidationContext);
DefaultICacheInvalidationContext() {}
~DefaultICacheInvalidationContext() {}
void set_has_modified_code() {}
};
#ifndef PD_ICACHE_INVALIDATION_CONTEXT
#define PD_ICACHE_INVALIDATION_CONTEXT DefaultICacheInvalidationContext
#endif // PD_ICACHE_INVALIDATION_CONTEXT
class ICacheInvalidationContext final : public PD_ICACHE_INVALIDATION_CONTEXT {
private:
NONCOPYABLE(ICacheInvalidationContext);
public:
using PD_ICACHE_INVALIDATION_CONTEXT::PD_ICACHE_INVALIDATION_CONTEXT;
};
#endif // SHARE_RUNTIME_ICACHE_HPP

434
test/hotspot/jtreg/compiler/runtime/TestDeferredICacheInvalidationCmdOptions.java Normal file
View File

@ -0,0 +1,434 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
package compiler.runtime;
import jdk.test.lib.process.OutputAnalyzer;
import jdk.test.lib.process.ProcessTools;
import jdk.test.whitebox.WhiteBox;
import jtreg.SkippedException;
/*
* @test
* @bug 8370947 8381003
* @summary Test command-line options for UseSingleICacheInvalidation and NeoverseN1ICacheErratumMitigation
* @library /test/lib
* @requires os.arch == "aarch64"
* @requires os.family == "linux"
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI compiler.runtime.TestDeferredICacheInvalidationCmdOptions
*/
public class TestDeferredICacheInvalidationCmdOptions {
// CPU identifiers
private static final int CPU_ARM = 0x41;
private static final int NEOVERSE_N1_MODEL = 0xd0c;
// Known ARM Neoverse models where we can predict UseSingleICacheInvalidation
// behavior.
private static final int[] KNOWN_NEOVERSE_MODELS = {
NEOVERSE_N1_MODEL,
0xd40, // Neoverse V1
0xd49, // Neoverse N2
0xd4f, // Neoverse V2
0xd83, // Neoverse V3AE
0xd84, // Neoverse V3
0xd8e, // Neoverse N3
};
private static boolean isAffected;
private static boolean isKnownModel;
public static void main(String[] args) throws Exception {
// This test does not depend on CPU identification — run it first.
testDisableBothFlags();
// Parse CPU features and print CPU info
parseCPUFeatures();
if (!isKnownModel) {
throw new SkippedException("Unknown CPU model - skipping remaining tests.");
}
if (isAffected) {
// Detect whether IC IVAU is trapped on this system.
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+NeoverseN1ICacheErratumMitigation",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
if (output.getExitValue() != 0) {
// Verify the failure is the expected probe error, not something else.
output.shouldContain("IC IVAU is not trapped");
throw new SkippedException("IC IVAU is not trapped - skipping remaining tests.");
} else {
System.out.println("IC IVAU trap active.");
}
}
if (isAffected) {
// Check defaults on Neoverse N1 pre-r4p1
testCase_DefaultsOnNeoverseN1();
// Check if NeoverseN1ICacheErratumMitigation is set to false on affected CPUs,
// UseSingleICacheInvalidation is also set to false
testCase_ExplicitlyDisableErrataAffectsDeferred();
// Check JVM error if UseSingleICacheInvalidation=true
// but NeoverseN1ICacheErratumMitigation=false on affected CPUs
testCase_ConflictingFlagsOnAffectedCPUs();
// Check explicit NeoverseN1ICacheErratumMitigation=true enables UseSingleICacheInvalidation
testCase_ExplicitlyEnableErrataEnablesDeferred();
// Check UseSingleICacheInvalidation=false with NeoverseN1ICacheErratumMitigation=true
testCase_ConflictingErrataWithoutDeferred();
} else {
// Check NeoverseN1ICacheErratumMitigation is false on unaffected CPUs
testCase_DefaultsOnUnaffectedCPUs();
// Check setting NeoverseN1ICacheErratumMitigation=true on unaffected CPU causes an error
testCase_EnablingErrataOnUnaffectedCPU();
}
System.out.println("All test cases passed!");
}
/**
* Parse CPU features string from WhiteBox.getCPUFeatures() to extract:
* - cpuFamily: 0x41 for ARM
* - cpuVariant: major revision
* - cpuModel: e.g., 0xd0c for Neoverse N1
* - cpuRevision: minor revision
* - cpuModel2: secondary model (if present, in parentheses)
*
* Sets the static fields isAffected and isKnownModel, and prints CPU info.
*
* Format: 0x%02x:0x%x:0x%03x:%d[(0x%03x)]
* Example: "0x41:0x3:0xd0c:0" or "0x41:0x3:0xd0c:0(0xd0c)"
*
* @throws SkippedException if not running on ARM CPU
*/
private static void parseCPUFeatures() {
WhiteBox wb = WhiteBox.getWhiteBox();
String cpuFeatures = wb.getCPUFeatures();
System.out.println("CPU Features string: " + cpuFeatures);
if (cpuFeatures == null || cpuFeatures.isEmpty()) {
throw new RuntimeException("No CPU features available");
}
int commaIndex = cpuFeatures.indexOf(",");
if (commaIndex == -1) {
throw new RuntimeException("Unexpected CPU features format (no comma): " + cpuFeatures);
}
String cpuPart = cpuFeatures.substring(0, commaIndex).trim();
String[] parts = cpuPart.split(":");
if (parts.length < 4) {
throw new RuntimeException("Unexpected CPU features format: " + cpuPart);
}
int cpuFamily = Integer.parseInt(parts[0].substring(2), 16);
if (cpuFamily != CPU_ARM) {
throw new SkippedException("Not running on ARM CPU (cpuFamily=0x" + Integer.toHexString(cpuFamily) + ")");
}
int cpuVariant = Integer.parseInt(parts[1].substring(2), 16);
int cpuModel = Integer.parseInt(parts[2].substring(2), 16);
int cpuModel2 = 0;
int model2Start = parts[3].indexOf("(");
String revisionStr = parts[3];
if (model2Start != -1) {
if (!parts[3].endsWith(")")) {
throw new RuntimeException("Unexpected CPU features format (missing closing parenthesis): " + parts[3]);
}
String model2Str = parts[3].substring(model2Start + 1, parts[3].length() - 1);
cpuModel2 = Integer.parseInt(model2Str.substring(2), 16);
revisionStr = parts[3].substring(0, model2Start);
}
int cpuRevision = Integer.parseInt(revisionStr);
// Neoverse N1 errata 1542419 affects r3p0, r3p1 and r4p0.
// It is fixed in r4p1 and later revisions.
if (cpuModel == NEOVERSE_N1_MODEL || cpuModel2 == NEOVERSE_N1_MODEL) {
isAffected = (cpuVariant == 3 && cpuRevision == 0) ||
(cpuVariant == 3 && cpuRevision == 1) ||
(cpuVariant == 4 && cpuRevision == 0);
}
// Check if this is a known Neoverse model.
isKnownModel = false;
for (int model : KNOWN_NEOVERSE_MODELS) {
if (cpuModel == model || cpuModel2 == model) {
isKnownModel = true;
break;
}
}
printCPUInfo(cpuFamily, cpuVariant, cpuModel, cpuModel2, cpuRevision);
}
private static void printCPUInfo(int cpuFamily, int cpuVariant, int cpuModel, int cpuModel2, int cpuRevision) {
boolean isNeoverseN1 = (cpuFamily == CPU_ARM) &&
(cpuModel == NEOVERSE_N1_MODEL || cpuModel2 == NEOVERSE_N1_MODEL);
System.out.println("\n=== CPU Information ===");
System.out.println("CPU Family: 0x" + Integer.toHexString(cpuFamily));
System.out.println("CPU Variant: 0x" + Integer.toHexString(cpuVariant));
System.out.println("CPU Model: 0x" + Integer.toHexString(cpuModel));
if (cpuModel2 != 0) {
System.out.println("CPU Model2: 0x" + Integer.toHexString(cpuModel2));
}
System.out.println("CPU Revision: " + cpuRevision);
System.out.println("Is Neoverse N1: " + isNeoverseN1);
System.out.println("Is affected by errata 1542419: " + isAffected);
System.out.println("Is known model: " + isKnownModel);
System.out.println("======================\n");
}
/**
* Test that UseSingleICacheInvalidation and NeoverseN1ICacheErratumMitigation flags
* can be explicitly set to false on any system.
*/
private static void testDisableBothFlags() throws Exception {
System.out.println("\nTest: Explicitly disable both flags");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:-UseSingleICacheInvalidation",
"-XX:-NeoverseN1ICacheErratumMitigation",
"-XX:+PrintFlagsFinal",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
output.shouldHaveExitValue(0);
String output_str = output.getOutput();
boolean hasSingleDisabled = output_str.matches("(?s).*bool\\s+UseSingleICacheInvalidation\\s*=\\s*false.*");
boolean hasErrataDisabled = output_str.matches("(?s).*bool\\s+NeoverseN1ICacheErratumMitigation\\s*=\\s*false.*");
System.out.println("UseSingleICacheInvalidation disabled: " + hasSingleDisabled);
System.out.println("NeoverseN1ICacheErratumMitigation disabled: " + hasErrataDisabled);
if (!hasErrataDisabled) {
throw new RuntimeException("Failed to disable NeoverseN1ICacheErratumMitigation via command line");
}
if (!hasSingleDisabled) {
throw new RuntimeException("Failed to disable UseSingleICacheInvalidation via command line");
}
System.out.println("Successfully disabled both flags");
}
/**
* Check defaults on Neoverse N1 affected revisions.
* UseSingleICacheInvalidation and NeoverseN1ICacheErratumMitigation flags should be true.
*/
private static void testCase_DefaultsOnNeoverseN1() throws Exception {
System.out.println("\nTest: Check defaults on Neoverse N1 affected revisions");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+PrintFlagsFinal",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
output.shouldHaveExitValue(0);
String output_str = output.getOutput();
boolean hasSingleEnabled = output_str.matches("(?s).*bool\\s+UseSingleICacheInvalidation\\s*=\\s*true.*");
boolean hasErrataEnabled = output_str.matches("(?s).*bool\\s+NeoverseN1ICacheErratumMitigation\\s*=\\s*true.*");
System.out.println("UseSingleICacheInvalidation enabled: " + hasSingleEnabled);
System.out.println("NeoverseN1ICacheErratumMitigation enabled: " + hasErrataEnabled);
if (!hasSingleEnabled || !hasErrataEnabled) {
throw new RuntimeException("On affected Neoverse N1 with trap active, " +
"UseSingleICacheInvalidation and NeoverseN1ICacheErratumMitigation flags should be enabled by default");
}
System.out.println("Correctly enabled on affected Neoverse N1");
}
/**
* Check NeoverseN1ICacheErratumMitigation is false on unaffected CPUs.
*/
private static void testCase_DefaultsOnUnaffectedCPUs() throws Exception {
System.out.println("\nTest: Check NeoverseN1ICacheErratumMitigation is false on unaffected CPUs");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+PrintFlagsFinal",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
output.shouldHaveExitValue(0);
String output_str = output.getOutput();
boolean hasErrataEnabled = output_str.matches("(?s).*bool\\s+NeoverseN1ICacheErratumMitigation\\s*=\\s*true.*");
System.out.println("NeoverseN1ICacheErratumMitigation enabled: " + hasErrataEnabled);
if (hasErrataEnabled) {
throw new RuntimeException("On unaffected CPUs, NeoverseN1ICacheErratumMitigation should be disabled");
}
System.out.println("Correctly disabled on unaffected CPU");
}
/**
* Check if NeoverseN1ICacheErratumMitigation is set to false via cmd on affected CPUs,
* UseSingleICacheInvalidation is set to false.
*/
private static void testCase_ExplicitlyDisableErrataAffectsDeferred() throws Exception {
System.out.println("\nTest: Explicitly disable NeoverseN1ICacheErratumMitigation, check UseSingleICacheInvalidation");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:-NeoverseN1ICacheErratumMitigation",
"-XX:+PrintFlagsFinal",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
output.shouldHaveExitValue(0);
String output_str = output.getOutput();
boolean hasSingleDisabled = output_str.matches("(?s).*bool\\s+UseSingleICacheInvalidation\\s*=\\s*false.*");
boolean hasErrataDisabled = output_str.matches("(?s).*bool\\s+NeoverseN1ICacheErratumMitigation\\s*=\\s*false.*");
System.out.println("NeoverseN1ICacheErratumMitigation disabled: " + hasErrataDisabled);
System.out.println("UseSingleICacheInvalidation disabled: " + hasSingleDisabled);
if (!hasErrataDisabled) {
throw new RuntimeException("Failed to disable NeoverseN1ICacheErratumMitigation via command line");
}
if (!hasSingleDisabled) {
throw new RuntimeException("On affected CPU, disabling NeoverseN1ICacheErratumMitigation should also disable UseSingleICacheInvalidation");
}
System.out.println("Correctly synchronized on affected CPU");
}
/**
* Check JVM reports an error if UseSingleICacheInvalidation is set to true
* but NeoverseN1ICacheErratumMitigation is set to false on affected CPUs.
*/
private static void testCase_ConflictingFlagsOnAffectedCPUs() throws Exception {
System.out.println("\nTest: Try to set UseSingleICacheInvalidation=true with NeoverseN1ICacheErratumMitigation=false");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+UseSingleICacheInvalidation",
"-XX:-NeoverseN1ICacheErratumMitigation",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
if (output.getExitValue() == 0) {
throw new RuntimeException("On affected CPU, setting UseSingleICacheInvalidation=true " +
"with NeoverseN1ICacheErratumMitigation=false should cause error");
}
output.shouldContain("Error");
System.out.println("JVM correctly rejected conflicting flags on affected CPU");
}
/**
* Check explicit NeoverseN1ICacheErratumMitigation=true enables UseSingleICacheInvalidation.
*/
private static void testCase_ExplicitlyEnableErrataEnablesDeferred() throws Exception {
System.out.println("\nTest: Explicitly enable NeoverseN1ICacheErratumMitigation, check UseSingleICacheInvalidation");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+NeoverseN1ICacheErratumMitigation",
"-XX:+PrintFlagsFinal",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
output.shouldHaveExitValue(0);
String output_str = output.getOutput();
boolean hasSingleEnabled = output_str.matches("(?s).*bool\\s+UseSingleICacheInvalidation\\s*=\\s*true.*");
boolean hasErrataEnabled = output_str.matches("(?s).*bool\\s+NeoverseN1ICacheErratumMitigation\\s*=\\s*true.*");
System.out.println("NeoverseN1ICacheErratumMitigation enabled: " + hasErrataEnabled);
System.out.println("UseSingleICacheInvalidation enabled: " + hasSingleEnabled);
if (!hasErrataEnabled) {
throw new RuntimeException("Failed to enable NeoverseN1ICacheErratumMitigation via command line");
}
if (!hasSingleEnabled) {
throw new RuntimeException("On affected CPU, enabling NeoverseN1ICacheErratumMitigation should also enable UseSingleICacheInvalidation");
}
System.out.println("Correctly synchronized on affected CPU");
}
/**
* Check JVM reports an error if UseSingleICacheInvalidation is set to false
* and NeoverseN1ICacheErratumMitigation is set to true on affected CPUs.
*/
private static void testCase_ConflictingErrataWithoutDeferred() throws Exception {
System.out.println("\nTest: Try to set NeoverseN1ICacheErratumMitigation=true with UseSingleICacheInvalidation=false");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:-UseSingleICacheInvalidation",
"-XX:+NeoverseN1ICacheErratumMitigation",
"-XX:+PrintFlagsFinal",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
if (output.getExitValue() == 0) {
throw new RuntimeException("On affected CPU, setting NeoverseN1ICacheErratumMitigation=true with UseSingleICacheInvalidation=false should cause an error");
}
output.shouldContain("Error");
System.out.println("JVM correctly rejected conflicting flags on affected CPU");
}
/**
* Check setting NeoverseN1ICacheErratumMitigation=true on unaffected CPU causes an error.
*/
private static void testCase_EnablingErrataOnUnaffectedCPU() throws Exception {
System.out.println("\nTest: Try to set NeoverseN1ICacheErratumMitigation=true on unaffected CPU");
ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+NeoverseN1ICacheErratumMitigation",
"-version");
OutputAnalyzer output = new OutputAnalyzer(pb.start());
if (output.getExitValue() == 0) {
throw new RuntimeException("On unaffected CPU, setting NeoverseN1ICacheErratumMitigation=true should cause error");
}
output.shouldContain("Error");
System.out.println("JVM correctly rejected enabling errata flag on unaffected CPU");
}
}

316
test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java Normal file
View File

@ -0,0 +1,316 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
package gc;
/*
* @test id=parallel
* @bug 8370947
* @summary Check no assertion is triggered when UseSingleICacheInvalidation is enabled for ParallelGC
* @library /test/lib
* @requires vm.debug
* @requires vm.gc.Parallel
* @requires os.arch == "aarch64"
* @requires os.family == "linux"
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2
*/
/*
* @test id=g1
* @bug 8370947
* @summary Check no assertion is triggered when UseSingleICacheInvalidation is enabled for G1GC
* @library /test/lib
* @requires vm.debug
* @requires vm.gc.G1
* @requires os.arch == "aarch64"
* @requires os.family == "linux"
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2
*/
/*
* @test id=shenandoah
* @bug 8370947
* @summary Check no assertion is triggered when UseSingleICacheInvalidation is enabled for ShenandoahGC
* @library /test/lib
* @requires vm.debug
* @requires vm.gc.Shenandoah
* @requires os.arch == "aarch64"
* @requires os.family == "linux"
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2
*/
/*
* @test id=genshen
* @bug 8370947
* @summary Check no assertion is triggered when UseSingleICacheInvalidation is enabled for generational ShenandoahGC
* @library /test/lib
* @requires vm.debug
* @requires vm.gc.Shenandoah
* @requires os.arch == "aarch64"
* @requires os.family == "linux"
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2
*/
/*
* @test id=z
* @bug 8370947
* @summary Check no assertion is triggered when UseSingleICacheInvalidation is enabled for ZGC
* @library /test/lib
* @requires vm.debug
* @requires vm.gc.Z
* @requires os.arch == "aarch64"
* @requires os.family == "linux"
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2
* @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2
*/
/*
* Nmethods have GC barriers and OOPs embedded into their code. GCs can patch nmethod's code
* which requires icache invalidation. Doing invalidation per instruction can be expensive.
* CPU can support hardware dcache and icache coherence. This would allow to defer cache
* invalidation.
*
* There are assertions for deferred cache invalidation. This test checks that all of them
* are passed.
*/
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import jdk.test.whitebox.WhiteBox;
import jtreg.SkippedException;
public class TestDeferredICacheInvalidation {
private static final WhiteBox WB = WhiteBox.getWhiteBox();
public static class A {
public String s1;
public String s2;
public String s3;
public String s4;
public String s5;
public String s6;
public String s7;
public String s8;
public String s9;
}
public static A a = new A();
private static int compLevel;
public static class B {
public static void test0() {
}
public static void test1() {
a.s1 = a.s1 + "1";
}
public static void test2() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
}
public static void test3() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
}
public static void test4() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
a.s4 = a.s4 + "4";
}
public static void test5() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
a.s4 = a.s4 + "4";
a.s5 = a.s5 + "5";
}
public static void test6() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
a.s4 = a.s4 + "4";
a.s5 = a.s5 + "5";
a.s6 = a.s6 + "6";
}
public static void test7() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
a.s4 = a.s4 + "4";
a.s5 = a.s5 + "5";
a.s6 = a.s6 + "6";
a.s7 = a.s7 + "7";
}
public static void test8() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
a.s4 = a.s4 + "4";
a.s5 = a.s5 + "5";
a.s6 = a.s6 + "6";
a.s7 = a.s7 + "7";
a.s8 = a.s8 + "8";
}
public static void test9() {
a.s1 = a.s1 + "1";
a.s2 = a.s2 + "2";
a.s3 = a.s3 + "3";
a.s4 = a.s4 + "4";
a.s5 = a.s5 + "5";
a.s6 = a.s6 + "6";
a.s7 = a.s7 + "7";
a.s8 = a.s8 + "8";
a.s9 = a.s9 + "9";
}
}
private static void compileMethods() throws Exception {
for (var m : B.class.getDeclaredMethods()) {
if (!m.getName().startsWith("test")) {
continue;
}
m.invoke(null);
WB.markMethodProfiled(m);
WB.enqueueMethodForCompilation(m, compLevel);
while (WB.isMethodQueuedForCompilation(m)) {
Thread.sleep(100);
}
if (WB.getMethodCompilationLevel(m) != compLevel) {
throw new IllegalStateException("Method " + m + " is not compiled at the compilation level: " + compLevel + ". Got: " + WB.getMethodCompilationLevel(m));
}
}
}
public static void youngGC() throws Exception {
a = null;
WB.youngGC();
}
public static void fullGC() throws Exception {
// Thread synchronization
final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
final ReentrantReadWriteLock.ReadLock readLock = rwLock.readLock();
final ReentrantReadWriteLock.WriteLock writeLock = rwLock.writeLock();
final AtomicBoolean running = new AtomicBoolean(true);
// Thread 1: GC thread that runs for 1 second with 100ms sleep intervals
Thread gcThread = new Thread(() -> {
final long startTime = System.currentTimeMillis();
final long duration = 1000;
try {
while (System.currentTimeMillis() - startTime < duration) {
writeLock.lock();
try {
a = new A();
WB.fullGC();
} finally {
writeLock.unlock();
}
Thread.sleep(100);
}
} catch (InterruptedException e) {
// Thread interrupted, exit
}
running.set(false);
});
// Threads 2-11: Test threads that execute test0() through test9()
Thread[] testThreads = new Thread[10];
for (int i = 0; i < 10; i++) {
final int testIdx = i;
testThreads[i] = new Thread(() -> {
try {
var method = B.class.getDeclaredMethod("test" + testIdx);
while (running.get()) {
readLock.lock();
try {
method.invoke(null);
} finally {
readLock.unlock();
}
}
} catch (Exception e) {
e.printStackTrace();
System.exit(10);
}
});
}
// Start all threads
gcThread.start();
for (Thread t : testThreads) {
t.start();
}
// Wait for all threads to complete
gcThread.join();
for (Thread t : testThreads) {
t.join();
}
}
public static void main(String[] args) throws Exception {
if (!Boolean.TRUE.equals(WB.getBooleanVMFlag("UseSingleICacheInvalidation"))) {
throw new SkippedException("Skip. Test requires UseSingleICacheInvalidation enabled.");
}
compLevel = (args[1].equals("C1")) ? 1 : 4;
compileMethods();
TestDeferredICacheInvalidation.class.getMethod(args[0]).invoke(null);
}
}

206
test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java Normal file
View File

@ -0,0 +1,206 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
package org.openjdk.bench.vm.gc;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.CompilerControl;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.bench.util.InMemoryJavaCompiler;
import jdk.test.whitebox.WhiteBox;
import jdk.test.whitebox.code.NMethod;
/*
* Nmethods have OOPs and GC barriers emmedded into their code.
* GCs patch them which causes invalidation of nmethods' code.
*
* This benchmark can be used to estimate the cost of patching
* OOPs and GC barriers.
*
* We create 5000 nmethods which access fields of a class.
* We measure the time of different GC cycles to see
* the impact of patching nmethods.
*
* The benchmark parameters are method count and accessed field count.
*/
@BenchmarkMode(Mode.SingleShotTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
@Fork(value = 1, jvmArgsAppend = {
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+UnlockExperimentalVMOptions",
"-XX:+WhiteBoxAPI",
"-Xbootclasspath/a:lib-test/wb.jar",
"-XX:-UseCodeCacheFlushing"
})
@Warmup(iterations = 5)
@Measurement(iterations = 5)
public class GCPatchingNmethodCost {
private static final int COMP_LEVEL = 1;
private static final String FIELD_USER = "FieldUser";
public static Fields fields;
private static TestMethod[] methods = {};
private static byte[] BYTE_CODE;
private static WhiteBox WB;
@Param({"5000"})
public int methodCount;
@Param({"0", "2", "4", "8"})
public int accessedFieldCount;
public static class Fields {
public String f1;
public String f2;
public String f3;
public String f4;
public String f5;
public String f6;
public String f7;
public String f8;
public String f9;
}
private static final class TestMethod {
private final Method method;
public TestMethod(Method method) throws Exception {
this.method = method;
WB.testSetDontInlineMethod(method, true);
}
public void profile() throws Exception {
method.invoke(null);
WB.markMethodProfiled(method);
}
public void invoke() throws Exception {
method.invoke(null);
}
public void compile() throws Exception {
WB.enqueueMethodForCompilation(method, COMP_LEVEL);
while (WB.isMethodQueuedForCompilation(method)) {
Thread.onSpinWait();
}
if (WB.getMethodCompilationLevel(method) != COMP_LEVEL) {
throw new IllegalStateException("Method " + method + " is not compiled at the compilation level: " + COMP_LEVEL + ". Got: " + WB.getMethodCompilationLevel(method));
}
}
public NMethod getNMethod() {
return NMethod.get(method, false);
}
}
private static ClassLoader createClassLoader() {
return new ClassLoader() {
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
if (!name.equals(FIELD_USER)) {
return super.loadClass(name);
}
return defineClass(name, BYTE_CODE, 0, BYTE_CODE.length);
}
};
}
private static void createTestMethods(int accessedFieldCount, int count) throws Exception {
String javaCode = "public class " + FIELD_USER + " {";
String field = GCPatchingNmethodCost.class.getName() + ".fields.f";
javaCode += "public static void accessFields() {";
for (int i = 1; i <= accessedFieldCount; i++) {
javaCode += field + i + "= " + field + i + " + " + i + ";";
}
javaCode += "}}";
BYTE_CODE = InMemoryJavaCompiler.compile(FIELD_USER, javaCode);
fields = new Fields();
methods = new TestMethod[count];
for (int i = 0; i < count; i++) {
var cl = createClassLoader().loadClass(FIELD_USER);
Method method = cl.getMethod("accessFields");
methods[i] = new TestMethod(method);
methods[i].profile();
methods[i].compile();
}
}
private static void initWhiteBox() {
WB = WhiteBox.getWhiteBox();
}
@Setup(Level.Trial)
public void setupCodeCache() throws Exception {
initWhiteBox();
createTestMethods(accessedFieldCount, methodCount);
System.gc();
}
@Setup(Level.Iteration)
public void setupIteration() {
fields = new Fields();
}
@Benchmark
public void youngGC() throws Exception {
fields = null;
WB.youngGC();
}
@Benchmark
public void fullGC() throws Exception {
fields = null;
WB.fullGC();
}
@Benchmark
public void systemGC() throws Exception {
fields = null;
System.gc();
}
}