diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 14d8366501e..4114db5563d 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -26,14 +26,10 @@ #include "pauth_aarch64.hpp" #include "register_aarch64.hpp" -#include "asm/codeBuffer.hpp" -#include "asm/macroAssembler.hpp" -#include "memory/resourceArea.hpp" #include "runtime/arguments.hpp" #include "runtime/globals_extension.hpp" #include "runtime/java.hpp" #include "runtime/os.inline.hpp" -#include "runtime/stubCodeGenerator.hpp" #include "runtime/vm_version.hpp" #include "utilities/formatBuffer.hpp" #include "utilities/macros.hpp" @@ -58,9 +54,7 @@ SpinWait VM_Version::_spin_wait; bool VM_Version::_cache_dic_enabled; bool VM_Version::_cache_idc_enabled; - -address VM_Version::_ic_ivau_probe_addr = nullptr; -address VM_Version::_ic_ivau_probe_cont = nullptr; +bool VM_Version::_ic_ivau_trapped; const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr }; @@ -86,64 +80,6 @@ static bool has_neoverse_n1_errata_1542419() { (major_rev_num == 4 && minor_rev_num == 0))); } -class VM_Version_StubGenerator : public StubCodeGenerator { -public: - VM_Version_StubGenerator(CodeBuffer *c) : StubCodeGenerator(c) {} - - // Returns: stub start address, and sets probe_addr/cont_addr via out params. - address generate_ic_ivau_probe(address* probe_addr, address* cont_addr) { - StubCodeMark mark(this, "VM_Version", "ic_ivau_probe"); -# define __ _masm-> - - address start = __ pc(); - - // Record the address of the IC IVAU instruction for the signal handler. - *probe_addr = __ pc(); - __ ic(Assembler::IVAU, as_Register(0b11111) /* XZR */); - - // If we reach here, IC IVAU did not fault — the trap is active. - __ mov(r0, 1); - __ ret(lr); - - // Signal handler redirects here on fault. - *cont_addr = __ pc(); - __ mov(r0, 0); - __ ret(lr); - -# undef __ - return start; - } -}; - -bool VM_Version::probe_ic_ivau_trap() { - ResourceMark rm; - BufferBlob* b = BufferBlob::create("ic_ivau_probe", 128); - if (b == nullptr) { - vm_exit_during_initialization("Unable to allocate ic_ivau_probe stub"); - } - - CodeBuffer c(b); - VM_Version_StubGenerator g(&c); - typedef int (*probe_fn_t)(void); - probe_fn_t probe_fn = (probe_fn_t)g.generate_ic_ivau_probe(&_ic_ivau_probe_addr, - &_ic_ivau_probe_cont); - - // StubCodeMark destructor calls flush() -> ICache::invalidate_range() - // with the stub's own (valid) address. NeoverseN1ICacheErratumMitigation - // is still false at this point, so invalidate_range() uses - // __builtin___clear_cache() — no chicken-and-egg problem. - int trapped = probe_fn(); - - BufferBlob::free(b); - - _ic_ivau_probe_addr = nullptr; - _ic_ivau_probe_cont = nullptr; - - // 1: 'ic ivau, xzr' trapped - // 0: not trapped (the insruction caused SIGSEGV) - return trapped == 1; -} - void VM_Version::initialize() { #define SET_CPU_FEATURE_NAME(id, name, bit) \ _features_names[bit] = XSTR(name); @@ -157,6 +93,7 @@ void VM_Version::initialize() { _cache_dic_enabled = false; _cache_idc_enabled = false; + _ic_ivau_trapped = false; get_os_cpu_info(); @@ -751,8 +688,7 @@ void VM_Version::initialize() { if (FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && has_neoverse_n1_errata_1542419() && is_cache_idc_enabled() && !is_cache_dic_enabled()) { - // Probe whether IC IVAU is trapped before enabling the mitigation. - if (probe_ic_ivau_trap()) { + if (_ic_ivau_trapped) { FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, true); } else { FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, false); @@ -766,7 +702,7 @@ void VM_Version::initialize() { vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set for the CPU not having Neoverse N1 errata 1542419"); } // If the user explicitly set the flag, verify the trap is active. - if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !probe_ic_ivau_trap()) { + if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !_ic_ivau_trapped) { vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but IC IVAU is not trapped. " "The optimization is not safe on this system."); } diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp index 0bb18a9b8a6..0c06ee561f9 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp @@ -62,8 +62,8 @@ protected: static bool _cache_idc_enabled; // IC IVAU trap probe for Neoverse N1 erratum 1542419. - static address _ic_ivau_probe_addr; // address of IC IVAU that may fault - static address _ic_ivau_probe_cont; // continuation if it faults + // Set by get_os_cpu_info() on Linux via ic_ivau_probe_linux_aarch64.S. + static bool _ic_ivau_trapped; static SpinWait _spin_wait; @@ -76,10 +76,6 @@ protected: static int set_and_get_current_sve_vector_length(int len); static int get_current_sve_vector_length(); - // Returns true if the IC IVAU is trapped, - // false otherwise. - static bool probe_ic_ivau_trap(); - public: // Initialization static void initialize(); @@ -265,9 +261,7 @@ public: static bool is_cache_dic_enabled() { return _cache_dic_enabled; } static bool is_cache_idc_enabled() { return _cache_idc_enabled; } - - static bool is_ic_ivau_probe_addr(address pc) { return _ic_ivau_probe_addr == pc; } - static address ic_ivau_probe_cont() { return _ic_ivau_probe_cont; } + static bool is_ic_ivau_trapped() { return _ic_ivau_trapped; } static void get_cpu_features_name(void* features_buffer, stringStream& ss); diff --git a/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S new file mode 100644 index 00000000000..b82053d37b9 --- /dev/null +++ b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "defs.S.inc" + + # Probe whether IC IVAU is trapped. + # + # Returns 1 if IC IVAU is trapped (did not fault), 0 if not trapped + # (faulted on VA 0x0, signal handler redirected to continuation). + # + # int ic_ivau_probe(void); +DECLARE_FUNC(ic_ivau_probe): +DECLARE_FUNC(_ic_ivau_probe_fault): + ic ivau, xzr + mov x0, #1 + ret +DECLARE_FUNC(_ic_ivau_probe_continuation): + mov x0, #0 + ret + +/* Emit .note.gnu.property section in case of PAC or BTI being enabled. */ +#ifdef __ARM_FEATURE_BTI_DEFAULT + #ifdef __ARM_FEATURE_PAC_DEFAULT + #define GNU_PROPERTY_AARCH64_FEATURE 3 + #else + #define GNU_PROPERTY_AARCH64_FEATURE 1 + #endif +#else + #ifdef __ARM_FEATURE_PAC_DEFAULT + #define GNU_PROPERTY_AARCH64_FEATURE 2 + #else + #define GNU_PROPERTY_AARCH64_FEATURE 0 + #endif +#endif + +#if (GNU_PROPERTY_AARCH64_FEATURE != 0) + .pushsection .note.gnu.property, "a" + .align 3 + .long 4 /* name length */ + .long 0x10 /* data length */ + .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */ + .string "GNU" /* vendor name */ + .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4 /* pr_datasze */ + .long GNU_PROPERTY_AARCH64_FEATURE + .long 0 + .popsection +#endif diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index b410aebed7b..444b3c3ebd6 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -29,6 +29,7 @@ #include "memory/allocation.hpp" #include "runtime/vm_version.hpp" #include "utilities/globalDefinitions.hpp" +#include "vm_version_aarch64.hpp" // Interface for updating the instruction cache. Whenever the VM // modifies code, part of the processor instruction cache potentially @@ -48,6 +49,7 @@ class ICache : public AbstractICache { assert(!VM_Version::is_cache_dic_enabled(), "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum " "1542419"); + assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped"); asm volatile("dsb ish \n" "ic ivau, xzr \n" "dsb ish \n" @@ -94,6 +96,7 @@ class AArch64ICacheInvalidationContext : StackObj { assert(!VM_Version::is_cache_dic_enabled(), "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum " "1542419"); + assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped"); // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature // may fetch stale instructions when software depends on diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp index 852dcea4892..67e0569bf31 100644 --- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp @@ -77,6 +77,11 @@ #define REG_LR 30 #define REG_BCP 22 +// IC IVAU trap probe. +// Defined in ic_ivau_probe_linux_aarch64.S. +extern "C" char _ic_ivau_probe_fault[] __attribute__ ((visibility ("hidden"))); +extern "C" char _ic_ivau_probe_continuation[] __attribute__ ((visibility ("hidden"))); + NOINLINE address os::current_stack_pointer() { return (address)__builtin_frame_address(0); } @@ -228,10 +233,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, } } - // 'IC IVAU, XZR' trap probe during VM_Version initialization. + // IC IVAU trap probe during VM_Version initialization. // If IC IVAU is not trapped, it faults on unmapped VA 0x0. - if (sig == SIGSEGV && VM_Version::is_ic_ivau_probe_addr(pc)) { - stub = VM_Version::ic_ivau_probe_cont(); + if (sig == SIGSEGV && pc == (address)_ic_ivau_probe_fault) { + stub = (address)_ic_ivau_probe_continuation; } if (thread->thread_state() == _thread_in_Java) { diff --git a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp index dad7161a3cb..5dbfffdef0f 100644 --- a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp @@ -31,6 +31,10 @@ #include #include +// IC IVAU trap probe. +// Defined in ic_ivau_probe_linux_aarch64.S. +extern "C" int ic_ivau_probe(void); + #ifndef HWCAP_AES #define HWCAP_AES (1<<3) #endif @@ -172,6 +176,10 @@ void VM_Version::get_os_cpu_info() { _cache_idc_enabled = ((ctr_el0 >> 28) & 0x1) != 0; _cache_dic_enabled = ((ctr_el0 >> 29) & 0x1) != 0; + // Probe whether IC IVAU is trapped. + // Must run before VM_Version::initialize() sets NeoverseN1ICacheErratumMitigation. + _ic_ivau_trapped = (ic_ivau_probe() == 1); + if (!(dczid_el0 & 0x10)) { _zva_length = 4 << (dczid_el0 & 0xf); }