infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-03-28 08:39:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reimplement ic_ivau_probe as static assembly function

Browse Source
This commit is contained in:
Evgeny Astigeevich 2026-03-27 20:32:58 +00:00
parent 9e6a0395a4
commit f90bac82b1
6 changed files with 95 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
View File

@ -26,14 +26,10 @@
#include "pauth_aarch64.hpp"
#include "register_aarch64.hpp"
#include "asm/codeBuffer.hpp"
#include "asm/macroAssembler.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/arguments.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/java.hpp"
#include "runtime/os.inline.hpp"
#include "runtime/stubCodeGenerator.hpp"
#include "runtime/vm_version.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/macros.hpp"
@ -58,9 +54,7 @@ SpinWait VM_Version::_spin_wait;
bool VM_Version::_cache_dic_enabled;
bool VM_Version::_cache_idc_enabled;
address VM_Version::_ic_ivau_probe_addr = nullptr;
address VM_Version::_ic_ivau_probe_cont = nullptr;
bool VM_Version::_ic_ivau_trapped;
const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr };
@ -86,64 +80,6 @@ static bool has_neoverse_n1_errata_1542419() {
(major_rev_num == 4 && minor_rev_num == 0)));
}
class VM_Version_StubGenerator : public StubCodeGenerator {
public:
VM_Version_StubGenerator(CodeBuffer *c) : StubCodeGenerator(c) {}
// Returns: stub start address, and sets probe_addr/cont_addr via out params.
address generate_ic_ivau_probe(address* probe_addr, address* cont_addr) {
StubCodeMark mark(this, "VM_Version", "ic_ivau_probe");
# define __ _masm->
address start = __ pc();
// Record the address of the IC IVAU instruction for the signal handler.
*probe_addr = __ pc();
__ ic(Assembler::IVAU, as_Register(0b11111) /* XZR */);
// If we reach here, IC IVAU did not fault — the trap is active.
__ mov(r0, 1);
__ ret(lr);
// Signal handler redirects here on fault.
*cont_addr = __ pc();
__ mov(r0, 0);
__ ret(lr);
# undef __
return start;
}
};
bool VM_Version::probe_ic_ivau_trap() {
ResourceMark rm;
BufferBlob* b = BufferBlob::create("ic_ivau_probe", 128);
if (b == nullptr) {
vm_exit_during_initialization("Unable to allocate ic_ivau_probe stub");
}
CodeBuffer c(b);
VM_Version_StubGenerator g(&c);
typedef int (*probe_fn_t)(void);
probe_fn_t probe_fn = (probe_fn_t)g.generate_ic_ivau_probe(&_ic_ivau_probe_addr,
&_ic_ivau_probe_cont);
// StubCodeMark destructor calls flush() -> ICache::invalidate_range()
// with the stub's own (valid) address. NeoverseN1ICacheErratumMitigation
// is still false at this point, so invalidate_range() uses
// __builtin___clear_cache() — no chicken-and-egg problem.
int trapped = probe_fn();
BufferBlob::free(b);
_ic_ivau_probe_addr = nullptr;
_ic_ivau_probe_cont = nullptr;
// 1: 'ic ivau, xzr' trapped
// 0: not trapped (the insruction caused SIGSEGV)
return trapped == 1;
}
void VM_Version::initialize() {
#define SET_CPU_FEATURE_NAME(id, name, bit) \
_features_names[bit] = XSTR(name);
@ -157,6 +93,7 @@ void VM_Version::initialize() {
_cache_dic_enabled = false;
_cache_idc_enabled = false;
_ic_ivau_trapped = false;
get_os_cpu_info();
@ -751,8 +688,7 @@ void VM_Version::initialize() {
if (FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && has_neoverse_n1_errata_1542419()
&& is_cache_idc_enabled() && !is_cache_dic_enabled()) {
// Probe whether IC IVAU is trapped before enabling the mitigation.
if (probe_ic_ivau_trap()) {
if (_ic_ivau_trapped) {
FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, true);
} else {
FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, false);
@ -766,7 +702,7 @@ void VM_Version::initialize() {
vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set for the CPU not having Neoverse N1 errata 1542419");
}
// If the user explicitly set the flag, verify the trap is active.
if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !probe_ic_ivau_trap()) {
if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !_ic_ivau_trapped) {
vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but IC IVAU is not trapped. "
"The optimization is not safe on this system.");
}

12
src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
View File

@ -62,8 +62,8 @@ protected:
static bool _cache_idc_enabled;
// IC IVAU trap probe for Neoverse N1 erratum 1542419.
static address _ic_ivau_probe_addr; // address of IC IVAU that may fault
static address _ic_ivau_probe_cont; // continuation if it faults
// Set by get_os_cpu_info() on Linux via ic_ivau_probe_linux_aarch64.S.
static bool _ic_ivau_trapped;
static SpinWait _spin_wait;
@ -76,10 +76,6 @@ protected:
static int set_and_get_current_sve_vector_length(int len);
static int get_current_sve_vector_length();
// Returns true if the IC IVAU is trapped,
// false otherwise.
static bool probe_ic_ivau_trap();
public:
// Initialization
static void initialize();
@ -265,9 +261,7 @@ public:
static bool is_cache_dic_enabled() { return _cache_dic_enabled; }
static bool is_cache_idc_enabled() { return _cache_idc_enabled; }
static bool is_ic_ivau_probe_addr(address pc) { return _ic_ivau_probe_addr == pc; }
static address ic_ivau_probe_cont() { return _ic_ivau_probe_cont; }
static bool is_ic_ivau_trapped() { return _ic_ivau_trapped; }
static void get_cpu_features_name(void* features_buffer, stringStream& ss);

69
src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "defs.S.inc"
# Probe whether IC IVAU is trapped.
#
# Returns 1 if IC IVAU is trapped (did not fault), 0 if not trapped
# (faulted on VA 0x0, signal handler redirected to continuation).
#
# int ic_ivau_probe(void);
DECLARE_FUNC(ic_ivau_probe):
DECLARE_FUNC(_ic_ivau_probe_fault):
ic ivau, xzr
mov x0, #1
ret
DECLARE_FUNC(_ic_ivau_probe_continuation):
mov x0, #0
ret
/* Emit .note.gnu.property section in case of PAC or BTI being enabled. */
#ifdef __ARM_FEATURE_BTI_DEFAULT
#ifdef __ARM_FEATURE_PAC_DEFAULT
#define GNU_PROPERTY_AARCH64_FEATURE 3
#else
#define GNU_PROPERTY_AARCH64_FEATURE 1
#endif
#else
#ifdef __ARM_FEATURE_PAC_DEFAULT
#define GNU_PROPERTY_AARCH64_FEATURE 2
#else
#define GNU_PROPERTY_AARCH64_FEATURE 0
#endif
#endif
#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
.pushsection .note.gnu.property, "a"
.align 3
.long 4 /* name length */
.long 0x10 /* data length */
.long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
.string "GNU" /* vendor name */
.long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
.long 4 /* pr_datasze */
.long GNU_PROPERTY_AARCH64_FEATURE
.long 0
.popsection
#endif

3
src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
View File

@ -29,6 +29,7 @@
#include "memory/allocation.hpp"
#include "runtime/vm_version.hpp"
#include "utilities/globalDefinitions.hpp"
#include "vm_version_aarch64.hpp"
// Interface for updating the instruction cache. Whenever the VM
// modifies code, part of the processor instruction cache potentially
@ -48,6 +49,7 @@ class ICache : public AbstractICache {
assert(!VM_Version::is_cache_dic_enabled(),
"Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
"1542419");
assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
asm volatile("dsb ish \n"
"ic ivau, xzr \n"
"dsb ish \n"
@ -94,6 +96,7 @@ class AArch64ICacheInvalidationContext : StackObj {
assert(!VM_Version::is_cache_dic_enabled(),
"Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
"1542419");
assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
// Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature
// may fetch stale instructions when software depends on

11
src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
View File

@ -77,6 +77,11 @@
#define REG_LR 30
#define REG_BCP 22
// IC IVAU trap probe.
// Defined in ic_ivau_probe_linux_aarch64.S.
extern "C" char _ic_ivau_probe_fault[] __attribute__ ((visibility ("hidden")));
extern "C" char _ic_ivau_probe_continuation[] __attribute__ ((visibility ("hidden")));
NOINLINE address os::current_stack_pointer() {
return (address)__builtin_frame_address(0);
}
@ -228,10 +233,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
}
}
// 'IC IVAU, XZR' trap probe during VM_Version initialization.
// IC IVAU trap probe during VM_Version initialization.
// If IC IVAU is not trapped, it faults on unmapped VA 0x0.
if (sig == SIGSEGV && VM_Version::is_ic_ivau_probe_addr(pc)) {
stub = VM_Version::ic_ivau_probe_cont();
if (sig == SIGSEGV && pc == (address)_ic_ivau_probe_fault) {
stub = (address)_ic_ivau_probe_continuation;
}
if (thread->thread_state() == _thread_in_Java) {

8
src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
View File

@ -31,6 +31,10 @@
#include <sys/auxv.h>
#include <sys/prctl.h>
// IC IVAU trap probe.
// Defined in ic_ivau_probe_linux_aarch64.S.
extern "C" int ic_ivau_probe(void);
#ifndef HWCAP_AES
#define HWCAP_AES (1<<3)
#endif
@ -172,6 +176,10 @@ void VM_Version::get_os_cpu_info() {
_cache_idc_enabled = ((ctr_el0 >> 28) & 0x1) != 0;
_cache_dic_enabled = ((ctr_el0 >> 29) & 0x1) != 0;
// Probe whether IC IVAU is trapped.
// Must run before VM_Version::initialize() sets NeoverseN1ICacheErratumMitigation.
_ic_ivau_trapped = (ic_ivau_probe() == 1);
if (!(dczid_el0 & 0x10)) {
_zva_length = 4 << (dczid_el0 & 0xf);
}