infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-01-28 03:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of http://github.com/openjdk/jdk into JDK-8303762

Browse Source
This commit is contained in:
Jatin Bhateja 2026-01-22 08:23:02 +00:00
commit 2c7eb96d6c
296 changed files with 9914 additions and 4262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
make/hotspot/lib/CompileGtest.gmk
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@ -61,7 +61,8 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
INCLUDE_FILES := gtest-all.cc gmock-all.cc, \
DISABLED_WARNINGS_gcc := format-nonliteral maybe-uninitialized undef \
unused-result zero-as-null-pointer-constant, \
DISABLED_WARNINGS_clang := format-nonliteral undef unused-result, \
DISABLED_WARNINGS_clang := format-nonliteral undef unused-result \
zero-as-null-pointer-constant, \
DISABLED_WARNINGS_microsoft := 4530, \
DEFAULT_CFLAGS := false, \
CFLAGS := $(JVM_CFLAGS) \

29
src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -722,22 +722,20 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ ldrh(rscratch1, Address(rmethod, Method::access_flags_offset()));
__ andsw(zr, rscratch1, JVM_ACC_STATIC);
__ br(Assembler::EQ, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ ldrh(rscratch1, Address(rmethod, Method::access_flags_offset()));
__ andsw(zr, rscratch1, JVM_ACC_STATIC);
__ br(Assembler::EQ, L_skip_barrier); // non-static
__ load_method_holder(rscratch2, rmethod);
__ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ load_method_holder(rscratch2, rmethod);
__ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
@ -1508,7 +1506,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// SVC, HVC, or SMC. Make it a NOP.
__ nop();
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
__ mov_metadata(rscratch2, method->method_holder()); // InstanceKlass*
__ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);

9
src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -2290,7 +2290,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ subs(zr, temp, (int) code); // have we resolved this bytecode?
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
__ br(Assembler::NE, L_clinit_barrier_slow);
__ ldr(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
__ load_method_holder(temp, temp);
@ -2340,8 +2341,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ subs(zr, temp, (int) code); // have we resolved this bytecode?
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = temp;
__ br(Assembler::NE, L_clinit_barrier_slow);

6
src/hotspot/cpu/arm/frame_arm.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -356,10 +356,10 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const {
bool frame::is_interpreted_frame_valid(JavaThread* thread) const {
assert(is_interpreted_frame(), "Not an interpreted frame");
// These are reasonable sanity checks
if (fp() == 0 || (intptr_t(fp()) & (wordSize-1)) != 0) {
if (fp() == nullptr || (intptr_t(fp()) & (wordSize-1)) != 0) {
return false;
}
if (sp() == 0 || (intptr_t(sp()) & (wordSize-1)) != 0) {
if (sp() == nullptr || (intptr_t(sp()) & (wordSize-1)) != 0) {
return false;
}
if (fp() + interpreter_frame_initial_sp_offset < sp()) {

6
src/hotspot/cpu/arm/nativeInst_arm_32.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -172,7 +172,7 @@ void NativeMovConstReg::set_data(intptr_t x, address pc) {
address addr = oop_addr != nullptr ? (address)oop_addr : (address)metadata_addr;
if(pc == 0) {
if (pc == nullptr) {
offset = addr - instruction_address() - 8;
} else {
offset = addr - pc - 8;
@ -228,7 +228,7 @@ void NativeMovConstReg::set_data(intptr_t x, address pc) {
void NativeMovConstReg::set_pc_relative_offset(address addr, address pc) {
int offset;
if (pc == 0) {
if (pc == nullptr) {
offset = addr - instruction_address() - 8;
} else {
offset = addr - pc - 8;

4
src/hotspot/cpu/arm/nativeInst_arm_32.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -371,7 +371,7 @@ class NativeMovConstReg: public NativeInstruction {
public:
intptr_t data() const;
void set_data(intptr_t x, address pc = 0);
void set_data(intptr_t x, address pc = nullptr);
bool is_pc_relative() {
return !is_movw();
}

4
src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
View File

@ -1109,11 +1109,11 @@ void InterpreterMacroAssembler::verify_method_data_pointer() {
lhz(R11_scratch1, in_bytes(DataLayout::bci_offset()), R28_mdx);
ld(R12_scratch2, in_bytes(Method::const_offset()), R19_method);
addi(R11_scratch1, R11_scratch1, in_bytes(ConstMethod::codes_offset()));
add(R11_scratch1, R12_scratch2, R12_scratch2);
add(R11_scratch1, R11_scratch1, R12_scratch2);
cmpd(CR0, R11_scratch1, R14_bcp);
beq(CR0, verify_continue);
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp ), R19_method, R14_bcp, R28_mdx);
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R19_method, R14_bcp, R28_mdx);
bind(verify_continue);
#endif

35
src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -1237,26 +1237,24 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ lhz(R0, in_bytes(Method::access_flags_offset()), R19_method);
__ andi_(R0, R0, JVM_ACC_STATIC);
__ beq(CR0, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ lhz(R0, in_bytes(Method::access_flags_offset()), R19_method);
__ andi_(R0, R0, JVM_ACC_STATIC);
__ beq(CR0, L_skip_barrier); // non-static
Register klass = R11_scratch1;
__ load_method_holder(klass, R19_method);
__ clinit_barrier(klass, R16_thread, &L_skip_barrier /*L_fast_path*/);
Register klass = R11_scratch1;
__ load_method_holder(klass, R19_method);
__ clinit_barrier(klass, R16_thread, &L_skip_barrier /*L_fast_path*/);
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub(), R0);
__ mtctr(klass);
__ bctr();
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub(), R0);
__ mtctr(klass);
__ bctr();
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm, /* tmp register*/ ic_klass, /* tmp register*/ receiver_klass, /* tmp register*/ code);
@ -2210,7 +2208,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// --------------------------------------------------------------------------
vep_start_pc = (intptr_t)__ pc();
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register klass = r_temp_1;
// Notify OOP recorder (don't need the relocation)

9
src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -2199,7 +2199,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca
__ isync(); // Order load wrt. succeeding loads.
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register method = Rscratch;
const Register klass = Rscratch;
@ -2244,8 +2245,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, Register Rcac
__ isync(); // Order load wrt. succeeding loads.
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = R4_ARG2;
// InterpreterRuntime::resolve_get_put sets field_holder and finally release-stores put_code.

31
src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -213,7 +213,7 @@ void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
// Is vector's size (in bytes) bigger than a size saved by default?
// riscv does not ovlerlay the floating-point registers on vector registers like aarch64.
bool SharedRuntime::is_wide_vector(int size) {
return UseRVV;
return UseRVV && size > 0;
}
// ---------------------------------------------------------------------------
@ -637,22 +637,20 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ load_unsigned_short(t0, Address(xmethod, Method::access_flags_offset()));
__ test_bit(t1, t0, exact_log2(JVM_ACC_STATIC));
__ beqz(t1, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ load_unsigned_short(t0, Address(xmethod, Method::access_flags_offset()));
__ test_bit(t1, t0, exact_log2(JVM_ACC_STATIC));
__ beqz(t1, L_skip_barrier); // non-static
__ load_method_holder(t1, xmethod);
__ clinit_barrier(t1, t0, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ load_method_holder(t1, xmethod);
__ clinit_barrier(t1, t0, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
@ -1443,7 +1441,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ nop(); // 4 bytes
}
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
__ mov_metadata(t1, method->method_holder()); // InstanceKlass*
__ clinit_barrier(t1, t0, &L_skip_barrier);

9
src/hotspot/cpu/riscv/templateTable_riscv.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -2192,7 +2192,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ mv(t0, (int) code);
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
__ bne(temp, t0, L_clinit_barrier_slow); // have we resolved this bytecode?
__ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
__ load_method_holder(temp, temp);
@ -2243,8 +2244,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ mv(t0, (int) code); // have we resolved this bytecode?
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = temp;
__ bne(temp, t0, L_clinit_barrier_slow);

31
src/hotspot/cpu/s390/sharedRuntime_s390.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -1567,7 +1567,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
//---------------------------------------------------------------------
wrapper_VEPStart = __ offset();
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register klass = Z_R1_scratch;
// Notify OOP recorder (don't need the relocation)
@ -2378,24 +2379,22 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ testbit_ushort(Address(Z_method, Method::access_flags_offset()), JVM_ACC_STATIC_BIT);
__ z_bfalse(L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ testbit_ushort(Address(Z_method, Method::access_flags_offset()), JVM_ACC_STATIC_BIT);
__ z_bfalse(L_skip_barrier); // non-static
Register klass = Z_R11;
__ load_method_holder(klass, Z_method);
__ clinit_barrier(klass, Z_thread, &L_skip_barrier /*L_fast_path*/);
Register klass = Z_R11;
__ load_method_holder(klass, Z_method);
__ clinit_barrier(klass, Z_thread, &L_skip_barrier /*L_fast_path*/);
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub());
__ z_br(klass);
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub());
__ z_br(klass);
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
return;

9
src/hotspot/cpu/s390/templateTable_s390.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -2377,7 +2377,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ z_cli(Address(Rcache, bc_offset), code);
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register method = Z_R1_scratch;
const Register klass = Z_R1_scratch;
__ z_brne(L_clinit_barrier_slow);
@ -2427,8 +2428,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ z_cli(Address(cache, code_offset), code);
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = index;
__ z_brne(L_clinit_barrier_slow);

36
src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1043,26 +1043,24 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
Register method = rbx;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register method = rbx;
{ // Bypass the barrier for non-static methods
Register flags = rscratch1;
__ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
__ testl(flags, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
Register flags = rscratch1;
__ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
__ testl(flags, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L_skip_barrier); // non-static
Register klass = rscratch1;
__ load_method_holder(klass, method);
__ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
Register klass = rscratch1;
__ load_method_holder(klass, method);
__ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
@ -1904,7 +1902,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
int vep_offset = ((intptr_t)__ pc()) - start;
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register klass = r10;
__ mov_metadata(klass, method->method_holder()); // InstanceKlass*
@ -3602,4 +3601,3 @@ RuntimeStub* SharedRuntime::generate_jfr_return_lease() {
}
#endif // INCLUDE_JFR

92
src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -64,6 +64,39 @@ static address kyberAvx512ConstsAddr(int offset) {
const Register scratch = r10;
ATTRIBUTE_ALIGNED(64) static const uint8_t kyberAvx512_12To16Dup[] = {
// 0 - 63
0, 1, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15, 16,
16, 17, 18, 19, 19, 20, 21, 22, 22, 23, 24, 25, 25, 26, 27, 28, 28, 29, 30,
31, 31, 32, 33, 34, 34, 35, 36, 37, 37, 38, 39, 40, 40, 41, 42, 43, 43, 44,
45, 46, 46, 47
};
static address kyberAvx512_12To16DupAddr() {
return (address) kyberAvx512_12To16Dup;
}
ATTRIBUTE_ALIGNED(64) static const uint16_t kyberAvx512_12To16Shift[] = {
// 0 - 31
0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0,
4, 0, 4, 0, 4, 0, 4
};
static address kyberAvx512_12To16ShiftAddr() {
return (address) kyberAvx512_12To16Shift;
}
ATTRIBUTE_ALIGNED(64) static const uint64_t kyberAvx512_12To16And[] = {
// 0 - 7
0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF,
0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF,
0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF
};
static address kyberAvx512_12To16AndAddr() {
return (address) kyberAvx512_12To16And;
}
ATTRIBUTE_ALIGNED(64) static const uint16_t kyberAvx512NttPerms[] = {
// 0
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
@ -822,10 +855,65 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
const Register perms = r11;
Label Loop;
Label Loop, VBMILoop;
__ addptr(condensed, condensedOffs);
if (VM_Version::supports_avx512_vbmi()) {
// mask load for the first 48 bytes of each vector
__ mov64(rax, 0x0000FFFFFFFFFFFF);
__ kmovql(k1, rax);
__ lea(perms, ExternalAddress(kyberAvx512_12To16DupAddr()));
__ evmovdqub(xmm20, Address(perms), Assembler::AVX_512bit);
__ lea(perms, ExternalAddress(kyberAvx512_12To16ShiftAddr()));
__ evmovdquw(xmm21, Address(perms), Assembler::AVX_512bit);
__ lea(perms, ExternalAddress(kyberAvx512_12To16AndAddr()));
__ evmovdquq(xmm22, Address(perms), Assembler::AVX_512bit);
__ align(OptoLoopAlignment);
__ BIND(VBMILoop);
__ evmovdqub(xmm0, k1, Address(condensed, 0), false,
Assembler::AVX_512bit);
__ evmovdqub(xmm1, k1, Address(condensed, 48), false,
Assembler::AVX_512bit);
__ evmovdqub(xmm2, k1, Address(condensed, 96), false,
Assembler::AVX_512bit);
__ evmovdqub(xmm3, k1, Address(condensed, 144), false,
Assembler::AVX_512bit);
__ evpermb(xmm4, k0, xmm20, xmm0, false, Assembler::AVX_512bit);
__ evpermb(xmm5, k0, xmm20, xmm1, false, Assembler::AVX_512bit);
__ evpermb(xmm6, k0, xmm20, xmm2, false, Assembler::AVX_512bit);
__ evpermb(xmm7, k0, xmm20, xmm3, false, Assembler::AVX_512bit);
__ evpsrlvw(xmm4, xmm4, xmm21, Assembler::AVX_512bit);
__ evpsrlvw(xmm5, xmm5, xmm21, Assembler::AVX_512bit);
__ evpsrlvw(xmm6, xmm6, xmm21, Assembler::AVX_512bit);
__ evpsrlvw(xmm7, xmm7, xmm21, Assembler::AVX_512bit);
__ evpandq(xmm0, xmm22, xmm4, Assembler::AVX_512bit);
__ evpandq(xmm1, xmm22, xmm5, Assembler::AVX_512bit);
__ evpandq(xmm2, xmm22, xmm6, Assembler::AVX_512bit);
__ evpandq(xmm3, xmm22, xmm7, Assembler::AVX_512bit);
store4regs(parsed, 0, xmm0_3, _masm);
__ addptr(condensed, 192);
__ addptr(parsed, 256);
__ subl(parsedLength, 128);
__ jcc(Assembler::greater, VBMILoop);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ mov64(rax, 0); // return 0
__ ret(0);
return start;
}
__ lea(perms, ExternalAddress(kyberAvx512_12To16PermsAddr()));
load4regs(xmm24_27, perms, 0, _masm);

9
src/hotspot/cpu/x86/templateTable_x86.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -2216,7 +2216,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ cmpl(temp, code); // have we resolved this bytecode?
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register method = temp;
const Register klass = temp;
@ -2264,8 +2265,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ cmpl(temp, code); // have we resolved this bytecode?
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = temp;
__ jcc(Assembler::notEqual, L_clinit_barrier_slow);

26
src/hotspot/os/aix/os_aix.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -258,10 +258,18 @@ bool os::free_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::available_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::Aix::available_memory(physical_memory_size_type& value) {
os::Aix::meminfo_t mi;
if (os::Aix::get_meminfo(&mi)) {
@ -273,6 +281,10 @@ bool os::Aix::available_memory(physical_memory_size_type& value) {
}
bool os::total_swap_space(physical_memory_size_type& value) {
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
perfstat_memory_total_t memory_info;
if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
return false;
@ -282,6 +294,10 @@ bool os::total_swap_space(physical_memory_size_type& value) {
}
bool os::free_swap_space(physical_memory_size_type& value) {
return Machine::free_swap_space(value);
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
perfstat_memory_total_t memory_info;
if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
return false;
@ -294,6 +310,10 @@ physical_memory_size_type os::physical_memory() {
return Aix::physical_memory();
}
physical_memory_size_type os::Machine::physical_memory() {
return Aix::physical_memory();
}
size_t os::rss() { return (size_t)0; }
// Cpu architecture string
@ -2264,6 +2284,10 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
int online_cpus = ::sysconf(_SC_NPROCESSORS_ONLN);
assert(online_cpus > 0 && online_cpus <= processor_count(), "sanity check");
return online_cpus;

2
src/hotspot/os/bsd/memMapPrinter_macosx.cpp
View File

@ -132,7 +132,7 @@ public:
static const char* tagToStr(uint32_t user_tag) {
switch (user_tag) {
case 0:
return 0;
return nullptr;
X1(MALLOC, malloc);
X1(MALLOC_SMALL, malloc_small);
X1(MALLOC_LARGE, malloc_large);

30
src/hotspot/os/bsd/os_bsd.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -137,10 +137,18 @@ bool os::available_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
bool os::free_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
// Available here means free. Note that this number is of no much use. As an estimate
// for future memory pressure it is far too conservative, since MacOS will use a lot
// of unused memory for caches, and return it willingly in case of needs.
@ -181,6 +189,10 @@ void os::Bsd::print_uptime_info(outputStream* st) {
}
bool os::total_swap_space(physical_memory_size_type& value) {
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
#if defined(__APPLE__)
struct xsw_usage vmusage;
size_t size = sizeof(vmusage);
@ -195,6 +207,10 @@ bool os::total_swap_space(physical_memory_size_type& value) {
}
bool os::free_swap_space(physical_memory_size_type& value) {
return Machine::free_swap_space(value);
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
#if defined(__APPLE__)
struct xsw_usage vmusage;
size_t size = sizeof(vmusage);
@ -212,6 +228,10 @@ physical_memory_size_type os::physical_memory() {
return Bsd::physical_memory();
}
physical_memory_size_type os::Machine::physical_memory() {
return Bsd::physical_memory();
}
size_t os::rss() {
size_t rss = 0;
#ifdef __APPLE__
@ -608,7 +628,7 @@ static void *thread_native_entry(Thread *thread) {
log_info(os, thread)("Thread finished (tid: %zu, pthread id: %zu).",
os::current_thread_id(), (uintx) pthread_self());
return 0;
return nullptr;
}
bool os::create_thread(Thread* thread, ThreadType thr_type,
@ -1400,7 +1420,7 @@ int os::get_loaded_modules_info(os::LoadedModulesCallbackFunc callback, void *pa
#elif defined(__APPLE__)
for (uint32_t i = 1; i < _dyld_image_count(); i++) {
// Value for top_address is returned as 0 since we don't have any information about module size
if (callback(_dyld_get_image_name(i), (address)_dyld_get_image_header(i), (address)0, param)) {
if (callback(_dyld_get_image_name(i), (address)_dyld_get_image_header(i), nullptr, param)) {
return 1;
}
}
@ -2189,6 +2209,10 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
return _processor_count;
}

22
src/hotspot/os/linux/cgroupSubsystem_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -631,22 +631,20 @@ void CgroupSubsystemFactory::cleanup(CgroupInfo* cg_infos) {
* return:
* true if there were no errors. false otherwise.
*/
bool CgroupSubsystem::active_processor_count(int& value) {
int cpu_count;
int result = -1;
bool CgroupSubsystem::active_processor_count(double& value) {
// We use a cache with a timeout to avoid performing expensive
// computations in the event this function is called frequently.
// [See 8227006].
CachingCgroupController<CgroupCpuController>* contrl = cpu_controller();
CachedMetric* cpu_limit = contrl->metrics_cache();
CachingCgroupController<CgroupCpuController, double>* contrl = cpu_controller();
CachedMetric<double>* cpu_limit = contrl->metrics_cache();
if (!cpu_limit->should_check_metric()) {
value = (int)cpu_limit->value();
log_trace(os, container)("CgroupSubsystem::active_processor_count (cached): %d", value);
value = cpu_limit->value();
log_trace(os, container)("CgroupSubsystem::active_processor_count (cached): %.2f", value);
return true;
}
cpu_count = os::Linux::active_processor_count();
int cpu_count = os::Linux::active_processor_count();
double result = -1;
if (!CgroupUtil::processor_count(contrl->controller(), cpu_count, result)) {
return false;
}
@ -671,8 +669,8 @@ bool CgroupSubsystem::active_processor_count(int& value) {
*/
bool CgroupSubsystem::memory_limit_in_bytes(physical_memory_size_type upper_bound,
physical_memory_size_type& value) {
CachingCgroupController<CgroupMemoryController>* contrl = memory_controller();
CachedMetric* memory_limit = contrl->metrics_cache();
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* contrl = memory_controller();
CachedMetric<physical_memory_size_type>* memory_limit = contrl->metrics_cache();
if (!memory_limit->should_check_metric()) {
value = memory_limit->value();
return true;

25
src/hotspot/os/linux/cgroupSubsystem_linux.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -181,20 +181,21 @@ class CgroupController: public CHeapObj<mtInternal> {
static bool limit_from_str(char* limit_str, physical_memory_size_type& value);
};
template <typename MetricType>
class CachedMetric : public CHeapObj<mtInternal>{
private:
volatile physical_memory_size_type _metric;
volatile MetricType _metric;
volatile jlong _next_check_counter;
public:
CachedMetric() {
_metric = value_unlimited;
_metric = static_cast<MetricType>(value_unlimited);
_next_check_counter = min_jlong;
}
bool should_check_metric() {
return os::elapsed_counter() > _next_check_counter;
}
physical_memory_size_type value() { return _metric; }
void set_value(physical_memory_size_type value, jlong timeout) {
MetricType value() { return _metric; }
void set_value(MetricType value, jlong timeout) {
_metric = value;
// Metric is unlikely to change, but we want to remain
// responsive to configuration changes. A very short grace time
@ -205,19 +206,19 @@ class CachedMetric : public CHeapObj<mtInternal>{
}
};
template <class T>
template <class T, typename MetricType>
class CachingCgroupController : public CHeapObj<mtInternal> {
private:
T* _controller;
CachedMetric* _metrics_cache;
CachedMetric<MetricType>* _metrics_cache;
public:
CachingCgroupController(T* cont) {
_controller = cont;
_metrics_cache = new CachedMetric();
_metrics_cache = new CachedMetric<MetricType>();
}
CachedMetric* metrics_cache() { return _metrics_cache; }
CachedMetric<MetricType>* metrics_cache() { return _metrics_cache; }
T* controller() { return _controller; }
};
@ -277,7 +278,7 @@ class CgroupMemoryController: public CHeapObj<mtInternal> {
class CgroupSubsystem: public CHeapObj<mtInternal> {
public:
bool memory_limit_in_bytes(physical_memory_size_type upper_bound, physical_memory_size_type& value);
bool active_processor_count(int& value);
bool active_processor_count(double& value);
virtual bool pids_max(uint64_t& value) = 0;
virtual bool pids_current(uint64_t& value) = 0;
@ -286,8 +287,8 @@ class CgroupSubsystem: public CHeapObj<mtInternal> {
virtual char * cpu_cpuset_cpus() = 0;
virtual char * cpu_cpuset_memory_nodes() = 0;
virtual const char * container_type() = 0;
virtual CachingCgroupController<CgroupMemoryController>* memory_controller() = 0;
virtual CachingCgroupController<CgroupCpuController>* cpu_controller() = 0;
virtual CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* memory_controller() = 0;
virtual CachingCgroupController<CgroupCpuController, double>* cpu_controller() = 0;
virtual CgroupCpuacctController* cpuacct_controller() = 0;
bool cpu_quota(int& value);

27
src/hotspot/os/linux/cgroupUtil_linux.cpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2024, 2025, Red Hat, Inc.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,9 +26,8 @@
#include "cgroupUtil_linux.hpp"
#include "os_linux.hpp"
bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, int& value) {
bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, double& value) {
assert(upper_bound > 0, "upper bound of cpus must be positive");
int limit_count = upper_bound;
int quota = -1;
int period = -1;
if (!cpu_ctrl->cpu_quota(quota)) {
@ -37,20 +37,15 @@ bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound,
return false;
}
int quota_count = 0;
int result = upper_bound;
double result = upper_bound;
if (quota > -1 && period > 0) {
quota_count = ceilf((float)quota / (float)period);
log_trace(os, container)("CPU Quota count based on quota/period: %d", quota_count);
if (quota > 0 && period > 0) { // Use quotas
double cpu_quota = static_cast<double>(quota) / period;
log_trace(os, container)("CPU Quota based on quota/period: %.2f", cpu_quota);
result = MIN2(result, cpu_quota);
}
// Use quotas
if (quota_count != 0) {
limit_count = quota_count;
}
result = MIN2(upper_bound, limit_count);
log_trace(os, container)("OSContainer::active_processor_count: %d", result);
log_trace(os, container)("OSContainer::active_processor_count: %.2f", result);
value = result;
return true;
}
@ -73,11 +68,11 @@ physical_memory_size_type CgroupUtil::get_updated_mem_limit(CgroupMemoryControll
// Get an updated cpu limit. The return value is strictly less than or equal to the
// passed in 'lowest' value.
int CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
double CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
int lowest,
int upper_bound) {
assert(lowest > 0 && lowest <= upper_bound, "invariant");
int cpu_limit_val = -1;
double cpu_limit_val = -1;
if (CgroupUtil::processor_count(cpu, upper_bound, cpu_limit_val) && cpu_limit_val != upper_bound) {
assert(cpu_limit_val <= upper_bound, "invariant");
if (lowest > cpu_limit_val) {
@ -172,7 +167,7 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
assert(cg_path[0] == '/', "cgroup path must start with '/'");
int host_cpus = os::Linux::active_processor_count();
int lowest_limit = host_cpus;
int cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
double cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
int orig_limit = lowest_limit != host_cpus ? lowest_limit : host_cpus;
char* limit_cg_path = nullptr;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {

7
src/hotspot/os/linux/cgroupUtil_linux.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2024, Red Hat, Inc.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,7 +32,7 @@
class CgroupUtil: AllStatic {
public:
static bool processor_count(CgroupCpuController* cpu, int upper_bound, int& value);
static bool processor_count(CgroupCpuController* cpu, int upper_bound, double& value);
// Given a memory controller, adjust its path to a point in the hierarchy
// that represents the closest memory limit.
static void adjust_controller(CgroupMemoryController* m);
@ -42,9 +43,7 @@ class CgroupUtil: AllStatic {
static physical_memory_size_type get_updated_mem_limit(CgroupMemoryController* m,
physical_memory_size_type lowest,
physical_memory_size_type upper_bound);
static int get_updated_cpu_limit(CgroupCpuController* c,
int lowest,
int upper_bound);
static double get_updated_cpu_limit(CgroupCpuController* c, int lowest, int upper_bound);
};
#endif // CGROUP_UTIL_LINUX_HPP

6
src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -328,8 +328,8 @@ CgroupV1Subsystem::CgroupV1Subsystem(CgroupV1Controller* cpuset,
_pids(pids) {
CgroupUtil::adjust_controller(memory);
CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController<CgroupMemoryController>(memory);
_cpu = new CachingCgroupController<CgroupCpuController>(cpu);
_memory = new CachingCgroupController<CgroupMemoryController, physical_memory_size_type>(memory);
_cpu = new CachingCgroupController<CgroupCpuController, double>(cpu);
}
bool CgroupV1Subsystem::is_containerized() {

10
src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -214,15 +214,15 @@ class CgroupV1Subsystem: public CgroupSubsystem {
const char * container_type() override {
return "cgroupv1";
}
CachingCgroupController<CgroupMemoryController>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController>* cpu_controller() override { return _cpu; }
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController, double>* cpu_controller() override { return _cpu; }
CgroupCpuacctController* cpuacct_controller() override { return _cpuacct; }
private:
/* controllers */
CachingCgroupController<CgroupMemoryController>* _memory = nullptr;
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* _memory = nullptr;
CgroupV1Controller* _cpuset = nullptr;
CachingCgroupController<CgroupCpuController>* _cpu = nullptr;
CachingCgroupController<CgroupCpuController, double>* _cpu = nullptr;
CgroupV1CpuacctController* _cpuacct = nullptr;
CgroupV1Controller* _pids = nullptr;

6
src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
View File

@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2025, Red Hat Inc.
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -156,8 +156,8 @@ CgroupV2Subsystem::CgroupV2Subsystem(CgroupV2MemoryController * memory,
_unified(unified) {
CgroupUtil::adjust_controller(memory);
CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController<CgroupMemoryController>(memory);
_cpu = new CachingCgroupController<CgroupCpuController>(cpu);
_memory = new CachingCgroupController<CgroupMemoryController, physical_memory_size_type>(memory);
_cpu = new CachingCgroupController<CgroupCpuController, double>(cpu);
_cpuacct = cpuacct;
}

10
src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
View File

@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2024, Red Hat Inc.
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -152,8 +152,8 @@ class CgroupV2Subsystem: public CgroupSubsystem {
/* One unified controller */
CgroupV2Controller _unified;
/* Caching wrappers for cpu/memory metrics */
CachingCgroupController<CgroupMemoryController>* _memory = nullptr;
CachingCgroupController<CgroupCpuController>* _cpu = nullptr;
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* _memory = nullptr;
CachingCgroupController<CgroupCpuController, double>* _cpu = nullptr;
CgroupCpuacctController* _cpuacct = nullptr;
@ -175,8 +175,8 @@ class CgroupV2Subsystem: public CgroupSubsystem {
const char * container_type() override {
return "cgroupv2";
}
CachingCgroupController<CgroupMemoryController>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController>* cpu_controller() override { return _cpu; }
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController, double>* cpu_controller() override { return _cpu; }
CgroupCpuacctController* cpuacct_controller() override { return _cpuacct; };
};

16
src/hotspot/os/linux/osContainer_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -86,8 +86,8 @@ void OSContainer::init() {
// 2.) On a physical Linux system with a limit enforced by other means (like systemd slice)
physical_memory_size_type mem_limit_val = value_unlimited;
(void)memory_limit_in_bytes(mem_limit_val); // discard error and use default
int host_cpus = os::Linux::active_processor_count();
int cpus = host_cpus;
double host_cpus = os::Linux::active_processor_count();
double cpus = host_cpus;
(void)active_processor_count(cpus); // discard error and use default
any_mem_cpu_limit_present = mem_limit_val != value_unlimited || host_cpus != cpus;
if (any_mem_cpu_limit_present) {
@ -127,8 +127,7 @@ bool OSContainer::available_memory_in_bytes(physical_memory_size_type& value) {
return false;
}
bool OSContainer::available_swap_in_bytes(physical_memory_size_type host_free_swap,
physical_memory_size_type& value) {
bool OSContainer::available_swap_in_bytes(physical_memory_size_type& value) {
physical_memory_size_type mem_limit = 0;
physical_memory_size_type mem_swap_limit = 0;
if (memory_limit_in_bytes(mem_limit) &&
@ -179,8 +178,7 @@ bool OSContainer::available_swap_in_bytes(physical_memory_size_type host_free_sw
assert(num < 25, "buffer too small");
mem_limit_buf[num] = '\0';
log_trace(os,container)("OSContainer::available_swap_in_bytes: container_swap_limit=%s"
" container_mem_limit=%s, host_free_swap: " PHYS_MEM_TYPE_FORMAT,
mem_swap_buf, mem_limit_buf, host_free_swap);
" container_mem_limit=%s", mem_swap_buf, mem_limit_buf);
}
return false;
}
@ -252,7 +250,7 @@ char * OSContainer::cpu_cpuset_memory_nodes() {
return cgroup_subsystem->cpu_cpuset_memory_nodes();
}
bool OSContainer::active_processor_count(int& value) {
bool OSContainer::active_processor_count(double& value) {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
return cgroup_subsystem->active_processor_count(value);
}
@ -291,11 +289,13 @@ template<typename T> struct metric_fmt;
template<> struct metric_fmt<unsigned long long int> { static constexpr const char* fmt = "%llu"; };
template<> struct metric_fmt<unsigned long int> { static constexpr const char* fmt = "%lu"; };
template<> struct metric_fmt<int> { static constexpr const char* fmt = "%d"; };
template<> struct metric_fmt<double> { static constexpr const char* fmt = "%.2f"; };
template<> struct metric_fmt<const char*> { static constexpr const char* fmt = "%s"; };
template void OSContainer::print_container_metric<unsigned long long int>(outputStream*, const char*, unsigned long long int, const char*);
template void OSContainer::print_container_metric<unsigned long int>(outputStream*, const char*, unsigned long int, const char*);
template void OSContainer::print_container_metric<int>(outputStream*, const char*, int, const char*);
template void OSContainer::print_container_metric<double>(outputStream*, const char*, double, const char*);
template void OSContainer::print_container_metric<const char*>(outputStream*, const char*, const char*, const char*);
template <typename T>

7
src/hotspot/os/linux/osContainer_linux.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -72,8 +72,7 @@ class OSContainer: AllStatic {
static const char * container_type();
static bool available_memory_in_bytes(physical_memory_size_type& value);
static bool available_swap_in_bytes(physical_memory_size_type host_free_swap,
physical_memory_size_type& value);
static bool available_swap_in_bytes(physical_memory_size_type& value);
static bool memory_limit_in_bytes(physical_memory_size_type& value);
static bool memory_and_swap_limit_in_bytes(physical_memory_size_type& value);
static bool memory_and_swap_usage_in_bytes(physical_memory_size_type& value);
@ -84,7 +83,7 @@ class OSContainer: AllStatic {
static bool rss_usage_in_bytes(physical_memory_size_type& value);
static bool cache_usage_in_bytes(physical_memory_size_type& value);
static bool active_processor_count(int& value);
static bool active_processor_count(double& value);
static char * cpu_cpuset_cpus();
static char * cpu_cpuset_memory_nodes();

150
src/hotspot/os/linux/os_linux.cpp
View File

@ -211,15 +211,58 @@ static bool suppress_primordial_thread_resolution = false;
// utility functions
bool os::is_containerized() {
return OSContainer::is_containerized();
}
bool os::Container::memory_limit(physical_memory_size_type& value) {
physical_memory_size_type result = 0;
if (OSContainer::memory_limit_in_bytes(result) && result != value_unlimited) {
value = result;
return true;
}
return false;
}
bool os::Container::memory_soft_limit(physical_memory_size_type& value) {
physical_memory_size_type result = 0;
if (OSContainer::memory_soft_limit_in_bytes(result) && result != 0 && result != value_unlimited) {
value = result;
return true;
}
return false;
}
bool os::Container::memory_throttle_limit(physical_memory_size_type& value) {
physical_memory_size_type result = 0;
if (OSContainer::memory_throttle_limit_in_bytes(result) && result != value_unlimited) {
value = result;
return true;
}
return false;
}
bool os::Container::used_memory(physical_memory_size_type& value) {
return OSContainer::memory_usage_in_bytes(value);
}
bool os::available_memory(physical_memory_size_type& value) {
if (OSContainer::is_containerized() && OSContainer::available_memory_in_bytes(value)) {
if (is_containerized() && Container::available_memory(value)) {
log_trace(os)("available container memory: " PHYS_MEM_TYPE_FORMAT, value);
return true;
}
return Machine::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return Linux::available_memory(value);
}
bool os::Container::available_memory(physical_memory_size_type& value) {
return OSContainer::available_memory_in_bytes(value);
}
bool os::Linux::available_memory(physical_memory_size_type& value) {
physical_memory_size_type avail_mem = 0;
@ -251,11 +294,15 @@ bool os::Linux::available_memory(physical_memory_size_type& value) {
}
bool os::free_memory(physical_memory_size_type& value) {
if (OSContainer::is_containerized() && OSContainer::available_memory_in_bytes(value)) {
if (is_containerized() && Container::available_memory(value)) {
log_trace(os)("free container memory: " PHYS_MEM_TYPE_FORMAT, value);
return true;
}
return Machine::free_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return Linux::free_memory(value);
}
@ -274,21 +321,30 @@ bool os::Linux::free_memory(physical_memory_size_type& value) {
}
bool os::total_swap_space(physical_memory_size_type& value) {
if (OSContainer::is_containerized()) {
physical_memory_size_type mem_swap_limit = value_unlimited;
physical_memory_size_type memory_limit = value_unlimited;
if (OSContainer::memory_and_swap_limit_in_bytes(mem_swap_limit) &&
OSContainer::memory_limit_in_bytes(memory_limit)) {
if (memory_limit != value_unlimited && mem_swap_limit != value_unlimited &&
mem_swap_limit >= memory_limit /* ensure swap is >= 0 */) {
value = mem_swap_limit - memory_limit;
return true;
}
}
} // fallback to the host swap space if the container returned unlimited
if (is_containerized() && Container::total_swap_space(value)) {
return true;
} // fallback to the host swap space if the container value fails
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
return Linux::host_swap(value);
}
bool os::Container::total_swap_space(physical_memory_size_type& value) {
physical_memory_size_type mem_swap_limit = value_unlimited;
physical_memory_size_type memory_limit = value_unlimited;
if (OSContainer::memory_and_swap_limit_in_bytes(mem_swap_limit) &&
OSContainer::memory_limit_in_bytes(memory_limit)) {
if (memory_limit != value_unlimited && mem_swap_limit != value_unlimited &&
mem_swap_limit >= memory_limit /* ensure swap is >= 0 */) {
value = mem_swap_limit - memory_limit;
return true;
}
}
return false;
}
static bool host_free_swap_f(physical_memory_size_type& value) {
struct sysinfo si;
int ret = sysinfo(&si);
@ -309,32 +365,45 @@ bool os::free_swap_space(physical_memory_size_type& value) {
return false;
}
physical_memory_size_type host_free_swap_val = MIN2(total_swap_space, host_free_swap);
if (OSContainer::is_containerized()) {
if (OSContainer::available_swap_in_bytes(host_free_swap_val, value)) {
if (is_containerized()) {
if (Container::free_swap_space(value)) {
return true;
}
// Fall through to use host value
log_trace(os,container)("os::free_swap_space: containerized value unavailable"
" returning host value: " PHYS_MEM_TYPE_FORMAT, host_free_swap_val);
}
value = host_free_swap_val;
return true;
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
return host_free_swap_f(value);
}
bool os::Container::free_swap_space(physical_memory_size_type& value) {
return OSContainer::available_swap_in_bytes(value);
}
physical_memory_size_type os::physical_memory() {
if (OSContainer::is_containerized()) {
if (is_containerized()) {
physical_memory_size_type mem_limit = value_unlimited;
if (OSContainer::memory_limit_in_bytes(mem_limit) && mem_limit != value_unlimited) {
if (Container::memory_limit(mem_limit) && mem_limit != value_unlimited) {
log_trace(os)("total container memory: " PHYS_MEM_TYPE_FORMAT, mem_limit);
return mem_limit;
}
}
physical_memory_size_type phys_mem = Linux::physical_memory();
physical_memory_size_type phys_mem = Machine::physical_memory();
log_trace(os)("total system memory: " PHYS_MEM_TYPE_FORMAT, phys_mem);
return phys_mem;
}
physical_memory_size_type os::Machine::physical_memory() {
return Linux::physical_memory();
}
// Returns the resident set size (RSS) of the process.
// Falls back to using VmRSS from /proc/self/status if /proc/self/smaps_rollup is unavailable.
// Note: On kernels with memory cgroups or shared memory, VmRSS may underreport RSS.
@ -2439,20 +2508,21 @@ bool os::Linux::print_container_info(outputStream* st) {
OSContainer::print_container_metric(st, "cpu_memory_nodes", p != nullptr ? p : "not supported");
free(p);
int i = -1;
bool supported = OSContainer::active_processor_count(i);
double cpus = -1;
bool supported = OSContainer::active_processor_count(cpus);
if (supported) {
assert(i > 0, "must be");
assert(cpus > 0, "must be");
if (ActiveProcessorCount > 0) {
OSContainer::print_container_metric(st, "active_processor_count", ActiveProcessorCount, "(from -XX:ActiveProcessorCount)");
} else {
OSContainer::print_container_metric(st, "active_processor_count", i);
OSContainer::print_container_metric(st, "active_processor_count", cpus);
}
} else {
OSContainer::print_container_metric(st, "active_processor_count", "not supported");
}
int i = -1;
supported = OSContainer::cpu_quota(i);
if (supported && i > 0) {
OSContainer::print_container_metric(st, "cpu_quota", i);
@ -4737,15 +4807,26 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
int active_cpus = -1;
if (OSContainer::is_containerized() && OSContainer::active_processor_count(active_cpus)) {
log_trace(os)("active_processor_count: determined by OSContainer: %d",
active_cpus);
} else {
active_cpus = os::Linux::active_processor_count();
if (is_containerized()) {
double cpu_quota;
if (Container::processor_count(cpu_quota)) {
int active_cpus = ceilf(cpu_quota); // Round fractional CPU quota up.
assert(active_cpus <= Machine::active_processor_count(), "must be");
log_trace(os)("active_processor_count: determined by OSContainer: %d",
active_cpus);
return active_cpus;
}
}
return active_cpus;
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
return os::Linux::active_processor_count();
}
bool os::Container::processor_count(double& value) {
return OSContainer::active_processor_count(value);
}
static bool should_warn_invalid_processor_id() {
@ -4882,9 +4963,14 @@ int os::open(const char *path, int oflag, int mode) {
oflag |= O_CLOEXEC;
int fd = ::open(path, oflag, mode);
if (fd == -1) return -1;
// No further checking is needed if open() returned an error or
// access mode is not read only.
if (fd == -1 || (oflag & O_ACCMODE) != O_RDONLY) {
return fd;
}
//If the open succeeded, the file might still be a directory
// If the open succeeded and is read only, the file might be a directory
// which the JVM doesn't allow to be read.
{
struct stat buf;
int ret = ::fstat(fd, &buf);

5
src/hotspot/os/posix/perfMemory_posix.cpp
View File

@ -112,6 +112,10 @@ static void save_memory_to_file(char* addr, size_t size) {
result = ::close(fd);
if (result == OS_ERR) {
warning("Could not close %s: %s\n", destfile, os::strerror(errno));
} else {
if (!successful_write) {
remove(destfile);
}
}
}
FREE_C_HEAP_ARRAY(char, destfile);
@ -949,6 +953,7 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
warning("Insufficient space for shared memory file: %s/%s\n", dirname, filename);
}
result = OS_ERR;
remove(filename);
break;
}
}

26
src/hotspot/os/windows/os_windows.cpp
View File

@ -839,10 +839,18 @@ bool os::available_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::free_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::win32::available_memory(physical_memory_size_type& value) {
// Use GlobalMemoryStatusEx() because GlobalMemoryStatus() may return incorrect
// value if total memory is larger than 4GB
@ -858,7 +866,11 @@ bool os::win32::available_memory(physical_memory_size_type& value) {
}
}
bool os::total_swap_space(physical_memory_size_type& value) {
bool os::total_swap_space(physical_memory_size_type& value) {
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
@ -872,6 +884,10 @@ bool os::total_swap_space(physical_memory_size_type& value) {
}
bool os::free_swap_space(physical_memory_size_type& value) {
return Machine::free_swap_space(value);
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
@ -888,6 +904,10 @@ physical_memory_size_type os::physical_memory() {
return win32::physical_memory();
}
physical_memory_size_type os::Machine::physical_memory() {
return win32::physical_memory();
}
size_t os::rss() {
size_t rss = 0;
PROCESS_MEMORY_COUNTERS_EX pmex;
@ -911,6 +931,10 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
bool schedules_all_processor_groups = win32::is_windows_11_or_greater() || win32::is_windows_server_2022_or_greater();
if (UseAllWindowsProcessorGroups && !schedules_all_processor_groups && !win32::processor_group_warning_displayed()) {
win32::set_processor_group_warning_displayed(true);

7
src/hotspot/share/classfile/stringTable.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -614,6 +614,10 @@ struct StringTableDeleteCheck : StackObj {
};
void StringTable::clean_dead_entries(JavaThread* jt) {
// BulkDeleteTask::prepare() may take ConcurrentHashTableResize_lock (nosafepoint-2).
// When NativeHeapTrimmer is enabled, SuspendMark may take NativeHeapTrimmer::_lock (nosafepoint).
// Take SuspendMark first to keep lock order and avoid deadlock.
NativeHeapTrimmer::SuspendMark sm("stringtable");
StringTableHash::BulkDeleteTask bdt(_local_table);
if (!bdt.prepare(jt)) {
return;
@ -621,7 +625,6 @@ void StringTable::clean_dead_entries(JavaThread* jt) {
StringTableDeleteCheck stdc;
StringTableDoDelete stdd;
NativeHeapTrimmer::SuspendMark sm("stringtable");
{
TraceTime timer("Clean", TRACETIME_LOG(Debug, stringtable, perf));
while(bdt.do_task(jt, stdc, stdd)) {

7
src/hotspot/share/classfile/symbolTable.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -763,6 +763,10 @@ struct SymbolTableDeleteCheck : StackObj {
};
void SymbolTable::clean_dead_entries(JavaThread* jt) {
// BulkDeleteTask::prepare() may take ConcurrentHashTableResize_lock (nosafepoint-2).
// When NativeHeapTrimmer is enabled, SuspendMark may take NativeHeapTrimmer::_lock (nosafepoint).
// Take SuspendMark first to keep lock order and avoid deadlock.
NativeHeapTrimmer::SuspendMark sm("symboltable");
SymbolTableHash::BulkDeleteTask bdt(_local_table);
if (!bdt.prepare(jt)) {
return;
@ -770,7 +774,6 @@ void SymbolTable::clean_dead_entries(JavaThread* jt) {
SymbolTableDeleteCheck stdc;
SymbolTableDoDelete stdd;
NativeHeapTrimmer::SuspendMark sm("symboltable");
{
TraceTime timer("Clean", TRACETIME_LOG(Debug, symboltable, perf));
while (bdt.do_task(jt, stdc, stdd)) {

13
src/hotspot/share/gc/g1/g1Arguments.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -209,6 +209,17 @@ void G1Arguments::initialize() {
FLAG_SET_DEFAULT(GCTimeRatio, 24);
}
// Do not interfere with GC-Pressure driven heap resizing unless the user
// explicitly sets otherwise. G1 heap sizing should be free to grow or shrink
// the heap based on GC pressure, rather than being forced to satisfy
// MinHeapFreeRatio or MaxHeapFreeRatio defaults that the user did not set.
if (FLAG_IS_DEFAULT(MinHeapFreeRatio)) {
FLAG_SET_DEFAULT(MinHeapFreeRatio, 0);
}
if (FLAG_IS_DEFAULT(MaxHeapFreeRatio)) {
FLAG_SET_DEFAULT(MaxHeapFreeRatio, 100);
}
// Below, we might need to calculate the pause time interval based on
// the pause target. When we do so we are going to give G1 maximum
// flexibility and allow it to do pauses when it needs to. So, we'll

8
src/hotspot/share/gc/g1/g1CardTableClaimTable.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -44,20 +44,20 @@ G1CardTableClaimTable::~G1CardTableClaimTable() {
void G1CardTableClaimTable::initialize(uint max_reserved_regions) {
assert(_card_claims == nullptr, "Must not be initialized twice");
_card_claims = NEW_C_HEAP_ARRAY(uint, max_reserved_regions, mtGC);
_card_claims = NEW_C_HEAP_ARRAY(Atomic<uint>, max_reserved_regions, mtGC);
_max_reserved_regions = max_reserved_regions;
reset_all_to_unclaimed();
}
void G1CardTableClaimTable::reset_all_to_unclaimed() {
for (uint i = 0; i < _max_reserved_regions; i++) {
_card_claims[i] = 0;
_card_claims[i].store_relaxed(0);
}
}
void G1CardTableClaimTable::reset_all_to_claimed() {
for (uint i = 0; i < _max_reserved_regions; i++) {
_card_claims[i] = (uint)G1HeapRegion::CardsPerRegion;
_card_claims[i].store_relaxed((uint)G1HeapRegion::CardsPerRegion);
}
}

5
src/hotspot/share/gc/g1/g1CardTableClaimTable.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,6 +27,7 @@
#include "gc/g1/g1CardTable.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
class G1HeapRegionClosure;
@ -45,7 +46,7 @@ class G1CardTableClaimTable : public CHeapObj<mtGC> {
// Card table iteration claim values for every heap region, from 0 (completely unclaimed)
// to (>=) G1HeapRegion::CardsPerRegion (completely claimed).
uint volatile* _card_claims;
Atomic<uint>* _card_claims;
uint _cards_per_chunk; // For conversion between card index and chunk index.

11
src/hotspot/share/gc/g1/g1CardTableClaimTable.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,26 +29,25 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "runtime/atomicAccess.hpp"
bool G1CardTableClaimTable::has_unclaimed_cards(uint region) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
return AtomicAccess::load(&_card_claims[region]) < G1HeapRegion::CardsPerRegion;
return _card_claims[region].load_relaxed() < G1HeapRegion::CardsPerRegion;
}
void G1CardTableClaimTable::reset_to_unclaimed(uint region) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
AtomicAccess::store(&_card_claims[region], 0u);
_card_claims[region].store_relaxed(0u);
}
uint G1CardTableClaimTable::claim_cards(uint region, uint increment) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
return AtomicAccess::fetch_then_add(&_card_claims[region], increment, memory_order_relaxed);
return _card_claims[region].fetch_then_add(increment, memory_order_relaxed);
}
uint G1CardTableClaimTable::claim_chunk(uint region) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
return AtomicAccess::fetch_then_add(&_card_claims[region], cards_per_chunk(), memory_order_relaxed);
return _card_claims[region].fetch_then_add(cards_per_chunk(), memory_order_relaxed);
}
uint G1CardTableClaimTable::claim_all_cards(uint region) {

14
src/hotspot/share/gc/g1/g1CodeRootSet.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +28,7 @@
#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/atomic.hpp"
#include "utilities/concurrentHashTable.inline.hpp"
#include "utilities/concurrentHashTableTasks.inline.hpp"
@ -60,7 +60,7 @@ class G1CodeRootSetHashTable : public CHeapObj<mtGC> {
HashTable _table;
HashTableScanTask _table_scanner;
size_t volatile _num_entries;
Atomic<size_t> _num_entries;
bool is_empty() const { return number_of_entries() == 0; }
@ -120,7 +120,7 @@ public:
bool grow_hint = false;
bool inserted = _table.insert(Thread::current(), lookup, method, &grow_hint);
if (inserted) {
AtomicAccess::inc(&_num_entries);
_num_entries.add_then_fetch(1u);
}
if (grow_hint) {
_table.grow(Thread::current());
@ -131,7 +131,7 @@ public:
HashTableLookUp lookup(method);
bool removed = _table.remove(Thread::current(), lookup);
if (removed) {
AtomicAccess::dec(&_num_entries);
_num_entries.sub_then_fetch(1u);
}
return removed;
}
@ -182,7 +182,7 @@ public:
guarantee(succeeded, "unable to clean table");
if (num_deleted != 0) {
size_t current_size = AtomicAccess::sub(&_num_entries, num_deleted);
size_t current_size = _num_entries.sub_then_fetch(num_deleted);
shrink_to_match(current_size);
}
}
@ -226,7 +226,7 @@ public:
size_t mem_size() { return sizeof(*this) + _table.get_mem_size(Thread::current()); }
size_t number_of_entries() const { return AtomicAccess::load(&_num_entries); }
size_t number_of_entries() const { return _num_entries.load_relaxed(); }
};
uintx G1CodeRootSetHashTable::HashTableLookUp::get_hash() const {

1
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -103,7 +103,6 @@
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/cpuTimeCounters.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/init.hpp"

5
src/hotspot/share/gc/g1/g1CollectedHeap.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -54,6 +54,7 @@
#include "memory/allocation.hpp"
#include "memory/iterator.hpp"
#include "memory/memRegion.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/threadSMR.hpp"
#include "utilities/bitMap.hpp"
@ -124,7 +125,7 @@ class G1JavaThreadsListClaimer : public StackObj {
ThreadsListHandle _list;
uint _claim_step;
volatile uint _cur_claim;
Atomic<uint> _cur_claim;
// Attempts to claim _claim_step JavaThreads, returning an array of claimed
// JavaThread* with count elements. Returns null (and a zero count) if there

7
src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -41,7 +41,6 @@
#include "gc/shared/markBitMap.inline.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "oops/stackChunkOop.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/threadSMR.inline.hpp"
#include "utilities/bitMap.inline.hpp"
@ -53,10 +52,10 @@ inline bool G1STWIsAliveClosure::do_object_b(oop p) {
inline JavaThread* const* G1JavaThreadsListClaimer::claim(uint& count) {
count = 0;
if (AtomicAccess::load(&_cur_claim) >= _list.length()) {
if (_cur_claim.load_relaxed() >= _list.length()) {
return nullptr;
}
uint claim = AtomicAccess::fetch_then_add(&_cur_claim, _claim_step);
uint claim = _cur_claim.fetch_then_add(_claim_step);
if (claim >= _list.length()) {
return nullptr;
}

17
src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,7 +27,7 @@
#include "gc/g1/g1CollectionSetChooser.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/space.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/atomic.hpp"
#include "utilities/quickSort.hpp"
// Determine collection set candidates (from marking): For all regions determine
@ -50,7 +50,7 @@ class G1BuildCandidateRegionsTask : public WorkerTask {
G1HeapRegion** _data;
uint volatile _cur_claim_idx;
Atomic<uint> _cur_claim_idx;
static int compare_region_gc_efficiency(G1HeapRegion** rr1, G1HeapRegion** rr2) {
G1HeapRegion* r1 = *rr1;
@ -105,7 +105,7 @@ class G1BuildCandidateRegionsTask : public WorkerTask {
// Claim a new chunk, returning its bounds [from, to[.
void claim_chunk(uint& from, uint& to) {
uint result = AtomicAccess::add(&_cur_claim_idx, _chunk_size);
uint result = _cur_claim_idx.add_then_fetch(_chunk_size);
assert(_max_size > result - 1,
"Array too small, is %u should be %u with chunk size %u.",
_max_size, result, _chunk_size);
@ -121,14 +121,15 @@ class G1BuildCandidateRegionsTask : public WorkerTask {
}
void sort_by_gc_efficiency() {
if (_cur_claim_idx == 0) {
uint length = _cur_claim_idx.load_relaxed();
if (length == 0) {
return;
}
for (uint i = _cur_claim_idx; i < _max_size; i++) {
for (uint i = length; i < _max_size; i++) {
assert(_data[i] == nullptr, "must be");
}
qsort(_data, _cur_claim_idx, sizeof(_data[0]), (_sort_Fn)compare_region_gc_efficiency);
for (uint i = _cur_claim_idx; i < _max_size; i++) {
qsort(_data, length, sizeof(_data[0]), (_sort_Fn)compare_region_gc_efficiency);
for (uint i = length; i < _max_size; i++) {
assert(_data[i] == nullptr, "must be");
}
}

134
src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -51,6 +51,9 @@
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/gcVMOperations.hpp"
#include "gc/shared/partialArraySplitter.inline.hpp"
#include "gc/shared/partialArrayState.hpp"
#include "gc/shared/partialArrayTaskStats.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
@ -67,7 +70,6 @@
#include "nmt/memTracker.hpp"
#include "oops/access.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/java.hpp"
@ -76,6 +78,7 @@
#include "runtime/prefetch.inline.hpp"
#include "runtime/threads.hpp"
#include "utilities/align.hpp"
#include "utilities/checkedCast.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/powerOfTwo.hpp"
@ -99,7 +102,7 @@ bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
// We move that task's local finger along.
_task->move_finger_to(addr);
_task->scan_task_entry(G1TaskQueueEntry::from_oop(cast_to_oop(addr)));
_task->process_entry(G1TaskQueueEntry(cast_to_oop(addr)), false /* stolen */);
// we only partially drain the local queue and global stack
_task->drain_local_queue(true);
_task->drain_global_stack(true);
@ -148,25 +151,25 @@ bool G1CMMarkStack::initialize() {
}
G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_chunk() {
if (_size >= _max_capacity) {
if (_size.load_relaxed() >= _max_capacity) {
return nullptr;
}
size_t cur_idx = AtomicAccess::fetch_then_add(&_size, 1u);
size_t cur_idx = _size.fetch_then_add(1u);
if (cur_idx >= _max_capacity) {
return nullptr;
}
size_t bucket = get_bucket(cur_idx);
if (AtomicAccess::load_acquire(&_buckets[bucket]) == nullptr) {
if (_buckets[bucket].load_acquire() == nullptr) {
if (!_should_grow) {
// Prefer to restart the CM.
return nullptr;
}
MutexLocker x(G1MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
if (AtomicAccess::load_acquire(&_buckets[bucket]) == nullptr) {
if (_buckets[bucket].load_acquire() == nullptr) {
size_t desired_capacity = bucket_size(bucket) * 2;
if (!try_expand_to(desired_capacity)) {
return nullptr;
@ -175,7 +178,7 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_
}
size_t bucket_idx = get_bucket_index(cur_idx);
TaskQueueEntryChunk* result = ::new (&_buckets[bucket][bucket_idx]) TaskQueueEntryChunk;
TaskQueueEntryChunk* result = ::new (&_buckets[bucket].load_relaxed()[bucket_idx]) TaskQueueEntryChunk;
result->next = nullptr;
return result;
}
@ -197,10 +200,10 @@ bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t m
_max_capacity = max_capacity;
_num_buckets = get_bucket(_max_capacity) + 1;
_buckets = NEW_C_HEAP_ARRAY(TaskQueueEntryChunk*, _num_buckets, mtGC);
_buckets = NEW_C_HEAP_ARRAY(Atomic<TaskQueueEntryChunk*>, _num_buckets, mtGC);
for (size_t i = 0; i < _num_buckets; i++) {
_buckets[i] = nullptr;
_buckets[i].store_relaxed(nullptr);
}
size_t new_capacity = bucket_size(0);
@ -240,9 +243,9 @@ G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
}
for (size_t i = 0; i < _num_buckets; i++) {
if (_buckets[i] != nullptr) {
MmapArrayAllocator<TaskQueueEntryChunk>::free(_buckets[i], bucket_size(i));
_buckets[i] = nullptr;
if (_buckets[i].load_relaxed() != nullptr) {
MmapArrayAllocator<TaskQueueEntryChunk>::free(_buckets[i].load_relaxed(), bucket_size(i));
_buckets[i].store_relaxed(nullptr);
}
}
@ -259,7 +262,7 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
// and the new capacity (new_capacity). This step ensures that there are no gaps in the
// array and that the capacity accurately reflects the reserved memory.
for (; i <= highest_bucket; i++) {
if (AtomicAccess::load_acquire(&_buckets[i]) != nullptr) {
if (_buckets[i].load_acquire() != nullptr) {
continue; // Skip over already allocated buckets.
}
@ -279,7 +282,7 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
return false;
}
_capacity += bucket_capacity;
AtomicAccess::release_store(&_buckets[i], bucket_base);
_buckets[i].release_store(bucket_base);
}
return true;
}
@ -490,6 +493,7 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
_task_queues(new G1CMTaskQueueSet(_max_num_tasks)),
_terminator(_max_num_tasks, _task_queues),
_partial_array_state_manager(new PartialArrayStateManager(_max_num_tasks)),
_first_overflow_barrier_sync(),
_second_overflow_barrier_sync(),
@ -556,6 +560,10 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
reset_at_marking_complete();
}
PartialArrayStateManager* G1ConcurrentMark::partial_array_state_manager() const {
return _partial_array_state_manager;
}
void G1ConcurrentMark::reset() {
_has_aborted = false;
@ -650,7 +658,26 @@ void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurr
}
}
#if TASKQUEUE_STATS
void G1ConcurrentMark::print_and_reset_taskqueue_stats() {
_task_queues->print_and_reset_taskqueue_stats("G1ConcurrentMark Oop Queue");
auto get_pa_stats = [&](uint i) {
return _tasks[i]->partial_array_task_stats();
};
PartialArrayTaskStats::log_set(_max_num_tasks, get_pa_stats,
"G1ConcurrentMark Partial Array Task Stats");
for (uint i = 0; i < _max_num_tasks; ++i) {
get_pa_stats(i)->reset();
}
}
#endif
void G1ConcurrentMark::reset_at_marking_complete() {
TASKQUEUE_STATS_ONLY(print_and_reset_taskqueue_stats());
// We set the global marking state to some default values when we're
// not doing marking.
reset_marking_for_restart();
@ -804,11 +831,25 @@ void G1ConcurrentMark::cleanup_for_next_mark() {
clear_bitmap(_concurrent_workers, true);
reset_partial_array_state_manager();
// Repeat the asserts from above.
guarantee(cm_thread()->in_progress(), "invariant");
guarantee(!_g1h->collector_state()->mark_or_rebuild_in_progress(), "invariant");
}
void G1ConcurrentMark::reset_partial_array_state_manager() {
for (uint i = 0; i < _max_num_tasks; ++i) {
_tasks[i]->unregister_partial_array_splitter();
}
partial_array_state_manager()->reset();
for (uint i = 0; i < _max_num_tasks; ++i) {
_tasks[i]->register_partial_array_splitter();
}
}
void G1ConcurrentMark::clear_bitmap(WorkerThreads* workers) {
assert_at_safepoint_on_vm_thread();
// To avoid fragmentation the full collection requesting to clear the bitmap
@ -1789,17 +1830,18 @@ public:
{ }
void operator()(G1TaskQueueEntry task_entry) const {
if (task_entry.is_array_slice()) {
guarantee(_g1h->is_in_reserved(task_entry.slice()), "Slice " PTR_FORMAT " must be in heap.", p2i(task_entry.slice()));
if (task_entry.is_partial_array_state()) {
oop obj = task_entry.to_partial_array_state()->source();
guarantee(_g1h->is_in_reserved(obj), "Partial Array " PTR_FORMAT " must be in heap.", p2i(obj));
return;
}
guarantee(oopDesc::is_oop(task_entry.obj()),
guarantee(oopDesc::is_oop(task_entry.to_oop()),
"Non-oop " PTR_FORMAT ", phase: %s, info: %d",
p2i(task_entry.obj()), _phase, _info);
G1HeapRegion* r = _g1h->heap_region_containing(task_entry.obj());
p2i(task_entry.to_oop()), _phase, _info);
G1HeapRegion* r = _g1h->heap_region_containing(task_entry.to_oop());
guarantee(!(r->in_collection_set() || r->has_index_in_opt_cset()),
"obj " PTR_FORMAT " from %s (%d) in region %u in (optional) collection set",
p2i(task_entry.obj()), _phase, _info, r->hrm_index());
p2i(task_entry.to_oop()), _phase, _info, r->hrm_index());
}
};
@ -2055,6 +2097,17 @@ void G1CMTask::reset(G1CMBitMap* mark_bitmap) {
_mark_stats_cache.reset();
}
void G1CMTask::register_partial_array_splitter() {
::new (&_partial_array_splitter) PartialArraySplitter(_cm->partial_array_state_manager(),
_cm->max_num_tasks(),
ObjArrayMarkingStride);
}
void G1CMTask::unregister_partial_array_splitter() {
_partial_array_splitter.~PartialArraySplitter();
}
bool G1CMTask::should_exit_termination() {
if (!regular_clock_call()) {
return true;
@ -2185,7 +2238,7 @@ bool G1CMTask::get_entries_from_global_stack() {
if (task_entry.is_null()) {
break;
}
assert(task_entry.is_array_slice() || oopDesc::is_oop(task_entry.obj()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj()));
assert(task_entry.is_partial_array_state() || oopDesc::is_oop(task_entry.to_oop()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.to_oop()));
bool success = _task_queue->push(task_entry);
// We only call this when the local queue is empty or under a
// given target limit. So, we do not expect this push to fail.
@ -2216,7 +2269,7 @@ void G1CMTask::drain_local_queue(bool partially) {
G1TaskQueueEntry entry;
bool ret = _task_queue->pop_local(entry);
while (ret) {
scan_task_entry(entry);
process_entry(entry, false /* stolen */);
if (_task_queue->size() <= target_size || has_aborted()) {
ret = false;
} else {
@ -2226,6 +2279,37 @@ void G1CMTask::drain_local_queue(bool partially) {
}
}
size_t G1CMTask::start_partial_array_processing(oop obj) {
assert(should_be_sliced(obj), "Must be an array object %d and large %zu", obj->is_objArray(), obj->size());
objArrayOop obj_array = objArrayOop(obj);
size_t array_length = obj_array->length();
size_t initial_chunk_size = _partial_array_splitter.start(_task_queue, obj_array, nullptr, array_length);
// Mark objArray klass metadata
if (_cm_oop_closure->do_metadata()) {
_cm_oop_closure->do_klass(obj_array->klass());
}
process_array_chunk(obj_array, 0, initial_chunk_size);
// Include object header size
return objArrayOopDesc::object_size(checked_cast<int>(initial_chunk_size));
}
size_t G1CMTask::process_partial_array(const G1TaskQueueEntry& task, bool stolen) {
PartialArrayState* state = task.to_partial_array_state();
// Access state before release by claim().
objArrayOop obj = objArrayOop(state->source());
PartialArraySplitter::Claim claim =
_partial_array_splitter.claim(state, _task_queue, stolen);
process_array_chunk(obj, claim._start, claim._end);
return heap_word_size((claim._end - claim._start) * heapOopSize);
}
void G1CMTask::drain_global_stack(bool partially) {
if (has_aborted()) {
return;
@ -2430,7 +2514,7 @@ void G1CMTask::attempt_stealing() {
while (!has_aborted()) {
G1TaskQueueEntry entry;
if (_cm->try_stealing(_worker_id, entry)) {
scan_task_entry(entry);
process_entry(entry, true /* stolen */);
// And since we're towards the end, let's totally drain the
// local queue and global stack.
@ -2759,12 +2843,12 @@ G1CMTask::G1CMTask(uint worker_id,
G1ConcurrentMark* cm,
G1CMTaskQueue* task_queue,
G1RegionMarkStats* mark_stats) :
_objArray_processor(this),
_worker_id(worker_id),
_g1h(G1CollectedHeap::heap()),
_cm(cm),
_mark_bitmap(nullptr),
_task_queue(task_queue),
_partial_array_splitter(_cm->partial_array_state_manager(), _cm->max_num_tasks(), ObjArrayMarkingStride),
_mark_stats_cache(mark_stats, G1RegionMarkStatsCache::RegionMarkStatsCacheSize),
_calls(0),
_time_target_ms(0.0),

98
src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -26,17 +26,20 @@
#define SHARE_GC_G1_G1CONCURRENTMARK_HPP
#include "gc/g1/g1ConcurrentMarkBitMap.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp"
#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1RegionMarkStatsCache.hpp"
#include "gc/shared/gcCause.hpp"
#include "gc/shared/partialArraySplitter.hpp"
#include "gc/shared/partialArrayState.hpp"
#include "gc/shared/partialArrayTaskStats.hpp"
#include "gc/shared/taskqueue.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shared/verifyOption.hpp"
#include "gc/shared/workerThread.hpp"
#include "gc/shared/workerUtils.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
#include "utilities/compilerWarnings.hpp"
#include "utilities/numberSeq.hpp"
@ -53,41 +56,7 @@ class G1RegionToSpaceMapper;
class G1SurvivorRegions;
class ThreadClosure;
// This is a container class for either an oop or a continuation address for
// mark stack entries. Both are pushed onto the mark stack.
class G1TaskQueueEntry {
private:
void* _holder;
static const uintptr_t ArraySliceBit = 1;
G1TaskQueueEntry(oop obj) : _holder(obj) {
assert(_holder != nullptr, "Not allowed to set null task queue element");
}
G1TaskQueueEntry(HeapWord* addr) : _holder((void*)((uintptr_t)addr | ArraySliceBit)) { }
public:
G1TaskQueueEntry() : _holder(nullptr) { }
// Trivially copyable, for use in GenericTaskQueue.
static G1TaskQueueEntry from_slice(HeapWord* what) { return G1TaskQueueEntry(what); }
static G1TaskQueueEntry from_oop(oop obj) { return G1TaskQueueEntry(obj); }
oop obj() const {
assert(!is_array_slice(), "Trying to read array slice " PTR_FORMAT " as oop", p2i(_holder));
return cast_to_oop(_holder);
}
HeapWord* slice() const {
assert(is_array_slice(), "Trying to read oop " PTR_FORMAT " as array slice", p2i(_holder));
return (HeapWord*)((uintptr_t)_holder & ~ArraySliceBit);
}
bool is_oop() const { return !is_array_slice(); }
bool is_array_slice() const { return ((uintptr_t)_holder & ArraySliceBit) != 0; }
bool is_null() const { return _holder == nullptr; }
};
typedef ScannerTask G1TaskQueueEntry;
typedef GenericTaskQueue<G1TaskQueueEntry, mtGC> G1CMTaskQueue;
typedef GenericTaskQueueSet<G1CMTaskQueue, mtGC> G1CMTaskQueueSet;
@ -172,9 +141,9 @@ private:
size_t _capacity;
size_t _num_buckets;
bool _should_grow;
TaskQueueEntryChunk* volatile* _buckets;
Atomic<TaskQueueEntryChunk*>* _buckets;
char _pad0[DEFAULT_PADDING_SIZE];
volatile size_t _size;
Atomic<size_t> _size;
char _pad4[DEFAULT_PADDING_SIZE - sizeof(size_t)];
size_t bucket_size(size_t bucket) {
@ -212,7 +181,7 @@ private:
bool initialize(size_t initial_capacity, size_t max_capacity);
void reset() {
_size = 0;
_size.store_relaxed(0);
_should_grow = false;
}
@ -411,6 +380,8 @@ class G1ConcurrentMark : public CHeapObj<mtGC> {
G1CMTaskQueueSet* _task_queues; // Task queue set
TaskTerminator _terminator; // For termination
PartialArrayStateManager* _partial_array_state_manager;
// Two sync barriers that are used to synchronize tasks when an
// overflow occurs. The algorithm is the following. All tasks enter
// the first one to ensure that they have all stopped manipulating
@ -488,6 +459,8 @@ class G1ConcurrentMark : public CHeapObj<mtGC> {
// Prints all gathered CM-related statistics
void print_stats();
void print_and_reset_taskqueue_stats();
HeapWord* finger() { return _finger; }
bool concurrent() { return _concurrent; }
uint active_tasks() { return _num_active_tasks; }
@ -556,14 +529,14 @@ public:
// mark_in_bitmap call. Updates various statistics data.
void add_to_liveness(uint worker_id, oop const obj, size_t size);
// Did the last marking find a live object between bottom and TAMS?
bool contains_live_object(uint region) const { return _region_mark_stats[region]._live_words != 0; }
bool contains_live_object(uint region) const { return _region_mark_stats[region].live_words() != 0; }
// Live bytes in the given region as determined by concurrent marking, i.e. the amount of
// live bytes between bottom and TAMS.
size_t live_bytes(uint region) const { return _region_mark_stats[region]._live_words * HeapWordSize; }
size_t live_bytes(uint region) const { return _region_mark_stats[region].live_words() * HeapWordSize; }
// Set live bytes for concurrent marking.
void set_live_bytes(uint region, size_t live_bytes) { _region_mark_stats[region]._live_words = live_bytes / HeapWordSize; }
void set_live_bytes(uint region, size_t live_bytes) { _region_mark_stats[region]._live_words.store_relaxed(live_bytes / HeapWordSize); }
// Approximate number of incoming references found during marking.
size_t incoming_refs(uint region) const { return _region_mark_stats[region]._incoming_refs; }
size_t incoming_refs(uint region) const { return _region_mark_stats[region].incoming_refs(); }
// Update the TAMS for the given region to the current top.
inline void update_top_at_mark_start(G1HeapRegion* r);
@ -582,6 +555,8 @@ public:
uint worker_id_offset() const { return _worker_id_offset; }
uint max_num_tasks() const {return _max_num_tasks; }
// Clear statistics gathered during the concurrent cycle for the given region after
// it has been reclaimed.
void clear_statistics(G1HeapRegion* r);
@ -631,6 +606,8 @@ public:
// Calculates the number of concurrent GC threads to be used in the marking phase.
uint calc_active_marking_workers();
PartialArrayStateManager* partial_array_state_manager() const;
// Resets the global marking data structures, as well as the
// task local ones; should be called during concurrent start.
void reset();
@ -642,6 +619,10 @@ public:
// to be called concurrently to the mutator. It will yield to safepoint requests.
void cleanup_for_next_mark();
// Recycle the memory that has been requested by allocators associated with
// this manager.
void reset_partial_array_state_manager();
// Clear the next marking bitmap during safepoint.
void clear_bitmap(WorkerThreads* workers);
@ -732,14 +713,13 @@ private:
refs_reached_period = 1024,
};
G1CMObjArrayProcessor _objArray_processor;
uint _worker_id;
G1CollectedHeap* _g1h;
G1ConcurrentMark* _cm;
G1CMBitMap* _mark_bitmap;
// the task queue of this task
G1CMTaskQueue* _task_queue;
PartialArraySplitter _partial_array_splitter;
G1RegionMarkStatsCache _mark_stats_cache;
// Number of calls to this task
@ -850,13 +830,24 @@ private:
// mark bitmap scan, and so needs to be pushed onto the mark stack.
bool is_below_finger(oop obj, HeapWord* global_finger) const;
template<bool scan> void process_grey_task_entry(G1TaskQueueEntry task_entry);
template<bool scan> void process_grey_task_entry(G1TaskQueueEntry task_entry, bool stolen);
static bool should_be_sliced(oop obj);
// Start processing the given objArrayOop by first pushing its continuations and
// then scanning the first chunk including the header.
size_t start_partial_array_processing(oop obj);
// Process the given continuation. Returns the number of words scanned.
size_t process_partial_array(const G1TaskQueueEntry& task, bool stolen);
// Apply the closure to the given range of elements in the objArray.
inline void process_array_chunk(objArrayOop obj, size_t start, size_t end);
public:
// Apply the closure on the given area of the objArray. Return the number of words
// scanned.
inline size_t scan_objArray(objArrayOop obj, MemRegion mr);
// Resets the task; should be called right at the beginning of a marking phase.
void reset(G1CMBitMap* mark_bitmap);
// Register/unregister Partial Array Splitter Allocator with the PartialArrayStateManager.
// This allows us to discard memory arenas used for partial object array states at the end
// of a concurrent mark cycle.
void register_partial_array_splitter();
void unregister_partial_array_splitter();
// Clears all the fields that correspond to a claimed region.
void clear_region_fields();
@ -912,7 +903,7 @@ public:
inline bool deal_with_reference(T* p);
// Scans an object and visits its children.
inline void scan_task_entry(G1TaskQueueEntry task_entry);
inline void process_entry(G1TaskQueueEntry task_entry, bool stolen);
// Pushes an object on the local queue.
inline void push(G1TaskQueueEntry task_entry);
@ -957,6 +948,11 @@ public:
Pair<size_t, size_t> flush_mark_stats_cache();
// Prints statistics associated with this task
void print_stats();
#if TASKQUEUE_STATS
PartialArrayTaskStats* partial_array_task_stats() {
return _partial_array_splitter.stats();
}
#endif
};
// Class that's used to to print out per-region liveness

49
src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,7 +29,6 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
@ -39,6 +38,7 @@
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/checkedCast.hpp"
inline bool G1CMIsAliveClosure::do_object_b(oop obj) {
// Check whether the passed in object is null. During discovery the referent
@ -107,13 +107,15 @@ inline void G1CMMarkStack::iterate(Fn fn) const {
#endif
// It scans an object and visits its children.
inline void G1CMTask::scan_task_entry(G1TaskQueueEntry task_entry) { process_grey_task_entry<true>(task_entry); }
inline void G1CMTask::process_entry(G1TaskQueueEntry task_entry, bool stolen) {
process_grey_task_entry<true>(task_entry, stolen);
}
inline void G1CMTask::push(G1TaskQueueEntry task_entry) {
assert(task_entry.is_array_slice() || _g1h->is_in_reserved(task_entry.obj()), "invariant");
assert(task_entry.is_array_slice() || !_g1h->is_on_master_free_list(
_g1h->heap_region_containing(task_entry.obj())), "invariant");
assert(task_entry.is_array_slice() || _mark_bitmap->is_marked(cast_from_oop<HeapWord*>(task_entry.obj())), "invariant");
assert(task_entry.is_partial_array_state() || _g1h->is_in_reserved(task_entry.to_oop()), "invariant");
assert(task_entry.is_partial_array_state() || !_g1h->is_on_master_free_list(
_g1h->heap_region_containing(task_entry.to_oop())), "invariant");
assert(task_entry.is_partial_array_state() || _mark_bitmap->is_marked(cast_from_oop<HeapWord*>(task_entry.to_oop())), "invariant");
if (!_task_queue->push(task_entry)) {
// The local task queue looks full. We need to push some entries
@ -159,29 +161,34 @@ inline bool G1CMTask::is_below_finger(oop obj, HeapWord* global_finger) const {
}
template<bool scan>
inline void G1CMTask::process_grey_task_entry(G1TaskQueueEntry task_entry) {
assert(scan || (task_entry.is_oop() && task_entry.obj()->is_typeArray()), "Skipping scan of grey non-typeArray");
assert(task_entry.is_array_slice() || _mark_bitmap->is_marked(cast_from_oop<HeapWord*>(task_entry.obj())),
inline void G1CMTask::process_grey_task_entry(G1TaskQueueEntry task_entry, bool stolen) {
assert(scan || (!task_entry.is_partial_array_state() && task_entry.to_oop()->is_typeArray()), "Skipping scan of grey non-typeArray");
assert(task_entry.is_partial_array_state() || _mark_bitmap->is_marked(cast_from_oop<HeapWord*>(task_entry.to_oop())),
"Any stolen object should be a slice or marked");
if (scan) {
if (task_entry.is_array_slice()) {
_words_scanned += _objArray_processor.process_slice(task_entry.slice());
if (task_entry.is_partial_array_state()) {
_words_scanned += process_partial_array(task_entry, stolen);
} else {
oop obj = task_entry.obj();
if (G1CMObjArrayProcessor::should_be_sliced(obj)) {
_words_scanned += _objArray_processor.process_obj(obj);
oop obj = task_entry.to_oop();
if (should_be_sliced(obj)) {
_words_scanned += start_partial_array_processing(obj);
} else {
_words_scanned += obj->oop_iterate_size(_cm_oop_closure);;
_words_scanned += obj->oop_iterate_size(_cm_oop_closure);
}
}
}
check_limits();
}
inline size_t G1CMTask::scan_objArray(objArrayOop obj, MemRegion mr) {
obj->oop_iterate(_cm_oop_closure, mr);
return mr.word_size();
inline bool G1CMTask::should_be_sliced(oop obj) {
return obj->is_objArray() && ((objArrayOop)obj)->length() >= (int)ObjArrayMarkingStride;
}
inline void G1CMTask::process_array_chunk(objArrayOop obj, size_t start, size_t end) {
obj->oop_iterate_elements_range(_cm_oop_closure,
checked_cast<int>(start),
checked_cast<int>(end));
}
inline void G1ConcurrentMark::update_top_at_mark_start(G1HeapRegion* r) {
@ -265,7 +272,7 @@ inline bool G1CMTask::make_reference_grey(oop obj) {
// be pushed on the stack. So, some duplicate work, but no
// correctness problems.
if (is_below_finger(obj, global_finger)) {
G1TaskQueueEntry entry = G1TaskQueueEntry::from_oop(obj);
G1TaskQueueEntry entry(obj);
if (obj->is_typeArray()) {
// Immediately process arrays of primitive types, rather
// than pushing on the mark stack. This keeps us from
@ -277,7 +284,7 @@ inline bool G1CMTask::make_reference_grey(oop obj) {
// by only doing a bookkeeping update and avoiding the
// actual scan of the object - a typeArray contains no
// references, and the metadata is built-in.
process_grey_task_entry<false>(entry);
process_grey_task_entry<false>(entry, false /* stolen */);
} else {
push(entry);
}

80
src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
View File

@ -1,80 +0,0 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/memRegion.hpp"
#include "utilities/globalDefinitions.hpp"
void G1CMObjArrayProcessor::push_array_slice(HeapWord* what) {
_task->push(G1TaskQueueEntry::from_slice(what));
}
size_t G1CMObjArrayProcessor::process_array_slice(objArrayOop obj, HeapWord* start_from, size_t remaining) {
size_t words_to_scan = MIN2(remaining, (size_t)ObjArrayMarkingStride);
if (remaining > ObjArrayMarkingStride) {
push_array_slice(start_from + ObjArrayMarkingStride);
}
// Then process current area.
MemRegion mr(start_from, words_to_scan);
return _task->scan_objArray(obj, mr);
}
size_t G1CMObjArrayProcessor::process_obj(oop obj) {
assert(should_be_sliced(obj), "Must be an array object %d and large %zu", obj->is_objArray(), obj->size());
return process_array_slice(objArrayOop(obj), cast_from_oop<HeapWord*>(obj), objArrayOop(obj)->size());
}
size_t G1CMObjArrayProcessor::process_slice(HeapWord* slice) {
// Find the start address of the objArrayOop.
// Shortcut the BOT access if the given address is from a humongous object. The BOT
// slide is fast enough for "smaller" objects in non-humongous regions, but is slower
// than directly using heap region table.
G1CollectedHeap* g1h = G1CollectedHeap::heap();
G1HeapRegion* r = g1h->heap_region_containing(slice);
HeapWord* const start_address = r->is_humongous() ?
r->humongous_start_region()->bottom() :
r->block_start(slice);
assert(cast_to_oop(start_address)->is_objArray(), "Address " PTR_FORMAT " does not refer to an object array ", p2i(start_address));
assert(start_address < slice,
"Object start address " PTR_FORMAT " must be smaller than decoded address " PTR_FORMAT,
p2i(start_address),
p2i(slice));
objArrayOop objArray = objArrayOop(cast_to_oop(start_address));
size_t already_scanned = pointer_delta(slice, start_address);
size_t remaining = objArray->size() - already_scanned;
return process_array_slice(objArray, slice, remaining);
}

59
src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp
View File

@ -1,59 +0,0 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP
#define SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP
#include "oops/oopsHierarchy.hpp"
class G1CMTask;
// Helper class to mark through large objArrays during marking in an efficient way.
// Instead of pushing large object arrays, we push continuations onto the
// mark stack. These continuations are identified by having their LSB set.
// This allows incremental processing of large objects.
class G1CMObjArrayProcessor {
private:
// Reference to the task for doing the actual work.
G1CMTask* _task;
// Push the continuation at the given address onto the mark stack.
void push_array_slice(HeapWord* addr);
// Process (apply the closure) on the given continuation of the given objArray.
size_t process_array_slice(objArrayOop const obj, HeapWord* start_from, size_t remaining);
public:
static bool should_be_sliced(oop obj);
G1CMObjArrayProcessor(G1CMTask* task) : _task(task) {
}
// Process the given continuation. Returns the number of words scanned.
size_t process_slice(HeapWord* slice);
// Start processing the given objArrayOop by scanning the header and pushing its
// continuation.
size_t process_obj(oop obj);
};
#endif // SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP

6
src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +29,6 @@
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/bitMap.inline.hpp"
G1EvacFailureRegions::G1EvacFailureRegions() :
@ -43,7 +43,7 @@ G1EvacFailureRegions::~G1EvacFailureRegions() {
}
void G1EvacFailureRegions::pre_collection(uint max_regions) {
AtomicAccess::store(&_num_regions_evac_failed, 0u);
_num_regions_evac_failed.store_relaxed(0u);
_regions_evac_failed.resize(max_regions);
_regions_pinned.resize(max_regions);
_regions_alloc_failed.resize(max_regions);
@ -69,6 +69,6 @@ void G1EvacFailureRegions::par_iterate(G1HeapRegionClosure* closure,
G1CollectedHeap::heap()->par_iterate_regions_array(closure,
hrclaimer,
_evac_failed_regions,
AtomicAccess::load(&_num_regions_evac_failed),
num_regions_evac_failed(),
worker_id);
}

6
src/hotspot/share/gc/g1/g1EvacFailureRegions.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,6 +26,7 @@
#ifndef SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP
#define SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP
#include "runtime/atomic.hpp"
#include "utilities/bitMap.hpp"
class G1AbstractSubTask;
@ -53,14 +55,14 @@ class G1EvacFailureRegions {
// Evacuation failed regions (indexes) in the current collection.
uint* _evac_failed_regions;
// Number of regions evacuation failed in the current collection.
volatile uint _num_regions_evac_failed;
Atomic<uint> _num_regions_evac_failed;
public:
G1EvacFailureRegions();
~G1EvacFailureRegions();
uint get_region_idx(uint idx) const {
assert(idx < _num_regions_evac_failed, "precondition");
assert(idx < _num_regions_evac_failed.load_relaxed(), "precondition");
return _evac_failed_regions[idx];
}

6
src/hotspot/share/gc/g1/g1EvacFailureRegions.inline.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,10 +30,9 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "runtime/atomicAccess.hpp"
uint G1EvacFailureRegions::num_regions_evac_failed() const {
return AtomicAccess::load(&_num_regions_evac_failed);
return _num_regions_evac_failed.load_relaxed();
}
bool G1EvacFailureRegions::has_regions_evac_failed() const {
@ -57,7 +57,7 @@ bool G1EvacFailureRegions::record(uint worker_id, uint region_idx, bool cause_pi
bool success = _regions_evac_failed.par_set_bit(region_idx,
memory_order_relaxed);
if (success) {
size_t offset = AtomicAccess::fetch_then_add(&_num_regions_evac_failed, 1u);
size_t offset = _num_regions_evac_failed.fetch_then_add(1u);
_evac_failed_regions[offset] = region_idx;
G1CollectedHeap* g1h = G1CollectedHeap::heap();

4
src/hotspot/share/gc/g1/g1FullCollector.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -122,7 +122,7 @@ public:
ReferenceProcessor* reference_processor();
size_t live_words(uint region_index) const {
assert(region_index < _heap->max_num_regions(), "sanity");
return _live_stats[region_index]._live_words;
return _live_stats[region_index].live_words();
}
void before_marking_update_attribute_table(G1HeapRegion* hr);

3
src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -37,7 +37,6 @@
#include "gc/shared/weakProcessor.inline.hpp"
#include "logging/log.hpp"
#include "memory/iterator.inline.hpp"
#include "runtime/atomicAccess.hpp"
class G1AdjustLiveClosure : public StackObj {
G1AdjustClosure* _adjust_closure;

3
src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,7 +31,6 @@
#include "memory/allocation.hpp"
#include "memory/padded.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/java.hpp"
#include "runtime/mutexLocker.hpp"

1
src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
View File

@ -30,7 +30,6 @@
#include "gc/g1/g1CodeRootSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1FromCardCache.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
#include "utilities/bitMap.hpp"

3
src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -30,7 +30,6 @@
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/bitMap.inline.hpp"
void G1HeapRegionRemSet::set_state_untracked() {

55
src/hotspot/share/gc/g1/g1MonotonicArena.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -24,7 +24,6 @@
#include "gc/g1/g1MonotonicArena.inline.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/vmOperations.hpp"
#include "utilities/globalCounter.inline.hpp"
@ -61,13 +60,13 @@ void G1MonotonicArena::SegmentFreeList::bulk_add(Segment& first,
size_t num,
size_t mem_size) {
_list.prepend(first, last);
AtomicAccess::add(&_num_segments, num, memory_order_relaxed);
AtomicAccess::add(&_mem_size, mem_size, memory_order_relaxed);
_num_segments.add_then_fetch(num, memory_order_relaxed);
_mem_size.add_then_fetch(mem_size, memory_order_relaxed);
}
void G1MonotonicArena::SegmentFreeList::print_on(outputStream* out, const char* prefix) {
out->print_cr("%s: segments %zu size %zu",
prefix, AtomicAccess::load(&_num_segments), AtomicAccess::load(&_mem_size));
prefix, _num_segments.load_relaxed(), _mem_size.load_relaxed());
}
G1MonotonicArena::Segment* G1MonotonicArena::SegmentFreeList::get_all(size_t& num_segments,
@ -75,12 +74,12 @@ G1MonotonicArena::Segment* G1MonotonicArena::SegmentFreeList::get_all(size_t& nu
GlobalCounter::CriticalSection cs(Thread::current());
Segment* result = _list.pop_all();
num_segments = AtomicAccess::load(&_num_segments);
mem_size = AtomicAccess::load(&_mem_size);
num_segments = _num_segments.load_relaxed();
mem_size = _mem_size.load_relaxed();
if (result != nullptr) {
AtomicAccess::sub(&_num_segments, num_segments, memory_order_relaxed);
AtomicAccess::sub(&_mem_size, mem_size, memory_order_relaxed);
_num_segments.sub_then_fetch(num_segments, memory_order_relaxed);
_mem_size.sub_then_fetch(mem_size, memory_order_relaxed);
}
return result;
}
@ -96,8 +95,8 @@ void G1MonotonicArena::SegmentFreeList::free_all() {
Segment::delete_segment(cur);
}
AtomicAccess::sub(&_num_segments, num_freed, memory_order_relaxed);
AtomicAccess::sub(&_mem_size, mem_size_freed, memory_order_relaxed);
_num_segments.sub_then_fetch(num_freed, memory_order_relaxed);
_mem_size.sub_then_fetch(mem_size_freed, memory_order_relaxed);
}
G1MonotonicArena::Segment* G1MonotonicArena::new_segment(Segment* const prev) {
@ -115,7 +114,7 @@ G1MonotonicArena::Segment* G1MonotonicArena::new_segment(Segment* const prev) {
}
// Install it as current allocation segment.
Segment* old = AtomicAccess::cmpxchg(&_first, prev, next);
Segment* old = _first.compare_exchange(prev, next);
if (old != prev) {
// Somebody else installed the segment, use that one.
Segment::delete_segment(next);
@ -126,9 +125,9 @@ G1MonotonicArena::Segment* G1MonotonicArena::new_segment(Segment* const prev) {
_last = next;
}
// Successfully installed the segment into the list.
AtomicAccess::inc(&_num_segments, memory_order_relaxed);
AtomicAccess::add(&_mem_size, next->mem_size(), memory_order_relaxed);
AtomicAccess::add(&_num_total_slots, next->num_slots(), memory_order_relaxed);
_num_segments.add_then_fetch(1u, memory_order_relaxed);
_mem_size.add_then_fetch(next->mem_size(), memory_order_relaxed);
_num_total_slots.add_then_fetch(next->num_slots(), memory_order_relaxed);
return next;
}
}
@ -155,7 +154,7 @@ uint G1MonotonicArena::slot_size() const {
}
void G1MonotonicArena::drop_all() {
Segment* cur = AtomicAccess::load_acquire(&_first);
Segment* cur = _first.load_acquire();
if (cur != nullptr) {
assert(_last != nullptr, "If there is at least one segment, there must be a last one.");
@ -175,25 +174,25 @@ void G1MonotonicArena::drop_all() {
cur = next;
}
#endif
assert(num_segments == _num_segments, "Segment count inconsistent %u %u", num_segments, _num_segments);
assert(mem_size == _mem_size, "Memory size inconsistent");
assert(num_segments == _num_segments.load_relaxed(), "Segment count inconsistent %u %u", num_segments, _num_segments.load_relaxed());
assert(mem_size == _mem_size.load_relaxed(), "Memory size inconsistent");
assert(last == _last, "Inconsistent last segment");
_segment_free_list->bulk_add(*first, *_last, _num_segments, _mem_size);
_segment_free_list->bulk_add(*first, *_last, _num_segments.load_relaxed(), _mem_size.load_relaxed());
}
_first = nullptr;
_first.store_relaxed(nullptr);
_last = nullptr;
_num_segments = 0;
_mem_size = 0;
_num_total_slots = 0;
_num_allocated_slots = 0;
_num_segments.store_relaxed(0);
_mem_size.store_relaxed(0);
_num_total_slots.store_relaxed(0);
_num_allocated_slots.store_relaxed(0);
}
void* G1MonotonicArena::allocate() {
assert(slot_size() > 0, "instance size not set.");
Segment* cur = AtomicAccess::load_acquire(&_first);
Segment* cur = _first.load_acquire();
if (cur == nullptr) {
cur = new_segment(cur);
}
@ -201,7 +200,7 @@ void* G1MonotonicArena::allocate() {
while (true) {
void* slot = cur->allocate_slot();
if (slot != nullptr) {
AtomicAccess::inc(&_num_allocated_slots, memory_order_relaxed);
_num_allocated_slots.add_then_fetch(1u, memory_order_relaxed);
guarantee(is_aligned(slot, _alloc_options->slot_alignment()),
"result " PTR_FORMAT " not aligned at %u", p2i(slot), _alloc_options->slot_alignment());
return slot;
@ -213,7 +212,7 @@ void* G1MonotonicArena::allocate() {
}
uint G1MonotonicArena::num_segments() const {
return AtomicAccess::load(&_num_segments);
return _num_segments.load_relaxed();
}
#ifdef ASSERT
@ -238,7 +237,7 @@ uint G1MonotonicArena::calculate_length() const {
template <typename SegmentClosure>
void G1MonotonicArena::iterate_segments(SegmentClosure& closure) const {
Segment* cur = AtomicAccess::load_acquire(&_first);
Segment* cur = _first.load_acquire();
assert((cur != nullptr) == (_last != nullptr),
"If there is at least one segment, there must be a last one");

47
src/hotspot/share/gc/g1/g1MonotonicArena.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -28,6 +28,7 @@
#include "gc/shared/freeListAllocator.hpp"
#include "nmt/memTag.hpp"
#include "runtime/atomic.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/lockFreeStack.hpp"
@ -65,27 +66,27 @@ private:
// AllocOptions provides parameters for Segment sizing and expansion.
const AllocOptions* _alloc_options;
Segment* volatile _first; // The (start of the) list of all segments.
Segment* _last; // The last segment of the list of all segments.
volatile uint _num_segments; // Number of assigned segments to this allocator.
volatile size_t _mem_size; // Memory used by all segments.
Atomic<Segment*> _first; // The (start of the) list of all segments.
Segment* _last; // The last segment of the list of all segments.
Atomic<uint> _num_segments; // Number of assigned segments to this allocator.
Atomic<size_t> _mem_size; // Memory used by all segments.
SegmentFreeList* _segment_free_list; // The global free segment list to preferentially
// get new segments from.
volatile uint _num_total_slots; // Number of slots available in all segments (allocated + not yet used).
volatile uint _num_allocated_slots; // Number of total slots allocated ever (including free and pending).
Atomic<uint> _num_total_slots; // Number of slots available in all segments (allocated + not yet used).
Atomic<uint> _num_allocated_slots; // Number of total slots allocated ever (including free and pending).
inline Segment* new_segment(Segment* const prev);
DEBUG_ONLY(uint calculate_length() const;)
public:
const Segment* first_segment() const { return AtomicAccess::load(&_first); }
const Segment* first_segment() const { return _first.load_relaxed(); }
uint num_total_slots() const { return AtomicAccess::load(&_num_total_slots); }
uint num_total_slots() const { return _num_total_slots.load_relaxed(); }
uint num_allocated_slots() const {
uint allocated = AtomicAccess::load(&_num_allocated_slots);
uint allocated = _num_allocated_slots.load_relaxed();
assert(calculate_length() == allocated, "Must be");
return allocated;
}
@ -116,11 +117,11 @@ static constexpr uint SegmentPayloadMaxAlignment = 8;
class alignas(SegmentPayloadMaxAlignment) G1MonotonicArena::Segment {
const uint _slot_size;
const uint _num_slots;
Segment* volatile _next;
Atomic<Segment*> _next;
// Index into the next free slot to allocate into. Full if equal (or larger)
// to _num_slots (can be larger because we atomically increment this value and
// check only afterwards if the allocation has been successful).
uint volatile _next_allocate;
Atomic<uint> _next_allocate;
const MemTag _mem_tag;
static size_t header_size() { return align_up(sizeof(Segment), SegmentPayloadMaxAlignment); }
@ -139,21 +140,21 @@ class alignas(SegmentPayloadMaxAlignment) G1MonotonicArena::Segment {
Segment(uint slot_size, uint num_slots, Segment* next, MemTag mem_tag);
~Segment() = default;
public:
Segment* volatile* next_addr() { return &_next; }
Atomic<Segment*>* next_addr() { return &_next; }
void* allocate_slot();
uint num_slots() const { return _num_slots; }
Segment* next() const { return _next; }
Segment* next() const { return _next.load_relaxed(); }
void set_next(Segment* next) {
assert(next != this, " loop condition");
_next = next;
_next.store_relaxed(next);
}
void reset(Segment* next) {
_next_allocate = 0;
_next_allocate.store_relaxed(0);
assert(next != this, " loop condition");
set_next(next);
memset(payload(0), 0, payload_size());
@ -166,7 +167,7 @@ public:
uint length() const {
// _next_allocate might grow larger than _num_slots in multi-thread environments
// due to races.
return MIN2(_next_allocate, _num_slots);
return MIN2(_next_allocate.load_relaxed(), _num_slots);
}
static size_t size_in_bytes(uint slot_size, uint num_slots) {
@ -176,7 +177,7 @@ public:
static Segment* create_segment(uint slot_size, uint num_slots, Segment* next, MemTag mem_tag);
static void delete_segment(Segment* segment);
bool is_full() const { return _next_allocate >= _num_slots; }
bool is_full() const { return _next_allocate.load_relaxed() >= _num_slots; }
};
static_assert(alignof(G1MonotonicArena::Segment) >= SegmentPayloadMaxAlignment, "assert alignment of Segment (and indirectly its payload)");
@ -186,15 +187,15 @@ static_assert(alignof(G1MonotonicArena::Segment) >= SegmentPayloadMaxAlignment,
// performed by multiple threads concurrently.
// Counts and memory usage are current on a best-effort basis if accessed concurrently.
class G1MonotonicArena::SegmentFreeList {
static Segment* volatile* next_ptr(Segment& segment) {
static Atomic<Segment*>* next_ptr(Segment& segment) {
return segment.next_addr();
}
using SegmentStack = LockFreeStack<Segment, &SegmentFreeList::next_ptr>;
SegmentStack _list;
volatile size_t _num_segments;
volatile size_t _mem_size;
Atomic<size_t> _num_segments;
Atomic<size_t> _mem_size;
public:
SegmentFreeList() : _list(), _num_segments(0), _mem_size(0) { }
@ -210,8 +211,8 @@ public:
void print_on(outputStream* out, const char* prefix = "");
size_t num_segments() const { return AtomicAccess::load(&_num_segments); }
size_t mem_size() const { return AtomicAccess::load(&_mem_size); }
size_t num_segments() const { return _num_segments.load_relaxed(); }
size_t mem_size() const { return _mem_size.load_relaxed(); }
};
// Configuration for G1MonotonicArena, e.g slot size, slot number of next Segment.

11
src/hotspot/share/gc/g1/g1MonotonicArena.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -28,14 +28,13 @@
#include "gc/g1/g1MonotonicArena.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/globalCounter.inline.hpp"
inline void* G1MonotonicArena::Segment::allocate_slot() {
if (_next_allocate >= _num_slots) {
if (_next_allocate.load_relaxed() >= _num_slots) {
return nullptr;
}
uint result = AtomicAccess::fetch_then_add(&_next_allocate, 1u, memory_order_relaxed);
uint result = _next_allocate.fetch_then_add(1u, memory_order_relaxed);
if (result >= _num_slots) {
return nullptr;
}
@ -48,8 +47,8 @@ inline G1MonotonicArena::Segment* G1MonotonicArena::SegmentFreeList::get() {
Segment* result = _list.pop();
if (result != nullptr) {
AtomicAccess::dec(&_num_segments, memory_order_relaxed);
AtomicAccess::sub(&_mem_size, result->mem_size(), memory_order_relaxed);
_num_segments.sub_then_fetch(1u, memory_order_relaxed);
_mem_size.sub_then_fetch(result->mem_size(), memory_order_relaxed);
}
return result;
}

3
src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +28,6 @@
#include "nmt/memTracker.hpp"
#include "oops/markWord.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/os.hpp"
#include "utilities/align.hpp"
#include "utilities/bitMap.inline.hpp"

3
src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -42,7 +42,6 @@
#include "memory/allocation.inline.hpp"
#include "oops/access.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/prefetch.inline.hpp"
#include "utilities/globalDefinitions.hpp"

11
src/hotspot/share/gc/g1/g1Policy.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -943,7 +943,7 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
phase_times()->sum_thread_work_items(G1GCPhaseTimes::MergePSS, G1GCPhaseTimes::MergePSSToYoungGenCards));
}
record_pause(this_pause, start_time_sec, end_time_sec, allocation_failure);
record_pause(this_pause, start_time_sec, end_time_sec);
if (G1GCPauseTypeHelper::is_last_young_pause(this_pause)) {
assert(!G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause),
@ -1389,16 +1389,13 @@ void G1Policy::update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_t
void G1Policy::record_pause(G1GCPauseType gc_type,
double start,
double end,
bool allocation_failure) {
double end) {
// Manage the MMU tracker. For some reason it ignores Full GCs.
if (gc_type != G1GCPauseType::FullGC) {
_mmu_tracker->add_pause(start, end);
}
if (!allocation_failure) {
update_gc_pause_time_ratios(gc_type, start, end);
}
update_gc_pause_time_ratios(gc_type, start, end);
update_time_to_mixed_tracking(gc_type, start, end);

5
src/hotspot/share/gc/g1/g1Policy.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -275,8 +275,7 @@ private:
// Record the given STW pause with the given start and end times (in s).
void record_pause(G1GCPauseType gc_type,
double start,
double end,
bool allocation_failure = false);
double end);
void update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_sec, double end_sec);

24
src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,6 +27,7 @@
#include "memory/allocation.hpp"
#include "oops/oop.hpp"
#include "runtime/atomic.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/pair.hpp"
@ -40,20 +41,23 @@
// * the number of incoming references found during marking. This is an approximate
// value because we do not mark through all objects.
struct G1RegionMarkStats {
size_t _live_words;
size_t _incoming_refs;
Atomic<size_t> _live_words;
Atomic<size_t> _incoming_refs;
// Clear all members.
void clear() {
_live_words = 0;
_incoming_refs = 0;
_live_words.store_relaxed(0);
_incoming_refs.store_relaxed(0);
}
// Clear all members after a marking overflow. Only needs to clear the number of
// incoming references as all objects will be rescanned, while the live words are
// gathered whenever a thread can mark an object, which is synchronized.
void clear_during_overflow() {
_incoming_refs = 0;
_incoming_refs.store_relaxed(0);
}
size_t live_words() const { return _live_words.load_relaxed(); }
size_t incoming_refs() const { return _incoming_refs.load_relaxed(); }
};
// Per-marking thread cache for the region mark statistics.
@ -112,12 +116,16 @@ public:
void add_live_words(oop obj);
void add_live_words(uint region_idx, size_t live_words) {
G1RegionMarkStatsCacheEntry* const cur = find_for_add(region_idx);
cur->_stats._live_words += live_words;
// This method is only ever called single-threaded, so we do not need atomic
// update here.
cur->_stats._live_words.store_relaxed(cur->_stats.live_words() + live_words);
}
void inc_incoming_refs(uint region_idx) {
G1RegionMarkStatsCacheEntry* const cur = find_for_add(region_idx);
cur->_stats._incoming_refs++;
// This method is only ever called single-threaded, so we do not need atomic
// update here.
cur->_stats._incoming_refs.store_relaxed(cur->_stats.incoming_refs() + 1u);
}
void reset(uint region_idx) {

12
src/hotspot/share/gc/g1/g1RegionMarkStatsCache.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,8 +27,6 @@
#include "gc/g1/g1RegionMarkStatsCache.hpp"
#include "runtime/atomicAccess.hpp"
inline G1RegionMarkStatsCache::G1RegionMarkStatsCacheEntry* G1RegionMarkStatsCache::find_for_add(uint region_idx) {
uint const cache_idx = hash(region_idx);
@ -46,12 +44,12 @@ inline G1RegionMarkStatsCache::G1RegionMarkStatsCacheEntry* G1RegionMarkStatsCac
inline void G1RegionMarkStatsCache::evict(uint idx) {
G1RegionMarkStatsCacheEntry* cur = &_cache[idx];
if (cur->_stats._live_words != 0) {
AtomicAccess::add(&_target[cur->_region_idx]._live_words, cur->_stats._live_words);
if (cur->_stats.live_words() != 0) {
_target[cur->_region_idx]._live_words.add_then_fetch(cur->_stats.live_words());
}
if (cur->_stats._incoming_refs != 0) {
AtomicAccess::add(&_target[cur->_region_idx]._incoming_refs, cur->_stats._incoming_refs);
if (cur->_stats.incoming_refs() != 0) {
_target[cur->_region_idx]._incoming_refs.add_then_fetch(cur->_stats.incoming_refs());
}
cur->clear();

6
src/hotspot/share/gc/shared/taskqueue.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -641,6 +641,10 @@ public:
return (raw_value() & PartialArrayTag) != 0;
}
bool is_null() const {
return _p == nullptr;
}
oop* to_oop_ptr() const {
return static_cast<oop*>(decode(OopTag));
}

16
src/hotspot/share/gc/shared/workerThread.cpp
View File

@ -96,8 +96,22 @@ void WorkerThreads::initialize_workers() {
}
}
bool WorkerThreads::allow_inject_creation_failure() const {
if (!is_init_completed()) {
// Never allow creation failures during VM init
return false;
}
if (_created_workers == 0) {
// Never allow creation failures of the first worker, it will cause the VM to exit
return false;
}
return true;
}
WorkerThread* WorkerThreads::create_worker(uint name_suffix) {
if (is_init_completed() && InjectGCWorkerCreationFailure) {
if (InjectGCWorkerCreationFailure && allow_inject_creation_failure()) {
return nullptr;
}

1
src/hotspot/share/gc/shared/workerThread.hpp
View File

@ -104,6 +104,7 @@ public:
WorkerThreads(const char* name, uint max_workers);
void initialize_workers();
bool allow_inject_creation_failure() const;
uint max_workers() const { return _max_workers; }
uint created_workers() const { return _created_workers; }

7
src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
View File

@ -129,6 +129,13 @@ protected:
#endif
}
inline void update_livedata(size_t live) {
_region_union._live_data = live;
#ifdef ASSERT
_union_tag = is_live_data;
#endif
}
inline ShenandoahHeapRegion* get_region() const {
assert(_union_tag != is_uninitialized, "Cannot fetch region from uninitialized RegionData");
return _region;

15
src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
View File

@ -89,6 +89,17 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll
return false;
}
// Between consecutive mixed-evacuation cycles, the live data within each candidate region may change due to
// promotions and old-gen evacuations. Re-sort the candidate regions in order to first evacuate regions that have
// the smallest amount of live data. These are easiest to evacuate with least effort. Doing these first allows
// us to more quickly replenish free memory with empty regions.
for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; i++) {
ShenandoahHeapRegion* r = _region_data[i].get_region();
_region_data[i].update_livedata(r->get_mixed_candidate_live_data_bytes());
}
QuickSort::sort<RegionData>(_region_data + _next_old_collection_candidate, unprocessed_old_collection_candidates(),
compare_by_live);
_first_pinned_candidate = NOT_FOUND;
uint included_old_regions = 0;
@ -414,6 +425,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
ShenandoahHeapRegion* r = candidates[i].get_region();
size_t region_garbage = r->garbage();
size_t region_free = r->free();
r->capture_mixed_candidate_garbage();
candidates_garbage += region_garbage;
unfragmented += region_free;
}
@ -456,6 +469,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
r->index(), ShenandoahHeapRegion::region_state_to_string(r->state()));
const size_t region_garbage = r->garbage();
const size_t region_free = r->free();
r->capture_mixed_candidate_garbage();
candidates_garbage += region_garbage;
unfragmented += region_free;
defrag_count++;

2
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
View File

@ -75,6 +75,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
_plab_allocs(0),
_live_data(0),
_critical_pins(0),
_mixed_candidate_garbage_words(0),
_update_watermark(start),
_age(0),
#ifdef SHENANDOAH_CENSUS_NOISE
@ -565,6 +566,7 @@ void ShenandoahHeapRegion::recycle_internal() {
assert(_recycling.is_set() && is_trash(), "Wrong state");
ShenandoahHeap* heap = ShenandoahHeap::heap();
_mixed_candidate_garbage_words = 0;
set_top(bottom());
clear_live_data();
reset_alloc_metadata();

11
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
View File

@ -43,6 +43,7 @@ class ShenandoahHeapRegion {
friend class VMStructs;
friend class ShenandoahHeapRegionStateConstant;
private:
/*
Region state is described by a state machine. Transitions are guarded by
heap lock, which allows changing the state of several regions atomically.
@ -259,6 +260,8 @@ private:
volatile size_t _live_data;
volatile size_t _critical_pins;
size_t _mixed_candidate_garbage_words;
HeapWord* volatile _update_watermark;
uint _age;
@ -398,6 +401,14 @@ public:
// above TAMS.
inline size_t get_live_data_words() const;
inline size_t get_mixed_candidate_live_data_bytes() const;
inline size_t get_mixed_candidate_live_data_words() const;
inline void capture_mixed_candidate_garbage();
// Returns garbage by calculating difference between used and get_live_data_words. The value returned is only
// meaningful immediately following completion of marking. If there have been subsequent allocations in this region,
// use a different approach to determine garbage, such as (used() - get_mixed_candidate_live_data_bytes())
inline size_t garbage() const;
void print_on(outputStream* st) const;

17
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
View File

@ -163,6 +163,23 @@ inline size_t ShenandoahHeapRegion::get_live_data_bytes() const {
return get_live_data_words() * HeapWordSize;
}
inline size_t ShenandoahHeapRegion::get_mixed_candidate_live_data_bytes() const {
shenandoah_assert_heaplocked_or_safepoint();
assert(used() >= _mixed_candidate_garbage_words * HeapWordSize, "used must exceed garbage");
return used() - _mixed_candidate_garbage_words * HeapWordSize;
}
inline size_t ShenandoahHeapRegion::get_mixed_candidate_live_data_words() const {
shenandoah_assert_heaplocked_or_safepoint();
assert(used() >= _mixed_candidate_garbage_words * HeapWordSize, "used must exceed garbage");
return used() / HeapWordSize - _mixed_candidate_garbage_words;
}
inline void ShenandoahHeapRegion::capture_mixed_candidate_garbage() {
shenandoah_assert_heaplocked_or_safepoint();
_mixed_candidate_garbage_words = garbage() / HeapWordSize;
}
inline bool ShenandoahHeapRegion::has_live() const {
return get_live_data_words() != 0;
}

27
src/hotspot/share/jfr/jni/jfrJniMethod.cpp
View File

@ -66,10 +66,6 @@
#include "runtime/mutexLocker.hpp"
#include "runtime/os.hpp"
#include "utilities/debug.hpp"
#ifdef LINUX
#include "os_linux.hpp"
#include "osContainer_linux.hpp"
#endif
#define NO_TRANSITION(result_type, header) extern "C" { result_type JNICALL header {
#define NO_TRANSITION_END } }
@ -400,35 +396,18 @@ JVM_ENTRY_NO_ENV(jboolean, jfr_is_class_instrumented(JNIEnv* env, jclass jvm, jc
JVM_END
JVM_ENTRY_NO_ENV(jboolean, jfr_is_containerized(JNIEnv* env, jclass jvm))
#ifdef LINUX
return OSContainer::is_containerized();
#else
return false;
#endif
return os::is_containerized();
JVM_END
JVM_ENTRY_NO_ENV(jlong, jfr_host_total_memory(JNIEnv* env, jclass jvm))
#ifdef LINUX
// We want the host memory, not the container limit.
// os::physical_memory() would return the container limit.
return static_cast<jlong>(os::Linux::physical_memory());
#else
return static_cast<jlong>(os::physical_memory());
#endif
return static_cast<jlong>(os::Machine::physical_memory());
JVM_END
JVM_ENTRY_NO_ENV(jlong, jfr_host_total_swap_memory(JNIEnv* env, jclass jvm))
#ifdef LINUX
// We want the host swap memory, not the container value.
physical_memory_size_type host_swap = 0;
(void)os::Linux::host_swap(host_swap); // Discard return value and treat as no swap
return static_cast<jlong>(host_swap);
#else
physical_memory_size_type total_swap_space = 0;
// Return value ignored - defaulting to 0 on failure.
(void)os::total_swap_space(total_swap_space);
(void)os::Machine::total_swap_space(total_swap_space);
return static_cast<jlong>(total_swap_space);
#endif
JVM_END
JVM_ENTRY_NO_ENV(void, jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes))

10
src/hotspot/share/jfr/utilities/jfrSet.hpp
View File

@ -26,6 +26,7 @@
#define SHARE_JFR_UTILITIES_JFRSET_HPP
#include "cppstdlib/new.hpp"
#include "cppstdlib/type_traits.hpp"
#include "jfr/utilities/jfrTypes.hpp"
#include "memory/allocation.hpp"
@ -110,7 +111,14 @@ class JfrSetStorage : public AnyObj {
}
void clear() {
memset(_table, 0, _table_size * sizeof(K));
for (unsigned i = 0; i < _table_size; ++i) {
if constexpr (std::is_copy_assignable_v<K>) {
_table[i] = K{};
} else {
_table[i].~K();
::new (&_table[i]) K{};
}
}
}
};

23
src/hotspot/share/opto/graphKit.cpp
View File

@ -1489,8 +1489,7 @@ Node* GraphKit::must_be_not_null(Node* value, bool do_replace_in_map) {
}
Node *if_f = _gvn.transform(new IfFalseNode(iff));
Node *frame = _gvn.transform(new ParmNode(C->start(), TypeFunc::FramePtr));
Node* halt = _gvn.transform(new HaltNode(if_f, frame, "unexpected null in intrinsic"));
C->root()->add_req(halt);
halt(if_f, frame, "unexpected null in intrinsic");
Node *if_t = _gvn.transform(new IfTrueNode(iff));
set_control(if_t);
return cast_not_null(value, do_replace_in_map);
@ -2073,6 +2072,12 @@ void GraphKit::increment_counter(Node* counter_addr) {
store_to_memory(ctrl, counter_addr, incr, T_LONG, MemNode::unordered);
}
void GraphKit::halt(Node* ctrl, Node* frameptr, const char* reason, bool generate_code_in_product) {
Node* halt = new HaltNode(ctrl, frameptr, reason
PRODUCT_ONLY(COMMA generate_code_in_product));
halt = _gvn.transform(halt);
root()->add_req(halt);
}
//------------------------------uncommon_trap----------------------------------
// Bail out to the interpreter in mid-method. Implemented by calling the
@ -2195,11 +2200,15 @@ Node* GraphKit::uncommon_trap(int trap_request,
// The debug info is the only real input to this call.
// Halt-and-catch fire here. The above call should never return!
HaltNode* halt = new HaltNode(control(), frameptr(), "uncommon trap returned which should never happen"
PRODUCT_ONLY(COMMA /*reachable*/false));
_gvn.set_type_bottom(halt);
root()->add_req(halt);
// We only emit code for the HaltNode in debug, which is enough for
// verifying correctness. In product, we don't want to emit it so
// that we can save on code space. HaltNode often get folded because
// the compiler can prove that the unreachable path is dead. But we
// cannot generally expect that for uncommon traps, which are often
// reachable and occasionally taken.
halt(control(), frameptr(),
"uncommon trap returned which should never happen",
false /* don't emit code in product */);
stop_and_kill_map();
return call;
}

2
src/hotspot/share/opto/graphKit.hpp
View File

@ -709,6 +709,8 @@ class GraphKit : public Phase {
void increment_counter(address counter_addr); // increment a debug counter
void increment_counter(Node* counter_addr); // increment a debug counter
void halt(Node* ctrl, Node* frameptr, const char* reason, bool generate_code_in_product = true);
// Bail out to the interpreter right now
// The optional klass is the one causing the trap.
// The optional reason is debug information written to the compile log.

4
src/hotspot/share/opto/matcher.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -327,6 +327,8 @@ public:
// e.g. Op_ vector nodes and other intrinsics while guarding with vlen
static bool match_rule_supported_vector(int opcode, int vlen, BasicType bt);
// Returns true if the platform efficiently implements the given masked vector
// operation using predicate features, false otherwise.
static bool match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt);
// Determines if a vector operation needs to be partially implemented with a mask

5
src/hotspot/share/opto/node.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2024, 2025, Alibaba Group Holding Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -845,7 +845,8 @@ public:
Flag_has_swapped_edges = 1ULL << 11,
Flag_is_scheduled = 1ULL << 12,
Flag_is_expensive = 1ULL << 13,
Flag_is_predicated_vector = 1ULL << 14,
Flag_is_predicated_vector = 1ULL << 14, // Marked on a vector node that has an additional
// mask input controlling the lane operations.
Flag_for_post_loop_opts_igvn = 1ULL << 15,
Flag_for_merge_stores_igvn = 1ULL << 16,
Flag_is_removed_by_peephole = 1ULL << 17,

3
src/hotspot/share/opto/parse1.cpp
View File

@ -1229,8 +1229,7 @@ void Parse::do_method_entry() {
Node* not_subtype_ctrl = gen_subtype_check(receiver_obj, holder_klass);
assert(!stopped(), "not a subtype");
Node* halt = _gvn.transform(new HaltNode(not_subtype_ctrl, frameptr(), "failed receiver subtype check"));
C->root()->add_req(halt);
halt(not_subtype_ctrl, frameptr(), "failed receiver subtype check");
}
}
#endif // ASSERT

41
src/hotspot/share/opto/rangeinference.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -147,7 +147,7 @@ public:
static const Type* int_type_xmeet(const CT* i1, const Type* t2);
template <class CTP>
static CTP int_type_union(CTP t1, CTP t2) {
static auto int_type_union(CTP t1, CTP t2) {
using CT = std::conditional_t<std::is_pointer_v<CTP>, std::remove_pointer_t<CTP>, CTP>;
using S = std::remove_const_t<decltype(CT::_lo)>;
using U = std::remove_const_t<decltype(CT::_ulo)>;
@ -209,7 +209,7 @@ public:
KnownBits<U> _bits;
int _widen = 0; // dummy field to mimic the same field in TypeInt, useful in testing
static TypeIntMirror make(const TypeIntPrototype<S, U>& t, int widen) {
static TypeIntMirror make(const TypeIntPrototype<S, U>& t, int widen = 0) {
auto canonicalized_t = t.canonicalize_constraints();
assert(!canonicalized_t.empty(), "must not be empty");
return TypeIntMirror{canonicalized_t._data._srange._lo, canonicalized_t._data._srange._hi,
@ -217,11 +217,15 @@ public:
canonicalized_t._data._bits};
}
TypeIntMirror meet(const TypeIntMirror& o) const {
return TypeIntHelper::int_type_union(this, &o);
}
// These allow TypeIntMirror to mimick the behaviors of TypeInt* and TypeLong*, so they can be
// passed into RangeInference methods. These are only used in testing, so they are implemented in
// the test file.
static TypeIntMirror make(const TypeIntMirror& t, int widen);
const TypeIntMirror* operator->() const;
TypeIntMirror meet(const TypeIntMirror& o) const;
bool contains(U u) const;
bool contains(const TypeIntMirror& o) const;
bool operator==(const TypeIntMirror& o) const;
@ -322,7 +326,7 @@ private:
// Infer a result given the input types of a binary operation
template <class CTP, class Inference>
static CTP infer_binary(CTP t1, CTP t2, Inference infer) {
CTP res;
TypeIntMirror<S<CTP>, U<CTP>> res;
bool is_init = false;
SimpleIntervalIterable<CTP> t1_simple_intervals(t1);
@ -330,10 +334,10 @@ private:
for (auto& st1 : t1_simple_intervals) {
for (auto& st2 : t2_simple_intervals) {
CTP current = infer(st1, st2);
TypeIntMirror<S<CTP>, U<CTP>> current = infer(st1, st2);
if (is_init) {
res = res->meet(current)->template cast<CT<CTP>>();
res = res.meet(current);
} else {
is_init = true;
res = current;
@ -342,7 +346,22 @@ private:
}
assert(is_init, "must be initialized");
return res;
// It is important that widen is computed on the whole result instead of during each step. This
// is because we normalize the widen of small Type instances to 0, so computing the widen value
// for each step and taking the union of them may return a widen value that conflicts with
// other computations, trigerring the monotonicity assert during CCP.
//
// For example, let us consider the operation r = x ^ y:
// - During the first step of CCP, type(x) = {0}, type(y) = [-2, 2], w = 3.
// Since x is a constant that is the identity element of the xor operation, type(r) = type(y) = [-2, 2], w = 3
// - During the second step, type(x) is widened to [0, 2], w = 0.
// We then compute the range for:
// r1 = x ^ y1, type(x) = [0, 2], w = 0, type(y1) = [0, 2], w = 0
// r2 = x ^ y2, type(x) = [0, 2], w = 0, type(y2) = [-2, -1], w = 0
// This results in type(r1) = [0, 3], w = 0, and type(r2) = [-4, -1], w = 0
// So the union of type(r1) and type(r2) is [-4, 3], w = 0. This widen value is smaller than
// that of the previous step, triggering the monotonicity assert.
return CT<CTP>::make(res, MAX2(t1->_widen, t2->_widen));
}
public:
@ -357,7 +376,7 @@ public:
U<CTP> uhi = MIN2(st1._uhi, st2._uhi);
U<CTP> zeros = st1._bits._zeros | st2._bits._zeros;
U<CTP> ones = st1._bits._ones & st2._bits._ones;
return CT<CTP>::make(TypeIntPrototype<S<CTP>, U<CTP>>{{lo, hi}, {ulo, uhi}, {zeros, ones}}, MAX2(t1->_widen, t2->_widen));
return TypeIntMirror<S<CTP>, U<CTP>>::make(TypeIntPrototype<S<CTP>, U<CTP>>{{lo, hi}, {ulo, uhi}, {zeros, ones}});
});
}
@ -372,7 +391,7 @@ public:
U<CTP> uhi = std::numeric_limits<U<CTP>>::max();
U<CTP> zeros = st1._bits._zeros & st2._bits._zeros;
U<CTP> ones = st1._bits._ones | st2._bits._ones;
return CT<CTP>::make(TypeIntPrototype<S<CTP>, U<CTP>>{{lo, hi}, {ulo, uhi}, {zeros, ones}}, MAX2(t1->_widen, t2->_widen));
return TypeIntMirror<S<CTP>, U<CTP>>::make(TypeIntPrototype<S<CTP>, U<CTP>>{{lo, hi}, {ulo, uhi}, {zeros, ones}});
});
}
@ -385,7 +404,7 @@ public:
U<CTP> uhi = std::numeric_limits<U<CTP>>::max();
U<CTP> zeros = (st1._bits._zeros & st2._bits._zeros) | (st1._bits._ones & st2._bits._ones);
U<CTP> ones = (st1._bits._zeros & st2._bits._ones) | (st1._bits._ones & st2._bits._zeros);
return CT<CTP>::make(TypeIntPrototype<S<CTP>, U<CTP>>{{lo, hi}, {ulo, uhi}, {zeros, ones}}, MAX2(t1->_widen, t2->_widen));
return TypeIntMirror<S<CTP>, U<CTP>>::make(TypeIntPrototype<S<CTP>, U<CTP>>{{lo, hi}, {ulo, uhi}, {zeros, ones}});
});
}
};

8
src/hotspot/share/opto/type.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -2443,6 +2443,12 @@ const TypeVect* TypeVect::make(BasicType elem_bt, uint length, bool is_mask) {
return nullptr;
}
// Create a vector mask type with the given element basic type and length.
// - Returns "TypeVectMask" (PVectMask) for platforms that support the predicate
// feature and it is implemented properly in the backend, allowing the mask to
// be stored in a predicate/mask register.
// - Returns a normal vector type "TypeVectA ~ TypeVectZ" (NVectMask) otherwise,
// where the vector mask is stored in a vector register.
const TypeVect* TypeVect::makemask(BasicType elem_bt, uint length) {
if (Matcher::has_predicated_vectors() &&
Matcher::match_rule_supported_vector_masked(Op_VectorLoadMask, length, elem_bt)) {

23
src/hotspot/share/opto/type.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -799,6 +799,9 @@ public:
static const TypeInt* make(jint lo, jint hi, int widen);
static const Type* make_or_top(const TypeIntPrototype<jint, juint>& t, int widen);
static const TypeInt* make(const TypeIntPrototype<jint, juint>& t, int widen) { return make_or_top(t, widen)->is_int(); }
static const TypeInt* make(const TypeIntMirror<jint, juint>& t, int widen) {
return (new TypeInt(TypeIntPrototype<jint, juint>{{t._lo, t._hi}, {t._ulo, t._uhi}, t._bits}, widen, false))->hashcons()->is_int();
}
// Check for single integer
bool is_con() const { return _lo == _hi; }
@ -881,6 +884,9 @@ public:
static const TypeLong* make(jlong lo, jlong hi, int widen);
static const Type* make_or_top(const TypeIntPrototype<jlong, julong>& t, int widen);
static const TypeLong* make(const TypeIntPrototype<jlong, julong>& t, int widen) { return make_or_top(t, widen)->is_long(); }
static const TypeLong* make(const TypeIntMirror<jlong, julong>& t, int widen) {
return (new TypeLong(TypeIntPrototype<jlong, julong>{{t._lo, t._hi}, {t._ulo, t._uhi}, t._bits}, widen, false))->hashcons()->is_long();
}
// Check for single integer
bool is_con() const { return _lo == _hi; }
@ -1012,7 +1018,7 @@ public:
};
//------------------------------TypeVect---------------------------------------
// Class of Vector Types
// Basic class of vector (mask) types.
class TypeVect : public Type {
const BasicType _elem_bt; // Vector's element type
const uint _length; // Elements in vector (power of 2)
@ -1052,6 +1058,16 @@ public:
#endif
};
// TypeVect subclasses representing vectors or vector masks with "BVectMask" or "NVectMask"
// layout (see vectornode.hpp for detailed notes on vector mask representations), mapped
// to vector registers and distinguished by vector register size:
//
// - TypeVectA: Scalable vector type (variable size, e.g., AArch64 SVE, RISC-V RVV)
// - TypeVectS: 32-bit vector type
// - TypeVectD: 64-bit vector type
// - TypeVectX: 128-bit vector type
// - TypeVectY: 256-bit vector type
// - TypeVectZ: 512-bit vector type
class TypeVectA : public TypeVect {
friend class TypeVect;
TypeVectA(BasicType elem_bt, uint length) : TypeVect(VectorA, elem_bt, length) {}
@ -1082,6 +1098,9 @@ class TypeVectZ : public TypeVect {
TypeVectZ(BasicType elem_bt, uint length) : TypeVect(VectorZ, elem_bt, length) {}
};
// Class of TypeVectMask, representing vector masks with "PVectMask" layout (see
// vectornode.hpp for detailed notes on vector mask representations), mapped to
// dedicated hardware predicate/mask registers.
class TypeVectMask : public TypeVect {
public:
friend class TypeVect;

99
src/hotspot/share/opto/vectorIntrinsics.cpp
View File

@ -25,6 +25,7 @@
#include "ci/ciSymbols.hpp"
#include "classfile/vmSymbols.hpp"
#include "opto/library_call.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/vectornode.hpp"
#include "prims/vectorSupport.hpp"
@ -73,6 +74,11 @@ static bool is_vector_mask(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
}
static Node* trace_vector(Node* operation) {
VectorNode::trace_new_vector(operation, "VectorAPI");
return operation;
}
bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
VectorMaskUseType mask_use_type, bool has_scalar_args) {
bool is_supported = true;
@ -457,7 +463,7 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
}
default: fatal("unsupported arity: %d", n);
}
trace_vector(operation);
if (is_masked_op && mask != nullptr) {
if (use_predicate) {
operation->add_req(mask);
@ -465,7 +471,7 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
} else {
operation->add_flag(Node::Flag_is_predicated_using_blend);
operation = gvn().transform(operation);
operation = new VectorBlendNode(opd1, operation, mask);
operation = trace_vector(new VectorBlendNode(opd1, operation, mask));
}
}
operation = gvn().transform(operation);
@ -626,7 +632,7 @@ bool LibraryCallKit::inline_vector_mask_operation() {
mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
}
const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
Node* maskoper = gvn().transform(trace_vector(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc)));
if (mopc != Op_VectorMaskToLong) {
maskoper = ConvI2L(maskoper);
}
@ -709,10 +715,10 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
if (opc == Op_VectorLongToMask) {
const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
if (Matcher::mask_op_prefers_predicate(opc, vt)) {
broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
broadcast = gvn().transform(trace_vector(new VectorLongToMaskNode(elem, vt)));
} else {
const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
broadcast = gvn().transform(trace_vector(new VectorLongToMaskNode(elem, mvt)));
broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
}
} else {
@ -740,7 +746,7 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
}
default: fatal("%s", type2name(elem_bt));
}
broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
broadcast = trace_vector(VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask));
broadcast = gvn().transform(broadcast);
}
@ -926,22 +932,22 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
if (is_mask) {
val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
}
Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
Node* vstore = gvn().transform(trace_vector(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem)));
set_memory(vstore, addr_type);
} else {
// When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
Node* vload = nullptr;
if (mismatched_ms) {
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt)));
const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
} else {
// Special handle for masks
if (is_mask) {
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN)));
vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
} else {
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt)));
}
}
Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
@ -1139,7 +1145,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
}
Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
Node* vstore = gvn().transform(trace_vector(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask)));
set_memory(vstore, addr_type);
} else {
Node* vload = nullptr;
@ -1154,13 +1160,13 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
if (supports_predicate) {
// Generate masked load vector node if predicate feature is supported.
const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
vload = gvn().transform(trace_vector(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask)));
} else {
// Use the vector blend to implement the masked load vector. The biased elements are zeros.
Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt)));
vload = gvn().transform(trace_vector(new VectorBlendNode(zero, vload, mask)));
}
if (mismatched_ms) {
@ -1364,17 +1370,17 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
Node* vstore = nullptr;
if (mask != nullptr) {
vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, indexes, mask));
vstore = gvn().transform(trace_vector(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, indexes, mask)));
} else {
vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, indexes));
vstore = gvn().transform(trace_vector(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, indexes)));
}
set_memory(vstore, addr_type);
} else {
Node* vload = nullptr;
if (mask != nullptr) {
vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, indexes, mask));
vload = gvn().transform(trace_vector(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, indexes, mask)));
} else {
vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, indexes));
vload = gvn().transform(trace_vector(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, indexes)));
}
Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
set_result(box);
@ -1492,7 +1498,7 @@ bool LibraryCallKit::inline_vector_reduction() {
// Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
// as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
value = trace_vector(ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false));
if (mask != nullptr && use_predicate) {
value->add_req(mask);
@ -1584,7 +1590,7 @@ bool LibraryCallKit::inline_vector_test() {
return false; // operand unboxing failed
}
Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
Node* cmp = gvn().transform(trace_vector(new VectorTestNode(opd1, opd2, booltest)));
BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
opd1->bottom_type()->isa_vectmask(), num_elem);
Node* bol = gvn().transform(new BoolNode(cmp, test));
@ -1652,7 +1658,7 @@ bool LibraryCallKit::inline_vector_blend() {
return false; // operand unboxing failed
}
Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
Node* blend = gvn().transform(trace_vector(new VectorBlendNode(v1, v2, mask)));
Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
set_result(box);
@ -1723,7 +1729,7 @@ bool LibraryCallKit::inline_vector_slice() {
// Defining origin in terms of number of bytes to make it type agnostic value.
Node* origin_node = gvn().intcon(origin->get_con() * type2aelembytes(elem_bt));
const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
Node* operation = gvn().transform(new VectorSliceNode(v1, v2, origin_node, vector_type));
Node* operation = gvn().transform(trace_vector(new VectorSliceNode(v1, v2, origin_node, vector_type)));
Node* box = box_vector(operation, vbox_type, elem_bt, num_elem);
set_result(box);
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
@ -1817,6 +1823,7 @@ bool LibraryCallKit::inline_vector_compare() {
const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
trace_vector(operation);
if (is_masked_op) {
if (use_predicate) {
@ -1956,6 +1963,7 @@ bool LibraryCallKit::inline_vector_rearrange() {
}
Node* rearrange = new VectorRearrangeNode(v1, shuffle);
trace_vector(rearrange);
if (is_masked_op) {
if (use_predicate) {
rearrange->add_req(mask);
@ -2103,6 +2111,7 @@ bool LibraryCallKit::inline_vector_select_from() {
// and finally rearrange
Node* rearrange = new VectorRearrangeNode(v2, shuffle);
trace_vector(rearrange);
if (is_masked_op) {
if (use_predicate) {
// masked rearrange is supported so use that directly
@ -2262,6 +2271,7 @@ bool LibraryCallKit::inline_vector_broadcast_int() {
}
Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
trace_vector(operation);
if (is_masked_op && mask != nullptr) {
if (use_predicate) {
operation->add_req(mask);
@ -2400,6 +2410,21 @@ bool LibraryCallKit::inline_vector_convert() {
Node* op = opd1;
if (is_cast) {
assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
// Make sure the precondition of VectorCastNode::opcode holds: we can only have
// unsigned casts for integral types (excluding long). VectorAPI code is not
// expected to violate this at runtime, but we may compile unreachable code
// where such impossible combinations arise.
if (is_ucast && (!is_integral_type(elem_bt_from) || elem_bt_from == T_LONG)) {
// Halt-and-catch fire here. This condition should never happen at runtime.
stringStream ss;
ss.print("impossible combination: unsigned vector cast from %s", type2name(elem_bt_from));
halt(control(), frameptr(), ss.as_string(C->comp_arena()));
stop_and_kill_map();
log_if_needed(" ** impossible combination: unsigned cast from %s", type2name(elem_bt_from));
return true;
}
int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
// Make sure that vector cast is implemented to particular type/size combination if it is
@ -2424,7 +2449,7 @@ bool LibraryCallKit::inline_vector_convert() {
return false;
}
op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
op = gvn().transform(trace_vector(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast)));
// Now ensure that the destination gets properly resized to needed size.
op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
} else if (num_elem_from > num_elem_to) {
@ -2445,7 +2470,7 @@ bool LibraryCallKit::inline_vector_convert() {
const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
op = gvn().transform(trace_vector(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)));
} else { // num_elem_from == num_elem_to
if (is_mask) {
// Make sure that cast for vector mask is implemented to particular type/size combination.
@ -2454,16 +2479,16 @@ bool LibraryCallKit::inline_vector_convert() {
num_elem_to, type2name(elem_bt_to), is_mask);
return false;
}
op = gvn().transform(new VectorMaskCastNode(op, dst_type));
op = gvn().transform(trace_vector(new VectorMaskCastNode(op, dst_type)));
} else {
// Since input and output number of elements match, and since we know this vector size is
// supported, simply do a cast with no resize needed.
op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
op = gvn().transform(trace_vector(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)));
}
}
} else if (!Type::equals(src_type, dst_type)) {
assert(!is_cast, "must be reinterpret");
op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
op = gvn().transform(trace_vector(new VectorReinterpretNode(op, src_type, dst_type)));
}
const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
@ -2548,7 +2573,7 @@ bool LibraryCallKit::inline_vector_insert() {
default: fatal("%s", type2name(elem_bt)); break;
}
Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
Node* operation = gvn().transform(trace_vector(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn())));
Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
set_result(vbox);
@ -2606,7 +2631,7 @@ bool LibraryCallKit::inline_vector_extract() {
if (opd == nullptr) {
return false;
}
opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
opd = gvn().transform(trace_vector(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem)));
opd = gvn().transform(new ExtractUBNode(opd, pos));
opd = gvn().transform(new ConvI2LNode(opd));
} else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
@ -2616,7 +2641,7 @@ bool LibraryCallKit::inline_vector_extract() {
}
// VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
if (!Matcher::mask_op_prefers_predicate(Op_VectorMaskToLong, opd->bottom_type()->is_vect())) {
opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
opd = gvn().transform(trace_vector(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem)));
}
// ((toLong() >>> pos) & 1L
opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
@ -2734,8 +2759,8 @@ static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec,
vmask_type = TypeVect::makemask(elem_bt, num_elem);
mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
Node* p1 = phase.transform(trace_vector(new VectorRearrangeNode(src1, wrapped_index_vec)));
Node* p2 = phase.transform(trace_vector(new VectorRearrangeNode(src2, wrapped_index_vec)));
return new VectorBlendNode(p2, p1, mask);
}
@ -2853,7 +2878,7 @@ bool LibraryCallKit::inline_vector_select_from_two_vectors() {
Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
operation = gvn().transform(trace_vector(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt)));
}
// Wrap it up in VectorBox to keep object type information.
@ -2938,7 +2963,7 @@ bool LibraryCallKit::inline_vector_compress_expand() {
}
const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
Node* operation = gvn().transform(trace_vector(VectorNode::make(opc, opd1, mask, vt)));
// Wrap it up in VectorBox to keep object type information.
const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
@ -3071,12 +3096,12 @@ bool LibraryCallKit::inline_index_vector() {
default: fatal("%s", type2name(elem_bt));
}
scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
index = gvn().transform(trace_vector(VectorNode::make(vmul_op, index, scale, vt)));
}
// Add "opd" if addition is needed.
if (needs_add) {
index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
index = gvn().transform(trace_vector(VectorNode::make(vadd_op, opd, index, vt)));
}
Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
set_result(vbox);
@ -3193,7 +3218,7 @@ bool LibraryCallKit::inline_index_partially_in_upper_range() {
// Compute the vector mask with "mask = iota < indexLimit".
ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
mask = gvn().transform(trace_vector(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type)));
}
Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
set_result(vbox);

344
src/hotspot/share/opto/vectornode.hpp
View File

File diff suppressed because it is too large Load Diff

9
src/hotspot/share/prims/jvm.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -115,9 +115,6 @@
#if INCLUDE_MANAGEMENT
#include "services/finalizerService.hpp"
#endif
#ifdef LINUX
#include "osContainer_linux.hpp"
#endif
#include <errno.h>
@ -500,11 +497,9 @@ JVM_LEAF(jboolean, JVM_IsUseContainerSupport(void))
JVM_END
JVM_LEAF(jboolean, JVM_IsContainerized(void))
#ifdef LINUX
if (OSContainer::is_containerized()) {
if (os::is_containerized()) {
return JNI_TRUE;
}
#endif
return JNI_FALSE;
JVM_END

9
src/hotspot/share/prims/whitebox.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -129,7 +129,6 @@
#ifdef LINUX
#include "cgroupSubsystem_linux.hpp"
#include "os_linux.hpp"
#include "osContainer_linux.hpp"
#endif
#define CHECK_JNI_EXCEPTION_(env, value) \
@ -2582,14 +2581,12 @@ WB_ENTRY(jboolean, WB_CheckLibSpecifiesNoexecstack(JNIEnv* env, jobject o, jstri
WB_END
WB_ENTRY(jboolean, WB_IsContainerized(JNIEnv* env, jobject o))
LINUX_ONLY(return OSContainer::is_containerized();)
return false;
return os::is_containerized();
WB_END
// Physical memory of the host machine (including containers)
WB_ENTRY(jlong, WB_HostPhysicalMemory(JNIEnv* env, jobject o))
LINUX_ONLY(return static_cast<jlong>(os::Linux::physical_memory());)
return static_cast<jlong>(os::physical_memory());
return static_cast<jlong>(os::Machine::physical_memory());
WB_END
// Available memory of the host machine (container-aware)

6
src/hotspot/share/runtime/handshake.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -521,8 +521,8 @@ HandshakeOperation* HandshakeState::get_op_for_self(bool allow_suspend, bool che
assert(_lock.owned_by_self(), "Lock must be held");
assert(allow_suspend || !check_async_exception, "invalid case");
#if INCLUDE_JVMTI
if (allow_suspend && _handshakee->is_disable_suspend()) {
// filter out suspend operations while JavaThread is in disable_suspend mode
if (allow_suspend && (_handshakee->is_disable_suspend() || _handshakee->is_vthread_transition_disabler())) {
// filter out suspend operations while JavaThread can not be suspended
allow_suspend = false;
}
#endif

12
src/hotspot/share/runtime/javaThread.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -499,7 +499,7 @@ JavaThread::JavaThread(MemTag mem_tag) :
_suspend_resume_manager(this, &_handshake._lock),
_is_in_vthread_transition(false),
DEBUG_ONLY(_is_vthread_transition_disabler(false) COMMA)
JVMTI_ONLY(_is_vthread_transition_disabler(false) COMMA)
DEBUG_ONLY(_is_disabler_at_start(false) COMMA)
_popframe_preserved_args(nullptr),
@ -1165,11 +1165,13 @@ void JavaThread::set_is_in_vthread_transition(bool val) {
AtomicAccess::store(&_is_in_vthread_transition, val);
}
#ifdef ASSERT
#if INCLUDE_JVMTI
void JavaThread::set_is_vthread_transition_disabler(bool val) {
_is_vthread_transition_disabler = val;
}
#endif
#ifdef ASSERT
void JavaThread::set_is_disabler_at_start(bool val) {
_is_disabler_at_start = val;
}
@ -1183,7 +1185,9 @@ void JavaThread::set_is_disabler_at_start(bool val) {
//
bool JavaThread::java_suspend(bool register_vthread_SR) {
// Suspending a vthread transition disabler can cause deadlocks.
assert(!is_vthread_transition_disabler(), "no suspend allowed for vthread transition disablers");
// The HandshakeState::has_operation does not allow such suspends.
// But the suspender thread is an exclusive transition disablers, so there can't be other disabers here.
JVMTI_ONLY(assert(!is_vthread_transition_disabler(), "suspender thread is an exclusive transition disabler");)
guarantee(Thread::is_JavaThread_protected(/* target */ this),
"target JavaThread is not protected in calling context.");

8
src/hotspot/share/runtime/javaThread.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -734,14 +734,14 @@ public:
private:
bool _is_in_vthread_transition; // thread is in virtual thread mount state transition
DEBUG_ONLY(bool _is_vthread_transition_disabler;) // thread currently disabled vthread transitions
JVMTI_ONLY(bool _is_vthread_transition_disabler;) // thread currently disabled vthread transitions
DEBUG_ONLY(bool _is_disabler_at_start;) // thread at process of disabling vthread transitions
public:
bool is_in_vthread_transition() const;
void set_is_in_vthread_transition(bool val);
JVMTI_ONLY(bool is_vthread_transition_disabler() const { return _is_vthread_transition_disabler; })
JVMTI_ONLY(void set_is_vthread_transition_disabler(bool val);)
#ifdef ASSERT
bool is_vthread_transition_disabler() const { return _is_vthread_transition_disabler; }
void set_is_vthread_transition_disabler(bool val);
bool is_disabler_at_start() const { return _is_disabler_at_start; }
void set_is_disabler_at_start(bool val);
#endif

15
src/hotspot/share/runtime/mountUnmountDisabler.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -129,7 +129,8 @@ bool MountUnmountDisabler::is_start_transition_disabled(JavaThread* thread, oop
int base_disable_count = notify_jvmti_events() ? 1 : 0;
return java_lang_Thread::vthread_transition_disable_count(vthread) > 0
|| global_vthread_transition_disable_count() > base_disable_count
JVMTI_ONLY(|| (JvmtiVTSuspender::is_vthread_suspended(java_lang_Thread::thread_id(vthread)) || thread->is_suspended()));
JVMTI_ONLY(|| (!thread->is_vthread_transition_disabler() &&
(JvmtiVTSuspender::is_vthread_suspended(java_lang_Thread::thread_id(vthread)) || thread->is_suspended())));
}
void MountUnmountDisabler::start_transition(JavaThread* current, oop vthread, bool is_mount, bool is_thread_end) {
@ -294,7 +295,7 @@ MountUnmountDisabler::disable_transition_for_one() {
// carrierThread to float up.
// This pairs with the release barrier in end_transition().
OrderAccess::acquire();
DEBUG_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(true);)
JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(true);)
}
// disable transitions for all virtual threads
@ -335,7 +336,7 @@ MountUnmountDisabler::disable_transition_for_all() {
// carrierThread to float up.
// This pairs with the release barrier in end_transition().
OrderAccess::acquire();
DEBUG_ONLY(thread->set_is_vthread_transition_disabler(true);)
JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(true);)
DEBUG_ONLY(thread->set_is_disabler_at_start(false);)
}
@ -358,14 +359,12 @@ MountUnmountDisabler::enable_transition_for_one() {
if (java_lang_Thread::vthread_transition_disable_count(_vthread()) == 0) {
ml.notify_all();
}
DEBUG_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(false);)
JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(false);)
}
// enable transitions for all virtual threads
void
MountUnmountDisabler::enable_transition_for_all() {
JavaThread* thread = JavaThread::current();
// End of the critical section. If some target was unmounted, we need a
// release barrier before decrementing _global_vthread_transition_disable_count
// to make sure any memory operations executed by the disabler are visible to
@ -384,7 +383,7 @@ MountUnmountDisabler::enable_transition_for_all() {
if (global_vthread_transition_disable_count() == base_disable_count || _is_exclusive) {
ml.notify_all();
}
DEBUG_ONLY(thread->set_is_vthread_transition_disabler(false);)
JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(false);)
}
int MountUnmountDisabler::global_vthread_transition_disable_count() {

55
src/hotspot/share/runtime/os.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -81,10 +81,6 @@
#include "utilities/permitForbiddenFunctions.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef LINUX
#include "osContainer_linux.hpp"
#endif
#ifndef _WINDOWS
# include <poll.h>
#endif
@ -2205,11 +2201,14 @@ static void assert_nonempty_range(const char* addr, size_t bytes) {
}
bool os::used_memory(physical_memory_size_type& value) {
#ifdef LINUX
if (OSContainer::is_containerized()) {
return OSContainer::memory_usage_in_bytes(value);
if (is_containerized()) {
return Container::used_memory(value);
}
#endif
return Machine::used_memory(value);
}
bool os::Machine::used_memory(physical_memory_size_type& value) {
physical_memory_size_type avail_mem = 0;
// Return value ignored - defaulting to 0 on failure.
(void)os::available_memory(avail_mem);
@ -2218,6 +2217,44 @@ bool os::used_memory(physical_memory_size_type& value) {
return true;
}
#ifndef LINUX
bool os::is_containerized() {
return false;
}
bool os::Container::processor_count(double& value) {
return false;
}
bool os::Container::available_memory(physical_memory_size_type& value) {
return false;
}
bool os::Container::used_memory(physical_memory_size_type& value) {
return false;
}
bool os::Container::total_swap_space(physical_memory_size_type& value) {
return false;
}
bool os::Container::free_swap_space(physical_memory_size_type& value) {
return false;
}
bool os::Container::memory_limit(physical_memory_size_type& value) {
return false;
}
bool os::Container::memory_soft_limit(physical_memory_size_type& value) {
return false;
}
bool os::Container::memory_throttle_limit(physical_memory_size_type& value) {
return false;
}
#endif
bool os::commit_memory(char* addr, size_t bytes, bool executable) {
assert_nonempty_range(addr, bytes);

48
src/hotspot/share/runtime/os.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -342,6 +342,52 @@ class os: AllStatic {
static bool is_server_class_machine();
static size_t rss();
// On platforms with container support (currently only Linux) we combine machine values with
// potential container values in os:: methods, abstracting which value is actually used.
// The os::Machine and os::Container classes and containing methods are used to get machine
// and container values (when available) separately.
static bool is_containerized();
// The os::Machine class reports system resource metrics from the perspective of the operating
// system, without considering container-imposed limits. The values returned by these methods
// reflect the resources visible to the process as reported by the OS, and may already be
// affected by mechanisms such as virtualization, hypervisor limits, or process affinity,
// but do NOT consider further restrictions imposed by container runtimes (e.g., cgroups)
class Machine : AllStatic {
public:
static int active_processor_count();
[[nodiscard]] static bool available_memory(physical_memory_size_type& value);
[[nodiscard]] static bool used_memory(physical_memory_size_type& value);
[[nodiscard]] static bool free_memory(physical_memory_size_type& value);
[[nodiscard]] static bool total_swap_space(physical_memory_size_type& value);
[[nodiscard]] static bool free_swap_space(physical_memory_size_type& value);
static physical_memory_size_type physical_memory();
};
// The os::Container class reports resource limits as imposed by a supported container runtime
// (currently only cgroup-based Linux runtimes). If the process is running inside a
// containerized environment, methods from this class report the effective limits imposed
// by the container, which may be more restrictive than what os::Machine reports.
// Methods return true and set the out-parameter if a limit is found,
// or false if no limit exists or it cannot be determined.
class Container : AllStatic {
public:
[[nodiscard]] static bool processor_count(double& value); // Returns the core-equivalent CPU quota
[[nodiscard]] static bool available_memory(physical_memory_size_type& value);
[[nodiscard]] static bool used_memory(physical_memory_size_type& value);
[[nodiscard]] static bool total_swap_space(physical_memory_size_type& value);
[[nodiscard]] static bool free_swap_space(physical_memory_size_type& value);
[[nodiscard]] static bool memory_limit(physical_memory_size_type& value);
[[nodiscard]] static bool memory_soft_limit(physical_memory_size_type& value);
[[nodiscard]] static bool memory_throttle_limit(physical_memory_size_type& value);
};
// Returns the id of the processor on which the calling thread is currently executing.
// The returned value is guaranteed to be between 0 and (os::processor_count() - 1).
static uint processor_id();

3
src/hotspot/share/runtime/suspendResumeManager.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -130,6 +130,7 @@ void SuspendResumeManager::do_owner_suspend() {
assert(_state_lock->owned_by_self(), "Lock must be held");
assert(!_target->has_last_Java_frame() || _target->frame_anchor()->walkable(), "should have walkable stack");
assert(_target->thread_state() == _thread_blocked, "Caller should have transitioned to _thread_blocked");
JVMTI_ONLY(assert(!_target->is_vthread_transition_disabler(), "attempt to suspend a vthread transition disabler");)
while (is_suspended()) {
log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " suspended", p2i(_target));

5
src/hotspot/share/runtime/thread.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -36,6 +36,7 @@
#include "memory/resourceArea.hpp"
#include "nmt/memTracker.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/javaThread.inline.hpp"
@ -82,7 +83,7 @@ Thread::Thread(MemTag mem_tag) {
_threads_hazard_ptr = nullptr;
_threads_list_ptr = nullptr;
_nested_threads_hazard_ptr_cnt = 0;
_rcu_counter = 0;
_rcu_counter.store_relaxed(0);
// the handle mark links itself to last_handle_mark
new HandleMark(this);

7
src/hotspot/share/runtime/thread.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -30,6 +30,7 @@
#include "gc/shared/threadLocalAllocBuffer.hpp"
#include "jni.h"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "runtime/os.hpp"
@ -238,9 +239,9 @@ class Thread: public ThreadShadow {
// Support for GlobalCounter
private:
volatile uintx _rcu_counter;
Atomic<uintx> _rcu_counter;
public:
volatile uintx* get_rcu_counter() {
Atomic<uintx>* get_rcu_counter() {
return &_rcu_counter;
}

6
src/hotspot/share/utilities/debug.cpp
View File

@ -429,10 +429,8 @@ extern "C" DEBUGEXPORT void pp(void* p) {
tty->print_cr("null");
return;
}
if (Universe::heap()->is_in(p)) {
oop obj = cast_to_oop(p);
obj->print();
} else {
if (!Universe::heap()->print_location(tty, p)) {
// Ask NMT about this pointer.
// GDB note: We will be using SafeFetch to access the supposed malloc header. If the address is
// not readable, this will generate a signal. That signal will trip up the debugger: gdb will

12
src/hotspot/share/utilities/globalCounter.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -23,7 +23,7 @@
*/
#include "memory/iterator.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/atomic.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/threadSMR.inline.hpp"
#include "runtime/vmThread.hpp"
@ -41,7 +41,7 @@ class GlobalCounter::CounterThreadCheck : public ThreadClosure {
SpinYield yield;
// Loops on this thread until it has exited the critical read section.
while(true) {
uintx cnt = AtomicAccess::load_acquire(thread->get_rcu_counter());
uintx cnt = thread->get_rcu_counter()->load_acquire();
// This checks if the thread's counter is active. And if so is the counter
// for a pre-existing reader (belongs to this grace period). A pre-existing
// reader will have a lower counter than the global counter version for this
@ -57,9 +57,9 @@ class GlobalCounter::CounterThreadCheck : public ThreadClosure {
};
void GlobalCounter::write_synchronize() {
assert((*Thread::current()->get_rcu_counter() & COUNTER_ACTIVE) == 0x0, "must be outside a critcal section");
// AtomicAccess::add must provide fence since we have storeload dependency.
uintx gbl_cnt = AtomicAccess::add(&_global_counter._counter, COUNTER_INCREMENT);
assert((Thread::current()->get_rcu_counter()->load_relaxed() & COUNTER_ACTIVE) == 0x0, "must be outside a critcal section");
// Atomic add must provide fence since we have storeload dependency.
uintx gbl_cnt = _global_counter._counter.add_then_fetch(COUNTER_INCREMENT);
// Do all RCU threads.
CounterThreadCheck ctc(gbl_cnt);

5
src/hotspot/share/utilities/globalCounter.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,6 +27,7 @@
#include "memory/allStatic.hpp"
#include "memory/padded.hpp"
#include "runtime/atomic.hpp"
class Thread;
@ -47,7 +48,7 @@ class GlobalCounter : public AllStatic {
// counter is on a separate cacheline.
struct PaddedCounter {
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_PADDING_SIZE, 0);
volatile uintx _counter;
Atomic<uintx> _counter;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(volatile uintx));
};

15
src/hotspot/share/utilities/globalCounter.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,30 +27,29 @@
#include "utilities/globalCounter.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/atomic.hpp"
#include "runtime/javaThread.hpp"
inline GlobalCounter::CSContext
GlobalCounter::critical_section_begin(Thread *thread) {
assert(thread == Thread::current(), "must be current thread");
uintx old_cnt = AtomicAccess::load(thread->get_rcu_counter());
uintx old_cnt = thread->get_rcu_counter()->load_relaxed();
// Retain the old counter value if already active, e.g. nested.
// Otherwise, set the counter to the current version + active bit.
uintx new_cnt = old_cnt;
if ((new_cnt & COUNTER_ACTIVE) == 0) {
new_cnt = AtomicAccess::load(&_global_counter._counter) | COUNTER_ACTIVE;
new_cnt = _global_counter._counter.load_relaxed() | COUNTER_ACTIVE;
}
AtomicAccess::release_store_fence(thread->get_rcu_counter(), new_cnt);
thread->get_rcu_counter()->release_store_fence(new_cnt);
return static_cast<CSContext>(old_cnt);
}
inline void
GlobalCounter::critical_section_end(Thread *thread, CSContext context) {
assert(thread == Thread::current(), "must be current thread");
assert((*thread->get_rcu_counter() & COUNTER_ACTIVE) == COUNTER_ACTIVE, "must be in critical section");
assert((thread->get_rcu_counter()->load_relaxed() & COUNTER_ACTIVE) == COUNTER_ACTIVE, "must be in critical section");
// Restore the counter value from before the associated begin.
AtomicAccess::release_store(thread->get_rcu_counter(),
static_cast<uintx>(context));
thread->get_rcu_counter()->release_store(static_cast<uintx>(context));
}
class GlobalCounter::CriticalSection {

Some files were not shown because too many files have changed in this diff Show More