infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-05-09 21:19:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge master

Browse Source
This commit is contained in:
Prasanta Sadhukhan 2026-01-21 07:20:35 +00:00
commit 9a2b4d9dbd
608 changed files with 25169 additions and 13401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
make/hotspot/lib/CompileGtest.gmk
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@ -61,7 +61,8 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
INCLUDE_FILES := gtest-all.cc gmock-all.cc, \
DISABLED_WARNINGS_gcc := format-nonliteral maybe-uninitialized undef \
unused-result zero-as-null-pointer-constant, \
DISABLED_WARNINGS_clang := format-nonliteral undef unused-result, \
DISABLED_WARNINGS_clang := format-nonliteral undef unused-result \
zero-as-null-pointer-constant, \
DISABLED_WARNINGS_microsoft := 4530, \
DEFAULT_CFLAGS := false, \
CFLAGS := $(JVM_CFLAGS) \

29
src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -722,22 +722,20 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ ldrh(rscratch1, Address(rmethod, Method::access_flags_offset()));
__ andsw(zr, rscratch1, JVM_ACC_STATIC);
__ br(Assembler::EQ, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ ldrh(rscratch1, Address(rmethod, Method::access_flags_offset()));
__ andsw(zr, rscratch1, JVM_ACC_STATIC);
__ br(Assembler::EQ, L_skip_barrier); // non-static
__ load_method_holder(rscratch2, rmethod);
__ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ load_method_holder(rscratch2, rmethod);
__ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
@ -1508,7 +1506,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// SVC, HVC, or SMC. Make it a NOP.
__ nop();
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
__ mov_metadata(rscratch2, method->method_holder()); // InstanceKlass*
__ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);

9
src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -2290,7 +2290,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ subs(zr, temp, (int) code); // have we resolved this bytecode?
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
__ br(Assembler::NE, L_clinit_barrier_slow);
__ ldr(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
__ load_method_holder(temp, temp);
@ -2340,8 +2341,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ subs(zr, temp, (int) code); // have we resolved this bytecode?
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = temp;
__ br(Assembler::NE, L_clinit_barrier_slow);

6
src/hotspot/cpu/arm/frame_arm.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -356,10 +356,10 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const {
bool frame::is_interpreted_frame_valid(JavaThread* thread) const {
assert(is_interpreted_frame(), "Not an interpreted frame");
// These are reasonable sanity checks
if (fp() == 0 || (intptr_t(fp()) & (wordSize-1)) != 0) {
if (fp() == nullptr || (intptr_t(fp()) & (wordSize-1)) != 0) {
return false;
}
if (sp() == 0 || (intptr_t(sp()) & (wordSize-1)) != 0) {
if (sp() == nullptr || (intptr_t(sp()) & (wordSize-1)) != 0) {
return false;
}
if (fp() + interpreter_frame_initial_sp_offset < sp()) {

6
src/hotspot/cpu/arm/nativeInst_arm_32.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -172,7 +172,7 @@ void NativeMovConstReg::set_data(intptr_t x, address pc) {
address addr = oop_addr != nullptr ? (address)oop_addr : (address)metadata_addr;
if(pc == 0) {
if (pc == nullptr) {
offset = addr - instruction_address() - 8;
} else {
offset = addr - pc - 8;
@ -228,7 +228,7 @@ void NativeMovConstReg::set_data(intptr_t x, address pc) {
void NativeMovConstReg::set_pc_relative_offset(address addr, address pc) {
int offset;
if (pc == 0) {
if (pc == nullptr) {
offset = addr - instruction_address() - 8;
} else {
offset = addr - pc - 8;

4
src/hotspot/cpu/arm/nativeInst_arm_32.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -371,7 +371,7 @@ class NativeMovConstReg: public NativeInstruction {
public:
intptr_t data() const;
void set_data(intptr_t x, address pc = 0);
void set_data(intptr_t x, address pc = nullptr);
bool is_pc_relative() {
return !is_movw();
}

4
src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
View File

@ -1109,11 +1109,11 @@ void InterpreterMacroAssembler::verify_method_data_pointer() {
lhz(R11_scratch1, in_bytes(DataLayout::bci_offset()), R28_mdx);
ld(R12_scratch2, in_bytes(Method::const_offset()), R19_method);
addi(R11_scratch1, R11_scratch1, in_bytes(ConstMethod::codes_offset()));
add(R11_scratch1, R12_scratch2, R12_scratch2);
add(R11_scratch1, R11_scratch1, R12_scratch2);
cmpd(CR0, R11_scratch1, R14_bcp);
beq(CR0, verify_continue);
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp ), R19_method, R14_bcp, R28_mdx);
call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R19_method, R14_bcp, R28_mdx);
bind(verify_continue);
#endif

4
src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
View File

@ -4535,7 +4535,7 @@ void MacroAssembler::push_cont_fastpath() {
Label done;
ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
cmpld(CR0, R1_SP, R0);
ble(CR0, done);
ble(CR0, done); // if (SP <= _cont_fastpath) goto done;
st_ptr(R1_SP, JavaThread::cont_fastpath_offset(), R16_thread);
bind(done);
}
@ -4546,7 +4546,7 @@ void MacroAssembler::pop_cont_fastpath() {
Label done;
ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
cmpld(CR0, R1_SP, R0);
ble(CR0, done);
blt(CR0, done); // if (SP < _cont_fastpath) goto done;
li(R0, 0);
st_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
bind(done);

35
src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -1237,26 +1237,24 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ lhz(R0, in_bytes(Method::access_flags_offset()), R19_method);
__ andi_(R0, R0, JVM_ACC_STATIC);
__ beq(CR0, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ lhz(R0, in_bytes(Method::access_flags_offset()), R19_method);
__ andi_(R0, R0, JVM_ACC_STATIC);
__ beq(CR0, L_skip_barrier); // non-static
Register klass = R11_scratch1;
__ load_method_holder(klass, R19_method);
__ clinit_barrier(klass, R16_thread, &L_skip_barrier /*L_fast_path*/);
Register klass = R11_scratch1;
__ load_method_holder(klass, R19_method);
__ clinit_barrier(klass, R16_thread, &L_skip_barrier /*L_fast_path*/);
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub(), R0);
__ mtctr(klass);
__ bctr();
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub(), R0);
__ mtctr(klass);
__ bctr();
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm, /* tmp register*/ ic_klass, /* tmp register*/ receiver_klass, /* tmp register*/ code);
@ -2210,7 +2208,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// --------------------------------------------------------------------------
vep_start_pc = (intptr_t)__ pc();
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register klass = r_temp_1;
// Notify OOP recorder (don't need the relocation)

9
src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -2199,7 +2199,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca
__ isync(); // Order load wrt. succeeding loads.
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register method = Rscratch;
const Register klass = Rscratch;
@ -2244,8 +2245,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, Register Rcac
__ isync(); // Order load wrt. succeeding loads.
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = R4_ARG2;
// InterpreterRuntime::resolve_get_put sets field_holder and finally release-stores put_code.

31
src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -213,7 +213,7 @@ void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
// Is vector's size (in bytes) bigger than a size saved by default?
// riscv does not ovlerlay the floating-point registers on vector registers like aarch64.
bool SharedRuntime::is_wide_vector(int size) {
return UseRVV;
return UseRVV && size > 0;
}
// ---------------------------------------------------------------------------
@ -637,22 +637,20 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ load_unsigned_short(t0, Address(xmethod, Method::access_flags_offset()));
__ test_bit(t1, t0, exact_log2(JVM_ACC_STATIC));
__ beqz(t1, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ load_unsigned_short(t0, Address(xmethod, Method::access_flags_offset()));
__ test_bit(t1, t0, exact_log2(JVM_ACC_STATIC));
__ beqz(t1, L_skip_barrier); // non-static
__ load_method_holder(t1, xmethod);
__ clinit_barrier(t1, t0, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ load_method_holder(t1, xmethod);
__ clinit_barrier(t1, t0, &L_skip_barrier);
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
@ -1443,7 +1441,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ nop(); // 4 bytes
}
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
__ mov_metadata(t1, method->method_holder()); // InstanceKlass*
__ clinit_barrier(t1, t0, &L_skip_barrier);

9
src/hotspot/cpu/riscv/templateTable_riscv.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -2192,7 +2192,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ mv(t0, (int) code);
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
__ bne(temp, t0, L_clinit_barrier_slow); // have we resolved this bytecode?
__ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
__ load_method_holder(temp, temp);
@ -2243,8 +2244,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ mv(t0, (int) code); // have we resolved this bytecode?
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = temp;
__ bne(temp, t0, L_clinit_barrier_slow);

10
src/hotspot/cpu/riscv/vm_version_riscv.cpp
View File

@ -167,11 +167,6 @@ void VM_Version::common_initialize() {
(unaligned_scalar.value() == MISALIGNED_SCALAR_FAST));
}
if (FLAG_IS_DEFAULT(AlignVector)) {
FLAG_SET_DEFAULT(AlignVector,
unaligned_vector.value() != MISALIGNED_VECTOR_FAST);
}
#ifdef __riscv_ztso
// Hotspot is compiled with TSO support, it will only run on hardware which
// supports Ztso
@ -242,6 +237,11 @@ void VM_Version::c2_initialize() {
}
}
if (FLAG_IS_DEFAULT(AlignVector)) {
FLAG_SET_DEFAULT(AlignVector,
unaligned_vector.value() != MISALIGNED_VECTOR_FAST);
}
// NOTE: Make sure codes dependent on UseRVV are put after MaxVectorSize initialize,
// as there are extra checks inside it which could disable UseRVV
// in some situations.

31
src/hotspot/cpu/s390/sharedRuntime_s390.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -1567,7 +1567,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
//---------------------------------------------------------------------
wrapper_VEPStart = __ offset();
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register klass = Z_R1_scratch;
// Notify OOP recorder (don't need the relocation)
@ -2378,24 +2379,22 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
{ // Bypass the barrier for non-static methods
__ testbit_ushort(Address(Z_method, Method::access_flags_offset()), JVM_ACC_STATIC_BIT);
__ z_bfalse(L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
__ testbit_ushort(Address(Z_method, Method::access_flags_offset()), JVM_ACC_STATIC_BIT);
__ z_bfalse(L_skip_barrier); // non-static
Register klass = Z_R11;
__ load_method_holder(klass, Z_method);
__ clinit_barrier(klass, Z_thread, &L_skip_barrier /*L_fast_path*/);
Register klass = Z_R11;
__ load_method_holder(klass, Z_method);
__ clinit_barrier(klass, Z_thread, &L_skip_barrier /*L_fast_path*/);
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub());
__ z_br(klass);
__ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub());
__ z_br(klass);
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
return;

9
src/hotspot/cpu/s390/templateTable_s390.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -2377,7 +2377,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ z_cli(Address(Rcache, bc_offset), code);
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register method = Z_R1_scratch;
const Register klass = Z_R1_scratch;
__ z_brne(L_clinit_barrier_slow);
@ -2427,8 +2428,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ z_cli(Address(cache, code_offset), code);
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = index;
__ z_brne(L_clinit_barrier_slow);

8
src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2024, 2025, Intel Corporation. All rights reserved.
* Copyright (c) 2024, 2026, Intel Corporation. All rights reserved.
*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -1330,10 +1330,12 @@ static void big_case_loop_helper(bool sizeKnown, int size, Label &noMatch, Label
// Clarification: The BYTE_K compare above compares haystack[(n-32):(n-1)]. We need to
// compare haystack[(k-1):(k-1+31)]. Subtracting either index gives shift value of
// (k + 31 - n): x = (k-1+31)-(n-1) = k-1+31-n+1 = k+31-n.
// When isU is set, similarly, shift is from haystack[(n-32):(n-1)] to [(k-2):(k-2+31)]
if (sizeKnown) {
__ movl(temp2, 31 + size);
__ movl(temp2, (isU ? 30 : 31) + size);
} else {
__ movl(temp2, 31);
__ movl(temp2, isU ? 30 : 31);
__ addl(temp2, needleLen);
}
__ subl(temp2, hsLength);

4
src/hotspot/cpu/x86/macroAssembler_x86.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -6086,7 +6086,7 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
subptr(count, 16 << shift);
jccb(Assembler::less, L_check_fill_32_bytes);
jcc(Assembler::less, L_check_fill_32_bytes);
align(16);
BIND(L_fill_64_bytes_loop_avx3);

8
src/hotspot/cpu/x86/register_x86.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,14 +32,10 @@ const KRegister::KRegisterImpl all_KRegisterImpls [KRegister::number_
const char * Register::RegisterImpl::name() const {
static const char *const names[number_of_registers] = {
#ifdef _LP64
"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
"r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
"r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
"r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"
#else
"eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"
#endif // _LP64
};
return is_valid() ? names[encoding()] : "noreg";
}
@ -54,11 +50,9 @@ const char* FloatRegister::FloatRegisterImpl::name() const {
const char* XMMRegister::XMMRegisterImpl::name() const {
static const char *const names[number_of_registers] = {
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
#ifdef _LP64
,"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"
,"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23"
,"xmm24", "xmm25", "xmm26", "xmm27", "xmm28", "xmm29", "xmm30", "xmm31"
#endif // _LP64
};
return is_valid() ? names[encoding()] : "xnoreg";
}

56
src/hotspot/cpu/x86/register_x86.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -34,7 +34,7 @@
class VMRegImpl;
typedef VMRegImpl* VMReg;
// The implementation of integer registers for the x86/x64 architectures.
// The implementation of integer registers for the x64 architectures.
class Register {
private:
int _encoding;
@ -44,11 +44,9 @@ private:
public:
inline friend constexpr Register as_Register(int encoding);
enum {
number_of_registers = LP64_ONLY( 32 ) NOT_LP64( 8 ),
number_of_byte_registers = LP64_ONLY( 32 ) NOT_LP64( 4 ),
max_slots_per_register = LP64_ONLY( 2 ) NOT_LP64( 1 )
};
static const int number_of_registers = 32;
static const int number_of_byte_registers = 32;
static const int max_slots_per_register = 2;
class RegisterImpl: public AbstractRegisterImpl {
friend class Register;
@ -79,11 +77,9 @@ public:
// Actually available GP registers for use, depending on actual CPU capabilities and flags.
static int available_gp_registers() {
#ifdef _LP64
if (!UseAPX) {
return number_of_registers / 2;
}
#endif // _LP64
return number_of_registers;
}
};
@ -116,9 +112,8 @@ constexpr Register rsp = as_Register(4);
constexpr Register rbp = as_Register(5);
constexpr Register rsi = as_Register(6);
constexpr Register rdi = as_Register(7);
#ifdef _LP64
constexpr Register r8 = as_Register( 8);
constexpr Register r9 = as_Register( 9);
constexpr Register r8 = as_Register(8);
constexpr Register r9 = as_Register(9);
constexpr Register r10 = as_Register(10);
constexpr Register r11 = as_Register(11);
constexpr Register r12 = as_Register(12);
@ -141,7 +136,6 @@ constexpr Register r28 = as_Register(28);
constexpr Register r29 = as_Register(29);
constexpr Register r30 = as_Register(30);
constexpr Register r31 = as_Register(31);
#endif // _LP64
// The implementation of x87 floating point registers for the ia32 architecture.
@ -154,10 +148,8 @@ private:
public:
inline friend constexpr FloatRegister as_FloatRegister(int encoding);
enum {
number_of_registers = 8,
max_slots_per_register = 2
};
static const int number_of_registers = 8;
static const int max_slots_per_register = 2;
class FloatRegisterImpl: public AbstractRegisterImpl {
friend class FloatRegister;
@ -217,10 +209,8 @@ private:
public:
inline friend constexpr XMMRegister as_XMMRegister(int encoding);
enum {
number_of_registers = LP64_ONLY( 32 ) NOT_LP64( 8 ),
max_slots_per_register = LP64_ONLY( 16 ) NOT_LP64( 16 ) // 512-bit
};
static const int number_of_registers = 32;
static const int max_slots_per_register = 16; // 512-bit
class XMMRegisterImpl: public AbstractRegisterImpl {
friend class XMMRegister;
@ -250,11 +240,9 @@ public:
// Actually available XMM registers for use, depending on actual CPU capabilities and flags.
static int available_xmm_registers() {
#ifdef _LP64
if (UseAVX < 3) {
return number_of_registers / 2;
}
#endif // _LP64
return number_of_registers;
}
};
@ -287,7 +275,6 @@ constexpr XMMRegister xmm4 = as_XMMRegister( 4);
constexpr XMMRegister xmm5 = as_XMMRegister( 5);
constexpr XMMRegister xmm6 = as_XMMRegister( 6);
constexpr XMMRegister xmm7 = as_XMMRegister( 7);
#ifdef _LP64
constexpr XMMRegister xmm8 = as_XMMRegister( 8);
constexpr XMMRegister xmm9 = as_XMMRegister( 9);
constexpr XMMRegister xmm10 = as_XMMRegister(10);
@ -312,7 +299,6 @@ constexpr XMMRegister xmm28 = as_XMMRegister(28);
constexpr XMMRegister xmm29 = as_XMMRegister(29);
constexpr XMMRegister xmm30 = as_XMMRegister(30);
constexpr XMMRegister xmm31 = as_XMMRegister(31);
#endif // _LP64
// The implementation of AVX-512 opmask registers.
@ -394,25 +380,17 @@ constexpr KRegister k7 = as_KRegister(7);
// Define a class that exports it.
class ConcreteRegisterImpl : public AbstractRegisterImpl {
public:
enum {
max_gpr = Register::number_of_registers * Register::max_slots_per_register,
max_fpr = max_gpr + FloatRegister::number_of_registers * FloatRegister::max_slots_per_register,
max_xmm = max_fpr + XMMRegister::number_of_registers * XMMRegister::max_slots_per_register,
max_kpr = max_xmm + KRegister::number_of_registers * KRegister::max_slots_per_register,
static const int max_gpr = Register::number_of_registers * Register::max_slots_per_register;
static const int max_fpr = max_gpr + FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
static const int max_xmm = max_fpr + XMMRegister::number_of_registers * XMMRegister::max_slots_per_register;
static const int max_kpr = max_xmm + KRegister::number_of_registers * KRegister::max_slots_per_register;
// A big enough number for C2: all the registers plus flags
// This number must be large enough to cover REG_COUNT (defined by c2) registers.
// There is no requirement that any ordering here matches any ordering c2 gives
// it's optoregs.
// x86_32.ad defines additional dummy FILL0-FILL7 registers, in order to tally
// REG_COUNT (computed by ADLC based on the number of reg_defs seen in .ad files)
// with ConcreteRegisterImpl::number_of_registers additional count of 8 is being
// added for 32 bit jvm.
number_of_registers = max_kpr + // gpr/fpr/xmm/kpr
NOT_LP64( 8 + ) // FILL0-FILL7 in x86_32.ad
1 // eflags
};
static const int number_of_registers = max_kpr + // gpr/fpr/xmm/kpr
1; // eflags
};
template <>

36
src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1043,26 +1043,24 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Class initialization barrier for static methods
entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
Register method = rbx;
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register method = rbx;
{ // Bypass the barrier for non-static methods
Register flags = rscratch1;
__ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
__ testl(flags, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L_skip_barrier); // non-static
}
// Bypass the barrier for non-static methods
Register flags = rscratch1;
__ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
__ testl(flags, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L_skip_barrier); // non-static
Register klass = rscratch1;
__ load_method_holder(klass, method);
__ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
Register klass = rscratch1;
__ load_method_holder(klass, method);
__ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
__ bind(L_skip_barrier);
entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
@ -1904,7 +1902,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
int vep_offset = ((intptr_t)__ pc()) - start;
if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
if (method->needs_clinit_barrier()) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
Label L_skip_barrier;
Register klass = r10;
__ mov_metadata(klass, method->method_holder()); // InstanceKlass*
@ -3602,4 +3601,3 @@ RuntimeStub* SharedRuntime::generate_jfr_return_lease() {
}
#endif // INCLUDE_JFR

92
src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -64,6 +64,39 @@ static address kyberAvx512ConstsAddr(int offset) {
const Register scratch = r10;
ATTRIBUTE_ALIGNED(64) static const uint8_t kyberAvx512_12To16Dup[] = {
// 0 - 63
0, 1, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15, 16,
16, 17, 18, 19, 19, 20, 21, 22, 22, 23, 24, 25, 25, 26, 27, 28, 28, 29, 30,
31, 31, 32, 33, 34, 34, 35, 36, 37, 37, 38, 39, 40, 40, 41, 42, 43, 43, 44,
45, 46, 46, 47
};
static address kyberAvx512_12To16DupAddr() {
return (address) kyberAvx512_12To16Dup;
}
ATTRIBUTE_ALIGNED(64) static const uint16_t kyberAvx512_12To16Shift[] = {
// 0 - 31
0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0,
4, 0, 4, 0, 4, 0, 4
};
static address kyberAvx512_12To16ShiftAddr() {
return (address) kyberAvx512_12To16Shift;
}
ATTRIBUTE_ALIGNED(64) static const uint64_t kyberAvx512_12To16And[] = {
// 0 - 7
0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF,
0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF,
0x0FFF0FFF0FFF0FFF, 0x0FFF0FFF0FFF0FFF
};
static address kyberAvx512_12To16AndAddr() {
return (address) kyberAvx512_12To16And;
}
ATTRIBUTE_ALIGNED(64) static const uint16_t kyberAvx512NttPerms[] = {
// 0
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
@ -822,10 +855,65 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
const Register perms = r11;
Label Loop;
Label Loop, VBMILoop;
__ addptr(condensed, condensedOffs);
if (VM_Version::supports_avx512_vbmi()) {
// mask load for the first 48 bytes of each vector
__ mov64(rax, 0x0000FFFFFFFFFFFF);
__ kmovql(k1, rax);
__ lea(perms, ExternalAddress(kyberAvx512_12To16DupAddr()));
__ evmovdqub(xmm20, Address(perms), Assembler::AVX_512bit);
__ lea(perms, ExternalAddress(kyberAvx512_12To16ShiftAddr()));
__ evmovdquw(xmm21, Address(perms), Assembler::AVX_512bit);
__ lea(perms, ExternalAddress(kyberAvx512_12To16AndAddr()));
__ evmovdquq(xmm22, Address(perms), Assembler::AVX_512bit);
__ align(OptoLoopAlignment);
__ BIND(VBMILoop);
__ evmovdqub(xmm0, k1, Address(condensed, 0), false,
Assembler::AVX_512bit);
__ evmovdqub(xmm1, k1, Address(condensed, 48), false,
Assembler::AVX_512bit);
__ evmovdqub(xmm2, k1, Address(condensed, 96), false,
Assembler::AVX_512bit);
__ evmovdqub(xmm3, k1, Address(condensed, 144), false,
Assembler::AVX_512bit);
__ evpermb(xmm4, k0, xmm20, xmm0, false, Assembler::AVX_512bit);
__ evpermb(xmm5, k0, xmm20, xmm1, false, Assembler::AVX_512bit);
__ evpermb(xmm6, k0, xmm20, xmm2, false, Assembler::AVX_512bit);
__ evpermb(xmm7, k0, xmm20, xmm3, false, Assembler::AVX_512bit);
__ evpsrlvw(xmm4, xmm4, xmm21, Assembler::AVX_512bit);
__ evpsrlvw(xmm5, xmm5, xmm21, Assembler::AVX_512bit);
__ evpsrlvw(xmm6, xmm6, xmm21, Assembler::AVX_512bit);
__ evpsrlvw(xmm7, xmm7, xmm21, Assembler::AVX_512bit);
__ evpandq(xmm0, xmm22, xmm4, Assembler::AVX_512bit);
__ evpandq(xmm1, xmm22, xmm5, Assembler::AVX_512bit);
__ evpandq(xmm2, xmm22, xmm6, Assembler::AVX_512bit);
__ evpandq(xmm3, xmm22, xmm7, Assembler::AVX_512bit);
store4regs(parsed, 0, xmm0_3, _masm);
__ addptr(condensed, 192);
__ addptr(parsed, 256);
__ subl(parsedLength, 128);
__ jcc(Assembler::greater, VBMILoop);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ mov64(rax, 0); // return 0
__ ret(0);
return start;
}
__ lea(perms, ExternalAddress(kyberAvx512_12To16PermsAddr()));
load4regs(xmm24_27, perms, 0, _masm);

9
src/hotspot/cpu/x86/templateTable_x86.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -2216,7 +2216,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ cmpl(temp, code); // have we resolved this bytecode?
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
if (bytecode() == Bytecodes::_invokestatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register method = temp;
const Register klass = temp;
@ -2264,8 +2265,8 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ cmpl(temp, code); // have we resolved this bytecode?
// Class initialization barrier for static fields
if (VM_Version::supports_fast_class_init_checks() &&
(bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
if (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic) {
assert(VM_Version::supports_fast_class_init_checks(), "sanity");
const Register field_holder = temp;
__ jcc(Assembler::notEqual, L_clinit_barrier_slow);

4
src/hotspot/cpu/x86/vmreg_x86.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2006, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2006, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,9 +32,7 @@ void VMRegImpl::set_regName() {
int i;
for (i = 0; i < ConcreteRegisterImpl::max_gpr ; ) {
regName[i++] = reg->name();
#ifdef AMD64
regName[i++] = reg->name();
#endif // AMD64
reg = reg->successor();
}

13
src/hotspot/cpu/x86/vmreg_x86.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2006, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2006, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -52,14 +52,8 @@ inline bool is_KRegister() {
}
inline Register as_Register() {
assert( is_Register(), "must be");
// Yuk
#ifdef AMD64
assert(is_Register(), "must be");
return ::as_Register(value() >> 1);
#else
return ::as_Register(value());
#endif // AMD64
}
inline FloatRegister as_FloatRegister() {
@ -82,9 +76,6 @@ inline KRegister as_KRegister() {
inline bool is_concrete() {
assert(is_reg(), "must be");
#ifndef AMD64
if (is_Register()) return true;
#endif // AMD64
// Do not use is_XMMRegister() here as it depends on the UseAVX setting.
if (value() >= ConcreteRegisterImpl::max_fpr && value() < ConcreteRegisterImpl::max_xmm) {
int base = value() - ConcreteRegisterImpl::max_fpr;

4
src/hotspot/cpu/x86/vmreg_x86.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2006, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -26,7 +26,7 @@
#define CPU_X86_VMREG_X86_INLINE_HPP
inline VMReg Register::RegisterImpl::as_VMReg() const {
return VMRegImpl::as_VMReg(encoding() LP64_ONLY( << 1 ));
return VMRegImpl::as_VMReg(encoding() << 1);
}
inline VMReg FloatRegister::FloatRegisterImpl::as_VMReg() const {

26
src/hotspot/os/aix/os_aix.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -258,10 +258,18 @@ bool os::free_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::available_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
bool os::Aix::available_memory(physical_memory_size_type& value) {
os::Aix::meminfo_t mi;
if (os::Aix::get_meminfo(&mi)) {
@ -273,6 +281,10 @@ bool os::Aix::available_memory(physical_memory_size_type& value) {
}
bool os::total_swap_space(physical_memory_size_type& value) {
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
perfstat_memory_total_t memory_info;
if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
return false;
@ -282,6 +294,10 @@ bool os::total_swap_space(physical_memory_size_type& value) {
}
bool os::free_swap_space(physical_memory_size_type& value) {
return Machine::free_swap_space(value);
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
perfstat_memory_total_t memory_info;
if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
return false;
@ -294,6 +310,10 @@ physical_memory_size_type os::physical_memory() {
return Aix::physical_memory();
}
physical_memory_size_type os::Machine::physical_memory() {
return Aix::physical_memory();
}
size_t os::rss() { return (size_t)0; }
// Cpu architecture string
@ -2264,6 +2284,10 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
int online_cpus = ::sysconf(_SC_NPROCESSORS_ONLN);
assert(online_cpus > 0 && online_cpus <= processor_count(), "sanity check");
return online_cpus;

2
src/hotspot/os/bsd/memMapPrinter_macosx.cpp
View File

@ -132,7 +132,7 @@ public:
static const char* tagToStr(uint32_t user_tag) {
switch (user_tag) {
case 0:
return 0;
return nullptr;
X1(MALLOC, malloc);
X1(MALLOC_SMALL, malloc_small);
X1(MALLOC_LARGE, malloc_large);

30
src/hotspot/os/bsd/os_bsd.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -137,10 +137,18 @@ bool os::available_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
bool os::free_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
// Available here means free. Note that this number is of no much use. As an estimate
// for future memory pressure it is far too conservative, since MacOS will use a lot
// of unused memory for caches, and return it willingly in case of needs.
@ -181,6 +189,10 @@ void os::Bsd::print_uptime_info(outputStream* st) {
}
bool os::total_swap_space(physical_memory_size_type& value) {
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
#if defined(__APPLE__)
struct xsw_usage vmusage;
size_t size = sizeof(vmusage);
@ -195,6 +207,10 @@ bool os::total_swap_space(physical_memory_size_type& value) {
}
bool os::free_swap_space(physical_memory_size_type& value) {
return Machine::free_swap_space(value);
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
#if defined(__APPLE__)
struct xsw_usage vmusage;
size_t size = sizeof(vmusage);
@ -212,6 +228,10 @@ physical_memory_size_type os::physical_memory() {
return Bsd::physical_memory();
}
physical_memory_size_type os::Machine::physical_memory() {
return Bsd::physical_memory();
}
size_t os::rss() {
size_t rss = 0;
#ifdef __APPLE__
@ -608,7 +628,7 @@ static void *thread_native_entry(Thread *thread) {
log_info(os, thread)("Thread finished (tid: %zu, pthread id: %zu).",
os::current_thread_id(), (uintx) pthread_self());
return 0;
return nullptr;
}
bool os::create_thread(Thread* thread, ThreadType thr_type,
@ -1400,7 +1420,7 @@ int os::get_loaded_modules_info(os::LoadedModulesCallbackFunc callback, void *pa
#elif defined(__APPLE__)
for (uint32_t i = 1; i < _dyld_image_count(); i++) {
// Value for top_address is returned as 0 since we don't have any information about module size
if (callback(_dyld_get_image_name(i), (address)_dyld_get_image_header(i), (address)0, param)) {
if (callback(_dyld_get_image_name(i), (address)_dyld_get_image_header(i), nullptr, param)) {
return 1;
}
}
@ -2189,6 +2209,10 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
return _processor_count;
}

22
src/hotspot/os/linux/cgroupSubsystem_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -631,22 +631,20 @@ void CgroupSubsystemFactory::cleanup(CgroupInfo* cg_infos) {
* return:
* true if there were no errors. false otherwise.
*/
bool CgroupSubsystem::active_processor_count(int& value) {
int cpu_count;
int result = -1;
bool CgroupSubsystem::active_processor_count(double& value) {
// We use a cache with a timeout to avoid performing expensive
// computations in the event this function is called frequently.
// [See 8227006].
CachingCgroupController<CgroupCpuController>* contrl = cpu_controller();
CachedMetric* cpu_limit = contrl->metrics_cache();
CachingCgroupController<CgroupCpuController, double>* contrl = cpu_controller();
CachedMetric<double>* cpu_limit = contrl->metrics_cache();
if (!cpu_limit->should_check_metric()) {
value = (int)cpu_limit->value();
log_trace(os, container)("CgroupSubsystem::active_processor_count (cached): %d", value);
value = cpu_limit->value();
log_trace(os, container)("CgroupSubsystem::active_processor_count (cached): %.2f", value);
return true;
}
cpu_count = os::Linux::active_processor_count();
int cpu_count = os::Linux::active_processor_count();
double result = -1;
if (!CgroupUtil::processor_count(contrl->controller(), cpu_count, result)) {
return false;
}
@ -671,8 +669,8 @@ bool CgroupSubsystem::active_processor_count(int& value) {
*/
bool CgroupSubsystem::memory_limit_in_bytes(physical_memory_size_type upper_bound,
physical_memory_size_type& value) {
CachingCgroupController<CgroupMemoryController>* contrl = memory_controller();
CachedMetric* memory_limit = contrl->metrics_cache();
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* contrl = memory_controller();
CachedMetric<physical_memory_size_type>* memory_limit = contrl->metrics_cache();
if (!memory_limit->should_check_metric()) {
value = memory_limit->value();
return true;

25
src/hotspot/os/linux/cgroupSubsystem_linux.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -181,20 +181,21 @@ class CgroupController: public CHeapObj<mtInternal> {
static bool limit_from_str(char* limit_str, physical_memory_size_type& value);
};
template <typename MetricType>
class CachedMetric : public CHeapObj<mtInternal>{
private:
volatile physical_memory_size_type _metric;
volatile MetricType _metric;
volatile jlong _next_check_counter;
public:
CachedMetric() {
_metric = value_unlimited;
_metric = static_cast<MetricType>(value_unlimited);
_next_check_counter = min_jlong;
}
bool should_check_metric() {
return os::elapsed_counter() > _next_check_counter;
}
physical_memory_size_type value() { return _metric; }
void set_value(physical_memory_size_type value, jlong timeout) {
MetricType value() { return _metric; }
void set_value(MetricType value, jlong timeout) {
_metric = value;
// Metric is unlikely to change, but we want to remain
// responsive to configuration changes. A very short grace time
@ -205,19 +206,19 @@ class CachedMetric : public CHeapObj<mtInternal>{
}
};
template <class T>
template <class T, typename MetricType>
class CachingCgroupController : public CHeapObj<mtInternal> {
private:
T* _controller;
CachedMetric* _metrics_cache;
CachedMetric<MetricType>* _metrics_cache;
public:
CachingCgroupController(T* cont) {
_controller = cont;
_metrics_cache = new CachedMetric();
_metrics_cache = new CachedMetric<MetricType>();
}
CachedMetric* metrics_cache() { return _metrics_cache; }
CachedMetric<MetricType>* metrics_cache() { return _metrics_cache; }
T* controller() { return _controller; }
};
@ -277,7 +278,7 @@ class CgroupMemoryController: public CHeapObj<mtInternal> {
class CgroupSubsystem: public CHeapObj<mtInternal> {
public:
bool memory_limit_in_bytes(physical_memory_size_type upper_bound, physical_memory_size_type& value);
bool active_processor_count(int& value);
bool active_processor_count(double& value);
virtual bool pids_max(uint64_t& value) = 0;
virtual bool pids_current(uint64_t& value) = 0;
@ -286,8 +287,8 @@ class CgroupSubsystem: public CHeapObj<mtInternal> {
virtual char * cpu_cpuset_cpus() = 0;
virtual char * cpu_cpuset_memory_nodes() = 0;
virtual const char * container_type() = 0;
virtual CachingCgroupController<CgroupMemoryController>* memory_controller() = 0;
virtual CachingCgroupController<CgroupCpuController>* cpu_controller() = 0;
virtual CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* memory_controller() = 0;
virtual CachingCgroupController<CgroupCpuController, double>* cpu_controller() = 0;
virtual CgroupCpuacctController* cpuacct_controller() = 0;
bool cpu_quota(int& value);

27
src/hotspot/os/linux/cgroupUtil_linux.cpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2024, 2025, Red Hat, Inc.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,9 +26,8 @@
#include "cgroupUtil_linux.hpp"
#include "os_linux.hpp"
bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, int& value) {
bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, double& value) {
assert(upper_bound > 0, "upper bound of cpus must be positive");
int limit_count = upper_bound;
int quota = -1;
int period = -1;
if (!cpu_ctrl->cpu_quota(quota)) {
@ -37,20 +37,15 @@ bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound,
return false;
}
int quota_count = 0;
int result = upper_bound;
double result = upper_bound;
if (quota > -1 && period > 0) {
quota_count = ceilf((float)quota / (float)period);
log_trace(os, container)("CPU Quota count based on quota/period: %d", quota_count);
if (quota > 0 && period > 0) { // Use quotas
double cpu_quota = static_cast<double>(quota) / period;
log_trace(os, container)("CPU Quota based on quota/period: %.2f", cpu_quota);
result = MIN2(result, cpu_quota);
}
// Use quotas
if (quota_count != 0) {
limit_count = quota_count;
}
result = MIN2(upper_bound, limit_count);
log_trace(os, container)("OSContainer::active_processor_count: %d", result);
log_trace(os, container)("OSContainer::active_processor_count: %.2f", result);
value = result;
return true;
}
@ -73,11 +68,11 @@ physical_memory_size_type CgroupUtil::get_updated_mem_limit(CgroupMemoryControll
// Get an updated cpu limit. The return value is strictly less than or equal to the
// passed in 'lowest' value.
int CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
double CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
int lowest,
int upper_bound) {
assert(lowest > 0 && lowest <= upper_bound, "invariant");
int cpu_limit_val = -1;
double cpu_limit_val = -1;
if (CgroupUtil::processor_count(cpu, upper_bound, cpu_limit_val) && cpu_limit_val != upper_bound) {
assert(cpu_limit_val <= upper_bound, "invariant");
if (lowest > cpu_limit_val) {
@ -172,7 +167,7 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
assert(cg_path[0] == '/', "cgroup path must start with '/'");
int host_cpus = os::Linux::active_processor_count();
int lowest_limit = host_cpus;
int cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
double cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
int orig_limit = lowest_limit != host_cpus ? lowest_limit : host_cpus;
char* limit_cg_path = nullptr;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {

7
src/hotspot/os/linux/cgroupUtil_linux.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2024, Red Hat, Inc.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,7 +32,7 @@
class CgroupUtil: AllStatic {
public:
static bool processor_count(CgroupCpuController* cpu, int upper_bound, int& value);
static bool processor_count(CgroupCpuController* cpu, int upper_bound, double& value);
// Given a memory controller, adjust its path to a point in the hierarchy
// that represents the closest memory limit.
static void adjust_controller(CgroupMemoryController* m);
@ -42,9 +43,7 @@ class CgroupUtil: AllStatic {
static physical_memory_size_type get_updated_mem_limit(CgroupMemoryController* m,
physical_memory_size_type lowest,
physical_memory_size_type upper_bound);
static int get_updated_cpu_limit(CgroupCpuController* c,
int lowest,
int upper_bound);
static double get_updated_cpu_limit(CgroupCpuController* c, int lowest, int upper_bound);
};
#endif // CGROUP_UTIL_LINUX_HPP

6
src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -328,8 +328,8 @@ CgroupV1Subsystem::CgroupV1Subsystem(CgroupV1Controller* cpuset,
_pids(pids) {
CgroupUtil::adjust_controller(memory);
CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController<CgroupMemoryController>(memory);
_cpu = new CachingCgroupController<CgroupCpuController>(cpu);
_memory = new CachingCgroupController<CgroupMemoryController, physical_memory_size_type>(memory);
_cpu = new CachingCgroupController<CgroupCpuController, double>(cpu);
}
bool CgroupV1Subsystem::is_containerized() {

10
src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -214,15 +214,15 @@ class CgroupV1Subsystem: public CgroupSubsystem {
const char * container_type() override {
return "cgroupv1";
}
CachingCgroupController<CgroupMemoryController>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController>* cpu_controller() override { return _cpu; }
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController, double>* cpu_controller() override { return _cpu; }
CgroupCpuacctController* cpuacct_controller() override { return _cpuacct; }
private:
/* controllers */
CachingCgroupController<CgroupMemoryController>* _memory = nullptr;
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* _memory = nullptr;
CgroupV1Controller* _cpuset = nullptr;
CachingCgroupController<CgroupCpuController>* _cpu = nullptr;
CachingCgroupController<CgroupCpuController, double>* _cpu = nullptr;
CgroupV1CpuacctController* _cpuacct = nullptr;
CgroupV1Controller* _pids = nullptr;

6
src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
View File

@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2025, Red Hat Inc.
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -156,8 +156,8 @@ CgroupV2Subsystem::CgroupV2Subsystem(CgroupV2MemoryController * memory,
_unified(unified) {
CgroupUtil::adjust_controller(memory);
CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController<CgroupMemoryController>(memory);
_cpu = new CachingCgroupController<CgroupCpuController>(cpu);
_memory = new CachingCgroupController<CgroupMemoryController, physical_memory_size_type>(memory);
_cpu = new CachingCgroupController<CgroupCpuController, double>(cpu);
_cpuacct = cpuacct;
}

10
src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
View File

@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2024, Red Hat Inc.
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -152,8 +152,8 @@ class CgroupV2Subsystem: public CgroupSubsystem {
/* One unified controller */
CgroupV2Controller _unified;
/* Caching wrappers for cpu/memory metrics */
CachingCgroupController<CgroupMemoryController>* _memory = nullptr;
CachingCgroupController<CgroupCpuController>* _cpu = nullptr;
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* _memory = nullptr;
CachingCgroupController<CgroupCpuController, double>* _cpu = nullptr;
CgroupCpuacctController* _cpuacct = nullptr;
@ -175,8 +175,8 @@ class CgroupV2Subsystem: public CgroupSubsystem {
const char * container_type() override {
return "cgroupv2";
}
CachingCgroupController<CgroupMemoryController>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController>* cpu_controller() override { return _cpu; }
CachingCgroupController<CgroupMemoryController, physical_memory_size_type>* memory_controller() override { return _memory; }
CachingCgroupController<CgroupCpuController, double>* cpu_controller() override { return _cpu; }
CgroupCpuacctController* cpuacct_controller() override { return _cpuacct; };
};

3
src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp
View File

@ -66,9 +66,6 @@
#endif
// open(2) flags
#ifndef O_CLOEXEC
#define O_CLOEXEC 02000000
#endif
#ifndef O_TMPFILE
#define O_TMPFILE (020000000 | O_DIRECTORY)
#endif

16
src/hotspot/os/linux/osContainer_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -86,8 +86,8 @@ void OSContainer::init() {
// 2.) On a physical Linux system with a limit enforced by other means (like systemd slice)
physical_memory_size_type mem_limit_val = value_unlimited;
(void)memory_limit_in_bytes(mem_limit_val); // discard error and use default
int host_cpus = os::Linux::active_processor_count();
int cpus = host_cpus;
double host_cpus = os::Linux::active_processor_count();
double cpus = host_cpus;
(void)active_processor_count(cpus); // discard error and use default
any_mem_cpu_limit_present = mem_limit_val != value_unlimited || host_cpus != cpus;
if (any_mem_cpu_limit_present) {
@ -127,8 +127,7 @@ bool OSContainer::available_memory_in_bytes(physical_memory_size_type& value) {
return false;
}
bool OSContainer::available_swap_in_bytes(physical_memory_size_type host_free_swap,
physical_memory_size_type& value) {
bool OSContainer::available_swap_in_bytes(physical_memory_size_type& value) {
physical_memory_size_type mem_limit = 0;
physical_memory_size_type mem_swap_limit = 0;
if (memory_limit_in_bytes(mem_limit) &&
@ -179,8 +178,7 @@ bool OSContainer::available_swap_in_bytes(physical_memory_size_type host_free_sw
assert(num < 25, "buffer too small");
mem_limit_buf[num] = '\0';
log_trace(os,container)("OSContainer::available_swap_in_bytes: container_swap_limit=%s"
" container_mem_limit=%s, host_free_swap: " PHYS_MEM_TYPE_FORMAT,
mem_swap_buf, mem_limit_buf, host_free_swap);
" container_mem_limit=%s", mem_swap_buf, mem_limit_buf);
}
return false;
}
@ -252,7 +250,7 @@ char * OSContainer::cpu_cpuset_memory_nodes() {
return cgroup_subsystem->cpu_cpuset_memory_nodes();
}
bool OSContainer::active_processor_count(int& value) {
bool OSContainer::active_processor_count(double& value) {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
return cgroup_subsystem->active_processor_count(value);
}
@ -291,11 +289,13 @@ template<typename T> struct metric_fmt;
template<> struct metric_fmt<unsigned long long int> { static constexpr const char* fmt = "%llu"; };
template<> struct metric_fmt<unsigned long int> { static constexpr const char* fmt = "%lu"; };
template<> struct metric_fmt<int> { static constexpr const char* fmt = "%d"; };
template<> struct metric_fmt<double> { static constexpr const char* fmt = "%.2f"; };
template<> struct metric_fmt<const char*> { static constexpr const char* fmt = "%s"; };
template void OSContainer::print_container_metric<unsigned long long int>(outputStream*, const char*, unsigned long long int, const char*);
template void OSContainer::print_container_metric<unsigned long int>(outputStream*, const char*, unsigned long int, const char*);
template void OSContainer::print_container_metric<int>(outputStream*, const char*, int, const char*);
template void OSContainer::print_container_metric<double>(outputStream*, const char*, double, const char*);
template void OSContainer::print_container_metric<const char*>(outputStream*, const char*, const char*, const char*);
template <typename T>

7
src/hotspot/os/linux/osContainer_linux.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -72,8 +72,7 @@ class OSContainer: AllStatic {
static const char * container_type();
static bool available_memory_in_bytes(physical_memory_size_type& value);
static bool available_swap_in_bytes(physical_memory_size_type host_free_swap,
physical_memory_size_type& value);
static bool available_swap_in_bytes(physical_memory_size_type& value);
static bool memory_limit_in_bytes(physical_memory_size_type& value);
static bool memory_and_swap_limit_in_bytes(physical_memory_size_type& value);
static bool memory_and_swap_usage_in_bytes(physical_memory_size_type& value);
@ -84,7 +83,7 @@ class OSContainer: AllStatic {
static bool rss_usage_in_bytes(physical_memory_size_type& value);
static bool cache_usage_in_bytes(physical_memory_size_type& value);
static bool active_processor_count(int& value);
static bool active_processor_count(double& value);
static char * cpu_cpuset_cpus();
static char * cpu_cpuset_memory_nodes();

190
src/hotspot/os/linux/os_linux.cpp
View File

@ -211,15 +211,58 @@ static bool suppress_primordial_thread_resolution = false;
// utility functions
bool os::is_containerized() {
return OSContainer::is_containerized();
}
bool os::Container::memory_limit(physical_memory_size_type& value) {
physical_memory_size_type result = 0;
if (OSContainer::memory_limit_in_bytes(result) && result != value_unlimited) {
value = result;
return true;
}
return false;
}
bool os::Container::memory_soft_limit(physical_memory_size_type& value) {
physical_memory_size_type result = 0;
if (OSContainer::memory_soft_limit_in_bytes(result) && result != 0 && result != value_unlimited) {
value = result;
return true;
}
return false;
}
bool os::Container::memory_throttle_limit(physical_memory_size_type& value) {
physical_memory_size_type result = 0;
if (OSContainer::memory_throttle_limit_in_bytes(result) && result != value_unlimited) {
value = result;
return true;
}
return false;
}
bool os::Container::used_memory(physical_memory_size_type& value) {
return OSContainer::memory_usage_in_bytes(value);
}
bool os::available_memory(physical_memory_size_type& value) {
if (OSContainer::is_containerized() && OSContainer::available_memory_in_bytes(value)) {
if (is_containerized() && Container::available_memory(value)) {
log_trace(os)("available container memory: " PHYS_MEM_TYPE_FORMAT, value);
return true;
}
return Machine::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return Linux::available_memory(value);
}
bool os::Container::available_memory(physical_memory_size_type& value) {
return OSContainer::available_memory_in_bytes(value);
}
bool os::Linux::available_memory(physical_memory_size_type& value) {
physical_memory_size_type avail_mem = 0;
@ -251,11 +294,15 @@ bool os::Linux::available_memory(physical_memory_size_type& value) {
}
bool os::free_memory(physical_memory_size_type& value) {
if (OSContainer::is_containerized() && OSContainer::available_memory_in_bytes(value)) {
if (is_containerized() && Container::available_memory(value)) {
log_trace(os)("free container memory: " PHYS_MEM_TYPE_FORMAT, value);
return true;
}
return Machine::free_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return Linux::free_memory(value);
}
@ -274,21 +321,30 @@ bool os::Linux::free_memory(physical_memory_size_type& value) {
}
bool os::total_swap_space(physical_memory_size_type& value) {
if (OSContainer::is_containerized()) {
physical_memory_size_type mem_swap_limit = value_unlimited;
physical_memory_size_type memory_limit = value_unlimited;
if (OSContainer::memory_and_swap_limit_in_bytes(mem_swap_limit) &&
OSContainer::memory_limit_in_bytes(memory_limit)) {
if (memory_limit != value_unlimited && mem_swap_limit != value_unlimited &&
mem_swap_limit >= memory_limit /* ensure swap is >= 0 */) {
value = mem_swap_limit - memory_limit;
return true;
}
}
} // fallback to the host swap space if the container returned unlimited
if (is_containerized() && Container::total_swap_space(value)) {
return true;
} // fallback to the host swap space if the container value fails
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
return Linux::host_swap(value);
}
bool os::Container::total_swap_space(physical_memory_size_type& value) {
physical_memory_size_type mem_swap_limit = value_unlimited;
physical_memory_size_type memory_limit = value_unlimited;
if (OSContainer::memory_and_swap_limit_in_bytes(mem_swap_limit) &&
OSContainer::memory_limit_in_bytes(memory_limit)) {
if (memory_limit != value_unlimited && mem_swap_limit != value_unlimited &&
mem_swap_limit >= memory_limit /* ensure swap is >= 0 */) {
value = mem_swap_limit - memory_limit;
return true;
}
}
return false;
}
static bool host_free_swap_f(physical_memory_size_type& value) {
struct sysinfo si;
int ret = sysinfo(&si);
@ -309,32 +365,45 @@ bool os::free_swap_space(physical_memory_size_type& value) {
return false;
}
physical_memory_size_type host_free_swap_val = MIN2(total_swap_space, host_free_swap);
if (OSContainer::is_containerized()) {
if (OSContainer::available_swap_in_bytes(host_free_swap_val, value)) {
if (is_containerized()) {
if (Container::free_swap_space(value)) {
return true;
}
// Fall through to use host value
log_trace(os,container)("os::free_swap_space: containerized value unavailable"
" returning host value: " PHYS_MEM_TYPE_FORMAT, host_free_swap_val);
}
value = host_free_swap_val;
return true;
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
return host_free_swap_f(value);
}
bool os::Container::free_swap_space(physical_memory_size_type& value) {
return OSContainer::available_swap_in_bytes(value);
}
physical_memory_size_type os::physical_memory() {
if (OSContainer::is_containerized()) {
if (is_containerized()) {
physical_memory_size_type mem_limit = value_unlimited;
if (OSContainer::memory_limit_in_bytes(mem_limit) && mem_limit != value_unlimited) {
if (Container::memory_limit(mem_limit) && mem_limit != value_unlimited) {
log_trace(os)("total container memory: " PHYS_MEM_TYPE_FORMAT, mem_limit);
return mem_limit;
}
}
physical_memory_size_type phys_mem = Linux::physical_memory();
physical_memory_size_type phys_mem = Machine::physical_memory();
log_trace(os)("total system memory: " PHYS_MEM_TYPE_FORMAT, phys_mem);
return phys_mem;
}
physical_memory_size_type os::Machine::physical_memory() {
return Linux::physical_memory();
}
// Returns the resident set size (RSS) of the process.
// Falls back to using VmRSS from /proc/self/status if /proc/self/smaps_rollup is unavailable.
// Note: On kernels with memory cgroups or shared memory, VmRSS may underreport RSS.
@ -2439,20 +2508,21 @@ bool os::Linux::print_container_info(outputStream* st) {
OSContainer::print_container_metric(st, "cpu_memory_nodes", p != nullptr ? p : "not supported");
free(p);
int i = -1;
bool supported = OSContainer::active_processor_count(i);
double cpus = -1;
bool supported = OSContainer::active_processor_count(cpus);
if (supported) {
assert(i > 0, "must be");
assert(cpus > 0, "must be");
if (ActiveProcessorCount > 0) {
OSContainer::print_container_metric(st, "active_processor_count", ActiveProcessorCount, "(from -XX:ActiveProcessorCount)");
} else {
OSContainer::print_container_metric(st, "active_processor_count", i);
OSContainer::print_container_metric(st, "active_processor_count", cpus);
}
} else {
OSContainer::print_container_metric(st, "active_processor_count", "not supported");
}
int i = -1;
supported = OSContainer::cpu_quota(i);
if (supported && i > 0) {
OSContainer::print_container_metric(st, "cpu_quota", i);
@ -4737,15 +4807,26 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
int active_cpus = -1;
if (OSContainer::is_containerized() && OSContainer::active_processor_count(active_cpus)) {
log_trace(os)("active_processor_count: determined by OSContainer: %d",
active_cpus);
} else {
active_cpus = os::Linux::active_processor_count();
if (is_containerized()) {
double cpu_quota;
if (Container::processor_count(cpu_quota)) {
int active_cpus = ceilf(cpu_quota); // Round fractional CPU quota up.
assert(active_cpus <= Machine::active_processor_count(), "must be");
log_trace(os)("active_processor_count: determined by OSContainer: %d",
active_cpus);
return active_cpus;
}
}
return active_cpus;
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
return os::Linux::active_processor_count();
}
bool os::Container::processor_count(double& value) {
return OSContainer::active_processor_count(value);
}
static bool should_warn_invalid_processor_id() {
@ -4878,36 +4959,18 @@ int os::open(const char *path, int oflag, int mode) {
// All file descriptors that are opened in the Java process and not
// specifically destined for a subprocess should have the close-on-exec
// flag set. If we don't set it, then careless 3rd party native code
// might fork and exec without closing all appropriate file descriptors,
// and this in turn might:
//
// - cause end-of-file to fail to be detected on some file
// descriptors, resulting in mysterious hangs, or
//
// - might cause an fopen in the subprocess to fail on a system
// suffering from bug 1085341.
//
// (Yes, the default setting of the close-on-exec flag is a Unix
// design flaw)
//
// See:
// 1085341: 32-bit stdio routines should support file descriptors >255
// 4843136: (process) pipe file descriptor from Runtime.exec not being closed
// 6339493: (process) Runtime.exec does not close all file descriptors on Solaris 9
//
// Modern Linux kernels (after 2.6.23 2007) support O_CLOEXEC with open().
// O_CLOEXEC is preferable to using FD_CLOEXEC on an open file descriptor
// because it saves a system call and removes a small window where the flag
// is unset. On ancient Linux kernels the O_CLOEXEC flag will be ignored
// and we fall back to using FD_CLOEXEC (see below).
#ifdef O_CLOEXEC
// might fork and exec without closing all appropriate file descriptors.
oflag |= O_CLOEXEC;
#endif
int fd = ::open(path, oflag, mode);
if (fd == -1) return -1;
// No further checking is needed if open() returned an error or
// access mode is not read only.
if (fd == -1 || (oflag & O_ACCMODE) != O_RDONLY) {
return fd;
}
//If the open succeeded, the file might still be a directory
// If the open succeeded and is read only, the file might be a directory
// which the JVM doesn't allow to be read.
{
struct stat buf;
int ret = ::fstat(fd, &buf);
@ -4925,21 +4988,6 @@ int os::open(const char *path, int oflag, int mode) {
}
}
#ifdef FD_CLOEXEC
// Validate that the use of the O_CLOEXEC flag on open above worked.
// With recent kernels, we will perform this check exactly once.
static sig_atomic_t O_CLOEXEC_is_known_to_work = 0;
if (!O_CLOEXEC_is_known_to_work) {
int flags = ::fcntl(fd, F_GETFD);
if (flags != -1) {
if ((flags & FD_CLOEXEC) != 0)
O_CLOEXEC_is_known_to_work = 1;
else
::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
}
}
#endif
return fd;
}

26
src/hotspot/os/windows/os_windows.cpp
View File

@ -839,10 +839,18 @@ bool os::available_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::Machine::available_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::free_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::Machine::free_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
bool os::win32::available_memory(physical_memory_size_type& value) {
// Use GlobalMemoryStatusEx() because GlobalMemoryStatus() may return incorrect
// value if total memory is larger than 4GB
@ -858,7 +866,11 @@ bool os::win32::available_memory(physical_memory_size_type& value) {
}
}
bool os::total_swap_space(physical_memory_size_type& value) {
bool os::total_swap_space(physical_memory_size_type& value) {
return Machine::total_swap_space(value);
}
bool os::Machine::total_swap_space(physical_memory_size_type& value) {
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
@ -872,6 +884,10 @@ bool os::total_swap_space(physical_memory_size_type& value) {
}
bool os::free_swap_space(physical_memory_size_type& value) {
return Machine::free_swap_space(value);
}
bool os::Machine::free_swap_space(physical_memory_size_type& value) {
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
@ -888,6 +904,10 @@ physical_memory_size_type os::physical_memory() {
return win32::physical_memory();
}
physical_memory_size_type os::Machine::physical_memory() {
return win32::physical_memory();
}
size_t os::rss() {
size_t rss = 0;
PROCESS_MEMORY_COUNTERS_EX pmex;
@ -911,6 +931,10 @@ int os::active_processor_count() {
return ActiveProcessorCount;
}
return Machine::active_processor_count();
}
int os::Machine::active_processor_count() {
bool schedules_all_processor_groups = win32::is_windows_11_or_greater() || win32::is_windows_server_2022_or_greater();
if (UseAllWindowsProcessorGroups && !schedules_all_processor_groups && !win32::processor_group_warning_displayed()) {
win32::set_processor_group_warning_displayed(true);

7
src/hotspot/share/classfile/stringTable.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -614,6 +614,10 @@ struct StringTableDeleteCheck : StackObj {
};
void StringTable::clean_dead_entries(JavaThread* jt) {
// BulkDeleteTask::prepare() may take ConcurrentHashTableResize_lock (nosafepoint-2).
// When NativeHeapTrimmer is enabled, SuspendMark may take NativeHeapTrimmer::_lock (nosafepoint).
// Take SuspendMark first to keep lock order and avoid deadlock.
NativeHeapTrimmer::SuspendMark sm("stringtable");
StringTableHash::BulkDeleteTask bdt(_local_table);
if (!bdt.prepare(jt)) {
return;
@ -621,7 +625,6 @@ void StringTable::clean_dead_entries(JavaThread* jt) {
StringTableDeleteCheck stdc;
StringTableDoDelete stdd;
NativeHeapTrimmer::SuspendMark sm("stringtable");
{
TraceTime timer("Clean", TRACETIME_LOG(Debug, stringtable, perf));
while(bdt.do_task(jt, stdc, stdd)) {

7
src/hotspot/share/classfile/symbolTable.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -763,6 +763,10 @@ struct SymbolTableDeleteCheck : StackObj {
};
void SymbolTable::clean_dead_entries(JavaThread* jt) {
// BulkDeleteTask::prepare() may take ConcurrentHashTableResize_lock (nosafepoint-2).
// When NativeHeapTrimmer is enabled, SuspendMark may take NativeHeapTrimmer::_lock (nosafepoint).
// Take SuspendMark first to keep lock order and avoid deadlock.
NativeHeapTrimmer::SuspendMark sm("symboltable");
SymbolTableHash::BulkDeleteTask bdt(_local_table);
if (!bdt.prepare(jt)) {
return;
@ -770,7 +774,6 @@ void SymbolTable::clean_dead_entries(JavaThread* jt) {
SymbolTableDeleteCheck stdc;
SymbolTableDoDelete stdd;
NativeHeapTrimmer::SuspendMark sm("symboltable");
{
TraceTime timer("Clean", TRACETIME_LOG(Debug, symboltable, perf));
while (bdt.do_task(jt, stdc, stdd)) {

3
src/hotspot/share/code/aotCodeCache.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1371,6 +1371,7 @@ void AOTCodeAddressTable::init_extrs() {
SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
#endif
#if INCLUDE_ZGC
SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr());
SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr());
#if defined(AMD64)
SET_ADDRESS(_extrs, &ZPointerLoadShift);

6
src/hotspot/share/gc/g1/g1CardSet.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1024,10 +1024,6 @@ size_t G1CardSet::num_containers() {
return cl._count;
}
G1CardSetCoarsenStats G1CardSet::coarsen_stats() {
return _coarsen_stats;
}
void G1CardSet::print_coarsen_stats(outputStream* out) {
_last_coarsen_stats.subtract_from(_coarsen_stats);

8
src/hotspot/share/gc/g1/g1CardTableClaimTable.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -44,20 +44,20 @@ G1CardTableClaimTable::~G1CardTableClaimTable() {
void G1CardTableClaimTable::initialize(uint max_reserved_regions) {
assert(_card_claims == nullptr, "Must not be initialized twice");
_card_claims = NEW_C_HEAP_ARRAY(uint, max_reserved_regions, mtGC);
_card_claims = NEW_C_HEAP_ARRAY(Atomic<uint>, max_reserved_regions, mtGC);
_max_reserved_regions = max_reserved_regions;
reset_all_to_unclaimed();
}
void G1CardTableClaimTable::reset_all_to_unclaimed() {
for (uint i = 0; i < _max_reserved_regions; i++) {
_card_claims[i] = 0;
_card_claims[i].store_relaxed(0);
}
}
void G1CardTableClaimTable::reset_all_to_claimed() {
for (uint i = 0; i < _max_reserved_regions; i++) {
_card_claims[i] = (uint)G1HeapRegion::CardsPerRegion;
_card_claims[i].store_relaxed((uint)G1HeapRegion::CardsPerRegion);
}
}

5
src/hotspot/share/gc/g1/g1CardTableClaimTable.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,6 +27,7 @@
#include "gc/g1/g1CardTable.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
class G1HeapRegionClosure;
@ -45,7 +46,7 @@ class G1CardTableClaimTable : public CHeapObj<mtGC> {
// Card table iteration claim values for every heap region, from 0 (completely unclaimed)
// to (>=) G1HeapRegion::CardsPerRegion (completely claimed).
uint volatile* _card_claims;
Atomic<uint>* _card_claims;
uint _cards_per_chunk; // For conversion between card index and chunk index.

11
src/hotspot/share/gc/g1/g1CardTableClaimTable.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,26 +29,25 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "runtime/atomicAccess.hpp"
bool G1CardTableClaimTable::has_unclaimed_cards(uint region) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
return AtomicAccess::load(&_card_claims[region]) < G1HeapRegion::CardsPerRegion;
return _card_claims[region].load_relaxed() < G1HeapRegion::CardsPerRegion;
}
void G1CardTableClaimTable::reset_to_unclaimed(uint region) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
AtomicAccess::store(&_card_claims[region], 0u);
_card_claims[region].store_relaxed(0u);
}
uint G1CardTableClaimTable::claim_cards(uint region, uint increment) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
return AtomicAccess::fetch_then_add(&_card_claims[region], increment, memory_order_relaxed);
return _card_claims[region].fetch_then_add(increment, memory_order_relaxed);
}
uint G1CardTableClaimTable::claim_chunk(uint region) {
assert(region < _max_reserved_regions, "Tried to access invalid region %u", region);
return AtomicAccess::fetch_then_add(&_card_claims[region], cards_per_chunk(), memory_order_relaxed);
return _card_claims[region].fetch_then_add(cards_per_chunk(), memory_order_relaxed);
}
uint G1CardTableClaimTable::claim_all_cards(uint region) {

5
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -103,7 +103,6 @@
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/cpuTimeCounters.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/init.hpp"
@ -687,8 +686,8 @@ HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size) {
// before the allocation is that we avoid having to keep track of the newly
// allocated memory while we do a GC.
// Only try that if we can actually perform a GC.
if (is_init_completed() && policy()->need_to_start_conc_mark("concurrent humongous allocation",
word_size)) {
if (is_init_completed() &&
policy()->need_to_start_conc_mark("concurrent humongous allocation", word_size)) {
try_collect(word_size, GCCause::_g1_humongous_allocation, collection_counters(this));
}

5
src/hotspot/share/gc/g1/g1CollectedHeap.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -54,6 +54,7 @@
#include "memory/allocation.hpp"
#include "memory/iterator.hpp"
#include "memory/memRegion.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/threadSMR.hpp"
#include "utilities/bitMap.hpp"
@ -124,7 +125,7 @@ class G1JavaThreadsListClaimer : public StackObj {
ThreadsListHandle _list;
uint _claim_step;
volatile uint _cur_claim;
Atomic<uint> _cur_claim;
// Attempts to claim _claim_step JavaThreads, returning an array of claimed
// JavaThread* with count elements. Returns null (and a zero count) if there

7
src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -41,7 +41,6 @@
#include "gc/shared/markBitMap.inline.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "oops/stackChunkOop.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/threadSMR.inline.hpp"
#include "utilities/bitMap.inline.hpp"
@ -53,10 +52,10 @@ inline bool G1STWIsAliveClosure::do_object_b(oop p) {
inline JavaThread* const* G1JavaThreadsListClaimer::claim(uint& count) {
count = 0;
if (AtomicAccess::load(&_cur_claim) >= _list.length()) {
if (_cur_claim.load_relaxed() >= _list.length()) {
return nullptr;
}
uint claim = AtomicAccess::fetch_then_add(&_cur_claim, _claim_step);
uint claim = _cur_claim.fetch_then_add(_claim_step);
if (claim >= _list.length()) {
return nullptr;
}

17
src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,7 +27,7 @@
#include "gc/g1/g1CollectionSetChooser.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/space.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/atomic.hpp"
#include "utilities/quickSort.hpp"
// Determine collection set candidates (from marking): For all regions determine
@ -50,7 +50,7 @@ class G1BuildCandidateRegionsTask : public WorkerTask {
G1HeapRegion** _data;
uint volatile _cur_claim_idx;
Atomic<uint> _cur_claim_idx;
static int compare_region_gc_efficiency(G1HeapRegion** rr1, G1HeapRegion** rr2) {
G1HeapRegion* r1 = *rr1;
@ -105,7 +105,7 @@ class G1BuildCandidateRegionsTask : public WorkerTask {
// Claim a new chunk, returning its bounds [from, to[.
void claim_chunk(uint& from, uint& to) {
uint result = AtomicAccess::add(&_cur_claim_idx, _chunk_size);
uint result = _cur_claim_idx.add_then_fetch(_chunk_size);
assert(_max_size > result - 1,
"Array too small, is %u should be %u with chunk size %u.",
_max_size, result, _chunk_size);
@ -121,14 +121,15 @@ class G1BuildCandidateRegionsTask : public WorkerTask {
}
void sort_by_gc_efficiency() {
if (_cur_claim_idx == 0) {
uint length = _cur_claim_idx.load_relaxed();
if (length == 0) {
return;
}
for (uint i = _cur_claim_idx; i < _max_size; i++) {
for (uint i = length; i < _max_size; i++) {
assert(_data[i] == nullptr, "must be");
}
qsort(_data, _cur_claim_idx, sizeof(_data[0]), (_sort_Fn)compare_region_gc_efficiency);
for (uint i = _cur_claim_idx; i < _max_size; i++) {
qsort(_data, length, sizeof(_data[0]), (_sort_Fn)compare_region_gc_efficiency);
for (uint i = length; i < _max_size; i++) {
assert(_data[i] == nullptr, "must be");
}
}

27
src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -67,7 +67,6 @@
#include "nmt/memTracker.hpp"
#include "oops/access.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/java.hpp"
@ -148,25 +147,25 @@ bool G1CMMarkStack::initialize() {
}
G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_chunk() {
if (_size >= _max_capacity) {
if (_size.load_relaxed() >= _max_capacity) {
return nullptr;
}
size_t cur_idx = AtomicAccess::fetch_then_add(&_size, 1u);
size_t cur_idx = _size.fetch_then_add(1u);
if (cur_idx >= _max_capacity) {
return nullptr;
}
size_t bucket = get_bucket(cur_idx);
if (AtomicAccess::load_acquire(&_buckets[bucket]) == nullptr) {
if (_buckets[bucket].load_acquire() == nullptr) {
if (!_should_grow) {
// Prefer to restart the CM.
return nullptr;
}
MutexLocker x(G1MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
if (AtomicAccess::load_acquire(&_buckets[bucket]) == nullptr) {
if (_buckets[bucket].load_acquire() == nullptr) {
size_t desired_capacity = bucket_size(bucket) * 2;
if (!try_expand_to(desired_capacity)) {
return nullptr;
@ -175,7 +174,7 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_
}
size_t bucket_idx = get_bucket_index(cur_idx);
TaskQueueEntryChunk* result = ::new (&_buckets[bucket][bucket_idx]) TaskQueueEntryChunk;
TaskQueueEntryChunk* result = ::new (&_buckets[bucket].load_relaxed()[bucket_idx]) TaskQueueEntryChunk;
result->next = nullptr;
return result;
}
@ -197,10 +196,10 @@ bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t m
_max_capacity = max_capacity;
_num_buckets = get_bucket(_max_capacity) + 1;
_buckets = NEW_C_HEAP_ARRAY(TaskQueueEntryChunk*, _num_buckets, mtGC);
_buckets = NEW_C_HEAP_ARRAY(Atomic<TaskQueueEntryChunk*>, _num_buckets, mtGC);
for (size_t i = 0; i < _num_buckets; i++) {
_buckets[i] = nullptr;
_buckets[i].store_relaxed(nullptr);
}
size_t new_capacity = bucket_size(0);
@ -240,9 +239,9 @@ G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
}
for (size_t i = 0; i < _num_buckets; i++) {
if (_buckets[i] != nullptr) {
MmapArrayAllocator<TaskQueueEntryChunk>::free(_buckets[i], bucket_size(i));
_buckets[i] = nullptr;
if (_buckets[i].load_relaxed() != nullptr) {
MmapArrayAllocator<TaskQueueEntryChunk>::free(_buckets[i].load_relaxed(), bucket_size(i));
_buckets[i].store_relaxed(nullptr);
}
}
@ -259,7 +258,7 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
// and the new capacity (new_capacity). This step ensures that there are no gaps in the
// array and that the capacity accurately reflects the reserved memory.
for (; i <= highest_bucket; i++) {
if (AtomicAccess::load_acquire(&_buckets[i]) != nullptr) {
if (_buckets[i].load_acquire() != nullptr) {
continue; // Skip over already allocated buckets.
}
@ -279,7 +278,7 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
return false;
}
_capacity += bucket_capacity;
AtomicAccess::release_store(&_buckets[i], bucket_base);
_buckets[i].release_store(bucket_base);
}
return true;
}

17
src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -37,6 +37,7 @@
#include "gc/shared/workerThread.hpp"
#include "gc/shared/workerUtils.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
#include "utilities/compilerWarnings.hpp"
#include "utilities/numberSeq.hpp"
@ -172,9 +173,9 @@ private:
size_t _capacity;
size_t _num_buckets;
bool _should_grow;
TaskQueueEntryChunk* volatile* _buckets;
Atomic<TaskQueueEntryChunk*>* _buckets;
char _pad0[DEFAULT_PADDING_SIZE];
volatile size_t _size;
Atomic<size_t> _size;
char _pad4[DEFAULT_PADDING_SIZE - sizeof(size_t)];
size_t bucket_size(size_t bucket) {
@ -212,7 +213,7 @@ private:
bool initialize(size_t initial_capacity, size_t max_capacity);
void reset() {
_size = 0;
_size.store_relaxed(0);
_should_grow = false;
}
@ -556,14 +557,14 @@ public:
// mark_in_bitmap call. Updates various statistics data.
void add_to_liveness(uint worker_id, oop const obj, size_t size);
// Did the last marking find a live object between bottom and TAMS?
bool contains_live_object(uint region) const { return _region_mark_stats[region]._live_words != 0; }
bool contains_live_object(uint region) const { return _region_mark_stats[region].live_words() != 0; }
// Live bytes in the given region as determined by concurrent marking, i.e. the amount of
// live bytes between bottom and TAMS.
size_t live_bytes(uint region) const { return _region_mark_stats[region]._live_words * HeapWordSize; }
size_t live_bytes(uint region) const { return _region_mark_stats[region].live_words() * HeapWordSize; }
// Set live bytes for concurrent marking.
void set_live_bytes(uint region, size_t live_bytes) { _region_mark_stats[region]._live_words = live_bytes / HeapWordSize; }
void set_live_bytes(uint region, size_t live_bytes) { _region_mark_stats[region]._live_words.store_relaxed(live_bytes / HeapWordSize); }
// Approximate number of incoming references found during marking.
size_t incoming_refs(uint region) const { return _region_mark_stats[region]._incoming_refs; }
size_t incoming_refs(uint region) const { return _region_mark_stats[region].incoming_refs(); }
// Update the TAMS for the given region to the current top.
inline void update_top_at_mark_start(G1HeapRegion* r);

6
src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +29,6 @@
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/bitMap.inline.hpp"
G1EvacFailureRegions::G1EvacFailureRegions() :
@ -43,7 +43,7 @@ G1EvacFailureRegions::~G1EvacFailureRegions() {
}
void G1EvacFailureRegions::pre_collection(uint max_regions) {
AtomicAccess::store(&_num_regions_evac_failed, 0u);
_num_regions_evac_failed.store_relaxed(0u);
_regions_evac_failed.resize(max_regions);
_regions_pinned.resize(max_regions);
_regions_alloc_failed.resize(max_regions);
@ -69,6 +69,6 @@ void G1EvacFailureRegions::par_iterate(G1HeapRegionClosure* closure,
G1CollectedHeap::heap()->par_iterate_regions_array(closure,
hrclaimer,
_evac_failed_regions,
AtomicAccess::load(&_num_regions_evac_failed),
num_regions_evac_failed(),
worker_id);
}

6
src/hotspot/share/gc/g1/g1EvacFailureRegions.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,6 +26,7 @@
#ifndef SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP
#define SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP
#include "runtime/atomic.hpp"
#include "utilities/bitMap.hpp"
class G1AbstractSubTask;
@ -53,14 +55,14 @@ class G1EvacFailureRegions {
// Evacuation failed regions (indexes) in the current collection.
uint* _evac_failed_regions;
// Number of regions evacuation failed in the current collection.
volatile uint _num_regions_evac_failed;
Atomic<uint> _num_regions_evac_failed;
public:
G1EvacFailureRegions();
~G1EvacFailureRegions();
uint get_region_idx(uint idx) const {
assert(idx < _num_regions_evac_failed, "precondition");
assert(idx < _num_regions_evac_failed.load_relaxed(), "precondition");
return _evac_failed_regions[idx];
}

6
src/hotspot/share/gc/g1/g1EvacFailureRegions.inline.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,10 +30,9 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "runtime/atomicAccess.hpp"
uint G1EvacFailureRegions::num_regions_evac_failed() const {
return AtomicAccess::load(&_num_regions_evac_failed);
return _num_regions_evac_failed.load_relaxed();
}
bool G1EvacFailureRegions::has_regions_evac_failed() const {
@ -57,7 +57,7 @@ bool G1EvacFailureRegions::record(uint worker_id, uint region_idx, bool cause_pi
bool success = _regions_evac_failed.par_set_bit(region_idx,
memory_order_relaxed);
if (success) {
size_t offset = AtomicAccess::fetch_then_add(&_num_regions_evac_failed, 1u);
size_t offset = _num_regions_evac_failed.fetch_then_add(1u);
_evac_failed_regions[offset] = region_idx;
G1CollectedHeap* g1h = G1CollectedHeap::heap();

4
src/hotspot/share/gc/g1/g1FullCollector.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -122,7 +122,7 @@ public:
ReferenceProcessor* reference_processor();
size_t live_words(uint region_index) const {
assert(region_index < _heap->max_num_regions(), "sanity");
return _live_stats[region_index]._live_words;
return _live_stats[region_index].live_words();
}
void before_marking_update_attribute_table(G1HeapRegion* hr);

3
src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -37,7 +37,6 @@
#include "gc/shared/weakProcessor.inline.hpp"
#include "logging/log.hpp"
#include "memory/iterator.inline.hpp"
#include "runtime/atomicAccess.hpp"
class G1AdjustLiveClosure : public StackObj {
G1AdjustClosure* _adjust_closure;

2
src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp
View File

@ -108,7 +108,7 @@ void G1FullGCMarker::follow_array_chunk(objArrayOop array, int index) {
push_objarray(array, end_index);
}
array->oop_iterate_range(mark_closure(), beg_index, end_index);
array->oop_iterate_elements_range(mark_closure(), beg_index, end_index);
}
inline void G1FullGCMarker::follow_object(oop obj) {

3
src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,7 +31,6 @@
#include "memory/allocation.hpp"
#include "memory/padded.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/java.hpp"
#include "runtime/mutexLocker.hpp"

6
src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -30,7 +30,6 @@
#include "gc/g1/g1CodeRootSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1FromCardCache.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
#include "utilities/bitMap.hpp"
@ -123,9 +122,6 @@ public:
static void initialize(MemRegion reserved);
// Coarsening statistics since VM start.
static G1CardSetCoarsenStats coarsen_stats() { return G1CardSet::coarsen_stats(); }
inline uintptr_t to_card(OopOrNarrowOopStar from) const;
private:

3
src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -30,7 +30,6 @@
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/bitMap.inline.hpp"
void G1HeapRegionRemSet::set_state_untracked() {

55
src/hotspot/share/gc/g1/g1MonotonicArena.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -24,7 +24,6 @@
#include "gc/g1/g1MonotonicArena.inline.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/vmOperations.hpp"
#include "utilities/globalCounter.inline.hpp"
@ -61,13 +60,13 @@ void G1MonotonicArena::SegmentFreeList::bulk_add(Segment& first,
size_t num,
size_t mem_size) {
_list.prepend(first, last);
AtomicAccess::add(&_num_segments, num, memory_order_relaxed);
AtomicAccess::add(&_mem_size, mem_size, memory_order_relaxed);
_num_segments.add_then_fetch(num, memory_order_relaxed);
_mem_size.add_then_fetch(mem_size, memory_order_relaxed);
}
void G1MonotonicArena::SegmentFreeList::print_on(outputStream* out, const char* prefix) {
out->print_cr("%s: segments %zu size %zu",
prefix, AtomicAccess::load(&_num_segments), AtomicAccess::load(&_mem_size));
prefix, _num_segments.load_relaxed(), _mem_size.load_relaxed());
}
G1MonotonicArena::Segment* G1MonotonicArena::SegmentFreeList::get_all(size_t& num_segments,
@ -75,12 +74,12 @@ G1MonotonicArena::Segment* G1MonotonicArena::SegmentFreeList::get_all(size_t& nu
GlobalCounter::CriticalSection cs(Thread::current());
Segment* result = _list.pop_all();
num_segments = AtomicAccess::load(&_num_segments);
mem_size = AtomicAccess::load(&_mem_size);
num_segments = _num_segments.load_relaxed();
mem_size = _mem_size.load_relaxed();
if (result != nullptr) {
AtomicAccess::sub(&_num_segments, num_segments, memory_order_relaxed);
AtomicAccess::sub(&_mem_size, mem_size, memory_order_relaxed);
_num_segments.sub_then_fetch(num_segments, memory_order_relaxed);
_mem_size.sub_then_fetch(mem_size, memory_order_relaxed);
}
return result;
}
@ -96,8 +95,8 @@ void G1MonotonicArena::SegmentFreeList::free_all() {
Segment::delete_segment(cur);
}
AtomicAccess::sub(&_num_segments, num_freed, memory_order_relaxed);
AtomicAccess::sub(&_mem_size, mem_size_freed, memory_order_relaxed);
_num_segments.sub_then_fetch(num_freed, memory_order_relaxed);
_mem_size.sub_then_fetch(mem_size_freed, memory_order_relaxed);
}
G1MonotonicArena::Segment* G1MonotonicArena::new_segment(Segment* const prev) {
@ -115,7 +114,7 @@ G1MonotonicArena::Segment* G1MonotonicArena::new_segment(Segment* const prev) {
}
// Install it as current allocation segment.
Segment* old = AtomicAccess::cmpxchg(&_first, prev, next);
Segment* old = _first.compare_exchange(prev, next);
if (old != prev) {
// Somebody else installed the segment, use that one.
Segment::delete_segment(next);
@ -126,9 +125,9 @@ G1MonotonicArena::Segment* G1MonotonicArena::new_segment(Segment* const prev) {
_last = next;
}
// Successfully installed the segment into the list.
AtomicAccess::inc(&_num_segments, memory_order_relaxed);
AtomicAccess::add(&_mem_size, next->mem_size(), memory_order_relaxed);
AtomicAccess::add(&_num_total_slots, next->num_slots(), memory_order_relaxed);
_num_segments.add_then_fetch(1u, memory_order_relaxed);
_mem_size.add_then_fetch(next->mem_size(), memory_order_relaxed);
_num_total_slots.add_then_fetch(next->num_slots(), memory_order_relaxed);
return next;
}
}
@ -155,7 +154,7 @@ uint G1MonotonicArena::slot_size() const {
}
void G1MonotonicArena::drop_all() {
Segment* cur = AtomicAccess::load_acquire(&_first);
Segment* cur = _first.load_acquire();
if (cur != nullptr) {
assert(_last != nullptr, "If there is at least one segment, there must be a last one.");
@ -175,25 +174,25 @@ void G1MonotonicArena::drop_all() {
cur = next;
}
#endif
assert(num_segments == _num_segments, "Segment count inconsistent %u %u", num_segments, _num_segments);
assert(mem_size == _mem_size, "Memory size inconsistent");
assert(num_segments == _num_segments.load_relaxed(), "Segment count inconsistent %u %u", num_segments, _num_segments.load_relaxed());
assert(mem_size == _mem_size.load_relaxed(), "Memory size inconsistent");
assert(last == _last, "Inconsistent last segment");
_segment_free_list->bulk_add(*first, *_last, _num_segments, _mem_size);
_segment_free_list->bulk_add(*first, *_last, _num_segments.load_relaxed(), _mem_size.load_relaxed());
}
_first = nullptr;
_first.store_relaxed(nullptr);
_last = nullptr;
_num_segments = 0;
_mem_size = 0;
_num_total_slots = 0;
_num_allocated_slots = 0;
_num_segments.store_relaxed(0);
_mem_size.store_relaxed(0);
_num_total_slots.store_relaxed(0);
_num_allocated_slots.store_relaxed(0);
}
void* G1MonotonicArena::allocate() {
assert(slot_size() > 0, "instance size not set.");
Segment* cur = AtomicAccess::load_acquire(&_first);
Segment* cur = _first.load_acquire();
if (cur == nullptr) {
cur = new_segment(cur);
}
@ -201,7 +200,7 @@ void* G1MonotonicArena::allocate() {
while (true) {
void* slot = cur->allocate_slot();
if (slot != nullptr) {
AtomicAccess::inc(&_num_allocated_slots, memory_order_relaxed);
_num_allocated_slots.add_then_fetch(1u, memory_order_relaxed);
guarantee(is_aligned(slot, _alloc_options->slot_alignment()),
"result " PTR_FORMAT " not aligned at %u", p2i(slot), _alloc_options->slot_alignment());
return slot;
@ -213,7 +212,7 @@ void* G1MonotonicArena::allocate() {
}
uint G1MonotonicArena::num_segments() const {
return AtomicAccess::load(&_num_segments);
return _num_segments.load_relaxed();
}
#ifdef ASSERT
@ -238,7 +237,7 @@ uint G1MonotonicArena::calculate_length() const {
template <typename SegmentClosure>
void G1MonotonicArena::iterate_segments(SegmentClosure& closure) const {
Segment* cur = AtomicAccess::load_acquire(&_first);
Segment* cur = _first.load_acquire();
assert((cur != nullptr) == (_last != nullptr),
"If there is at least one segment, there must be a last one");

47
src/hotspot/share/gc/g1/g1MonotonicArena.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -28,6 +28,7 @@
#include "gc/shared/freeListAllocator.hpp"
#include "nmt/memTag.hpp"
#include "runtime/atomic.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/lockFreeStack.hpp"
@ -65,27 +66,27 @@ private:
// AllocOptions provides parameters for Segment sizing and expansion.
const AllocOptions* _alloc_options;
Segment* volatile _first; // The (start of the) list of all segments.
Segment* _last; // The last segment of the list of all segments.
volatile uint _num_segments; // Number of assigned segments to this allocator.
volatile size_t _mem_size; // Memory used by all segments.
Atomic<Segment*> _first; // The (start of the) list of all segments.
Segment* _last; // The last segment of the list of all segments.
Atomic<uint> _num_segments; // Number of assigned segments to this allocator.
Atomic<size_t> _mem_size; // Memory used by all segments.
SegmentFreeList* _segment_free_list; // The global free segment list to preferentially
// get new segments from.
volatile uint _num_total_slots; // Number of slots available in all segments (allocated + not yet used).
volatile uint _num_allocated_slots; // Number of total slots allocated ever (including free and pending).
Atomic<uint> _num_total_slots; // Number of slots available in all segments (allocated + not yet used).
Atomic<uint> _num_allocated_slots; // Number of total slots allocated ever (including free and pending).
inline Segment* new_segment(Segment* const prev);
DEBUG_ONLY(uint calculate_length() const;)
public:
const Segment* first_segment() const { return AtomicAccess::load(&_first); }
const Segment* first_segment() const { return _first.load_relaxed(); }
uint num_total_slots() const { return AtomicAccess::load(&_num_total_slots); }
uint num_total_slots() const { return _num_total_slots.load_relaxed(); }
uint num_allocated_slots() const {
uint allocated = AtomicAccess::load(&_num_allocated_slots);
uint allocated = _num_allocated_slots.load_relaxed();
assert(calculate_length() == allocated, "Must be");
return allocated;
}
@ -116,11 +117,11 @@ static constexpr uint SegmentPayloadMaxAlignment = 8;
class alignas(SegmentPayloadMaxAlignment) G1MonotonicArena::Segment {
const uint _slot_size;
const uint _num_slots;
Segment* volatile _next;
Atomic<Segment*> _next;
// Index into the next free slot to allocate into. Full if equal (or larger)
// to _num_slots (can be larger because we atomically increment this value and
// check only afterwards if the allocation has been successful).
uint volatile _next_allocate;
Atomic<uint> _next_allocate;
const MemTag _mem_tag;
static size_t header_size() { return align_up(sizeof(Segment), SegmentPayloadMaxAlignment); }
@ -139,21 +140,21 @@ class alignas(SegmentPayloadMaxAlignment) G1MonotonicArena::Segment {
Segment(uint slot_size, uint num_slots, Segment* next, MemTag mem_tag);
~Segment() = default;
public:
Segment* volatile* next_addr() { return &_next; }
Atomic<Segment*>* next_addr() { return &_next; }
void* allocate_slot();
uint num_slots() const { return _num_slots; }
Segment* next() const { return _next; }
Segment* next() const { return _next.load_relaxed(); }
void set_next(Segment* next) {
assert(next != this, " loop condition");
_next = next;
_next.store_relaxed(next);
}
void reset(Segment* next) {
_next_allocate = 0;
_next_allocate.store_relaxed(0);
assert(next != this, " loop condition");
set_next(next);
memset(payload(0), 0, payload_size());
@ -166,7 +167,7 @@ public:
uint length() const {
// _next_allocate might grow larger than _num_slots in multi-thread environments
// due to races.
return MIN2(_next_allocate, _num_slots);
return MIN2(_next_allocate.load_relaxed(), _num_slots);
}
static size_t size_in_bytes(uint slot_size, uint num_slots) {
@ -176,7 +177,7 @@ public:
static Segment* create_segment(uint slot_size, uint num_slots, Segment* next, MemTag mem_tag);
static void delete_segment(Segment* segment);
bool is_full() const { return _next_allocate >= _num_slots; }
bool is_full() const { return _next_allocate.load_relaxed() >= _num_slots; }
};
static_assert(alignof(G1MonotonicArena::Segment) >= SegmentPayloadMaxAlignment, "assert alignment of Segment (and indirectly its payload)");
@ -186,15 +187,15 @@ static_assert(alignof(G1MonotonicArena::Segment) >= SegmentPayloadMaxAlignment,
// performed by multiple threads concurrently.
// Counts and memory usage are current on a best-effort basis if accessed concurrently.
class G1MonotonicArena::SegmentFreeList {
static Segment* volatile* next_ptr(Segment& segment) {
static Atomic<Segment*>* next_ptr(Segment& segment) {
return segment.next_addr();
}
using SegmentStack = LockFreeStack<Segment, &SegmentFreeList::next_ptr>;
SegmentStack _list;
volatile size_t _num_segments;
volatile size_t _mem_size;
Atomic<size_t> _num_segments;
Atomic<size_t> _mem_size;
public:
SegmentFreeList() : _list(), _num_segments(0), _mem_size(0) { }
@ -210,8 +211,8 @@ public:
void print_on(outputStream* out, const char* prefix = "");
size_t num_segments() const { return AtomicAccess::load(&_num_segments); }
size_t mem_size() const { return AtomicAccess::load(&_mem_size); }
size_t num_segments() const { return _num_segments.load_relaxed(); }
size_t mem_size() const { return _mem_size.load_relaxed(); }
};
// Configuration for G1MonotonicArena, e.g slot size, slot number of next Segment.

11
src/hotspot/share/gc/g1/g1MonotonicArena.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -28,14 +28,13 @@
#include "gc/g1/g1MonotonicArena.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/globalCounter.inline.hpp"
inline void* G1MonotonicArena::Segment::allocate_slot() {
if (_next_allocate >= _num_slots) {
if (_next_allocate.load_relaxed() >= _num_slots) {
return nullptr;
}
uint result = AtomicAccess::fetch_then_add(&_next_allocate, 1u, memory_order_relaxed);
uint result = _next_allocate.fetch_then_add(1u, memory_order_relaxed);
if (result >= _num_slots) {
return nullptr;
}
@ -48,8 +47,8 @@ inline G1MonotonicArena::Segment* G1MonotonicArena::SegmentFreeList::get() {
Segment* result = _list.pop();
if (result != nullptr) {
AtomicAccess::dec(&_num_segments, memory_order_relaxed);
AtomicAccess::sub(&_mem_size, result->mem_size(), memory_order_relaxed);
_num_segments.sub_then_fetch(1u, memory_order_relaxed);
_mem_size.sub_then_fetch(result->mem_size(), memory_order_relaxed);
}
return result;
}

3
src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +28,6 @@
#include "nmt/memTracker.hpp"
#include "oops/markWord.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/os.hpp"
#include "utilities/align.hpp"
#include "utilities/bitMap.inline.hpp"

11
src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -42,7 +42,6 @@
#include "memory/allocation.inline.hpp"
#include "oops/access.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/prefetch.inline.hpp"
#include "utilities/globalDefinitions.hpp"
@ -238,9 +237,9 @@ void G1ParScanThreadState::do_partial_array(PartialArrayState* state, bool stole
G1HeapRegionAttr dest_attr = _g1h->region_attr(to_array);
G1SkipCardMarkSetter x(&_scanner, dest_attr.is_new_survivor());
// Process claimed task.
to_array->oop_iterate_range(&_scanner,
checked_cast<int>(claim._start),
checked_cast<int>(claim._end));
to_array->oop_iterate_elements_range(&_scanner,
checked_cast<int>(claim._start),
checked_cast<int>(claim._end));
}
MAYBE_INLINE_EVACUATION
@ -260,7 +259,7 @@ void G1ParScanThreadState::start_partial_objarray(oop from_obj,
// Process the initial chunk. No need to process the type in the
// klass, as it will already be handled by processing the built-in
// module.
to_array->oop_iterate_range(&_scanner, 0, checked_cast<int>(initial_chunk_size));
to_array->oop_iterate_elements_range(&_scanner, 0, checked_cast<int>(initial_chunk_size));
}
MAYBE_INLINE_EVACUATION

11
src/hotspot/share/gc/g1/g1Policy.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -943,7 +943,7 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
phase_times()->sum_thread_work_items(G1GCPhaseTimes::MergePSS, G1GCPhaseTimes::MergePSSToYoungGenCards));
}
record_pause(this_pause, start_time_sec, end_time_sec, allocation_failure);
record_pause(this_pause, start_time_sec, end_time_sec);
if (G1GCPauseTypeHelper::is_last_young_pause(this_pause)) {
assert(!G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause),
@ -1389,16 +1389,13 @@ void G1Policy::update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_t
void G1Policy::record_pause(G1GCPauseType gc_type,
double start,
double end,
bool allocation_failure) {
double end) {
// Manage the MMU tracker. For some reason it ignores Full GCs.
if (gc_type != G1GCPauseType::FullGC) {
_mmu_tracker->add_pause(start, end);
}
if (!allocation_failure) {
update_gc_pause_time_ratios(gc_type, start, end);
}
update_gc_pause_time_ratios(gc_type, start, end);
update_time_to_mixed_tracking(gc_type, start, end);

5
src/hotspot/share/gc/g1/g1Policy.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -275,8 +275,7 @@ private:
// Record the given STW pause with the given start and end times (in s).
void record_pause(G1GCPauseType gc_type,
double start,
double end,
bool allocation_failure = false);
double end);
void update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_sec, double end_sec);

24
src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,6 +27,7 @@
#include "memory/allocation.hpp"
#include "oops/oop.hpp"
#include "runtime/atomic.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/pair.hpp"
@ -40,20 +41,23 @@
// * the number of incoming references found during marking. This is an approximate
// value because we do not mark through all objects.
struct G1RegionMarkStats {
size_t _live_words;
size_t _incoming_refs;
Atomic<size_t> _live_words;
Atomic<size_t> _incoming_refs;
// Clear all members.
void clear() {
_live_words = 0;
_incoming_refs = 0;
_live_words.store_relaxed(0);
_incoming_refs.store_relaxed(0);
}
// Clear all members after a marking overflow. Only needs to clear the number of
// incoming references as all objects will be rescanned, while the live words are
// gathered whenever a thread can mark an object, which is synchronized.
void clear_during_overflow() {
_incoming_refs = 0;
_incoming_refs.store_relaxed(0);
}
size_t live_words() const { return _live_words.load_relaxed(); }
size_t incoming_refs() const { return _incoming_refs.load_relaxed(); }
};
// Per-marking thread cache for the region mark statistics.
@ -112,12 +116,16 @@ public:
void add_live_words(oop obj);
void add_live_words(uint region_idx, size_t live_words) {
G1RegionMarkStatsCacheEntry* const cur = find_for_add(region_idx);
cur->_stats._live_words += live_words;
// This method is only ever called single-threaded, so we do not need atomic
// update here.
cur->_stats._live_words.store_relaxed(cur->_stats.live_words() + live_words);
}
void inc_incoming_refs(uint region_idx) {
G1RegionMarkStatsCacheEntry* const cur = find_for_add(region_idx);
cur->_stats._incoming_refs++;
// This method is only ever called single-threaded, so we do not need atomic
// update here.
cur->_stats._incoming_refs.store_relaxed(cur->_stats.incoming_refs() + 1u);
}
void reset(uint region_idx) {

12
src/hotspot/share/gc/g1/g1RegionMarkStatsCache.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,8 +27,6 @@
#include "gc/g1/g1RegionMarkStatsCache.hpp"
#include "runtime/atomicAccess.hpp"
inline G1RegionMarkStatsCache::G1RegionMarkStatsCacheEntry* G1RegionMarkStatsCache::find_for_add(uint region_idx) {
uint const cache_idx = hash(region_idx);
@ -46,12 +44,12 @@ inline G1RegionMarkStatsCache::G1RegionMarkStatsCacheEntry* G1RegionMarkStatsCac
inline void G1RegionMarkStatsCache::evict(uint idx) {
G1RegionMarkStatsCacheEntry* cur = &_cache[idx];
if (cur->_stats._live_words != 0) {
AtomicAccess::add(&_target[cur->_region_idx]._live_words, cur->_stats._live_words);
if (cur->_stats.live_words() != 0) {
_target[cur->_region_idx]._live_words.add_then_fetch(cur->_stats.live_words());
}
if (cur->_stats._incoming_refs != 0) {
AtomicAccess::add(&_target[cur->_region_idx]._incoming_refs, cur->_stats._incoming_refs);
if (cur->_stats.incoming_refs() != 0) {
_target[cur->_region_idx]._incoming_refs.add_then_fetch(cur->_stats.incoming_refs());
}
cur->clear();

2
src/hotspot/share/gc/serial/serialFullGC.cpp
View File

@ -412,7 +412,7 @@ void SerialFullGC::follow_array_chunk(objArrayOop array, int index) {
const int stride = MIN2(len - beg_index, (int) ObjArrayMarkingStride);
const int end_index = beg_index + stride;
array->oop_iterate_range(&mark_and_push_closure, beg_index, end_index);
array->oop_iterate_elements_range(&mark_and_push_closure, beg_index, end_index);
if (end_index < len) {
SerialFullGC::push_objarray(array, end_index); // Push the continuation.

16
src/hotspot/share/gc/shared/workerThread.cpp
View File

@ -96,8 +96,22 @@ void WorkerThreads::initialize_workers() {
}
}
bool WorkerThreads::allow_inject_creation_failure() const {
if (!is_init_completed()) {
// Never allow creation failures during VM init
return false;
}
if (_created_workers == 0) {
// Never allow creation failures of the first worker, it will cause the VM to exit
return false;
}
return true;
}
WorkerThread* WorkerThreads::create_worker(uint name_suffix) {
if (is_init_completed() && InjectGCWorkerCreationFailure) {
if (InjectGCWorkerCreationFailure && allow_inject_creation_failure()) {
return nullptr;
}

1
src/hotspot/share/gc/shared/workerThread.hpp
View File

@ -104,6 +104,7 @@ public:
WorkerThreads(const char* name, uint max_workers);
void initialize_workers();
bool allow_inject_creation_failure() const;
uint max_workers() const { return _max_workers; }
uint created_workers() const { return _created_workers; }

25
src/hotspot/share/gc/shared/workerUtils.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -22,8 +22,9 @@
*
*/
#include "cppstdlib/new.hpp"
#include "gc/shared/workerUtils.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
// *** WorkerThreadsBarrierSync
@ -80,21 +81,21 @@ void WorkerThreadsBarrierSync::abort() {
SubTasksDone::SubTasksDone(uint n) :
_tasks(nullptr), _n_tasks(n) {
_tasks = NEW_C_HEAP_ARRAY(bool, n, mtInternal);
_tasks = NEW_C_HEAP_ARRAY(Atomic<bool>, n, mtInternal);
for (uint i = 0; i < _n_tasks; i++) {
_tasks[i] = false;
::new (&_tasks[i]) Atomic<bool>(false);
}
}
#ifdef ASSERT
void SubTasksDone::all_tasks_claimed_impl(uint skipped[], size_t skipped_size) {
if (AtomicAccess::cmpxchg(&_verification_done, false, true)) {
if (!_verification_done.compare_set(false, true)) {
// another thread has done the verification
return;
}
// all non-skipped tasks are claimed
for (uint i = 0; i < _n_tasks; ++i) {
if (!_tasks[i]) {
if (!_tasks[i].load_relaxed()) {
auto is_skipped = false;
for (size_t j = 0; j < skipped_size; ++j) {
if (i == skipped[j]) {
@ -109,27 +110,27 @@ void SubTasksDone::all_tasks_claimed_impl(uint skipped[], size_t skipped_size) {
for (size_t i = 0; i < skipped_size; ++i) {
auto task_index = skipped[i];
assert(task_index < _n_tasks, "Array in range.");
assert(!_tasks[task_index], "%d is both claimed and skipped.", task_index);
assert(!_tasks[task_index].load_relaxed(), "%d is both claimed and skipped.", task_index);
}
}
#endif
bool SubTasksDone::try_claim_task(uint t) {
assert(t < _n_tasks, "bad task id.");
return !_tasks[t] && !AtomicAccess::cmpxchg(&_tasks[t], false, true);
return !_tasks[t].load_relaxed() && _tasks[t].compare_set(false, true);
}
SubTasksDone::~SubTasksDone() {
assert(_verification_done, "all_tasks_claimed must have been called.");
FREE_C_HEAP_ARRAY(bool, _tasks);
assert(_verification_done.load_relaxed(), "all_tasks_claimed must have been called.");
FREE_C_HEAP_ARRAY(Atomic<bool>, _tasks);
}
// *** SequentialSubTasksDone
bool SequentialSubTasksDone::try_claim_task(uint& t) {
t = _num_claimed;
t = _num_claimed.load_relaxed();
if (t < _num_tasks) {
t = AtomicAccess::add(&_num_claimed, 1u) - 1;
t = _num_claimed.fetch_then_add(1u);
}
return t < _num_tasks;
}

12
src/hotspot/share/gc/shared/workerUtils.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,6 +28,7 @@
#include "cppstdlib/type_traits.hpp"
#include "memory/allocation.hpp"
#include "metaprogramming/enableIf.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutex.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
@ -79,11 +80,11 @@ public:
// enumeration type.
class SubTasksDone: public CHeapObj<mtInternal> {
volatile bool* _tasks;
Atomic<bool>* _tasks;
uint _n_tasks;
// make sure verification logic is run exactly once to avoid duplicate assertion failures
DEBUG_ONLY(volatile bool _verification_done = false;)
DEBUG_ONLY(Atomic<bool> _verification_done;)
void all_tasks_claimed_impl(uint skipped[], size_t skipped_size) NOT_DEBUG_RETURN;
NONCOPYABLE(SubTasksDone);
@ -127,7 +128,7 @@ public:
class SequentialSubTasksDone : public CHeapObj<mtInternal> {
uint _num_tasks; // Total number of tasks available.
volatile uint _num_claimed; // Number of tasks claimed.
Atomic<uint> _num_claimed; // Number of tasks claimed.
NONCOPYABLE(SequentialSubTasksDone);
@ -135,7 +136,8 @@ public:
SequentialSubTasksDone(uint num_tasks) : _num_tasks(num_tasks), _num_claimed(0) { }
~SequentialSubTasksDone() {
// Claiming may try to claim more tasks than there are.
assert(_num_claimed >= _num_tasks, "Claimed %u tasks of %u", _num_claimed, _num_tasks);
assert(_num_claimed.load_relaxed() >= _num_tasks,
"Claimed %u tasks of %u", _num_claimed.load_relaxed(), _num_tasks);
}
// Attempt to claim the next unclaimed task in the sequence,

7
src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
View File

@ -129,6 +129,13 @@ protected:
#endif
}
inline void update_livedata(size_t live) {
_region_union._live_data = live;
#ifdef ASSERT
_union_tag = is_live_data;
#endif
}
inline ShenandoahHeapRegion* get_region() const {
assert(_union_tag != is_uninitialized, "Cannot fetch region from uninitialized RegionData");
return _region;

15
src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
View File

@ -89,6 +89,17 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll
return false;
}
// Between consecutive mixed-evacuation cycles, the live data within each candidate region may change due to
// promotions and old-gen evacuations. Re-sort the candidate regions in order to first evacuate regions that have
// the smallest amount of live data. These are easiest to evacuate with least effort. Doing these first allows
// us to more quickly replenish free memory with empty regions.
for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; i++) {
ShenandoahHeapRegion* r = _region_data[i].get_region();
_region_data[i].update_livedata(r->get_mixed_candidate_live_data_bytes());
}
QuickSort::sort<RegionData>(_region_data + _next_old_collection_candidate, unprocessed_old_collection_candidates(),
compare_by_live);
_first_pinned_candidate = NOT_FOUND;
uint included_old_regions = 0;
@ -414,6 +425,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
ShenandoahHeapRegion* r = candidates[i].get_region();
size_t region_garbage = r->garbage();
size_t region_free = r->free();
r->capture_mixed_candidate_garbage();
candidates_garbage += region_garbage;
unfragmented += region_free;
}
@ -456,6 +469,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
r->index(), ShenandoahHeapRegion::region_state_to_string(r->state()));
const size_t region_garbage = r->garbage();
const size_t region_free = r->free();
r->capture_mixed_candidate_garbage();
candidates_garbage += region_garbage;
unfragmented += region_free;
defrag_count++;

11
src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
View File

@ -338,7 +338,7 @@ void ShenandoahRegionPartitions::make_all_regions_unavailable() {
_empty_region_counts[partition_id] = 0;
_used[partition_id] = 0;
_humongous_waste[partition_id] = 0;
_available[partition_id] = FreeSetUnderConstruction;
_available[partition_id] = 0;
}
}
@ -2495,6 +2495,10 @@ void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_r
void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_trashed_regions, size_t &old_trashed_regions,
size_t &first_old_region, size_t &last_old_region, size_t &old_region_count) {
shenandoah_assert_heaplocked();
assert(rebuild_lock() != nullptr, "sanity");
rebuild_lock()->lock(false);
// This resets all state information, removing all regions from all sets.
clear();
log_debug(gc, free)("Rebuilding FreeSet");
// This places regions that have alloc_capacity into the old_collector set if they identify as is_old() or the
@ -2524,6 +2528,9 @@ void ShenandoahFreeSet::finish_rebuild(size_t young_trashed_regions, size_t old_
_total_young_regions = _heap->num_regions() - old_region_count;
_total_global_regions = _heap->num_regions();
establish_old_collector_alloc_bias();
// Release the rebuild lock now. What remains in this function is read-only
rebuild_lock()->unlock();
_partitions.assert_bounds(true);
log_status();
}
@ -3058,7 +3065,7 @@ void ShenandoahFreeSet::log_status() {
size_t max_humongous = max_contig * ShenandoahHeapRegion::region_size_bytes();
// capacity() is capacity of mutator
// used() is used of mutator
size_t free = capacity() - used();
size_t free = capacity_holding_lock() - used_holding_lock();
// Since certain regions that belonged to the Mutator free partition at the time of most recent rebuild may have been
// retired, the sum of used and capacities within regions that are still in the Mutator free partition may not match
// my internally tracked values of used() and free().

74
src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
View File

@ -28,9 +28,13 @@
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
#include "logging/logStream.hpp"
typedef ShenandoahLock ShenandoahRebuildLock;
typedef ShenandoahLocker ShenandoahRebuildLocker;
// Each ShenandoahHeapRegion is associated with a ShenandoahFreeSetPartitionId.
enum class ShenandoahFreeSetPartitionId : uint8_t {
Mutator, // Region is in the Mutator free set: available memory is available to mutators.
@ -139,8 +143,6 @@ public:
ShenandoahRegionPartitions(size_t max_regions, ShenandoahFreeSet* free_set);
~ShenandoahRegionPartitions() {}
static const size_t FreeSetUnderConstruction = SIZE_MAX;
inline idx_t max() const { return _max; }
// At initialization, reset OldCollector tallies
@ -352,6 +354,16 @@ public:
return _available[int(which_partition)];
}
// Return available_in assuming caller does not hold the heap lock but does hold the rebuild_lock.
// The returned value may be "slightly stale" because we do not assure that every fetch of this value
// sees the most recent update of this value. Requiring the caller to hold the rebuild_lock assures
// that we don't see "bogus" values that are "worse than stale". During rebuild of the freeset, the
// value of _available is not reliable.
inline size_t available_in_locked_for_rebuild(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
return _available[int(which_partition)];
}
// Returns bytes of humongous waste
inline size_t humongous_waste(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
@ -359,23 +371,6 @@ public:
return _humongous_waste[int(which_partition)];
}
// Return available_in assuming caller does not hold the heap lock. In production builds, available is
// returned without acquiring the lock. In debug builds, the global heap lock is acquired in order to
// enforce a consistency assert.
inline size_t available_in_not_locked(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
shenandoah_assert_not_heaplocked();
#ifdef ASSERT
ShenandoahHeapLocker locker(ShenandoahHeap::heap()->lock());
assert((_available[int(which_partition)] == FreeSetUnderConstruction) ||
(_available[int(which_partition)] == _capacity[int(which_partition)] - _used[int(which_partition)]),
"Expect available (%zu) equals capacity (%zu) - used (%zu) for partition %s",
_available[int(which_partition)], _capacity[int(which_partition)], _used[int(which_partition)],
partition_membership_name(idx_t(which_partition)));
#endif
return _available[int(which_partition)];
}
inline void set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value);
inline void set_used_by(ShenandoahFreeSetPartitionId which_partition, size_t value) {
@ -440,6 +435,15 @@ private:
ShenandoahHeap* const _heap;
ShenandoahRegionPartitions _partitions;
// This locks the rebuild process (in combination with the global heap lock). Whenever we rebuild the free set,
// we first acquire the global heap lock and then we acquire this _rebuild_lock in a nested context. Threads that
// need to check available, acquire only the _rebuild_lock to make sure that they are not obtaining the value of
// available for a partially reconstructed free-set.
//
// Note that there is rank ordering of nested locks to prevent deadlock. All threads that need to acquire both
// locks will acquire them in the same order: first the global heap lock and then the rebuild lock.
ShenandoahRebuildLock _rebuild_lock;
size_t _total_humongous_waste;
HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r);
@ -635,10 +639,12 @@ private:
void log_status();
public:
static const size_t FreeSetUnderConstruction = ShenandoahRegionPartitions::FreeSetUnderConstruction;
ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions);
ShenandoahRebuildLock* rebuild_lock() {
return &_rebuild_lock;
}
inline size_t max_regions() const { return _partitions.max(); }
ShenandoahFreeSetPartitionId membership(size_t index) const { return _partitions.membership(index); }
inline void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition,
@ -776,9 +782,29 @@ public:
// adjusts available with respect to lock holders. However, sequential calls to these three functions may produce
// inconsistent data: available may not equal capacity - used because the intermediate states of any "atomic"
// locked action can be seen by these unlocked functions.
inline size_t capacity() const { return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator); }
inline size_t used() const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator); }
inline size_t available() const { return _partitions.available_in_not_locked(ShenandoahFreeSetPartitionId::Mutator); }
inline size_t capacity_holding_lock() const {
shenandoah_assert_heaplocked();
return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator);
}
inline size_t capacity_not_holding_lock() {
shenandoah_assert_not_heaplocked();
ShenandoahRebuildLocker locker(rebuild_lock());
return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator);
}
inline size_t used_holding_lock() const {
shenandoah_assert_heaplocked();
return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator);
}
inline size_t used_not_holding_lock() {
shenandoah_assert_not_heaplocked();
ShenandoahRebuildLocker locker(rebuild_lock());
return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator);
}
inline size_t available() {
shenandoah_assert_not_heaplocked();
ShenandoahRebuildLocker locker(rebuild_lock());
return _partitions.available_in_locked_for_rebuild(ShenandoahFreeSetPartitionId::Mutator);
}
inline size_t total_humongous_waste() const { return _total_humongous_waste; }
inline size_t humongous_waste_in_mutator() const { return _partitions.humongous_waste(ShenandoahFreeSetPartitionId::Mutator); }

21
src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
View File

@ -1113,18 +1113,17 @@ void ShenandoahFullGC::phase5_epilog() {
ShenandoahPostCompactClosure post_compact;
heap->heap_region_iterate(&post_compact);
heap->collection_set()->clear();
size_t young_cset_regions, old_cset_regions;
size_t first_old, last_old, num_old;
heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
// We also do not expand old generation size following Full GC because we have scrambled age populations and
// no longer have objects separated by age into distinct regions.
if (heap->mode()->is_generational()) {
ShenandoahGenerationalFullGC::compute_balances();
size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old;
ShenandoahFreeSet* free_set = heap->free_set();
{
free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
// We also do not expand old generation size following Full GC because we have scrambled age populations and
// no longer have objects separated by age into distinct regions.
if (heap->mode()->is_generational()) {
ShenandoahGenerationalFullGC::compute_balances();
}
free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
}
heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
// Set mark incomplete because the marking bitmaps have been reset except pinned regions.
_generation->set_mark_incomplete();

3
src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
View File

@ -815,10 +815,9 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
ShenandoahHeapLocker locker(heap->lock());
size_t young_cset_regions, old_cset_regions;
// We are preparing for evacuation. At this time, we ignore cset region tallies.
size_t first_old, last_old, num_old;
size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old;
_free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
if (heap->mode()->is_generational()) {

11
src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp
View File

@ -104,17 +104,6 @@ void ShenandoahGenerationalHeap::initialize_heuristics() {
// Initialize global generation and heuristics even in generational mode.
ShenandoahHeap::initialize_heuristics();
// Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity
// for old would be total heap - minimum capacity of young. This means the sum of the maximum
// allowed for old and young could exceed the total heap size. It remains the case that the
// _actual_ capacity of young + old = total.
size_t region_count = num_regions();
size_t max_young_regions = MAX2((region_count * ShenandoahMaxYoungPercentage) / 100, (size_t) 1U);
size_t initial_capacity_young = max_young_regions * ShenandoahHeapRegion::region_size_bytes();
size_t max_capacity_young = initial_capacity_young;
size_t initial_capacity_old = max_capacity() - max_capacity_young;
size_t max_capacity_old = max_capacity() - initial_capacity_young;
_young_generation = new ShenandoahYoungGeneration(max_workers());
_old_generation = new ShenandoahOldGeneration(max_workers());
_young_generation->initialize_heuristics(mode());

16
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
View File

@ -426,8 +426,6 @@ jint ShenandoahHeap::initialize() {
_affiliations[i] = ShenandoahAffiliation::FREE;
}
_free_set = new ShenandoahFreeSet(this, _num_regions);
post_initialize_heuristics();
// We are initializing free set. We ignore cset region tallies.
size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old;
@ -1658,7 +1656,7 @@ void ShenandoahHeap::verify(VerifyOption vo) {
}
}
size_t ShenandoahHeap::tlab_capacity() const {
return _free_set->capacity();
return _free_set->capacity_not_holding_lock();
}
class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
@ -2138,7 +2136,7 @@ GCTracer* ShenandoahHeap::tracer() {
}
size_t ShenandoahHeap::tlab_used() const {
return _free_set->used();
return _free_set->used_not_holding_lock();
}
bool ShenandoahHeap::try_cancel_gc(GCCause::Cause cause) {
@ -2528,8 +2526,7 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) {
ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
ShenandoahHeapLocker locker(lock());
size_t young_cset_regions, old_cset_regions;
size_t first_old_region, last_old_region, old_region_count;
size_t young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count;
_free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
// If there are no old regions, first_old_region will be greater than last_old_region
assert((first_old_region > last_old_region) ||
@ -2548,13 +2545,14 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) {
// The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
// available for transfer to old. Note that transfer of humongous regions does not impact available.
ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
size_t allocation_runway = gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
size_t allocation_runway =
gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions);
// Total old_available may have been expanded to hold anticipated promotions. We trigger if the fragmented available
// memory represents more than 16 regions worth of data. Note that fragmentation may increase when we promote regular
// regions in place when many of these regular regions have an abundant amount of available memory within them. Fragmentation
// will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
// regions in place when many of these regular regions have an abundant amount of available memory within them.
// Fragmentation will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
//
// We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
// within partially consumed regions of memory.

2
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
View File

@ -75,6 +75,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
_plab_allocs(0),
_live_data(0),
_critical_pins(0),
_mixed_candidate_garbage_words(0),
_update_watermark(start),
_age(0),
#ifdef SHENANDOAH_CENSUS_NOISE
@ -565,6 +566,7 @@ void ShenandoahHeapRegion::recycle_internal() {
assert(_recycling.is_set() && is_trash(), "Wrong state");
ShenandoahHeap* heap = ShenandoahHeap::heap();
_mixed_candidate_garbage_words = 0;
set_top(bottom());
clear_live_data();
reset_alloc_metadata();

11
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
View File

@ -43,6 +43,7 @@ class ShenandoahHeapRegion {
friend class VMStructs;
friend class ShenandoahHeapRegionStateConstant;
private:
/*
Region state is described by a state machine. Transitions are guarded by
heap lock, which allows changing the state of several regions atomically.
@ -259,6 +260,8 @@ private:
volatile size_t _live_data;
volatile size_t _critical_pins;
size_t _mixed_candidate_garbage_words;
HeapWord* volatile _update_watermark;
uint _age;
@ -398,6 +401,14 @@ public:
// above TAMS.
inline size_t get_live_data_words() const;
inline size_t get_mixed_candidate_live_data_bytes() const;
inline size_t get_mixed_candidate_live_data_words() const;
inline void capture_mixed_candidate_garbage();
// Returns garbage by calculating difference between used and get_live_data_words. The value returned is only
// meaningful immediately following completion of marking. If there have been subsequent allocations in this region,
// use a different approach to determine garbage, such as (used() - get_mixed_candidate_live_data_bytes())
inline size_t garbage() const;
void print_on(outputStream* st) const;

17
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
View File

@ -163,6 +163,23 @@ inline size_t ShenandoahHeapRegion::get_live_data_bytes() const {
return get_live_data_words() * HeapWordSize;
}
inline size_t ShenandoahHeapRegion::get_mixed_candidate_live_data_bytes() const {
shenandoah_assert_heaplocked_or_safepoint();
assert(used() >= _mixed_candidate_garbage_words * HeapWordSize, "used must exceed garbage");
return used() - _mixed_candidate_garbage_words * HeapWordSize;
}
inline size_t ShenandoahHeapRegion::get_mixed_candidate_live_data_words() const {
shenandoah_assert_heaplocked_or_safepoint();
assert(used() >= _mixed_candidate_garbage_words * HeapWordSize, "used must exceed garbage");
return used() / HeapWordSize - _mixed_candidate_garbage_words;
}
inline void ShenandoahHeapRegion::capture_mixed_candidate_garbage() {
shenandoah_assert_heaplocked_or_safepoint();
_mixed_candidate_garbage_words = garbage() / HeapWordSize;
}
inline bool ShenandoahHeapRegion::has_live() const {
return get_live_data_words() != 0;
}

6
src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
View File

@ -167,7 +167,7 @@ inline void ShenandoahMark::do_chunked_array_start(ShenandoahObjToScanQueue* q,
if (len <= (int) ObjArrayMarkingStride*2) {
// A few slices only, process directly
array->oop_iterate_range(cl, 0, len);
array->oop_iterate_elements_range(cl, 0, len);
} else {
int bits = log2i_graceful(len);
// Compensate for non-power-of-two arrays, cover the array in excess:
@ -216,7 +216,7 @@ inline void ShenandoahMark::do_chunked_array_start(ShenandoahObjToScanQueue* q,
// Process the irregular tail, if present
int from = last_idx;
if (from < len) {
array->oop_iterate_range(cl, from, len);
array->oop_iterate_elements_range(cl, from, len);
}
}
}
@ -248,7 +248,7 @@ inline void ShenandoahMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl,
assert (0 < to && to <= len, "to is sane: %d/%d", to, len);
#endif
array->oop_iterate_range(cl, from, to);
array->oop_iterate_elements_range(cl, from, to);
}
template <ShenandoahGenerationType GENERATION>

5
src/hotspot/share/gc/shenandoah/shenandoahMetrics.cpp
View File

@ -30,7 +30,7 @@
ShenandoahMetricsSnapshot::ShenandoahMetricsSnapshot(ShenandoahFreeSet* free_set)
: _free_set(free_set)
, _used_before(free_set->used())
, _used_before(free_set->used_not_holding_lock())
, _if_before(free_set->internal_fragmentation())
, _ef_before(free_set->external_fragmentation()) {
}
@ -38,7 +38,6 @@ ShenandoahMetricsSnapshot::ShenandoahMetricsSnapshot(ShenandoahFreeSet* free_set
bool ShenandoahMetricsSnapshot::is_good_progress() const {
// Under the critical threshold?
const size_t free_actual = _free_set->available();
assert(free_actual != ShenandoahFreeSet::FreeSetUnderConstruction, "Avoid this race");
// ShenandoahCriticalFreeThreshold is expressed as a percentage. We multiply this percentage by 1/100th
// of the soft max capacity to determine whether the available memory within the mutator partition of the
@ -52,7 +51,7 @@ bool ShenandoahMetricsSnapshot::is_good_progress() const {
}
// Freed up enough?
const size_t used_after = _free_set->used();
const size_t used_after = _free_set->used_not_holding_lock();
const size_t progress_actual = (_used_before > used_after) ? _used_before - used_after : 0;
const size_t progress_expected = ShenandoahHeapRegion::region_size_bytes();
const bool prog_used = progress_actual >= progress_expected;

7
src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp
View File

@ -412,9 +412,12 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent
ShenandoahGCPhase phase(concurrent ?
ShenandoahPhaseTimings::final_rebuild_freeset :
ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
ShenandoahFreeSet* free_set = heap->free_set();
ShenandoahHeapLocker locker(heap->lock());
size_t young_trash_regions, old_trash_regions;
size_t first_old, last_old, num_old;
// This is completion of old-gen marking. We rebuild in order to reclaim immediate garbage and to
// prepare for subsequent mixed evacuations.
size_t young_trash_regions, old_trash_regions, first_old, last_old, num_old;
heap->free_set()->prepare_to_rebuild(young_trash_regions, old_trash_regions, first_old, last_old, num_old);
// At the end of old-gen, we may find that we have reclaimed immediate garbage, allowing a longer allocation runway.
// We may also find that we have accumulated canddiate regions for mixed evacuation. If so, we will want to expand

12
src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
View File

@ -430,18 +430,6 @@
"by thread type (worker or mutator) and evacuation type (young, " \
"old, or promotion.") \
\
product(uintx, ShenandoahMinYoungPercentage, 20, EXPERIMENTAL, \
"The minimum percentage of the heap to use for the young " \
"generation. Heuristics will not adjust the young generation " \
"to be less than this.") \
range(0, 100) \
\
product(uintx, ShenandoahMaxYoungPercentage, 100, EXPERIMENTAL, \
"The maximum percentage of the heap to use for the young " \
"generation. Heuristics will not adjust the young generation " \
"to be more than this.") \
range(0, 100) \
\
product(uintx, ShenandoahCriticalFreeThreshold, 1, EXPERIMENTAL, \
"How much of the heap needs to be free after recovery cycles, " \
"either Degenerated or Full GC to be claimed successful. If this "\

2
src/hotspot/share/gc/z/zHeapIterator.cpp
View File

@ -456,7 +456,7 @@ void ZHeapIterator::follow_array_chunk(const ZHeapIteratorContext& context, cons
// Follow array chunk
ZHeapIteratorOopClosure<false /* VisitReferents */> cl(this, context, obj);
ZIterator::oop_iterate_range(obj, &cl, start, end);
ZIterator::oop_iterate_elements_range(obj, &cl, start, end);
}
template <bool VisitWeaks>

2
src/hotspot/share/gc/z/zIterator.hpp
View File

@ -41,7 +41,7 @@ public:
static void oop_iterate(oop obj, OopClosureT* cl);
template <typename OopClosureT>
static void oop_iterate_range(objArrayOop obj, OopClosureT* cl, int start, int end);
static void oop_iterate_elements_range(objArrayOop obj, OopClosureT* cl, int start, int end);
// This function skips invisible roots
template <typename Function>

4
src/hotspot/share/gc/z/zIterator.inline.hpp
View File

@ -66,9 +66,9 @@ void ZIterator::oop_iterate(oop obj, OopClosureT* cl) {
}
template <typename OopClosureT>
void ZIterator::oop_iterate_range(objArrayOop obj, OopClosureT* cl, int start, int end) {
void ZIterator::oop_iterate_elements_range(objArrayOop obj, OopClosureT* cl, int start, int end) {
assert(!is_invisible_object_array(obj), "not safe");
obj->oop_iterate_range(cl, start, end);
obj->oop_iterate_elements_range(cl, start, end);
}
template <typename Function>

14
src/hotspot/share/jfr/jfr.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,6 +32,7 @@
#include "jfr/recorder/repository/jfrEmergencyDump.hpp"
#include "jfr/recorder/repository/jfrRepository.hpp"
#include "jfr/recorder/service/jfrOptionSet.hpp"
#include "jfr/recorder/service/jfrRecorderService.hpp"
#include "jfr/support/jfrClassDefineEvent.hpp"
#include "jfr/support/jfrKlassExtension.hpp"
#include "jfr/support/jfrResolution.hpp"
@ -43,6 +44,7 @@
#include "runtime/java.hpp"
#include "runtime/javaThread.hpp"
bool Jfr::is_enabled() {
return JfrRecorder::is_enabled();
}
@ -153,9 +155,9 @@ void Jfr::on_resolution(const Method* caller, const Method* target, TRAPS) {
}
#endif
void Jfr::on_vm_shutdown(bool emit_old_object_samples, bool emit_event_shutdown, bool halt) {
void Jfr::on_vm_shutdown(bool exception_handler /* false */, bool halt /* false */, bool oom /* false */) {
if (!halt && JfrRecorder::is_recording()) {
JfrEmergencyDump::on_vm_shutdown(emit_old_object_samples, emit_event_shutdown);
JfrEmergencyDump::on_vm_shutdown(exception_handler, oom);
}
}
@ -173,6 +175,12 @@ bool Jfr::on_start_flight_recording_option(const JavaVMOption** option, char* de
return JfrOptionSet::parse_start_flight_recording_option(option, delimiter);
}
void Jfr::on_report_java_out_of_memory() {
if (CrashOnOutOfMemoryError && JfrRecorder::is_recording()) {
JfrRecorderService::emit_leakprofiler_events_on_oom();
}
}
#if INCLUDE_CDS
void Jfr::on_restoration(const Klass* k, JavaThread* jt) {
assert(k != nullptr, "invariant");

5
src/hotspot/share/jfr/jfr.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -71,7 +71,7 @@ class Jfr : AllStatic {
static void on_resolution(const Method* caller, const Method* target, TRAPS);
static void on_java_thread_start(JavaThread* starter, JavaThread* startee);
static void on_set_current_thread(JavaThread* jt, oop thread);
static void on_vm_shutdown(bool emit_old_object_samples, bool emit_event_shutdown, bool halt = false);
static void on_vm_shutdown(bool exception_handler = false, bool halt = false, bool oom = false);
static void on_vm_error_report(outputStream* st);
static bool on_flight_recorder_option(const JavaVMOption** option, char* delimiter);
static bool on_start_flight_recording_option(const JavaVMOption** option, char* delimiter);
@ -79,6 +79,7 @@ class Jfr : AllStatic {
static void initialize_main_thread(JavaThread* jt);
static bool has_sample_request(JavaThread* jt);
static void check_and_process_sample_request(JavaThread* jt);
static void on_report_java_out_of_memory();
CDS_ONLY(static void on_restoration(const Klass* k, JavaThread* jt);)
};

32
src/hotspot/share/jfr/jni/jfrJniMethod.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -66,10 +66,6 @@
#include "runtime/mutexLocker.hpp"
#include "runtime/os.hpp"
#include "utilities/debug.hpp"
#ifdef LINUX
#include "os_linux.hpp"
#include "osContainer_linux.hpp"
#endif
#define NO_TRANSITION(result_type, header) extern "C" { result_type JNICALL header {
#define NO_TRANSITION_END } }
@ -364,8 +360,7 @@ JVM_ENTRY_NO_ENV(void, jfr_set_force_instrumentation(JNIEnv* env, jclass jvm, jb
JVM_END
NO_TRANSITION(void, jfr_emit_old_object_samples(JNIEnv* env, jclass jvm, jlong cutoff_ticks, jboolean emit_all, jboolean skip_bfs))
JfrRecorderService service;
service.emit_leakprofiler_events(cutoff_ticks, emit_all == JNI_TRUE, skip_bfs == JNI_TRUE);
JfrRecorderService::emit_leakprofiler_events(cutoff_ticks, emit_all == JNI_TRUE, skip_bfs == JNI_TRUE);
NO_TRANSITION_END
JVM_ENTRY_NO_ENV(void, jfr_exclude_thread(JNIEnv* env, jclass jvm, jobject t))
@ -401,35 +396,18 @@ JVM_ENTRY_NO_ENV(jboolean, jfr_is_class_instrumented(JNIEnv* env, jclass jvm, jc
JVM_END
JVM_ENTRY_NO_ENV(jboolean, jfr_is_containerized(JNIEnv* env, jclass jvm))
#ifdef LINUX
return OSContainer::is_containerized();
#else
return false;
#endif
return os::is_containerized();
JVM_END
JVM_ENTRY_NO_ENV(jlong, jfr_host_total_memory(JNIEnv* env, jclass jvm))
#ifdef LINUX
// We want the host memory, not the container limit.
// os::physical_memory() would return the container limit.
return static_cast<jlong>(os::Linux::physical_memory());
#else
return static_cast<jlong>(os::physical_memory());
#endif
return static_cast<jlong>(os::Machine::physical_memory());
JVM_END
JVM_ENTRY_NO_ENV(jlong, jfr_host_total_swap_memory(JNIEnv* env, jclass jvm))
#ifdef LINUX
// We want the host swap memory, not the container value.
physical_memory_size_type host_swap = 0;
(void)os::Linux::host_swap(host_swap); // Discard return value and treat as no swap
return static_cast<jlong>(host_swap);
#else
physical_memory_size_type total_swap_space = 0;
// Return value ignored - defaulting to 0 on failure.
(void)os::total_swap_space(total_swap_space);
(void)os::Machine::total_swap_space(total_swap_space);
return static_cast<jlong>(total_swap_space);
#endif
JVM_END
JVM_ENTRY_NO_ENV(void, jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes))

99
src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -232,41 +232,50 @@ void JfrSamplerThread::task_stacktrace(JfrSampleRequestType type, JavaThread** l
JavaThread* start = nullptr;
elapsedTimer sample_time;
sample_time.start();
ThreadsListHandle tlh;
// Resolve a sample session relative start position index into the thread list array.
// In cases where the last sampled thread is null or not-null but stale, find_index() returns -1.
_cur_index = tlh.list()->find_index_of_JavaThread(*last_thread);
JavaThread* current = _cur_index != -1 ? *last_thread : nullptr;
{
/*
* Take the Threads_lock for three purposes:
*
* 1) Avoid sampling right through a safepoint,
* which could result in touching oops in case of virtual threads.
* 2) Prevent JFR from issuing an epoch rotation while the sampler thread
* is actively processing a thread in state native, as both threads are outside the safepoint protocol.
* 3) Some operating systems (BSD / Mac) require a process lock when sending a signal with pthread_kill.
* Holding the Threads_lock prevents a JavaThread from calling os::create_thread(), which also takes the process lock.
* In a sense, we provide a coarse signal mask, so we can always send the resume signal.
*/
MutexLocker tlock(Threads_lock);
ThreadsListHandle tlh;
// Resolve a sample session relative start position index into the thread list array.
// In cases where the last sampled thread is null or not-null but stale, find_index() returns -1.
_cur_index = tlh.list()->find_index_of_JavaThread(*last_thread);
JavaThread* current = _cur_index != -1 ? *last_thread : nullptr;
while (num_samples < sample_limit) {
current = next_thread(tlh.list(), start, current);
if (current == nullptr) {
break;
}
if (is_excluded(current)) {
continue;
}
if (start == nullptr) {
start = current; // remember the thread where we started to attempt sampling
}
bool success;
if (JAVA_SAMPLE == type) {
success = sample_java_thread(current);
} else {
assert(type == NATIVE_SAMPLE, "invariant");
success = sample_native_thread(current);
}
if (success) {
num_samples++;
}
if (SafepointSynchronize::is_at_safepoint()) {
// For _thread_in_native, we cannot get the Threads_lock.
// For _thread_in_Java, well, there are none.
break;
while (num_samples < sample_limit) {
current = next_thread(tlh.list(), start, current);
if (current == nullptr) {
break;
}
if (is_excluded(current)) {
continue;
}
if (start == nullptr) {
start = current; // remember the thread where we started to attempt sampling
}
bool success;
if (JAVA_SAMPLE == type) {
success = sample_java_thread(current);
} else {
assert(type == NATIVE_SAMPLE, "invariant");
success = sample_native_thread(current);
}
if (success) {
num_samples++;
}
}
*last_thread = current; // remember the thread we last attempted to sample
}
*last_thread = current; // remember the thread we last attempted to sample
sample_time.stop();
log_trace(jfr)("JFR thread sampling done in %3.7f secs with %d java %d native samples",
sample_time.seconds(), type == JAVA_SAMPLE ? num_samples : 0, type == NATIVE_SAMPLE ? num_samples : 0);
@ -297,6 +306,7 @@ class OSThreadSampler : public SuspendedThreadTask {
// Sampling a thread in state _thread_in_Java
// involves a platform-specific thread suspend and CPU context retrieval.
bool JfrSamplerThread::sample_java_thread(JavaThread* jt) {
assert_lock_strong(Threads_lock);
if (jt->thread_state() != _thread_in_Java) {
return false;
}
@ -328,6 +338,7 @@ static JfrSamplerThread* _sampler_thread = nullptr;
// without thread suspension and CPU context retrieval,
// if we carefully order the loads of the thread state.
bool JfrSamplerThread::sample_native_thread(JavaThread* jt) {
assert_lock_strong(Threads_lock);
if (jt->thread_state() != _thread_in_native) {
return false;
}
@ -343,24 +354,14 @@ bool JfrSamplerThread::sample_native_thread(JavaThread* jt) {
SafepointMechanism::arm_local_poll_release(jt);
// Take the Threads_lock for two purposes:
// 1) Avoid sampling through a safepoint which could result
// in touching oops in case of virtual threads.
// 2) Prevent JFR from issuing an epoch rotation while the sampler thread
// is actively processing a thread in native, as both threads are now
// outside the safepoint protocol.
// OrderAccess::fence() as part of acquiring the lock prevents loads from floating up.
JfrMutexTryLock threads_lock(Threads_lock);
if (!threads_lock.acquired() || !jt->has_last_Java_frame()) {
// Remove the native sample request and release the potentially waiting thread.
JfrSampleMonitor jsm(tl);
return false;
// Separate the arming of the poll (above) from the reading of JavaThread state (below).
if (UseSystemMemoryBarrier) {
SystemMemoryBarrier::emit();
} else {
OrderAccess::fence();
}
if (jt->thread_state() != _thread_in_native) {
assert_lock_strong(Threads_lock);
if (jt->thread_state() != _thread_in_native || !jt->has_last_Java_frame()) {
JfrSampleMonitor jsm(tl);
if (jsm.is_waiting()) {
// The thread has already returned from native,

56
src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -38,6 +38,8 @@
#include "runtime/mutexLocker.hpp"
#include "runtime/os.hpp"
#include "runtime/thread.inline.hpp"
#include "runtime/vmOperations.hpp"
#include "runtime/vmThread.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/ostream.hpp"
@ -460,15 +462,6 @@ static void release_locks(Thread* thread) {
assert(thread != nullptr, "invariant");
assert(!thread->is_Java_thread() || JavaThread::cast(thread)->thread_state() == _thread_in_vm, "invariant");
#ifdef ASSERT
Mutex* owned_lock = thread->owned_locks();
while (owned_lock != nullptr) {
Mutex* next = owned_lock->next();
owned_lock->unlock();
owned_lock = next;
}
#endif // ASSERT
if (Threads_lock->owned_by_self()) {
Threads_lock->unlock();
}
@ -550,17 +543,14 @@ class JavaThreadInVMAndNative : public StackObj {
}
};
static void post_events(bool emit_old_object_samples, bool emit_event_shutdown, Thread* thread) {
if (emit_old_object_samples) {
LeakProfiler::emit_events(max_jlong, false, false);
}
if (emit_event_shutdown) {
static void post_events(bool exception_handler, bool oom, Thread * thread) {
if (exception_handler) {
EventShutdown e;
e.set_reason("VM Error");
e.set_reason(oom ? "CrashOnOutOfMemoryError" : "VM Error");
e.commit();
}
EventDumpReason event;
event.set_reason(emit_old_object_samples ? "Out of Memory" : "Crash");
event.set_reason(exception_handler && oom ? "CrashOnOutOfMemoryError" : exception_handler ? "Crash" : "Out of Memory");
event.set_recordingId(-1);
event.commit();
}
@ -594,20 +584,40 @@ static bool guard_reentrancy() {
return false;
}
void JfrEmergencyDump::on_vm_shutdown(bool emit_old_object_samples, bool emit_event_shutdown) {
void JfrEmergencyDump::on_vm_shutdown(bool exception_handler, bool oom) {
if (!guard_reentrancy()) {
return;
}
Thread* const thread = Thread::current_or_null_safe();
assert(thread != nullptr, "invariant");
if (thread->is_Watcher_thread()) {
log_info(jfr, system)("The Watcher thread crashed so no jfr emergency dump will be generated.");
return;
}
// Ensure a JavaThread is _thread_in_vm when we make this call
JavaThreadInVMAndNative jtivm(thread);
post_events(exception_handler, oom, thread);
if (thread->is_Watcher_thread()) {
// We cannot attempt an emergency dump using the Watcher thread
// because we rely on the WatcherThread task "is_error_reported()",
// to exit the VM after a hardcoded timeout, should the relatively
// risky operation of an emergency dump fail (deadlock, livelock).
log_warning(jfr, system)
("The Watcher thread crashed so no jfr emergency dump will be generated.");
return;
}
if (thread->is_VM_thread()) {
const VM_Operation* const operation = VMThread::vm_operation();
if (operation != nullptr && operation->type() == VM_Operation::VMOp_JFROldObject) {
// We will not be able to issue a rotation because the rotation lock
// is held by the JFR Recorder Thread that issued the VM_Operation.
log_warning(jfr, system)
("The VM Thread crashed as part of emitting leak profiler events so no jfr emergency dump will be generated.");
return;
}
}
release_locks(thread);
post_events(emit_old_object_samples, emit_event_shutdown, thread);
// if JavaThread, transition to _thread_in_native to issue a final flushpoint
NoHandleMark nhm;

Some files were not shown because too many files have changed in this diff Show More