From d618008b33b3b6e9fef9e370693439ac2ca9acfd Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Tue, 28 Apr 2026 16:04:51 +0000 Subject: [PATCH] 8382711: Shenandoah: Support AOT code dumping Reviewed-by: kvn, iveresov, wkemper, adinn --- .../shenandoahBarrierSetAssembler_aarch64.cpp | 44 +++++++++++++------ .../shenandoahBarrierSetAssembler_x86.cpp | 42 +++++++++++++++--- src/hotspot/share/code/aotCodeCache.cpp | 7 ++- src/hotspot/share/code/aotCodeCache.hpp | 3 ++ .../gc/shenandoah/c2/shenandoahSupport.cpp | 24 +++++++++- 5 files changed, 97 insertions(+), 23 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index 2f7707227b4..83c309e7257 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -261,8 +261,16 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, // Test for in-cset if (is_strong) { - __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); - __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + if (AOTCodeCache::is_on_for_dump()) { + __ lea(rscratch2, ExternalAddress(AOTRuntimeConstants::cset_base_address())); + __ ldr(rscratch2, Address(rscratch2)); + __ lea(rscratch1, ExternalAddress(AOTRuntimeConstants::grain_shift_address())); + __ ldrw(rscratch1, Address(rscratch1)); + __ lsrv(rscratch1, r0, rscratch1); + } else { + __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); + __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + } __ ldrb(rscratch2, Address(rscratch2, rscratch1)); __ tbz(rscratch2, 0, not_cset); } @@ -270,15 +278,15 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, __ push_call_clobbered_registers(); if (is_strong) { if (is_narrow) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow))); } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong))); } } else if (is_weak) { if (is_narrow) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow))); } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak))); } } else { assert(is_phantom, "only remaining strength"); @@ -709,8 +717,16 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble if (is_strong) { // Check for object in cset. - __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); - __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + if (AOTCodeCache::is_on_for_dump()) { + __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address())); + __ ldr(tmp2, Address(tmp2)); + __ lea(tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address())); + __ ldrw(tmp1, Address(tmp1)); + __ lsrv(tmp1, res, tmp1); + } else { + __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); + __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + } __ ldrb(tmp2, Address(tmp2, tmp1)); __ cbz(tmp2, *stub->continuation()); } @@ -795,25 +811,25 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s bool is_native = ShenandoahBarrierSet::is_native_access(decorators); if (is_strong) { if (is_native) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong))); } else { if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow))); } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong))); } } } else if (is_weak) { assert(!is_native, "weak must not be called off-heap"); if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow))); } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak))); } } else { assert(is_phantom, "only remaining strength"); assert(is_native, "phantom must only be called off-heap"); - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)); + __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom))); } __ blr(lr); __ mov(rscratch1, r0); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 67510fac58f..7d92bfab2fe 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -312,9 +312,9 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, if (is_strong) { // Test for object in cset // Allocate temporary registers - for (int i = 0; i < 8; i++) { + for (int i = 0; i < Register::available_gp_registers(); i++) { Register r = as_Register(i); - if (r != rsp && r != rbp && r != dst && r != src.base() && r != src.index()) { + if (r != rsp && r != rbp && r != rcx && r != dst && r != src.base() && r != src.index() ) { if (tmp1 == noreg) { tmp1 = r; } else { @@ -333,8 +333,19 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, // Optimized cset-test __ movptr(tmp1, dst); - __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + if (AOTCodeCache::is_on_for_dump()) { + assert_different_registers(tmp1, tmp2, rcx); + __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address())); + __ push(rcx); + __ movb(rcx, Address(tmp2)); + __ shrptr(tmp1); + __ pop(rcx); + __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address())); + __ movptr(tmp2, Address(tmp2)); + } else { + __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + } __ movbool(tmp1, Address(tmp1, tmp2, Address::times_1)); __ testbool(tmp1); __ jcc(Assembler::zero, not_cset); @@ -886,8 +897,27 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble if (is_strong) { // Check for object being in the collection set. __ mov(tmp1, res); - __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + if (AOTCodeCache::is_on_for_dump()) { + __ push(rcx); + __ lea(rcx, ExternalAddress(AOTRuntimeConstants::grain_shift_address())); + __ movl(rcx, Address(rcx)); + if (tmp1 != rcx) { + __ mov(tmp1, res); + __ shrptr(tmp1); + __ pop(rcx); + } else { + assert_different_registers(tmp2, rcx); + __ mov(tmp2, res); + __ shrptr(tmp2); + __ pop(rcx); + __ movptr(tmp1, tmp2); + } + __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address())); + __ movptr(tmp2, Address(tmp2)); + } else { + __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + } __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); __ testbool(tmp2); __ jcc(Assembler::zero, *stub->continuation()); diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 1ef1142d5e7..e1c9184faca 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -63,6 +63,7 @@ #include "gc/g1/g1HeapRegion.hpp" #endif #if INCLUDE_SHENANDOAHGC +#include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahRuntime.hpp" #endif #if INCLUDE_ZGC @@ -2476,6 +2477,7 @@ void AOTRuntimeConstants::initialize_from_runtime() { BarrierSet* bs = BarrierSet::barrier_set(); address card_table_base = nullptr; uint grain_shift = 0; + address cset_base = nullptr; #if INCLUDE_G1GC if (bs->is_a(BarrierSet::G1BarrierSet)) { grain_shift = G1HeapRegion::LogOfHRGrainBytes; @@ -2483,7 +2485,8 @@ void AOTRuntimeConstants::initialize_from_runtime() { #endif #if INCLUDE_SHENANDOAHGC if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) { - grain_shift = 0; + grain_shift = ShenandoahHeapRegion::region_size_bytes_shift_jint(); + cset_base = ShenandoahHeap::in_cset_fast_test_addr(); } else #endif if (bs->is_a(BarrierSet::CardTableBarrierSet)) { @@ -2495,11 +2498,13 @@ void AOTRuntimeConstants::initialize_from_runtime() { } _aot_runtime_constants._card_table_base = card_table_base; _aot_runtime_constants._grain_shift = grain_shift; + _aot_runtime_constants._cset_base = cset_base; } address AOTRuntimeConstants::_field_addresses_list[] = { ((address)&_aot_runtime_constants._card_table_base), ((address)&_aot_runtime_constants._grain_shift), + ((address)&_aot_runtime_constants._cset_base), nullptr }; diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 296464533f0..999fa5fb658 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -688,6 +688,7 @@ class AOTRuntimeConstants { private: address _card_table_base; uint _grain_shift; + address _cset_base; static address _field_addresses_list[]; static AOTRuntimeConstants _aot_runtime_constants; // private constructor for unique singleton @@ -703,6 +704,7 @@ class AOTRuntimeConstants { } static address card_table_base_address(); static address grain_shift_address() { return (address)&_aot_runtime_constants._grain_shift; } + static address cset_base_address() { return (address)&_aot_runtime_constants._cset_base; } static address* field_addresses_list() { return _field_addresses_list; } @@ -710,6 +712,7 @@ class AOTRuntimeConstants { static bool contains(address adr) { return false; } static address card_table_base_address() { return nullptr; } static address grain_shift_address() { return nullptr; } + static address cset_base_address() { return nullptr; } static address* field_addresses_list() { return nullptr; } #endif }; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 015276feb5c..86de9eef459 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -25,6 +25,7 @@ #include "classfile/javaClasses.hpp" +#include "code/aotCodeCache.hpp" #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #include "gc/shenandoah/c2/shenandoahSupport.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" @@ -928,12 +929,31 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, PhaseIterGVN& igvn = phase->igvn(); Node* raw_val = new CastP2XNode(old_ctrl, val); - Node* cset_idx = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); + Node* region_size_shift = nullptr; + if (AOTCodeCache::is_on_for_dump()) { + Node* aot_addr = igvn.makecon(TypeRawPtr::make(AOTRuntimeConstants::grain_shift_address())); + region_size_shift = new LoadINode(old_ctrl, raw_mem, aot_addr, + DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr), + TypeInt::INT, MemNode::unordered); + phase->register_new_node(region_size_shift, old_ctrl); + } else { + region_size_shift = igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()); + } + Node* cset_idx = new URShiftXNode(raw_val, region_size_shift); // Figure out the target cset address with raw pointer math. // This avoids matching AddP+LoadB that would emit inefficient code. // See JDK-8245465. - Node* cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); + Node* cset_addr_ptr = nullptr; + if (AOTCodeCache::is_on_for_dump()) { + Node* aot_addr = igvn.makecon(TypeRawPtr::make(AOTRuntimeConstants::cset_base_address())); + cset_addr_ptr = new LoadPNode(old_ctrl, raw_mem, aot_addr, + DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr), + TypeRawPtr::NOTNULL, MemNode::unordered); + phase->register_new_node(cset_addr_ptr, old_ctrl); + } else { + cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); + } Node* cset_addr = new CastP2XNode(old_ctrl, cset_addr_ptr); Node* cset_load_addr = new AddXNode(cset_addr, cset_idx); Node* cset_load_ptr = new CastX2PNode(cset_load_addr);