From 4bee207d0ad948734e04516dd8d7c9504cb83665 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Fri, 27 Feb 2026 21:49:55 +0000 Subject: [PATCH] 8377554: Load card table base and other values via AOTRuntimeConstants in AOT code Reviewed-by: kvn, asmehra --- src/hotspot/cpu/aarch64/aarch64.ad | 30 +++++++++- .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 10 ++++ .../gc/g1/g1BarrierSetAssembler_aarch64.cpp | 23 +++++++- .../cpu/aarch64/macroAssembler_aarch64.cpp | 22 ++++++++ .../cpu/aarch64/macroAssembler_aarch64.hpp | 3 + src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 10 ++++ .../x86/gc/g1/g1BarrierSetAssembler_x86.cpp | 41 ++++++++++++-- .../cardTableBarrierSetAssembler_x86.cpp | 24 ++++++-- .../cardTableBarrierSetAssembler_x86.hpp | 2 +- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 14 +++++ src/hotspot/cpu/x86/macroAssembler_x86.hpp | 1 + src/hotspot/cpu/x86/x86.ad | 25 +++++++++ src/hotspot/share/adlc/main.cpp | 2 + src/hotspot/share/code/aotCodeCache.cpp | 55 +++++++++++++++++++ src/hotspot/share/code/aotCodeCache.hpp | 33 +++++++++++ src/hotspot/share/gc/g1/g1BarrierSet.hpp | 3 + .../gc/shared/c1/cardTableBarrierSetC1.cpp | 13 +++++ .../gc/shared/c2/cardTableBarrierSetC2.cpp | 14 ++++- .../gc/shared/c2/cardTableBarrierSetC2.hpp | 2 +- .../share/gc/shared/cardTableBarrierSet.hpp | 4 ++ src/hotspot/share/opto/idealKit.cpp | 11 ++++ src/hotspot/share/opto/idealKit.hpp | 3 + src/hotspot/share/opto/type.cpp | 2 +- 23 files changed, 329 insertions(+), 18 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index a9ca91d9309..9734c6845ea 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -3403,11 +3403,13 @@ encode %{ } else if (rtype == relocInfo::metadata_type) { __ mov_metadata(dst_reg, (Metadata*)con); } else { - assert(rtype == relocInfo::none, "unexpected reloc type"); + assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type"); + // load fake address constants using a normal move if (! __ is_valid_AArch64_address(con) || con < (address)(uintptr_t)os::vm_page_size()) { __ mov(dst_reg, con); } else { + // no reloc so just use adrp and add uint64_t offset; __ adrp(dst_reg, con, offset); __ add(dst_reg, dst_reg, offset); @@ -4535,6 +4537,18 @@ operand immP_1() interface(CONST_INTER); %} +// AOT Runtime Constants Address +operand immAOTRuntimeConstantsAddress() +%{ + // Check if the address is in the range of AOT Runtime Constants + predicate(AOTRuntimeConstants::contains((address)(n->get_ptr()))); + match(ConP); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + // Float and Double operands // Double Immediate operand immD() @@ -6898,6 +6912,20 @@ instruct loadConP1(iRegPNoSp dst, immP_1 con) ins_pipe(ialu_imm); %} +instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con) +%{ + match(Set dst con); + + ins_cost(INSN_COST); + format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %} + + ins_encode %{ + __ load_aotrc_address($dst$$Register, (address)$con$$constant); + %} + + ins_pipe(ialu_imm); +%} + // Load Narrow Pointer Constant instruct loadConN(iRegNNoSp dst, immN con) diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index c0621cbd5c2..30048a2079d 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -33,6 +33,7 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" +#include "code/aotCodeCache.hpp" #include "code/compiledIC.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/gc_globals.hpp" @@ -532,6 +533,15 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod case T_LONG: { assert(patch_code == lir_patch_none, "no patching handled here"); +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + address b = c->as_pointer(); + if (AOTRuntimeConstants::contains(b)) { + __ load_aotrc_address(dest->as_register_lo(), b); + break; + } + } +#endif __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong()); break; } diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp index d7884c27a2c..68291720208 100644 --- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp @@ -23,6 +23,7 @@ */ #include "asm/macroAssembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "gc/g1/g1BarrierSet.hpp" #include "gc/g1/g1BarrierSetAssembler.hpp" #include "gc/g1/g1BarrierSetRuntime.hpp" @@ -243,9 +244,25 @@ static void generate_post_barrier(MacroAssembler* masm, assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1); // Does store cross heap regions? - __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value - __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) - __ cbz(tmp1, done); + #if INCLUDE_CDS + // AOT code needs to load the barrier grain shift from the aot + // runtime constants area in the code cache otherwise we can compile + // it as an immediate operand + if (AOTCodeCache::is_on_for_dump()) { + address grain_shift_address = (address)AOTRuntimeConstants::grain_shift_address(); + __ eor(tmp1, store_addr, new_val); + __ lea(tmp2, ExternalAddress(grain_shift_address)); + __ ldrb(tmp2, tmp2); + __ lsrv(tmp1, tmp1, tmp2); + __ cbz(tmp1, done); + } else +#endif + { + __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value + __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) + __ cbz(tmp1, done); + } + // Crosses regions, storing null? if (new_val_may_be_null) { __ cbz(new_val, done); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 35e90d296c9..3e3e95be07e 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -5754,6 +5754,14 @@ void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_off } void MacroAssembler::load_byte_map_base(Register reg) { +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + address byte_map_base_adr = AOTRuntimeConstants::card_table_base_address(); + lea(reg, ExternalAddress(byte_map_base_adr)); + ldr(reg, Address(reg)); + return; + } +#endif CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set(); // Strictly speaking the card table base isn't an address at all, and it might @@ -5761,6 +5769,20 @@ void MacroAssembler::load_byte_map_base(Register reg) { mov(reg, (uint64_t)ctbs->card_table_base_const()); } +void MacroAssembler::load_aotrc_address(Register reg, address a) { +#if INCLUDE_CDS + assert(AOTRuntimeConstants::contains(a), "address out of range for data area"); + if (AOTCodeCache::is_on_for_dump()) { + // all aotrc field addresses should be registered in the AOTCodeCache address table + lea(reg, ExternalAddress(a)); + } else { + mov(reg, (uint64_t)a); + } +#else + ShouldNotReachHere(); +#endif +} + void MacroAssembler::build_frame(int framesize) { assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR"); assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index 7b5af532ca1..fa32f3055b9 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -1476,6 +1476,9 @@ public: // Load the base of the cardtable byte map into reg. void load_byte_map_base(Register reg); + // Load a constant address in the AOT Runtime Constants area + void load_aotrc_address(Register reg, address a); + // Prolog generator routines to support switch between x86 code and // generated ARM code diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 37ee9451405..d9be0fdcc8d 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -32,6 +32,7 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" +#include "code/aotCodeCache.hpp" #include "compiler/oopMap.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/gc_globals.hpp" @@ -535,6 +536,15 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod case T_LONG: { assert(patch_code == lir_patch_none, "no patching handled here"); +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + address b = c->as_pointer(); + if (AOTRuntimeConstants::contains(b)) { + __ load_aotrc_address(dest->as_register_lo(), b); + break; + } + } +#endif __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong()); break; } diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp index 34de9403ccf..b20d7b5cd07 100644 --- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp @@ -23,6 +23,7 @@ */ #include "asm/macroAssembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "gc/g1/g1BarrierSet.hpp" #include "gc/g1/g1BarrierSetAssembler.hpp" #include "gc/g1/g1BarrierSetRuntime.hpp" @@ -268,6 +269,16 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ bind(done); } +#if INCLUDE_CDS +// return a register that differs from reg1, reg2, reg3 and reg4 + +static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) { + RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx - + RegSet::of(reg1, reg2, reg3, reg4)); + return *(available.begin()); +} +#endif // INCLUDE_CDS + static void generate_post_barrier(MacroAssembler* masm, const Register store_addr, const Register new_val, @@ -280,10 +291,32 @@ static void generate_post_barrier(MacroAssembler* masm, Label L_done; // Does store cross heap regions? - __ movptr(tmp1, store_addr); // tmp1 := store address - __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value - __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0? - __ jccb(Assembler::equal, L_done); +#if INCLUDE_CDS + // AOT code needs to load the barrier grain shift from the aot + // runtime constants area in the code cache otherwise we can compile + // it as an immediate operand + + if (AOTCodeCache::is_on_for_dump()) { + address grain_shift_addr = AOTRuntimeConstants::grain_shift_address(); + Register save = pick_different_reg(rcx, tmp1, new_val, store_addr); + __ push(save); + __ movptr(save, store_addr); + __ xorptr(save, new_val); + __ push(rcx); + __ lea(rcx, ExternalAddress(grain_shift_addr)); + __ movl(rcx, Address(rcx, 0)); + __ shrptr(save); + __ pop(rcx); + __ pop(save); + __ jcc(Assembler::equal, L_done); + } else +#endif // INCLUDE_CDS + { + __ movptr(tmp1, store_addr); // tmp1 := store address + __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value + __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0? + __ jccb(Assembler::equal, L_done); + } // Crosses regions, storing null? if (new_val_may_be_null) { diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp index 65e6b4e01fc..0ea769dd488 100644 --- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp @@ -23,6 +23,7 @@ */ #include "asm/macroAssembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/cardTable.hpp" #include "gc/shared/cardTableBarrierSet.hpp" @@ -111,7 +112,15 @@ void CardTableBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembl __ shrptr(end, CardTable::card_shift()); __ subptr(end, addr); // end --> cards count - __ mov64(tmp, (intptr_t)ctbs->card_table_base_const()); +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + __ lea(tmp, ExternalAddress(AOTRuntimeConstants::card_table_base_address())); + __ movq(tmp, Address(tmp, 0)); + } else +#endif + { + __ mov64(tmp, (intptr_t)ctbs->card_table_base_const()); + } __ addptr(addr, tmp); __ BIND(L_loop); __ movb(Address(addr, count, Address::times_1), 0); @@ -121,7 +130,7 @@ __ BIND(L_loop); __ BIND(L_done); } -void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) { +void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch) { // Does a store check for the oop in register obj. The content of // register obj is destroyed afterwards. CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set(); @@ -136,6 +145,13 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob // never need to be relocated. On 64bit however the value may be too // large for a 32bit displacement. intptr_t byte_map_base = (intptr_t)ctbs->card_table_base_const(); +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + __ lea(rscratch, ExternalAddress(AOTRuntimeConstants::card_table_base_address())); + __ movq(rscratch, Address(rscratch, 0)); + card_addr = Address(rscratch, obj, Address::times_1, 0); + } else +#endif if (__ is_simm32(byte_map_base)) { card_addr = Address(noreg, obj, Address::times_1, byte_map_base); } else { @@ -174,10 +190,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS if (needs_post_barrier) { // flatten object address if needed if (!precise || (dst.index() == noreg && dst.disp() == 0)) { - store_check(masm, dst.base(), dst); + store_check(masm, dst.base(), dst, tmp2); } else { __ lea(tmp1, dst); - store_check(masm, tmp1, dst); + store_check(masm, tmp1, dst, tmp2); } } } diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp index 0a36571c757..201c11062f2 100644 --- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp @@ -33,7 +33,7 @@ protected: virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count) {} - void store_check(MacroAssembler* masm, Register obj, Address dst); + void store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch); virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 83169df3456..b54f6adc263 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -10034,6 +10034,20 @@ void MacroAssembler::restore_legacy_gprs() { addq(rsp, 16 * wordSize); } +void MacroAssembler::load_aotrc_address(Register reg, address a) { +#if INCLUDE_CDS + assert(AOTRuntimeConstants::contains(a), "address out of range for data area"); + if (AOTCodeCache::is_on_for_dump()) { + // all aotrc field addresses should be registered in the AOTCodeCache address table + lea(reg, ExternalAddress(a)); + } else { + mov64(reg, (uint64_t)a); + } +#else + ShouldNotReachHere(); +#endif +} + void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) { if (VM_Version::supports_apx_f()) { esetzucc(comparison, dst); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index eb23199ca63..5c049f710e2 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -2070,6 +2070,7 @@ public: void save_legacy_gprs(); void restore_legacy_gprs(); + void load_aotrc_address(Register reg, address a); void setcc(Assembler::Condition comparison, Register dst); }; diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index aed54fe93d4..0ffa4c2031c 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -5187,6 +5187,18 @@ operand immL_65535() interface(CONST_INTER); %} +// AOT Runtime Constants Address +operand immAOTRuntimeConstantsAddress() +%{ + // Check if the address is in the range of AOT Runtime Constants + predicate(AOTRuntimeConstants::contains((address)(n->get_ptr()))); + match(ConP); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + operand kReg() %{ constraint(ALLOC_IN_RC(vectmask_reg)); @@ -7332,6 +7344,19 @@ instruct loadD(regD dst, memory mem) ins_pipe(pipe_slow); // XXX %} +instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con) +%{ + match(Set dst con); + + format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %} + + ins_encode %{ + __ load_aotrc_address($dst$$Register, (address)$con$$constant); + %} + + ins_pipe(ialu_reg_fat); +%} + // max = java.lang.Math.max(float a, float b) instruct maxF_reg_avx10_2(regF dst, regF a, regF b) %{ predicate(VM_Version::supports_avx10_2()); diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp index 4e8a96617e8..8e6ea5bbec9 100644 --- a/src/hotspot/share/adlc/main.cpp +++ b/src/hotspot/share/adlc/main.cpp @@ -213,6 +213,7 @@ int main(int argc, char *argv[]) AD.addInclude(AD._CPP_file, "adfiles", get_basename(AD._VM_file._name)); AD.addInclude(AD._CPP_file, "adfiles", get_basename(AD._HPP_file._name)); AD.addInclude(AD._CPP_file, "memory/allocation.inline.hpp"); + AD.addInclude(AD._CPP_file, "code/aotCodeCache.hpp"); AD.addInclude(AD._CPP_file, "code/codeCache.hpp"); AD.addInclude(AD._CPP_file, "code/compiledIC.hpp"); AD.addInclude(AD._CPP_file, "code/nativeInst.hpp"); @@ -257,6 +258,7 @@ int main(int argc, char *argv[]) AD.addInclude(AD._CPP_PEEPHOLE_file, "adfiles", get_basename(AD._HPP_file._name)); AD.addInclude(AD._CPP_PIPELINE_file, "adfiles", get_basename(AD._HPP_file._name)); AD.addInclude(AD._DFA_file, "adfiles", get_basename(AD._HPP_file._name)); + AD.addInclude(AD._DFA_file, "code/aotCodeCache.hpp"); AD.addInclude(AD._DFA_file, "oops/compressedOops.hpp"); AD.addInclude(AD._DFA_file, "opto/cfgnode.hpp"); // Use PROB_MAX in predicate. AD.addInclude(AD._DFA_file, "opto/intrinsicnode.hpp"); diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 32a53691f3f..e5f68afc51d 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -29,9 +29,11 @@ #include "cds/cds_globals.hpp" #include "cds/cdsConfig.hpp" #include "cds/heapShared.hpp" +#include "ci/ciUtilities.hpp" #include "classfile/javaAssertions.hpp" #include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" +#include "gc/shared/cardTableBarrierSet.hpp" #include "gc/shared/gcConfig.hpp" #include "logging/logStream.hpp" #include "memory/memoryReserver.hpp" @@ -53,6 +55,7 @@ #endif #if INCLUDE_G1GC #include "gc/g1/g1BarrierSetRuntime.hpp" +#include "gc/g1/g1HeapRegion.hpp" #endif #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/shenandoahRuntime.hpp" @@ -258,6 +261,9 @@ void AOTCodeCache::init2() { return; } + // initialize aot runtime constants as appropriate to this runtime + AOTRuntimeConstants::initialize_from_runtime(); + // initialize the table of external routines so we can save // generated code blobs that reference them AOTCodeAddressTable* table = opened_cache->_table; @@ -1447,6 +1453,12 @@ void AOTCodeAddressTable::init_extrs() { #endif #endif // ZERO + // addresses of fields in AOT runtime constants area + address* p = AOTRuntimeConstants::field_addresses_list(); + while (*p != nullptr) { + SET_ADDRESS(_extrs, *p++); + } + _extrs_complete = true; log_debug(aot, codecache, init)("External addresses recorded"); } @@ -1729,6 +1741,11 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB if (addr == (address)-1) { // Static call stub has jump to itself return id; } + // Check card_table_base address first since it can point to any address + BarrierSet* bs = BarrierSet::barrier_set(); + bool is_const_card_table_base = !UseG1GC && !UseShenandoahGC && bs->is_a(BarrierSet::CardTableBarrierSet); + guarantee(!is_const_card_table_base || addr != ci_card_table_address_const(), "sanity"); + // Seach for C string id = id_for_C_string(addr); if (id >= 0) { @@ -1798,6 +1815,44 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB return id; } +AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants; + +void AOTRuntimeConstants::initialize_from_runtime() { + BarrierSet* bs = BarrierSet::barrier_set(); + address card_table_base = nullptr; + uint grain_shift = 0; +#if INCLUDE_G1GC + if (bs->is_a(BarrierSet::G1BarrierSet)) { + grain_shift = G1HeapRegion::LogOfHRGrainBytes; + } else +#endif +#if INCLUDE_SHENANDOAHGC + if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) { + grain_shift = 0; + } else +#endif + if (bs->is_a(BarrierSet::CardTableBarrierSet)) { + CardTable::CardValue* base = ci_card_table_address_const(); + assert(base != nullptr, "unexpected byte_map_base"); + card_table_base = base; + CardTableBarrierSet* ctbs = barrier_set_cast(bs); + grain_shift = ctbs->grain_shift(); + } + _aot_runtime_constants._card_table_base = card_table_base; + _aot_runtime_constants._grain_shift = grain_shift; +} + +address AOTRuntimeConstants::_field_addresses_list[] = { + ((address)&_aot_runtime_constants._card_table_base), + ((address)&_aot_runtime_constants._grain_shift), + nullptr +}; + +address AOTRuntimeConstants::card_table_base_address() { + assert(UseSerialGC || UseParallelGC, "Only these GCs have constant card table base"); + return (address)&_aot_runtime_constants._card_table_base; +} + // This is called after initialize() but before init2() // and _cache is not set yet. void AOTCodeCache::print_on(outputStream* st) { diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 45b4a15510d..85f8b47920f 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_CODE_AOTCODECACHE_HPP #define SHARE_CODE_AOTCODECACHE_HPP +#include "gc/shared/gc_globals.hpp" #include "runtime/stubInfo.hpp" /* @@ -422,4 +423,36 @@ public: #endif // PRODUCT }; +// code cache internal runtime constants area used by AOT code +class AOTRuntimeConstants { + friend class AOTCodeCache; + private: + address _card_table_base; + uint _grain_shift; + static address _field_addresses_list[]; + static AOTRuntimeConstants _aot_runtime_constants; + // private constructor for unique singleton + AOTRuntimeConstants() { } + // private for use by friend class AOTCodeCache + static void initialize_from_runtime(); + public: +#if INCLUDE_CDS + static bool contains(address adr) { + address base = (address)&_aot_runtime_constants; + address hi = base + sizeof(AOTRuntimeConstants); + return (base <= adr && adr < hi); + } + static address card_table_base_address(); + static address grain_shift_address() { return (address)&_aot_runtime_constants._grain_shift; } + static address* field_addresses_list() { + return _field_addresses_list; + } +#else + static bool contains(address adr) { return false; } + static address card_table_base_address() { return nullptr; } + static address grain_shift_address() { return nullptr; } + static address* field_addresses_list() { return nullptr; } +#endif +}; + #endif // SHARE_CODE_AOTCODECACHE_HPP diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.hpp b/src/hotspot/share/gc/g1/g1BarrierSet.hpp index 406096acf10..c5c7058471c 100644 --- a/src/hotspot/share/gc/g1/g1BarrierSet.hpp +++ b/src/hotspot/share/gc/g1/g1BarrierSet.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_GC_G1_G1BARRIERSET_HPP #define SHARE_GC_G1_G1BARRIERSET_HPP +#include "gc/g1/g1HeapRegion.hpp" #include "gc/g1/g1SATBMarkQueueSet.hpp" #include "gc/shared/bufferNode.hpp" #include "gc/shared/cardTable.hpp" @@ -116,6 +117,8 @@ class G1BarrierSet: public CardTableBarrierSet { virtual void print_on(outputStream* st) const; + virtual uint grain_shift() { return G1HeapRegion::LogOfHRGrainBytes; } + // Callbacks for runtime accesses. template class AccessBarrier: public CardTableBarrierSet::AccessBarrier { diff --git a/src/hotspot/share/gc/shared/c1/cardTableBarrierSetC1.cpp b/src/hotspot/share/gc/shared/c1/cardTableBarrierSetC1.cpp index 914358760aa..86b74aa5736 100644 --- a/src/hotspot/share/gc/shared/c1/cardTableBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shared/c1/cardTableBarrierSetC1.cpp @@ -22,6 +22,7 @@ * */ +#include "code/aotCodeCache.hpp" #include "gc/shared/c1/cardTableBarrierSetC1.hpp" #include "gc/shared/cardTable.hpp" #include "gc/shared/cardTableBarrierSet.hpp" @@ -123,6 +124,7 @@ void CardTableBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Op assert(addr->is_register(), "must be a register at this point"); #ifdef CARDTABLEBARRIERSET_POST_BARRIER_HELPER + assert(!AOTCodeCache::is_on(), "this path is not implemented"); gen->CardTableBarrierSet_post_barrier_helper(addr, card_table_base); #else LIR_Opr tmp = gen->new_pointer_register(); @@ -135,6 +137,17 @@ void CardTableBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Op } LIR_Address* card_addr; +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + // load the card table address from the AOT Runtime Constants area + LIR_Opr byte_map_base_adr = LIR_OprFact::intptrConst(AOTRuntimeConstants::card_table_base_address()); + LIR_Opr byte_map_base_reg = gen->new_pointer_register(); + __ move(byte_map_base_adr, byte_map_base_reg); + LIR_Address* byte_map_base_indirect = new LIR_Address(byte_map_base_reg, 0, T_LONG); + __ move(byte_map_base_indirect, byte_map_base_reg); + card_addr = new LIR_Address(tmp, byte_map_base_reg, T_BYTE); + } else +#endif if (gen->can_inline_as_constant(card_table_base)) { card_addr = new LIR_Address(tmp, card_table_base->as_jint(), T_BYTE); } else { diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp index 42af77ebdf4..f7445ff254f 100644 --- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp @@ -23,6 +23,7 @@ */ #include "ci/ciUtilities.hpp" +#include "code/aotCodeCache.hpp" #include "gc/shared/c2/cardTableBarrierSetC2.hpp" #include "gc/shared/cardTable.hpp" #include "gc/shared/cardTableBarrierSet.hpp" @@ -114,13 +115,20 @@ Node* CardTableBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access return result; } -Node* CardTableBarrierSetC2::byte_map_base_node(GraphKit* kit) const { +Node* CardTableBarrierSetC2::byte_map_base_node(IdealKit* kit) const { // Get base of card map +#if INCLUDE_CDS + if (AOTCodeCache::is_on_for_dump()) { + // load the card table address from the AOT Runtime Constants area + Node* byte_map_base_adr = kit->makecon(TypeRawPtr::make(AOTRuntimeConstants::card_table_base_address())); + return kit->load_aot_const(byte_map_base_adr, TypeRawPtr::NOTNULL); + } +#endif CardTable::CardValue* card_table_base = ci_card_table_address_const(); if (card_table_base != nullptr) { return kit->makecon(TypeRawPtr::make((address)card_table_base)); } else { - return kit->null(); + return kit->makecon(Type::get_zero_type(T_ADDRESS)); } } @@ -168,7 +176,7 @@ void CardTableBarrierSetC2::post_barrier(GraphKit* kit, Node* card_offset = __ URShiftX(cast, __ ConI(CardTable::card_shift())); // Combine card table base and card offset - Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset); + Node* card_adr = __ AddP(__ top(), byte_map_base_node(&ideal), card_offset); // Get the alias_index for raw card-mark memory int adr_type = Compile::AliasIdxRaw; diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp index 84876808f0d..8f5bae8c6dd 100644 --- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp @@ -43,7 +43,7 @@ protected: Node* new_val, const Type* value_type) const; virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const; - Node* byte_map_base_node(GraphKit* kit) const; + Node* byte_map_base_node(IdealKit* kit) const; public: virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; diff --git a/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp b/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp index 3a9b46d9df8..5d355318b21 100644 --- a/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp +++ b/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp @@ -103,6 +103,10 @@ public: virtual void print_on(outputStream* st) const; + // The AOT code cache manager needs to know the region grain size + // shift for some barrier sets. + virtual uint grain_shift() { return 0; } + template class AccessBarrier: public BarrierSet::AccessBarrier { typedef BarrierSet::AccessBarrier Raw; diff --git a/src/hotspot/share/opto/idealKit.cpp b/src/hotspot/share/opto/idealKit.cpp index dd7e9ae52b7..fbb61bfdf72 100644 --- a/src/hotspot/share/opto/idealKit.cpp +++ b/src/hotspot/share/opto/idealKit.cpp @@ -360,6 +360,17 @@ Node* IdealKit::load(Node* ctl, return transform(ld); } +// Load AOT runtime constant +Node* IdealKit::load_aot_const(Node* adr, const Type* t) { + BasicType bt = t->basic_type(); + const TypePtr* adr_type = nullptr; // debug-mode-only argument + DEBUG_ONLY(adr_type = C->get_adr_type(Compile::AliasIdxRaw)); + Node* ctl = (Node*)C->root(); // Raw memory access needs control + Node* ld = LoadNode::make(_gvn, ctl, C->immutable_memory(), adr, adr_type, t, bt, MemNode::unordered, + LoadNode::DependsOnlyOnTest, false, false, false, false, 0); + return transform(ld); +} + Node* IdealKit::store(Node* ctl, Node* adr, Node *val, BasicType bt, int adr_idx, MemNode::MemOrd mo, bool require_atomic_access, diff --git a/src/hotspot/share/opto/idealKit.hpp b/src/hotspot/share/opto/idealKit.hpp index 280f61fd9a9..518c3b92136 100644 --- a/src/hotspot/share/opto/idealKit.hpp +++ b/src/hotspot/share/opto/idealKit.hpp @@ -224,6 +224,9 @@ class IdealKit: public StackObj { MemNode::MemOrd mo = MemNode::unordered, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest); + // Load AOT runtime constant + Node* load_aot_const(Node* adr, const Type* t); + // Return the new StoreXNode Node* store(Node* ctl, Node* adr, diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index c637737eef9..1a0872ee0e6 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -97,7 +97,7 @@ const Type::TypeInfo Type::_type_info[Type::lastype] = { { Bad, T_ILLEGAL, "vectorz:", false, Op_VecZ, relocInfo::none }, // VectorZ #endif { Bad, T_ADDRESS, "anyptr:", false, Op_RegP, relocInfo::none }, // AnyPtr - { Bad, T_ADDRESS, "rawptr:", false, Op_RegP, relocInfo::none }, // RawPtr + { Bad, T_ADDRESS, "rawptr:", false, Op_RegP, relocInfo::external_word_type }, // RawPtr { Bad, T_OBJECT, "oop:", true, Op_RegP, relocInfo::oop_type }, // OopPtr { Bad, T_OBJECT, "inst:", true, Op_RegP, relocInfo::oop_type }, // InstPtr { Bad, T_OBJECT, "ary:", true, Op_RegP, relocInfo::oop_type }, // AryPtr