From 94d3aecfed043f0e8825922a17ef1d50e6bc86f7 Mon Sep 17 00:00:00 2001 From: Richard Reingruber Date: Fri, 22 May 2026 06:23:00 +0000 Subject: [PATCH] 8384161: [PPC64] Consolidate code related to calls in nmethods that use trampoline stubs Reviewed-by: mdoerr, dbriemann --- src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp | 20 +- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 60 +--- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp | 10 +- src/hotspot/cpu/ppc/macroAssembler_ppc.cpp | 131 ++++---- src/hotspot/cpu/ppc/macroAssembler_ppc.hpp | 15 +- src/hotspot/cpu/ppc/ppc.ad | 338 ++------------------ src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp | 51 +-- 7 files changed, 139 insertions(+), 486 deletions(-) diff --git a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp index 61780a73969..65e9505c812 100644 --- a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2025 SAP SE. All rights reserved. + * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2026 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -443,15 +443,13 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) { return; // CodeCache is full } - bool success = ce->emit_trampoline_stub_for_call(SharedRuntime::get_resolve_static_call_stub()); - if (!success) { return; } - - __ relocate(relocInfo::static_call_type); - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ code()->set_insts_mark(); - __ bl(__ pc()); + AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(), + relocInfo::static_call_type); + address call_pc = __ trampoline_call(resolve); + if (call_pc == nullptr) { + ce->bailout("const/stub overflow in call with trampoline"); + return; + } ce->add_call_info_here(info()); ce->verify_oop_map(info()); diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 777b41577be..a9f34b148c6 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2025 SAP SE. All rights reserved. + * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2026 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -609,67 +609,25 @@ void LIR_Assembler::align_call(LIR_Code) { // do nothing since all instructions are word aligned on ppc } - -bool LIR_Assembler::emit_trampoline_stub_for_call(address target, Register Rtoc) { - int start_offset = __ offset(); - // Put the entry point as a constant into the constant pool. - const address entry_point_toc_addr = __ address_constant(target, RelocationHolder::none); - if (entry_point_toc_addr == nullptr) { - bailout("const section overflow"); - return false; - } - const int entry_point_toc_offset = __ offset_to_method_toc(entry_point_toc_addr); - - // Emit the trampoline stub which will be related to the branch-and-link below. - address stub = __ emit_trampoline_stub(entry_point_toc_offset, start_offset, Rtoc); - if (!stub) { - bailout("no space for trampoline stub"); - return false; - } - return true; -} - - void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { assert(rtype==relocInfo::opt_virtual_call_type || rtype==relocInfo::static_call_type, "unexpected rtype"); - bool success = emit_trampoline_stub_for_call(op->addr()); - if (!success) { return; } - - __ relocate(rtype); - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ code()->set_insts_mark(); - __ bl(__ pc()); + address call_pc = __ trampoline_call(AddressLiteral(op->addr(), rtype)); + if (call_pc == nullptr) { + bailout("const/stub overflow in call with trampoline"); + return; + } add_call_info(code_offset(), op->info()); __ post_call_nop(); } - void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { __ calculate_address_from_global_toc(R2_TOC, __ method_toc()); - - // Virtual call relocation will point to ic load. - address virtual_call_meta_addr = __ pc(); - // Load a clear inline cache. - AddressLiteral empty_ic((address) Universe::non_oop_word()); - bool success = __ load_const_from_method_toc(R19_inline_cache_reg, empty_ic, R2_TOC); + bool success = __ ic_call(R2_TOC, op->addr()); if (!success) { - bailout("const section overflow"); + bailout("const/stub overflow in ic_call with trampoline"); return; } - // Call to fixup routine. Fixup routine uses ScopeDesc info - // to determine who we intended to call. - __ relocate(virtual_call_Relocation::spec(virtual_call_meta_addr)); - - success = emit_trampoline_stub_for_call(op->addr(), R2_TOC); - if (!success) { return; } - - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ bl(__ pc()); add_call_info(code_offset(), op->info()); __ post_call_nop(); } diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp index dea055710cd..7399a4544e6 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2015 SAP SE. All rights reserved. + * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2026 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,9 @@ #ifndef CPU_PPC_C1_LIRASSEMBLER_PPC_HPP #define CPU_PPC_C1_LIRASSEMBLER_PPC_HPP +// ArrayCopyStub needs access to bailout +friend class ArrayCopyStub; + private: ////////////////////////////////////////////////////////////////////////////// @@ -56,9 +59,6 @@ public: static const ConditionRegister BOOL_RESULT; - // Emit trampoline stub for call. Call bailout() if failed. Return true on success. - bool emit_trampoline_stub_for_call(address target, Register Rtoc = noreg); - enum { _static_call_stub_size = 4 * BytesPerInstWord + MacroAssembler::b64_patchable_size, // or smaller _call_stub_size = _static_call_stub_size + MacroAssembler::trampoline_stub_size, // or smaller diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp index 24c314b8be3..95d58d470c8 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp @@ -1199,6 +1199,75 @@ address MacroAssembler::call_c_using_toc(const FunctionDescriptor* fd, } #endif // ABI_ELFv2 +bool MacroAssembler::ic_call(Register Rmethod_toc, + address target, + jint method_index, + bool scratch_emit, + bool fixed_size) { + AddressLiteral target_al(target, virtual_call_Relocation::spec(pc(), method_index)); + DEBUG_ONLY(int ic_load_offset = offset()); + + // Load a clear inline cache. + AddressLiteral empty_ic((address) Universe::non_oop_word()); + bool success = load_const_from_method_toc(R19_inline_cache_reg, empty_ic, Rmethod_toc, fixed_size); + if (!success) return false; + + assert(MacroAssembler::is_load_const_from_method_toc_at(addr_at(ic_load_offset)), + "should be load from TOC"); + + address call_pc = trampoline_call(target_al, Rmethod_toc, scratch_emit); + return call_pc != nullptr; +} + +address MacroAssembler::trampoline_call(AddressLiteral target, + Register Rmethod_toc, + bool scratch_emit) { + // First, emit the trampoline stub + if (!scratch_emit) { + RelocationHolder rh = trampoline_stub_Relocation::spec(pc() /* of the bl below */); + + // Put the target's entry point as a constant into the constant pool. + const address target_toc_addr = address_constant((address)target.value()); + if (target_toc_addr == nullptr) return nullptr; + + const int target_toc_offset = offset_to_method_toc(target_toc_addr); + address stub = start_a_stub(64); + if (stub == nullptr) return nullptr; + + // Annotate the stub with a relocation that points to the owning call instruction. + relocate(rh); + DEBUG_ONLY(int stub_start_offset = offset()); + + // For java_to_interp stubs we use R11_scratch1 as scratch register + // and in call trampoline stubs we use R12_scratch2. This way we + // can distinguish them (see is_NativeCallTrampolineStub_at()). + Register reg_scratch = R12_scratch2; + + if (Rmethod_toc == noreg) { + calculate_address_from_global_toc(reg_scratch, method_toc()); + Rmethod_toc = reg_scratch; + } + + ld_largeoffset_unchecked(reg_scratch, target_toc_offset, Rmethod_toc, false); + mtctr(reg_scratch); + bctr(); + + assert(target_toc_offset == NativeCallTrampolineStub_at(addr_at(stub_start_offset))->destination_toc_offset(), + "encoded offset into the constant pool must match"); + assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size"); + assert(is_NativeCallTrampolineStub_at(addr_at(stub_start_offset)), "doesn't look like a trampoline"); + + // End the stub. + end_a_stub(); + } + + // The call will be resolved / patched later. + address call_pc = pc(); + relocate(target.rspec()); + bl(call_pc); + return call_pc; +} + void MacroAssembler::post_call_nop() { // Make inline again when loom is always enabled. if (!Continuations::enabled()) { @@ -2615,50 +2684,6 @@ void MacroAssembler::tlab_allocate( //verify_tlab(); not implemented } -address MacroAssembler::emit_trampoline_stub(int destination_toc_offset, - int insts_call_instruction_offset, Register Rtoc) { - // Start the stub. - address stub = start_a_stub(64); - if (stub == nullptr) { return nullptr; } // CodeCache full: bail out - - // Create a trampoline stub relocation which relates this trampoline stub - // with the call instruction at insts_call_instruction_offset in the - // instructions code-section. - relocate(trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset)); - const int stub_start_offset = offset(); - - // For java_to_interp stubs we use R11_scratch1 as scratch register - // and in call trampoline stubs we use R12_scratch2. This way we - // can distinguish them (see is_NativeCallTrampolineStub_at()). - Register reg_scratch = R12_scratch2; - - // Now, create the trampoline stub's code: - // - load the TOC - // - load the call target from the constant pool - // - call - if (Rtoc == noreg) { - calculate_address_from_global_toc(reg_scratch, method_toc()); - Rtoc = reg_scratch; - } - - ld_largeoffset_unchecked(reg_scratch, destination_toc_offset, Rtoc, false); - mtctr(reg_scratch); - bctr(); - - const address stub_start_addr = addr_at(stub_start_offset); - - // Assert that the encoded destination_toc_offset can be identified and that it is correct. - assert(destination_toc_offset == NativeCallTrampolineStub_at(stub_start_addr)->destination_toc_offset(), - "encoded offset into the constant pool must match"); - // Trampoline_stub_size should be good. - assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size"); - assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline"); - - // End the stub. - end_a_stub(); - return stub; -} - // "The box" is the space on the stack where we copy the object mark. void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register obj, Register box, Register tmp1, Register tmp2, Register tmp3) { @@ -3202,24 +3227,6 @@ void MacroAssembler::store_klass_gap(Register dst_oop, Register val) { stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); } -int MacroAssembler::instr_size_for_load_klass() { - static int computed_size = -1; - - // Not yet computed? - if (computed_size == -1) { - - // Determine by scratch emit. - ResourceMark rm; - int code_size = 16 * BytesPerInstWord; - CodeBuffer cb("load_klass scratch buffer", code_size, 0); - MacroAssembler* a = new MacroAssembler(&cb); - a->load_klass(R11_scratch1, R11_scratch1); - computed_size = a->offset(); - } - - return computed_size; -} - void MacroAssembler::decode_klass_not_null(Register dst, Register src) { assert(dst != R0, "Dst reg may not be R0, as R0 is used here."); if (src == noreg) src = dst; diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp index b2f5e8f0b60..21ab192373f 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp @@ -380,9 +380,20 @@ class MacroAssembler: public Assembler { Register toc); #endif + // CompiledIC call + bool ic_call(Register Rmethod_toc, + address target, + jint method_index = 0, + bool scratch_emit = false, + bool fixed_size = false); static int ic_check_size(); int ic_check(int end_alignment); + enum { trampoline_stub_size = 6 * 4 }; + address trampoline_call(AddressLiteral target, + Register Rmethod_toc = noreg, + bool scratch_emit = false); + protected: // It is imperative that all calls into the VM are handled via the @@ -702,9 +713,6 @@ class MacroAssembler: public Assembler { Label& slow_case // continuation point if fast allocation fails ); - enum { trampoline_stub_size = 6 * 4 }; - address emit_trampoline_stub(int destination_toc_offset, int insts_call_instruction_offset, Register Rtoc = noreg); - void compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box, Register tmp1, Register tmp2, Register tmp3); @@ -802,7 +810,6 @@ class MacroAssembler: public Assembler { MacroAssembler::PreservationLevel preservation_level); void load_method_holder(Register holder, Register method); - static int instr_size_for_load_klass(); void decode_klass_not_null(Register dst, Register src = noreg); Register encode_klass_not_null(Register dst, Register src = noreg); diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 00549ac8508..66538271232 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -1177,18 +1177,7 @@ int MachCallStaticJavaNode::ret_addr_offset() { } int MachCallDynamicJavaNode::ret_addr_offset() { - // Offset is 4 with postalloc expanded calls (bl is one instruction). We use - // postalloc expanded calls if we use inline caches and do not update method data. - if (UseInlineCaches) return 4; - - int vtable_index = this->_vtable_index; - if (vtable_index < 0) { - // Must be invalid_vtable_index, not nonvirtual_vtable_index. - assert(vtable_index == Method::invalid_vtable_index, "correct sentinel value"); - return 12; - } else { - return 20 + MacroAssembler::instr_size_for_load_klass(); - } + return 12; } int MachCallRuntimeNode::ret_addr_offset() { @@ -1311,9 +1300,6 @@ class CallStubImpl { public: - // Emit call stub, compiled java to interpreter. - static void emit_trampoline_stub(C2_MacroAssembler *masm, int destination_toc_offset, int insts_call_instruction_offset); - // Size of call trampoline stub. // This doesn't need to be accurate to the byte, but it // must be larger than or equal to the real size of the stub. @@ -1332,81 +1318,6 @@ class CallStubImpl { source %{ -// Emit a trampoline stub for a call to a target which is too far away. -// -// code sequences: -// -// call-site: -// branch-and-link to or -// -// Related trampoline stub for this call-site in the stub section: -// load the call target from the constant pool -// branch via CTR (LR/link still points to the call-site above) - -void CallStubImpl::emit_trampoline_stub(C2_MacroAssembler *masm, int destination_toc_offset, int insts_call_instruction_offset) { - address stub = __ emit_trampoline_stub(destination_toc_offset, insts_call_instruction_offset); - if (stub == nullptr) { - ciEnv::current()->record_out_of_memory_failure(); - } -} - -//============================================================================= - -// Emit an inline branch-and-link call and a related trampoline stub. -// -// code sequences: -// -// call-site: -// branch-and-link to or -// -// Related trampoline stub for this call-site in the stub section: -// load the call target from the constant pool -// branch via CTR (LR/link still points to the call-site above) -// - -typedef struct { - int insts_call_instruction_offset; - int ret_addr_offset; -} EmitCallOffsets; - -// Emit a branch-and-link instruction that branches to a trampoline. -// - Remember the offset of the branch-and-link instruction. -// - Add a relocation at the branch-and-link instruction. -// - Emit a branch-and-link. -// - Remember the return pc offset. -EmitCallOffsets emit_call_with_trampoline_stub(C2_MacroAssembler *masm, address entry_point, relocInfo::relocType rtype) { - EmitCallOffsets offsets = { -1, -1 }; - const int start_offset = __ offset(); - offsets.insts_call_instruction_offset = __ offset(); - - // No entry point given, use the current pc. - if (entry_point == nullptr) entry_point = __ pc(); - - // Put the entry point as a constant into the constant pool. - const address entry_point_toc_addr = __ address_constant(entry_point, RelocationHolder::none); - if (entry_point_toc_addr == nullptr) { - ciEnv::current()->record_out_of_memory_failure(); - return offsets; - } - const int entry_point_toc_offset = __ offset_to_method_toc(entry_point_toc_addr); - - // Emit the trampoline stub which will be related to the branch-and-link below. - CallStubImpl::emit_trampoline_stub(masm, entry_point_toc_offset, offsets.insts_call_instruction_offset); - if (ciEnv::current()->failing()) { return offsets; } // Code cache may be full. - __ relocate(rtype); - - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ bl((address) __ pc()); - - offsets.ret_addr_offset = __ offset() - start_offset; - - return offsets; -} - -//============================================================================= - // Factory for creating loadConL* nodes for large/small constant pool. static inline jlong replicate_immF(float con) { @@ -3321,45 +3232,27 @@ encode %{ // Usage of r1 and r2 in the stubs allows to distinguish them. enc_class enc_java_static_call(method meth) %{ address entry_point = (address)$meth$$method; + address call_pc; if (!_method) { // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap. - emit_call_with_trampoline_stub(masm, entry_point, relocInfo::runtime_call_type); - if (ciEnv::current()->failing()) { return; } // Code cache may be full. - } else { - // Remember the offset not the address. - const int start_offset = __ offset(); - - // The trampoline stub. - // No entry point given, use the current pc. - // Make sure branch fits into - if (entry_point == nullptr) entry_point = __ pc(); - - // Put the entry point as a constant into the constant pool. - const address entry_point_toc_addr = __ address_constant(entry_point, RelocationHolder::none); - if (entry_point_toc_addr == nullptr) { - ciEnv::current()->record_out_of_memory_failure(); + call_pc = __ trampoline_call(AddressLiteral(entry_point, relocInfo::runtime_call_type)); + if (call_pc == nullptr) { + ciEnv::current()->record_failure("CodeCache is full"); return; } - const int entry_point_toc_offset = __ offset_to_method_toc(entry_point_toc_addr); - - // Emit the trampoline stub which will be related to the branch-and-link below. - CallStubImpl::emit_trampoline_stub(masm, entry_point_toc_offset, start_offset); - if (ciEnv::current()->failing()) { return; } // Code cache may be full. + } else { int method_index = resolved_method_index(masm); - __ relocate(_optimized_virtual ? opt_virtual_call_Relocation::spec(method_index) - : static_call_Relocation::spec(method_index)); + RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index) + : static_call_Relocation::spec(method_index); + call_pc = __ trampoline_call(AddressLiteral(entry_point, rspec)); + if (call_pc == nullptr) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } - // The real call. - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ set_inst_mark(); - __ bl(__ pc()); // Emits a relocation. - - // The stub for call to interpreter. - address stub = CompiledDirectCall::emit_to_interp_stub(masm); - __ clear_inst_mark(); + // Emit stub for static call + address stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc); if (stub == nullptr) { ciEnv::current()->record_failure("CodeCache is full"); return; @@ -3368,157 +3261,21 @@ encode %{ __ post_call_nop(); %} - // Second node of expanded dynamic call - the call. - enc_class enc_java_dynamic_call_sched(method meth) %{ - if (!ra_->C->output()->in_scratch_emit_size()) { - // Create a call trampoline stub for the given method. - const address entry_point = !($meth$$method) ? nullptr : (address)$meth$$method; - const address entry_point_const = __ address_constant(entry_point, RelocationHolder::none); - if (entry_point_const == nullptr) { - ciEnv::current()->record_out_of_memory_failure(); - return; - } - const int entry_point_const_toc_offset = __ offset_to_method_toc(entry_point_const); - CallStubImpl::emit_trampoline_stub(masm, entry_point_const_toc_offset, __ offset()); - if (ra_->C->env()->failing()) { return; } // Code cache may be full. - - // Build relocation at call site with ic position as data. - assert((_load_ic_hi_node != nullptr && _load_ic_node == nullptr) || - (_load_ic_hi_node == nullptr && _load_ic_node != nullptr), - "must have one, but can't have both"); - assert((_load_ic_hi_node != nullptr && _load_ic_hi_node->_cbuf_insts_offset != -1) || - (_load_ic_node != nullptr && _load_ic_node->_cbuf_insts_offset != -1), - "must contain instruction offset"); - const int virtual_call_oop_addr_offset = _load_ic_hi_node != nullptr - ? _load_ic_hi_node->_cbuf_insts_offset - : _load_ic_node->_cbuf_insts_offset; - const address virtual_call_oop_addr = __ addr_at(virtual_call_oop_addr_offset); - assert(MacroAssembler::is_load_const_from_method_toc_at(virtual_call_oop_addr), - "should be load from TOC"); - int method_index = resolved_method_index(masm); - __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr, method_index)); - } - - // At this point I do not have the address of the trampoline stub, - // and the entry point might be too far away for bl. Pc() serves - // as dummy and bl will be patched later. - __ bl((address) __ pc()); - __ post_call_nop(); - %} - - // postalloc expand emitter for virtual calls. - enc_class postalloc_expand_java_dynamic_call_sched(method meth, iRegLdst toc) %{ - - // Create the nodes for loading the IC from the TOC. - loadConLNodesTuple loadConLNodes_IC = - loadConLNodesTuple_create(ra_, n_toc, new immLOper((jlong) Universe::non_oop_word()), - OptoReg::Name(R19_H_num), OptoReg::Name(R19_num)); - - // Create the call node. - CallDynamicJavaDirectSchedNode *call = new CallDynamicJavaDirectSchedNode(); - call->_vtable_index = _vtable_index; - call->_method = _method; - call->_optimized_virtual = _optimized_virtual; - call->_tf = _tf; - call->_entry_point = _entry_point; - call->_cnt = _cnt; - call->_guaranteed_safepoint = true; - call->_oop_map = _oop_map; - call->_jvms = _jvms; - call->_jvmadj = _jvmadj; - call->_has_ea_local_in_scope = _has_ea_local_in_scope; - call->_in_rms = _in_rms; - call->_nesting = _nesting; - call->_override_symbolic_info = _override_symbolic_info; - call->_arg_escape = _arg_escape; - - // New call needs all inputs of old call. - // Req... - for (uint i = 0; i < req(); ++i) { - // The expanded node does not need toc any more. - // Add the inline cache constant here instead. This expresses the - // register of the inline cache must be live at the call. - // Else we would have to adapt JVMState by -1. - if (i == mach_constant_base_node_input()) { - call->add_req(loadConLNodes_IC._last); - } else { - call->add_req(in(i)); - } - } - // ...as well as prec - for (uint i = req(); i < len(); ++i) { - call->add_prec(in(i)); - } - - // Remember nodes loading the inline cache into r19. - call->_load_ic_hi_node = loadConLNodes_IC._large_hi; - call->_load_ic_node = loadConLNodes_IC._small; - - // Operands for new nodes. - call->_opnds[0] = _opnds[0]; - call->_opnds[1] = _opnds[1]; - - // Only the inline cache is associated with a register. - assert(Matcher::inline_cache_reg() == OptoReg::Name(R19_num), "ic reg should be R19"); - - // Push new nodes. - if (loadConLNodes_IC._large_hi) nodes->push(loadConLNodes_IC._large_hi); - if (loadConLNodes_IC._last) nodes->push(loadConLNodes_IC._last); - nodes->push(call); - %} - // Compound version of call dynamic // Toc is only passed so that it can be used in ins_encode statement. // In the code we have to use $constanttablebase. enc_class enc_java_dynamic_call(method meth, iRegLdst toc) %{ int start_offset = __ offset(); - - Register Rtoc = (ra_) ? $constanttablebase : R2_TOC; - - int vtable_index = this->_vtable_index; - if (vtable_index < 0) { - // Must be invalid_vtable_index, not nonvirtual_vtable_index. - assert(vtable_index == Method::invalid_vtable_index, "correct sentinel value"); - Register ic_reg = as_Register(Matcher::inline_cache_reg_encode()); - - // Virtual call relocation will point to ic load. - address virtual_call_meta_addr = __ pc(); - // Load a clear inline cache. - AddressLiteral empty_ic((address) Universe::non_oop_word()); - bool success = __ load_const_from_method_toc(ic_reg, empty_ic, Rtoc, /*fixed_size*/ true); - if (!success) { - ciEnv::current()->record_out_of_memory_failure(); - return; - } - // CALL to fixup routine. Fixup routine uses ScopeDesc info - // to determine who we intended to call. - __ relocate(virtual_call_Relocation::spec(virtual_call_meta_addr)); - emit_call_with_trampoline_stub(masm, (address)$meth$$method, relocInfo::none); - if (ciEnv::current()->failing()) { return; } // Code cache may be full. - assert(((MachCallDynamicJavaNode*)this)->ret_addr_offset() == __ offset() - start_offset, - "Fix constant in ret_addr_offset(), expected %d", __ offset() - start_offset); - } else { - assert(!UseInlineCaches, "expect vtable calls only if not using ICs"); - // Go thru the vtable. Get receiver klass. Receiver already - // checked for non-null. If we'll go thru a C2I adapter, the - // interpreter expects method in R19_method. - - __ load_klass(R11_scratch1, R3); - - int entry_offset = in_bytes(Klass::vtable_start_offset()) + vtable_index * vtableEntry::size_in_bytes(); - int v_off = entry_offset + in_bytes(vtableEntry::method_offset()); - __ li(R19_method, v_off); - __ ldx(R19_method/*method*/, R19_method/*method offset*/, R11_scratch1/*class*/); - // NOTE: for vtable dispatches, the vtable entry will never be - // null. However it may very well end up in handle_wrong_method - // if the method is abstract for the particular class. - __ ld(R11_scratch1, in_bytes(Method::from_compiled_offset()), R19_method); - // Call target. Either compiled code or C2I adapter. - __ mtctr(R11_scratch1); - __ bctrl(); - assert(((MachCallDynamicJavaNode*)this)->ret_addr_offset() == __ offset() - start_offset, - "Fix constant in ret_addr_offset(), expected %d", __ offset() - start_offset); + int method_index = resolved_method_index(masm); + bool scratch_emit = ra_ == nullptr; + Register Rtoc = scratch_emit ? R2_TOC : $constanttablebase; + bool success = __ ic_call(Rtoc, (address)$meth$$method, method_index, scratch_emit, true /*fixed_size*/); + if (!success) { + ciEnv::current()->record_failure("CodeCache is full"); + return; } + assert(((MachCallDynamicJavaNode*)this)->ret_addr_offset() == __ offset() - start_offset, + "Fix constant in ret_addr_offset(), expected %d", __ offset() - start_offset); __ post_call_nop(); %} @@ -14028,15 +13785,14 @@ instruct safePoint_poll(iRegPdst poll) %{ // ============================================================================ // Call Instructions -// Call Java Static Instruction - source %{ #include "runtime/continuation.hpp" %} -// Schedulable version of call static node. +// Call Java Static Instruction + instruct CallStaticJavaDirect(method meth) %{ match(CallStaticJava); effect(USE meth); @@ -14052,51 +13808,9 @@ instruct CallStaticJavaDirect(method meth) %{ // Call Java Dynamic Instruction -// Used by postalloc expand of CallDynamicJavaDirectSchedEx (actual call). -// Loading of IC was postalloc expanded. The nodes loading the IC are reachable -// via fields ins_field_load_ic_hi_node and ins_field_load_ic_node. -// The call destination must still be placed in the constant pool. -instruct CallDynamicJavaDirectSched(method meth) %{ - match(CallDynamicJava); // To get all the data fields we need ... - effect(USE meth); - predicate(false); // ... but never match. - - ins_field_load_ic_hi_node(loadConL_hiNode*); - ins_field_load_ic_node(loadConLNode*); - ins_num_consts(1 /* 1 patchable constant: call destination */); - - format %{ "BL \t// dynamic $meth ==> " %} - size((Continuations::enabled() ? 8 : 4)); - ins_encode( enc_java_dynamic_call_sched(meth) ); - ins_pipe(pipe_class_call); -%} - -// Schedulable (i.e. postalloc expanded) version of call dynamic java. -// We use postalloc expanded calls if we use inline caches -// and do not update method data. -// -// This instruction has two constants: inline cache (IC) and call destination. -// Loading the inline cache will be postalloc expanded, thus leaving a call with -// one constant. -instruct CallDynamicJavaDirectSched_Ex(method meth) %{ - match(CallDynamicJava); - effect(USE meth); - predicate(UseInlineCaches); - ins_cost(CALL_COST); - - ins_num_consts(2 /* 2 patchable constants: inline cache, call destination. */); - - format %{ "CALL,dynamic $meth \t// postalloc expanded" %} - postalloc_expand( postalloc_expand_java_dynamic_call_sched(meth, constanttablebase) ); -%} - -// Compound version of call dynamic java -// We use postalloc expanded calls if we use inline caches -// and do not update method data. instruct CallDynamicJavaDirect(method meth) %{ match(CallDynamicJava); effect(USE meth); - predicate(!UseInlineCaches); ins_cost(CALL_COST); // Enc_java_to_runtime_call needs up to 4 constants (method data oop). diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index 54336e9f62b..fc444cb6923 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -1713,10 +1713,8 @@ static void gen_continuation_enter(MacroAssembler* masm, check_continuation_enter_argument(regs[pos_is_cont].first(), reg_is_cont, "isContinue"); check_continuation_enter_argument(regs[pos_is_virtual].first(), reg_is_virtual, "isVirtualThread"); - address resolve_static_call = SharedRuntime::get_resolve_static_call_stub(); - + AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(), relocInfo::static_call_type); address start = __ pc(); - Label L_thaw, L_exit; // i2i entry used at interp_only_mode only @@ -1753,33 +1751,17 @@ static void gen_continuation_enter(MacroAssembler* masm, // Emit compiled static call. The call will be always resolved to the c2i // entry of Continuation.enter(Continuation c, boolean isContinue). - // There are special cases in SharedRuntime::resolve_static_call_C() and - // SharedRuntime::resolve_sub_helper_internal() to achieve this - // See also corresponding call below. - address c2i_call_pc = __ pc(); - int start_offset = __ offset(); - // Put the entry point as a constant into the constant pool. - const address entry_point_toc_addr = __ address_constant(resolve_static_call, RelocationHolder::none); - const int entry_point_toc_offset = __ offset_to_method_toc(entry_point_toc_addr); - guarantee(entry_point_toc_addr != nullptr, "const section overflow"); + address c2i_call_pc = __ trampoline_call(resolve); + guarantee(c2i_call_pc != nullptr, "CodeCache is full at gen_continuation_enter"); - // Emit the trampoline stub which will be related to the branch-and-link below. - address stub = __ emit_trampoline_stub(entry_point_toc_offset, start_offset); - guarantee(stub != nullptr, "no space for trampoline stub"); + // Emit stub for static call + address stub = CompiledDirectCall::emit_to_interp_stub(masm, c2i_call_pc); + guarantee(stub != nullptr, "CodeCache is full at gen_continuation_enter"); - __ relocate(relocInfo::static_call_type); - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ bl(__ pc()); oop_maps->add_gc_map(__ pc() - start, map); __ post_call_nop(); __ b(L_exit); - - // static stub for the call above - stub = CompiledDirectCall::emit_to_interp_stub(masm, c2i_call_pc); - guarantee(stub != nullptr, "no space for static stub"); } // compiled entry @@ -1804,22 +1786,9 @@ static void gen_continuation_enter(MacroAssembler* masm, // SharedRuntime::find_callee_info_helper() which calls // LinkResolver::resolve_continuation_enter() which resolves the call to // Continuation.enter(Continuation c, boolean isContinue). - address call_pc = __ pc(); - int start_offset = __ offset(); - // Put the entry point as a constant into the constant pool. - const address entry_point_toc_addr = __ address_constant(resolve_static_call, RelocationHolder::none); - const int entry_point_toc_offset = __ offset_to_method_toc(entry_point_toc_addr); - guarantee(entry_point_toc_addr != nullptr, "const section overflow"); + address call_pc = __ trampoline_call(resolve); + guarantee(call_pc != nullptr, "CodeCache is full at gen_continuation_enter"); - // Emit the trampoline stub which will be related to the branch-and-link below. - address stub = __ emit_trampoline_stub(entry_point_toc_offset, start_offset); - guarantee(stub != nullptr, "no space for trampoline stub"); - - __ relocate(relocInfo::static_call_type); - // Note: At this point we do not have the address of the trampoline - // stub, and the entry point might be too far away for bl, so __ pc() - // serves as dummy and the bl will be patched later. - __ bl(__ pc()); oop_maps->add_gc_map(__ pc() - start, map); __ post_call_nop(); @@ -1872,8 +1841,8 @@ static void gen_continuation_enter(MacroAssembler* masm, __ blr(); // static stub for the call above - stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc); - guarantee(stub != nullptr, "no space for static stub"); + address stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc); + guarantee(stub != nullptr, "CodeCache is full at gen_continuation_enter"); } static void gen_continuation_yield(MacroAssembler* masm,