diff --git a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp index 1a24f78ad12..b7e1b7863ef 100644 --- a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp @@ -318,7 +318,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) { } Address resolve(SharedRuntime::get_resolve_static_call_stub(), relocInfo::static_call_type); - address call = __ trampoline_call(resolve); + address call = __ reloc_call(resolve); if (call == nullptr) { ce->bailout("trampoline stub overflow"); return; diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index b2489268611..798679185d3 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -1346,7 +1346,7 @@ void LIR_Assembler::align_call(LIR_Code code) { } void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { - address call = __ trampoline_call(Address(op->addr(), rtype)); + address call = __ reloc_call(Address(op->addr(), rtype)); if (call == nullptr) { bailout("trampoline stub overflow"); return; diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp index ce23213776c..4388d0cf4d7 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp @@ -69,15 +69,15 @@ private: enum { // See emit_static_call_stub for detail // CompiledDirectCall::to_interp_stub_size() (14) + CompiledDirectCall::to_trampoline_stub_size() (1 + 3 + address) - _call_stub_size = 14 * NativeInstruction::instruction_size + - (NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size), + _call_stub_size = 14 * MacroAssembler::instruction_size + + (MacroAssembler::instruction_size + MacroAssembler::NativeShortCall::trampoline_size), // See emit_exception_handler for detail // verify_not_null_oop + far_call + should_not_reach_here + invalidate_registers(DEBUG_ONLY) _exception_handler_size = DEBUG_ONLY(584) NOT_DEBUG(548), // or smaller // See emit_deopt_handler for detail // auipc (1) + far_jump (6 or 2) - _deopt_handler_size = 1 * NativeInstruction::instruction_size + - 6 * NativeInstruction::instruction_size // or smaller + _deopt_handler_size = 1 * MacroAssembler::instruction_size + + 6 * MacroAssembler::instruction_size // or smaller }; void check_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp index 3cb2e52c8cb..d88e4bf320d 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp @@ -1040,7 +1040,7 @@ void C2_MacroAssembler::string_indexof(Register haystack, Register needle, stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_uu()); assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated"); } - address call = trampoline_call(stub); + address call = reloc_call(stub); if (call == nullptr) { DEBUG_ONLY(reset_labels(LINEARSEARCH, DONE, NOMATCH)); ciEnv::current()->record_failure("CodeCache is full"); @@ -1478,7 +1478,7 @@ void C2_MacroAssembler::string_compare(Register str1, Register str2, ShouldNotReachHere(); } assert(stub.target() != nullptr, "compare_long_string stub has not been generated"); - address call = trampoline_call(stub); + address call = reloc_call(stub); if (call == nullptr) { DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START)); ciEnv::current()->record_failure("CodeCache is full"); diff --git a/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp b/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp index d62d595e4bc..e772959ed08 100644 --- a/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp +++ b/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp @@ -50,17 +50,18 @@ static bool emit_shared_trampolines(CodeBuffer* cb, CodeBuffer::SharedTrampoline if (requests == nullptr) { return true; } + assert(UseTrampolines, "We are not using trampolines"); MacroAssembler masm(cb); auto emit = [&](address dest, const CodeBuffer::Offsets &offsets) { - assert(cb->stubs()->remaining() >= MacroAssembler::max_trampoline_stub_size(), "pre-allocated trampolines"); + assert(cb->stubs()->remaining() >= MacroAssembler::max_reloc_call_stub_size(), "pre-allocated trampolines"); LinkedListIterator it(offsets.head()); int offset = *it.next(); address stub = __ emit_trampoline_stub(offset, dest); assert(stub, "pre-allocated trampolines"); - address reloc_pc = cb->stubs()->end() - NativeCallTrampolineStub::instruction_size; + address reloc_pc = cb->stubs()->end() - MacroAssembler::NativeShortCall::trampoline_size; while (!it.is_empty()) { offset = *it.next(); address caller_pc = cb->insts()->start() + offset; @@ -70,7 +71,7 @@ static bool emit_shared_trampolines(CodeBuffer* cb, CodeBuffer::SharedTrampoline }; assert(requests->number_of_entries() >= 1, "at least one"); - const int total_requested_size = MacroAssembler::max_trampoline_stub_size() * requests->number_of_entries(); + const int total_requested_size = MacroAssembler::max_reloc_call_stub_size() * requests->number_of_entries(); if (cb->stubs()->maybe_expand_to_ensure_remaining(total_requested_size) && cb->blob() == nullptr) { return false; } diff --git a/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp b/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp index 73a7f1cb89f..de70bc2ecc9 100644 --- a/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp +++ b/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp @@ -33,7 +33,7 @@ private: public: void flush_bundle(bool start_new_bundle) {} - static constexpr bool supports_shared_stubs() { return true; } + static bool supports_shared_stubs() { return UseTrampolines; } void share_trampoline_for(address dest, int caller_offset); diff --git a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp index 60dceb3ada7..4bbea8f356f 100644 --- a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp +++ b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp @@ -69,10 +69,9 @@ int CompiledDirectCall::to_interp_stub_size() { } int CompiledDirectCall::to_trampoline_stub_size() { - // Somewhat pessimistically, we count 4 instructions here (although - // there are only 3) because we sometimes emit an alignment nop. + // We count instructions and an additional alignment nop. // Trampoline stubs are always word aligned. - return MacroAssembler::max_trampoline_stub_size(); + return MacroAssembler::max_reloc_call_stub_size(); } // Relocation entries for call stub, compiled java to interpreter. diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp index c39b29f6ee4..c2585f2d161 100644 --- a/src/hotspot/cpu/riscv/globals_riscv.hpp +++ b/src/hotspot/cpu/riscv/globals_riscv.hpp @@ -120,6 +120,8 @@ define_pd_global(intx, InlineSmallCode, 1000); product(bool, UseZvkn, false, EXPERIMENTAL, \ "Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt") \ product(bool, UseRVVForBigIntegerShiftIntrinsics, true, \ - "Use RVV instructions for left/right shift of BigInteger") + "Use RVV instructions for left/right shift of BigInteger") \ + product(bool, UseTrampolines, false, EXPERIMENTAL, \ + "Far calls uses jal to trampoline.") #endif // CPU_RISCV_GLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp b/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp index ba3d9c99ace..a366ef6171d 100644 --- a/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp +++ b/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp @@ -39,7 +39,7 @@ jint CodeInstaller::pd_next_offset(NativeInstruction* inst, jint pc_offset, JVMCI_TRAPS) { address pc = (address) inst; if (inst->is_call()) { - return pc_offset + NativeCall::instruction_size; + return pc_offset + NativeCall::byte_size(); } else if (inst->is_jump()) { return pc_offset + NativeJump::instruction_size; } else if (inst->is_movptr1()) { diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index b3ae5fbcdd0..e349eab3177 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -978,36 +978,23 @@ void MacroAssembler::li(Register Rd, int64_t imm) { } } -void MacroAssembler::jump_link(const address dest, Register temp) { - assert_cond(dest != nullptr); - int64_t distance = dest - pc(); - if (is_simm21(distance) && ((distance % 2) == 0)) { - Assembler::jal(x1, distance); - } else { - assert(temp != noreg && temp != x0, "expecting a register"); - int32_t offset = 0; - la(temp, dest, offset); - jalr(temp, offset); - } +void MacroAssembler::load_link_jump(const address source, Register temp) { + assert(temp != noreg && temp != x0, "expecting a register"); + assert_cond(source != nullptr); + int64_t distance = source - pc(); + assert(is_simm32(distance), "Must be"); + auipc(temp, (int32_t)distance + 0x800); + ld(temp, Address(temp, ((int32_t)distance << 20) >> 20)); + jalr(temp); } -void MacroAssembler::jump_link(const Address &adr, Register temp) { - switch (adr.getMode()) { - case Address::literal: { - relocate(adr.rspec(), [&] { - jump_link(adr.target(), temp); - }); - break; - } - case Address::base_plus_offset: { - int32_t offset = ((int32_t)adr.offset() << 20) >> 20; - la(temp, Address(adr.base(), adr.offset() - offset)); - jalr(temp, offset); - break; - } - default: - ShouldNotReachHere(); - } +void MacroAssembler::jump_link(const address dest, Register temp) { + assert(UseTrampolines, "Must be"); + assert_cond(dest != nullptr); + int64_t distance = dest - pc(); + assert(is_simm21(distance), "Must be"); + assert((distance % 2) == 0, "Must be"); + jal(x1, distance); } void MacroAssembler::j(const address dest, Register temp) { @@ -3941,15 +3928,7 @@ bool MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass, // The next slot to be inspected, by the stub we're about to call, // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap // have been checked. - Address stub = RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()); - if (stub_is_near) { - jump_link(stub, t0); - } else { - address call = trampoline_call(stub); - if (call == nullptr) { - return false; // trampoline allocation failed - } - } + rt_call(StubRoutines::lookup_secondary_supers_table_slow_path_stub()); BLOCK_COMMENT("} lookup_secondary_supers_table"); @@ -4258,12 +4237,42 @@ address MacroAssembler::trampoline_call(Address entry) { return call_pc; } +address MacroAssembler::load_and_call(Address entry) { + assert(entry.rspec().type() == relocInfo::runtime_call_type || + entry.rspec().type() == relocInfo::opt_virtual_call_type || + entry.rspec().type() == relocInfo::static_call_type || + entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); + + address target = entry.target(); + + if (!in_scratch_emit_size()) { + address stub = emit_address_stub(offset(), target); + if (stub == nullptr) { + postcond(pc() == badAddress); + return nullptr; // CodeCache is full + } + } + + address call_pc = pc(); +#ifdef ASSERT + if (entry.rspec().type() != relocInfo::runtime_call_type) { + assert_alignment(call_pc); + } +#endif + relocate(entry.rspec(), [&] { + load_link_jump(target); + }); + + postcond(pc() != badAddress); + return call_pc; +} + address MacroAssembler::ic_call(address entry, jint method_index) { RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); IncompressibleRegion ir(this); // relocations movptr(t1, (address)Universe::non_oop_word(), t0); assert_cond(entry != nullptr); - return trampoline_call(Address(entry, rh)); + return reloc_call(Address(entry, rh)); } int MacroAssembler::ic_check_size() { @@ -4308,6 +4317,34 @@ int MacroAssembler::ic_check(int end_alignment) { return uep_offset; } +address MacroAssembler::emit_address_stub(int insts_call_instruction_offset, address dest) { + address stub = start_a_stub(max_reloc_call_stub_size()); + if (stub == nullptr) { + return nullptr; // CodeBuffer::expand failed + } + + // We are always 4-byte aligned here. + assert_alignment(pc()); + + // Make sure the address of destination 8-byte aligned. + align(wordSize, 0); + + RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + + insts_call_instruction_offset); + const int stub_start_offset = offset(); + relocate(rh, [&] { + assert(offset() - stub_start_offset == 0, + "%ld - %ld == %ld : should be", (long)offset(), (long)stub_start_offset, (long)0); + assert(offset() % wordSize == 0, "bad alignment"); + emit_int64((int64_t)dest); + }); + + const address stub_start_addr = addr_at(stub_start_offset); + end_a_stub(); + + return stub_start_addr; +} + // Emit a trampoline stub for a call to a target which is too far away. // // code sequences: @@ -4322,11 +4359,13 @@ int MacroAssembler::ic_check(int end_alignment) { address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, address dest) { // Max stub size: alignment nop, TrampolineStub. - address stub = start_a_stub(max_trampoline_stub_size()); + address stub = start_a_stub(max_reloc_call_stub_size()); if (stub == nullptr) { return nullptr; // CodeBuffer::expand failed } + assert(UseTrampolines, "Must be using trampos."); + // We are always 4-byte aligned here. assert_alignment(pc()); @@ -4335,7 +4374,7 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, // instructions code-section. // Make sure the address of destination 8-byte aligned after 3 instructions. - align(wordSize, MacroAssembler::trampoline_stub_data_offset); + align(wordSize, MacroAssembler::NativeShortCall::trampoline_data_offset); RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset); @@ -4348,7 +4387,7 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, ld(t0, target); // auipc + ld jr(t0); // jalr bind(target); - assert(offset() - stub_start_offset == MacroAssembler::trampoline_stub_data_offset, + assert(offset() - stub_start_offset == MacroAssembler::NativeShortCall::trampoline_data_offset, "should be"); assert(offset() % wordSize == 0, "bad alignment"); emit_int64((int64_t)dest); @@ -4356,15 +4395,17 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, const address stub_start_addr = addr_at(stub_start_offset); - assert(MacroAssembler::is_trampoline_stub_at(stub_start_addr), "doesn't look like a trampoline"); - end_a_stub(); + return stub_start_addr; } -int MacroAssembler::max_trampoline_stub_size() { +int MacroAssembler::max_reloc_call_stub_size() { // Max stub size: alignment nop, TrampolineStub. - return MacroAssembler::instruction_size + MacroAssembler::trampoline_stub_instruction_size; + if (UseTrampolines) { + return instruction_size + MacroAssembler::NativeShortCall::trampoline_size; + } + return instruction_size + wordSize; } int MacroAssembler::static_call_stub_size() { @@ -5083,14 +5124,14 @@ address MacroAssembler::zero_words(Register ptr, Register cnt) { RuntimeAddress zero_blocks(StubRoutines::riscv::zero_blocks()); assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); if (StubRoutines::riscv::complete()) { - address tpc = trampoline_call(zero_blocks); + address tpc = reloc_call(zero_blocks); if (tpc == nullptr) { DEBUG_ONLY(reset_labels(around)); postcond(pc() == badAddress); return nullptr; } } else { - jump_link(zero_blocks, t0); + rt_call(zero_blocks.target()); } } bind(around); diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp index ea2b9229eab..3c1add90367 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -466,8 +466,10 @@ class MacroAssembler: public Assembler { return false; } + address emit_address_stub(int insts_call_instruction_offset, address target); address emit_trampoline_stub(int insts_call_instruction_offset, address target); - static int max_trampoline_stub_size(); + static int max_reloc_call_stub_size(); + void emit_static_call_stub(); static int static_call_stub_size(); @@ -623,8 +625,8 @@ class MacroAssembler: public Assembler { void bgtz(Register Rs, const address dest); private: + void load_link_jump(const address source, Register temp = t0); void jump_link(const address dest, Register temp); - void jump_link(const Address &adr, Register temp); public: // We try to follow risc-v asm menomics. // But as we don't layout a reachable GOT, @@ -1205,7 +1207,27 @@ public: // be used instead. // All instructions are embedded at a call site. // - // - trampoline call: + // - indirect call: movptr + jalr + // This too can reach anywhere in the address space, but it cannot be + // patched while code is running, so it must only be modified at a safepoint. + // This form of call is most suitable for targets at fixed addresses, which + // will never be patched. + // + // - reloc call: + // This is only available in C1/C2-generated code (nmethod). + // + // [Main code section] + // auipc + // ld + // jalr + // [Stub section] + // trampoline: + // <64-bit destination address> + // + // To change the destination we simply atomically store the new + // address in the stub section. + // + // - trampoline call (old reloc call / -XX:+UseTrampolines): // This is only available in C1/C2-generated code (nmethod). It is a combination // of a direct call, which is used if the destination of a call is in range, // and a register-indirect call. It has the advantages of reaching anywhere in @@ -1224,18 +1246,11 @@ public: // cache, 'jal trampoline' is replaced with 'jal destination' and the trampoline // is not used. // The optimization does not remove the trampoline from the stub section. - + // // This is necessary because the trampoline may well be redirected later when // code is patched, and the new destination may not be reachable by a simple JAL // instruction. // - // - indirect call: movptr + jalr - // This too can reach anywhere in the address space, but it cannot be - // patched while code is running, so it must only be modified at a safepoint. - // This form of call is most suitable for targets at fixed addresses, which - // will never be patched. - // - // // To patch a trampoline call when the JAL can't reach, we first modify // the 64-bit destination address in the trampoline, then modify the // JAL to point to the trampoline, then flush the instruction cache to @@ -1248,9 +1263,10 @@ public: // invalidated, so there will be a trap at its start. // For this to work, the destination address in the trampoline is // always updated, even if we're not using the trampoline. + // -- // Emit a direct call if the entry address will always be in range, - // otherwise a trampoline call. + // otherwise a reloc call. // Supported entry.rspec(): // - relocInfo::runtime_call_type // - relocInfo::opt_virtual_call_type @@ -1258,7 +1274,13 @@ public: // - relocInfo::virtual_call_type // // Return: the call PC or null if CodeCache is full. + address reloc_call(Address entry) { + return UseTrampolines ? trampoline_call(entry) : load_and_call(entry); + } + private: address trampoline_call(Address entry); + address load_and_call(Address entry); + public: address ic_call(address entry, jint method_index = 0); static int ic_check_size(); @@ -1585,51 +1607,20 @@ public: public: enum { - // Refer to function emit_trampoline_stub. - trampoline_stub_instruction_size = 3 * instruction_size + wordSize, // auipc + ld + jr + target address - trampoline_stub_data_offset = 3 * instruction_size, // auipc + ld + jr - // movptr movptr1_instruction_size = 6 * instruction_size, // lui, addi, slli, addi, slli, addi. See movptr1(). movptr2_instruction_size = 5 * instruction_size, // lui, lui, slli, add, addi. See movptr2(). load_pc_relative_instruction_size = 2 * instruction_size // auipc, ld }; + enum NativeShortCall { + trampoline_size = 3 * instruction_size + wordSize, + trampoline_data_offset = 3 * instruction_size + }; + static bool is_load_pc_relative_at(address branch); static bool is_li16u_at(address instr); - static bool is_trampoline_stub_at(address addr) { - // Ensure that the stub is exactly - // ld t0, L--->auipc + ld - // jr t0 - // L: - - // judge inst + register + imm - // 1). check the instructions: auipc + ld + jalr - // 2). check if auipc[11:7] == t0 and ld[11:7] == t0 and ld[19:15] == t0 && jr[19:15] == t0 - // 3). check if the offset in ld[31:20] equals the data_offset - assert_cond(addr != nullptr); - const int instr_size = instruction_size; - if (is_auipc_at(addr) && - is_ld_at(addr + instr_size) && - is_jalr_at(addr + 2 * instr_size) && - (extract_rd(addr) == x5) && - (extract_rd(addr + instr_size) == x5) && - (extract_rs1(addr + instr_size) == x5) && - (extract_rs1(addr + 2 * instr_size) == x5) && - (Assembler::extract(Assembler::ld_instr(addr + 4), 31, 20) == trampoline_stub_data_offset)) { - return true; - } - return false; - } - - static bool is_call_at(address instr) { - if (is_jal_at(instr) || is_jalr_at(instr)) { - return true; - } - return false; - } - static bool is_jal_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b1101111; } static bool is_jalr_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b1100111 && extract_funct3(instr) == 0b000; } static bool is_branch_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b1100011; } @@ -1664,7 +1655,6 @@ public: static bool is_lwu_to_zr(address instr); -private: static Register extract_rs1(address instr); static Register extract_rs2(address instr); static Register extract_rd(address instr); diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp index 348a00b6767..f0357f1cd30 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp @@ -39,71 +39,228 @@ #include "c1/c1_Runtime1.hpp" #endif -void NativeCall::verify() { - assert(MacroAssembler::is_call_at((address)this), "unexpected code at call site"); +//----------------------------------------------------------------------------- +// NativeInstruction + +bool NativeInstruction::is_call_at(address addr) { + return NativeCall::is_at(addr); } -address NativeCall::destination() const { - address addr = (address)this; +//----------------------------------------------------------------------------- +// NativeShortCallTrampoline +// +// Implements the trampoline part of reloc call - trampoline call. + +class NativeShortCall; + +class NativeShortCallTrampolineStub : public NativeInstruction { + private: + friend NativeShortCall; + enum RISCV_specific_constants { + trampoline_data_offset = 3 * NativeInstruction::instruction_size // auipc + ld + jr + }; + + address destination() const; + void set_destination(address new_destination); + + static bool is_at(address addr); + static NativeShortCallTrampolineStub* at(address addr); +}; + +address NativeShortCallTrampolineStub::destination() const { + return ptr_at(trampoline_data_offset); +} + +void NativeShortCallTrampolineStub::set_destination(address new_destination) { + set_ptr_at(trampoline_data_offset, new_destination); + OrderAccess::release(); +} + +bool NativeShortCallTrampolineStub::is_at(address addr) { + // Ensure that the stub is exactly + // ld t0, L--->auipc + ld + // jr t0 + // L: + + // judge inst + register + imm + // 1). check the instructions: auipc + ld + jalr + // 2). check if auipc[11:7] == t0 and ld[11:7] == t0 and ld[19:15] == t0 && jr[19:15] == t0 + // 3). check if the offset in ld[31:20] equals the data_offset + assert_cond(addr != nullptr); + const int instr_size = NativeInstruction::instruction_size; + if (MacroAssembler::is_auipc_at(addr) && + MacroAssembler::is_ld_at(addr + instr_size) && + MacroAssembler::is_jalr_at(addr + 2 * instr_size) && + (MacroAssembler::extract_rd(addr) == x5) && + (MacroAssembler::extract_rd(addr + instr_size) == x5) && + (MacroAssembler::extract_rs1(addr + instr_size) == x5) && + (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x5) && + (Assembler::extract(Assembler::ld_instr(addr + 4), 31, 20) == trampoline_data_offset)) { + return true; + } + return false; +} + +NativeShortCallTrampolineStub* NativeShortCallTrampolineStub::at(address addr) { + assert_cond(addr != nullptr); + assert(NativeShortCallTrampolineStub::is_at(addr), "no call trampoline found"); + return (NativeShortCallTrampolineStub*)addr; +} + +//----------------------------------------------------------------------------- +// NativeShortCall +// +// Implements the trampoline call, a short call with a trampoline, version of reloc call. +// Enabled by setting the experimental UseTrampolines to true. + +class NativeShortCall: private NativeInstruction { + public: + enum RISCV_specific_constants { + return_address_offset = 1 * NativeInstruction::instruction_size // jal + }; + + address instruction_address() const { return addr_at(0); } + address next_instruction_address() const { return addr_at(return_address_offset); } + address return_address() const { return addr_at(return_address_offset); } + address destination() const; + address reloc_destination(address orig_address); + + void set_destination(address dest); + void verify(); + void print(); + + bool set_destination_mt_safe(address dest, bool assert_lock = true); + bool reloc_set_destination(address dest); + + private: + address get_trampoline(); + bool has_trampoline(); + address trampoline_destination(); + public: + + static NativeShortCall* at(address addr); + static bool is_at(address addr); + static bool is_call_before(address return_address); +}; + +address NativeShortCall::destination() const { + address addr = instruction_address(); assert(MacroAssembler::is_jal_at(instruction_address()), "inst must be jal."); + address destination = MacroAssembler::target_addr_for_insn(instruction_address()); // Do we use a trampoline stub for this call? CodeBlob* cb = CodeCache::find_blob(addr); assert(cb && cb->is_nmethod(), "sanity"); nmethod *nm = (nmethod *)cb; - if (nm != nullptr && nm->stub_contains(destination) && MacroAssembler::is_trampoline_stub_at(destination)) { + if (nm != nullptr && nm->stub_contains(destination) && NativeShortCallTrampolineStub::is_at(destination)) { // Yes we do, so get the destination from the trampoline stub. const address trampoline_stub_addr = destination; - destination = nativeCallTrampolineStub_at(trampoline_stub_addr)->destination(); + destination = NativeShortCallTrampolineStub::at(trampoline_stub_addr)->destination(); } return destination; } -// Similar to replace_mt_safe, but just changes the destination. The -// important thing is that free-running threads are able to execute this -// call instruction at all times. +address NativeShortCall::reloc_destination(address orig_address) { + address addr = instruction_address(); + if (NativeShortCall::is_at(addr)) { + NativeShortCall* call = NativeShortCall::at(addr); + if (call->has_trampoline()) { + return call->trampoline_destination(); + } + } + if (orig_address != nullptr) { + // the extracted address from the instructions in address orig_addr + address new_addr = MacroAssembler::pd_call_destination(orig_address); + // If call is branch to self, don't try to relocate it, just leave it + // as branch to self. This happens during code generation if the code + // buffer expands. It will be relocated to the trampoline above once + // code generation is complete. + new_addr = (new_addr == orig_address) ? addr : new_addr; + return new_addr; + } + return MacroAssembler::pd_call_destination(addr); +} + +void NativeShortCall::set_destination(address dest) { + assert(NativeShortCall::is_at(instruction_address()), "unexpected code at call site"); + assert(is_jal(), "Should be jal instruction!"); + intptr_t offset = (intptr_t)(dest - instruction_address()); + assert((offset & 0x1) == 0, "bad alignment"); + assert(Assembler::is_simm21(offset), "encoding constraint"); + unsigned int insn = 0b1101111; // jal + address pInsn = (address)(&insn); + Assembler::patch(pInsn, 31, 31, (offset >> 20) & 0x1); + Assembler::patch(pInsn, 30, 21, (offset >> 1) & 0x3ff); + Assembler::patch(pInsn, 20, 20, (offset >> 11) & 0x1); + Assembler::patch(pInsn, 19, 12, (offset >> 12) & 0xff); + Assembler::patch(pInsn, 11, 7, ra->encoding()); // Rd must be x1, need ra + set_int_at(0, insn); +} + +void NativeShortCall::verify() { + assert(NativeShortCall::is_at(instruction_address()), + "unexpected code at call site: %p", instruction_address()); +} + +void NativeShortCall::print() { + assert(NativeShortCall::is_at(instruction_address()), "unexpected code at call site"); + tty->print_cr(PTR_FORMAT ": jal/auipc,ld,jalr x1, offset/reg", p2i(instruction_address())); +} + +// The important thing is that threads are able to execute this +// call instruction at all times. (cmodx) // // Used in the runtime linkage of calls; see class CompiledIC. // // Add parameter assert_lock to switch off assertion // during code generation, where no patching lock is needed. -void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) { +bool NativeShortCall::set_destination_mt_safe(address dest, bool assert_lock) { assert(!assert_lock || (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) || - CompiledICLocker::is_safe(addr_at(0)), + CompiledICLocker::is_safe(instruction_address()), "concurrent code patching"); - address addr_call = addr_at(0); - assert(MacroAssembler::is_call_at(addr_call), "unexpected code at call site"); + address call_addr = instruction_address(); + assert(NativeCall::is_at(call_addr), "unexpected code at call site"); + + reloc_set_destination(dest); + + ICache::invalidate_range(call_addr, instruction_size); + return true; +} + +bool NativeShortCall::reloc_set_destination(address dest) { + address call_addr = instruction_address(); + assert(NativeCall::is_at(call_addr), "unexpected code at call site"); // Patch the constant in the call's trampoline stub. address trampoline_stub_addr = get_trampoline(); if (trampoline_stub_addr != nullptr) { - assert (!MacroAssembler::is_trampoline_stub_at(dest), "chained trampolines"); - nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest); + assert(!NativeShortCallTrampolineStub::is_at(dest), "chained trampolines"); + NativeShortCallTrampolineStub::at(trampoline_stub_addr)->set_destination(dest); } // Patch the call. - if (Assembler::reachable_from_branch_at(addr_call, dest)) { + if (Assembler::reachable_from_branch_at(call_addr, dest)) { set_destination(dest); } else { assert (trampoline_stub_addr != nullptr, "we need a trampoline"); set_destination(trampoline_stub_addr); } - ICache::invalidate_range(addr_call, instruction_size); + return true; } -address NativeCall::get_trampoline() { - address call_addr = addr_at(0); +address NativeShortCall::get_trampoline() { + address call_addr = instruction_address(); CodeBlob *code = CodeCache::find_blob(call_addr); assert(code != nullptr, "Could not find the containing code blob"); address jal_destination = MacroAssembler::pd_call_destination(call_addr); - if (code != nullptr && code->contains(jal_destination) && MacroAssembler::is_trampoline_stub_at(jal_destination)) { + if (code != nullptr && code->contains(jal_destination) && NativeShortCallTrampolineStub::is_at(jal_destination)) { return jal_destination; } @@ -114,8 +271,326 @@ address NativeCall::get_trampoline() { return nullptr; } -// Inserts a native call instruction at a given pc -void NativeCall::insert(address code_pos, address entry) { Unimplemented(); } +bool NativeShortCall::has_trampoline() { + return NativeShortCall::get_trampoline() != nullptr; +} + +address NativeShortCall::trampoline_destination() { + return NativeShortCallTrampolineStub::at(get_trampoline())->destination(); +} + +NativeShortCall* NativeShortCall::at(address addr) { + assert_cond(addr != nullptr); + assert(NativeShortCall::is_at(addr), "unexpected code at call site: %p", addr); + NativeShortCall* call = (NativeShortCall*)(addr); + return call; +} + +bool NativeShortCall::is_at(address addr) { + if (MacroAssembler::is_jal_at(addr)) { + if (MacroAssembler::extract_rd(addr) == x1) { + return true; + } + } + return false; +} + +bool NativeShortCall::is_call_before(address return_address) { + return NativeShortCall::is_at(return_address - instruction_size); +} + +//----------------------------------------------------------------------------- +// NativeFarCall +// +// Implements direct far calling loading an address from the stub section version of reloc call. +// This is the default (experimental flag UseTrampolines, default false). + +class NativeFarCall: public NativeInstruction { + public: + enum RISCV_specific_constants { + return_address_offset = 3 * NativeInstruction::instruction_size, // auipc + ld + jalr + }; + + address instruction_address() const { return addr_at(0); } + address next_instruction_address() const { return addr_at(return_address_offset); } + address return_address() const { return addr_at(return_address_offset); } + address destination() const; + address reloc_destination(address orig_address); + + void set_destination(address dest); + void verify(); + void print(); + + bool set_destination_mt_safe(address dest, bool assert_lock = true); + bool reloc_set_destination(address dest); + + private: + address stub_address(); + + static void set_stub_address_destination_at(address dest, address value); + static address stub_address_destination_at(address src); + public: + + static NativeFarCall* at(address addr); + static bool is_at(address addr); + static bool is_call_before(address return_address); +}; + +address NativeFarCall::destination() const { + address addr = instruction_address(); + assert(NativeFarCall::is_at(addr), "unexpected code at call site"); + + address destination = MacroAssembler::target_addr_for_insn(addr); + + CodeBlob* cb = CodeCache::find_blob(addr); + assert(cb && cb->is_nmethod(), "sanity"); + nmethod *nm = (nmethod *)cb; + assert(nm != nullptr, "Sanity"); + assert(nm->stub_contains(destination), "Sanity"); + assert(destination != nullptr, "Sanity"); + return stub_address_destination_at(destination); +} + +address NativeFarCall::reloc_destination(address orig_address) { + address call_addr = instruction_address(); + + CodeBlob *code = CodeCache::find_blob(call_addr); + assert(code != nullptr, "Could not find the containing code blob"); + + address stub_addr = nullptr; + if (code != nullptr && code->is_nmethod()) { + stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code); + } + + if (stub_addr != nullptr) { + stub_addr = MacroAssembler::target_addr_for_insn(call_addr); + } + return stub_addr; +} + +void NativeFarCall::set_destination(address dest) { + address addr = instruction_address(); + assert(NativeFarCall::is_at(addr), "unexpected code at call site"); + Unimplemented(); +} + +void NativeFarCall::verify() { + assert(NativeFarCall::is_at(instruction_address()), "unexpected code at call site"); +} + +void NativeFarCall::print() { + assert(NativeFarCall::is_at(instruction_address()), "unexpected code at call site"); + tty->print_cr(PTR_FORMAT ": auipc,ld,jalr x1, offset/reg, ", p2i(addr_at(0))); +} + +bool NativeFarCall::set_destination_mt_safe(address dest, bool assert_lock) { + assert(NativeFarCall::is_at(addr_at(0)), "unexpected code at call site"); + assert(!assert_lock || + (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) || + CompiledICLocker::is_safe(addr_at(0)), + "concurrent code patching"); + + address call_addr = addr_at(0); + assert(NativeFarCall::is_at(call_addr), "unexpected code at call site"); + + address stub_addr = stub_address(); + + if (stub_addr != nullptr) { + set_stub_address_destination_at(stub_addr, dest); + return true; + } + + return false; +} + +bool NativeFarCall::reloc_set_destination(address dest) { + address call_addr = addr_at(0); + assert(NativeFarCall::is_at(call_addr), "unexpected code at call site"); + + CodeBlob *code = CodeCache::find_blob(call_addr); + assert(code != nullptr, "Could not find the containing code blob"); + + address stub_addr = nullptr; + if (code != nullptr && code->is_nmethod()) { + stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code); + } + + if (stub_addr != nullptr) { + MacroAssembler::pd_patch_instruction_size(call_addr, stub_addr); + } + + return true; +} + +void NativeFarCall::set_stub_address_destination_at(address dest, address value) { + assert_cond(dest != nullptr); + assert_cond(value != nullptr); + + set_data64_at(dest, (uint64_t)value); + OrderAccess::release(); +} + +address NativeFarCall::stub_address_destination_at(address src) { + assert_cond(src != nullptr); + address dest = (address)get_data64_at(src); + return dest; +} + +address NativeFarCall::stub_address() { + address call_addr = addr_at(0); + + CodeBlob *code = CodeCache::find_blob(call_addr); + assert(code != nullptr, "Could not find the containing code blob"); + + address dest = MacroAssembler::pd_call_destination(call_addr); + assert(code->contains(dest), "Sanity"); + return dest; +} + +NativeFarCall* NativeFarCall::at(address addr) { + assert_cond(addr != nullptr); + assert(NativeFarCall::is_at(addr), "unexpected code at call site: %p", addr); + NativeFarCall* call = (NativeFarCall*)(addr); + return call; +} + +bool NativeFarCall::is_at(address addr) { + assert_cond(addr != nullptr); + const int instr_size = NativeInstruction::instruction_size; + if (MacroAssembler::is_auipc_at(addr) && + MacroAssembler::is_ld_at(addr + instr_size) && + MacroAssembler::is_jalr_at(addr + 2 * instr_size) && + (MacroAssembler::extract_rd(addr) == x5) && + (MacroAssembler::extract_rd(addr + instr_size) == x5) && + (MacroAssembler::extract_rs1(addr + instr_size) == x5) && + (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x5) && + (MacroAssembler::extract_rd(addr + 2 * instr_size) == x1)) { + return true; + } + return false; +} + +bool NativeFarCall::is_call_before(address return_address) { + return NativeFarCall::is_at(return_address - return_address_offset); +} + +//----------------------------------------------------------------------------- +// NativeCall + +address NativeCall::instruction_address() const { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->instruction_address(); + } else { + return NativeFarCall::at(addr_at(0))->instruction_address(); + } +} + +address NativeCall::next_instruction_address() const { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->next_instruction_address(); + } else { + return NativeFarCall::at(addr_at(0))->next_instruction_address(); + } +} + +address NativeCall::return_address() const { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->return_address(); + } else { + return NativeFarCall::at(addr_at(0))->return_address(); + } +} + +address NativeCall::destination() const { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->destination(); + } else { + return NativeFarCall::at(addr_at(0))->destination(); + } +} + +address NativeCall::reloc_destination(address orig_address) { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->reloc_destination(orig_address); + } else { + return NativeFarCall::at(addr_at(0))->reloc_destination(orig_address); + } +} + +void NativeCall::set_destination(address dest) { + if (UseTrampolines) { + NativeShortCall::at(addr_at(0))->set_destination(dest); + } else { + NativeFarCall::at(addr_at(0))->set_destination(dest); + } +} + +void NativeCall::verify() { + if (UseTrampolines) { + NativeShortCall::at(addr_at(0))->verify(); + } else { + NativeFarCall::at(addr_at(0))->verify();; + } +} + +void NativeCall::print() { + if (UseTrampolines) { + NativeShortCall::at(addr_at(0))->print(); + } else { + NativeFarCall::at(addr_at(0))->print();; + } +} + +bool NativeCall::set_destination_mt_safe(address dest, bool assert_lock) { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->set_destination_mt_safe(dest, assert_lock); + } else { + return NativeFarCall::at(addr_at(0))->set_destination_mt_safe(dest, assert_lock); + } +} + +bool NativeCall::reloc_set_destination(address dest) { + if (UseTrampolines) { + return NativeShortCall::at(addr_at(0))->reloc_set_destination(dest); + } else { + return NativeFarCall::at(addr_at(0))->reloc_set_destination(dest); + } +} + +bool NativeCall::is_at(address addr) { + if (UseTrampolines) { + return NativeShortCall::is_at(addr); + } else { + return NativeFarCall::is_at(addr); + } +} + +bool NativeCall::is_call_before(address return_address) { + if (UseTrampolines) { + return NativeShortCall::is_call_before(return_address); + } else { + return NativeFarCall::is_call_before(return_address); + } +} + +NativeCall* nativeCall_at(address addr) { + assert_cond(addr != nullptr); + NativeCall* call = (NativeCall*)(addr); + DEBUG_ONLY(call->verify()); + return call; +} + +NativeCall* nativeCall_before(address return_address) { + assert_cond(return_address != nullptr); + NativeCall* call = nullptr; + if (UseTrampolines) { + call = (NativeCall*)(return_address - NativeShortCall::return_address_offset); + } else { + call = (NativeCall*)(return_address - NativeFarCall::return_address_offset); + } + DEBUG_ONLY(call->verify()); + return call; +} //------------------------------------------------------------------- @@ -327,15 +802,6 @@ void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) //------------------------------------------------------------------- -address NativeCallTrampolineStub::destination(nmethod *nm) const { - return ptr_at(data_offset); -} - -void NativeCallTrampolineStub::set_destination(address new_destination) { - set_ptr_at(data_offset, new_destination); - OrderAccess::release(); -} - void NativePostCallNop::make_deopt() { MacroAssembler::assert_alignment(addr_at(0)); NativeDeoptInstruction::insert(addr_at(0)); diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp index f925f8950aa..0ac3a89969a 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -41,8 +41,6 @@ // - - NativeJump // - - NativeGeneralJump // - - NativeIllegalInstruction -// - - NativeCallTrampolineStub -// - - NativeMembar // - - NativePostCallNop // - - NativeDeoptInstruction @@ -69,30 +67,31 @@ class NativeInstruction { bool is_movptr1() const { return MacroAssembler::is_movptr1_at(addr_at(0)); } bool is_movptr2() const { return MacroAssembler::is_movptr2_at(addr_at(0)); } bool is_auipc() const { return MacroAssembler::is_auipc_at(addr_at(0)); } - bool is_call() const { return MacroAssembler::is_call_at(addr_at(0)); } bool is_jump() const { return MacroAssembler::is_jump_at(addr_at(0)); } + bool is_call() const { return is_call_at(addr_at(0)); } + static bool is_call_at(address addr); - inline bool is_nop() const; - inline bool is_jump_or_nop(); + bool is_nop() const; + bool is_jump_or_nop(); bool is_safepoint_poll(); bool is_sigill_not_entrant(); bool is_stop(); protected: - address addr_at(int offset) const { return address(this) + offset; } - - jint int_at(int offset) const { return (jint)Bytes::get_native_u4(addr_at(offset)); } - juint uint_at(int offset) const { return Bytes::get_native_u4(addr_at(offset)); } - - address ptr_at(int offset) const { return (address)Bytes::get_native_u8(addr_at(offset)); } - - oop oop_at (int offset) const { return cast_to_oop(Bytes::get_native_u8(addr_at(offset))); } + address addr_at(int offset) const { return address(this) + offset; } + jint int_at(int offset) const { return (jint) Bytes::get_native_u4(addr_at(offset)); } + juint uint_at(int offset) const { return Bytes::get_native_u4(addr_at(offset)); } + address ptr_at(int offset) const { return (address) Bytes::get_native_u8(addr_at(offset)); } + oop oop_at(int offset) const { return cast_to_oop(Bytes::get_native_u8(addr_at(offset))); } - void set_int_at(int offset, jint i) { Bytes::put_native_u4(addr_at(offset), i); } - void set_uint_at(int offset, jint i) { Bytes::put_native_u4(addr_at(offset), i); } - void set_ptr_at (int offset, address ptr) { Bytes::put_native_u8(addr_at(offset), (u8)ptr); } - void set_oop_at (int offset, oop o) { Bytes::put_native_u8(addr_at(offset), cast_from_oop(o)); } + void set_int_at(int offset, jint i) { Bytes::put_native_u4(addr_at(offset), i); } + void set_uint_at(int offset, jint i) { Bytes::put_native_u4(addr_at(offset), i); } + void set_ptr_at(int offset, address ptr) { Bytes::put_native_u8(addr_at(offset), (u8)ptr); } + void set_oop_at(int offset, oop o) { Bytes::put_native_u8(addr_at(offset), cast_from_oop(o)); } + + static void set_data64_at(address dest, uint64_t data) { Bytes::put_native_u8(dest, (u8)data); } + static uint64_t get_data64_at(address src) { return Bytes::get_native_u8(src); } public: @@ -103,99 +102,56 @@ class NativeInstruction { } }; -inline NativeInstruction* nativeInstruction_at(address addr) { +NativeInstruction* nativeInstruction_at(address addr) { return (NativeInstruction*)addr; } -// The natural type of an RISCV instruction is uint32_t -inline NativeInstruction* nativeInstruction_at(uint32_t *addr) { - return (NativeInstruction*)addr; -} +NativeCall* nativeCall_at(address addr); +NativeCall* nativeCall_before(address return_address); -inline NativeCall* nativeCall_at(address addr); // The NativeCall is an abstraction for accessing/manipulating native // call instructions (used to manipulate inline caches, primitive & // DSO calls, etc.). - -class NativeCall: public NativeInstruction { - public: - enum RISCV_specific_constants { - instruction_size = 4, - instruction_offset = 0, - displacement_offset = 0, - return_address_offset = 4 +class NativeCall: private NativeInstruction { + // private: when common code is using byte_size() + private: + enum { + // Use byte_size() as it can be changed in runtime + // Since instruction_size exists on NativeInstruction we need + // to overload and hide it. + instruction_size = 3 * Assembler::instruction_size // auipc + ld + jalr }; + public: - static int byte_size() { return instruction_size; } - address instruction_address() const { return addr_at(instruction_offset); } - address next_instruction_address() const { return addr_at(return_address_offset); } - address return_address() const { return addr_at(return_address_offset); } - address destination() const; - - void set_destination(address dest) { - assert(is_jal(), "Should be jal instruction!"); - intptr_t offset = (intptr_t)(dest - instruction_address()); - assert((offset & 0x1) == 0, "bad alignment"); - assert(Assembler::is_simm21(offset), "encoding constraint"); - unsigned int insn = 0b1101111; // jal - address pInsn = (address)(&insn); - Assembler::patch(pInsn, 31, 31, (offset >> 20) & 0x1); - Assembler::patch(pInsn, 30, 21, (offset >> 1) & 0x3ff); - Assembler::patch(pInsn, 20, 20, (offset >> 11) & 0x1); - Assembler::patch(pInsn, 19, 12, (offset >> 12) & 0xff); - Assembler::patch(pInsn, 11, 7, ra->encoding()); // Rd must be x1, need ra - set_int_at(displacement_offset, insn); + static int byte_size() { + if (UseTrampolines) { + return NativeInstruction::instruction_size; // jal + } else { + return 3 * NativeInstruction::instruction_size; // auipc + ld + jalr + } } + // Creation + friend NativeCall* nativeCall_at(address addr); + friend NativeCall* nativeCall_before(address return_address); + + address instruction_address() const; + address next_instruction_address() const; + address return_address() const; + address destination() const; + address reloc_destination(address orig_address); void verify_alignment() {} // do nothing on riscv void verify(); void print(); - // Creation - inline friend NativeCall* nativeCall_at(address addr); - inline friend NativeCall* nativeCall_before(address return_address); + void set_destination(address dest); + bool set_destination_mt_safe(address dest, bool assert_lock = true); + bool reloc_set_destination(address dest); - static bool is_call_before(address return_address) { - return MacroAssembler::is_call_at(return_address - NativeCall::return_address_offset); - } - - // MT-safe patching of a call instruction. - static void insert(address code_pos, address entry); - - static void replace_mt_safe(address instr_addr, address code_buffer); - - // Similar to replace_mt_safe, but just changes the destination. The - // important thing is that free-running threads are able to execute - // this call instruction at all times. If the call is an immediate BL - // instruction we can simply rely on atomicity of 32-bit writes to - // make sure other threads will see no intermediate states. - - // We cannot rely on locks here, since the free-running threads must run at - // full speed. - // - // Used in the runtime linkage of calls; see class CompiledIC. - // (Cf. 4506997 and 4479829, where threads witnessed garbage displacements.) - - // The parameter assert_lock disables the assertion during code generation. - void set_destination_mt_safe(address dest, bool assert_lock = true); - - address get_trampoline(); + static bool is_at(address addr); + static bool is_call_before(address return_address); }; -inline NativeCall* nativeCall_at(address addr) { - assert_cond(addr != nullptr); - NativeCall* call = (NativeCall*)(addr - NativeCall::instruction_offset); - DEBUG_ONLY(call->verify()); - return call; -} - -inline NativeCall* nativeCall_before(address return_address) { - assert_cond(return_address != nullptr); - NativeCall* call = (NativeCall*)(return_address - NativeCall::return_address_offset); - DEBUG_ONLY(call->verify()); - return call; -} - // An interface for accessing/manipulating native mov reg, imm instructions. // (used to manipulate inlined 64-bit data calls, etc.) class NativeMovConstReg: public NativeInstruction { @@ -366,27 +322,6 @@ inline bool NativeInstruction::is_jump_or_nop() { return is_nop() || is_jump(); } -// Call trampoline stubs. -class NativeCallTrampolineStub : public NativeInstruction { - public: - - enum RISCV_specific_constants { - // Refer to function emit_trampoline_stub. - instruction_size = MacroAssembler::trampoline_stub_instruction_size, // auipc + ld + jr + target address - data_offset = MacroAssembler::trampoline_stub_data_offset, // auipc + ld + jr - }; - - address destination(nmethod *nm = nullptr) const; - void set_destination(address new_destination); - ptrdiff_t destination_offset() const; -}; - -inline NativeCallTrampolineStub* nativeCallTrampolineStub_at(address addr) { - assert_cond(addr != nullptr); - assert(MacroAssembler::is_trampoline_stub_at(addr), "no call trampoline found"); - return (NativeCallTrampolineStub*)addr; -} - // A NativePostCallNop takes the form of three instructions: // nop; lui zr, hi20; addiw zr, lo12 // diff --git a/src/hotspot/cpu/riscv/relocInfo_riscv.cpp b/src/hotspot/cpu/riscv/relocInfo_riscv.cpp index 43a9b8f89ae..d0903c96e22 100644 --- a/src/hotspot/cpu/riscv/relocInfo_riscv.cpp +++ b/src/hotspot/cpu/riscv/relocInfo_riscv.cpp @@ -60,12 +60,10 @@ void Relocation::pd_set_data_value(address x, bool verify_only) { address Relocation::pd_call_destination(address orig_addr) { assert(is_call(), "should be an address instruction here"); - if (MacroAssembler::is_call_at(addr())) { - address trampoline = nativeCall_at(addr())->get_trampoline(); - if (trampoline != nullptr) { - return nativeCallTrampolineStub_at(trampoline)->destination(); - } + if (NativeCall::is_at(addr())) { + return nativeCall_at(addr())->reloc_destination(orig_addr); } + // Non call reloc if (orig_addr != nullptr) { // the extracted address from the instructions in address orig_addr address new_addr = MacroAssembler::pd_call_destination(orig_addr); @@ -81,10 +79,9 @@ address Relocation::pd_call_destination(address orig_addr) { void Relocation::pd_set_call_destination(address x) { assert(is_call(), "should be an address instruction here"); - if (MacroAssembler::is_call_at(addr())) { - address trampoline = nativeCall_at(addr())->get_trampoline(); - if (trampoline != nullptr) { - nativeCall_at(addr())->set_destination_mt_safe(x, /* assert_lock */false); + if (NativeCall::is_at(addr())) { + NativeCall* nc = nativeCall_at(addr()); + if (nc->reloc_set_destination(x)) { return; } } diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index e2a1fcf621f..b176c6e9cb7 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1238,17 +1238,24 @@ bool needs_acquiring_load_reserved(const Node *n) int MachCallStaticJavaNode::ret_addr_offset() { - // jal - return 1 * NativeInstruction::instruction_size; + if (UseTrampolines) { + return 1 * NativeInstruction::instruction_size; // jal + } + return 3 * NativeInstruction::instruction_size; // auipc + ld + jalr } int MachCallDynamicJavaNode::ret_addr_offset() { - return NativeMovConstReg::movptr2_instruction_size + NativeInstruction::instruction_size; // movptr2, jal + if (UseTrampolines) { + return NativeMovConstReg::movptr2_instruction_size + NativeInstruction::instruction_size; // movptr2, jal + } + return NativeMovConstReg::movptr2_instruction_size + (3 * NativeInstruction::instruction_size); // movptr2, auipc + ld + jal } int MachCallRuntimeNode::ret_addr_offset() { - // for generated stubs the call will be + // For generated stubs the call will be: + // auipc + ld + jalr + // Using trampos: // jal(addr) // or with far branches // jal(trampoline_stub) @@ -1261,7 +1268,10 @@ int MachCallRuntimeNode::ret_addr_offset() { // jalr(t0) -> jalr CodeBlob *cb = CodeCache::find_blob(_entry_point); if (cb != nullptr) { - return 1 * NativeInstruction::instruction_size; + if (UseTrampolines) { + return 1 * NativeInstruction::instruction_size; + } + return 3 * NativeInstruction::instruction_size; } else { return 11 * NativeInstruction::instruction_size; } @@ -2403,7 +2413,7 @@ encode %{ assert_cond(addr != nullptr); if (!_method) { // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap. - call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type)); + call = __ reloc_call(Address(addr, relocInfo::runtime_call_type)); if (call == nullptr) { ciEnv::current()->record_failure("CodeCache is full"); return; @@ -2412,12 +2422,16 @@ encode %{ // The NOP here is purely to ensure that eliding a call to // JVM_EnsureMaterializedForStackWalk doesn't change the code size. __ nop(); + if (!UseTrampolines) { + __ nop(); + __ nop(); + } __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)"); } else { int method_index = resolved_method_index(masm); RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index) : static_call_Relocation::spec(method_index); - call = __ trampoline_call(Address(addr, rspec)); + call = __ reloc_call(Address(addr, rspec)); if (call == nullptr) { ciEnv::current()->record_failure("CodeCache is full"); return; @@ -2469,7 +2483,7 @@ encode %{ address entry = (address)$meth$$method; CodeBlob *cb = CodeCache::find_blob(entry); if (cb != nullptr) { - address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type)); + address call = __ reloc_call(Address(entry, relocInfo::runtime_call_type)); if (call == nullptr) { ciEnv::current()->record_failure("CodeCache is full"); return; @@ -10098,7 +10112,7 @@ instruct partialSubtypeCheckConstSuper(iRegP_R14 sub, iRegP_R10 super_reg, immP $tmpR11$$Register, $tmpR12$$Register, $tmpR13$$Register, $tmpR16$$Register, super_klass_slot); } else { - address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot))); + address call = __ reloc_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot))); success = (call != nullptr); } if (!success) { diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp index 3af203a95b0..01ab3d5c274 100644 --- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -1005,7 +1005,7 @@ static void gen_continuation_enter(MacroAssembler* masm, // Make sure the call is patchable __ align(NativeInstruction::instruction_size); - const address tr_call = __ trampoline_call(resolve); + const address tr_call = __ reloc_call(resolve); if (tr_call == nullptr) { fatal("CodeCache is full at gen_continuation_enter"); } @@ -1037,7 +1037,7 @@ static void gen_continuation_enter(MacroAssembler* masm, // Make sure the call is patchable __ align(NativeInstruction::instruction_size); - const address tr_call = __ trampoline_call(resolve); + const address tr_call = __ reloc_call(resolve); if (tr_call == nullptr) { fatal("CodeCache is full at gen_continuation_enter"); }