From 07ecc93dbd0b74e2362d369e22b5141289eb1f76 Mon Sep 17 00:00:00 2001 From: Robbin Ehn Date: Tue, 30 Sep 2025 15:10:30 +0000 Subject: [PATCH] 8367692: RISC-V: Align post call nop Reviewed-by: fyang, fjiang, mli --- src/hotspot/cpu/riscv/assembler_riscv.hpp | 29 +++++--- .../cpu/riscv/c1_LIRAssembler_riscv.cpp | 6 ++ .../cpu/riscv/macroAssembler_riscv.cpp | 13 ++-- src/hotspot/cpu/riscv/nativeInst_riscv.cpp | 25 ++++--- src/hotspot/cpu/riscv/nativeInst_riscv.hpp | 15 +--- src/hotspot/cpu/riscv/riscv.ad | 26 ++++++- src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp | 69 ++++++++++++------- 7 files changed, 115 insertions(+), 68 deletions(-) diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp index 0828342ee65..0712b60fb2a 100644 --- a/src/hotspot/cpu/riscv/assembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp @@ -914,6 +914,17 @@ protected: public: + static uint32_t encode_csrrw(Register Rd, const uint32_t csr, Register Rs1) { + guarantee(is_uimm12(csr), "csr is invalid"); + uint32_t insn = 0; + patch((address)&insn, 6, 0, 0b1110011); + patch((address)&insn, 14, 12, 0b001); + patch_reg((address)&insn, 7, Rd); + patch_reg((address)&insn, 15, Rs1); + patch((address)&insn, 31, 20, csr); + return insn; + } + static uint32_t encode_jal(Register Rd, const int32_t offset) { guarantee(is_simm21(offset) && ((offset % 2) == 0), "offset is invalid."); uint32_t insn = 0; @@ -3693,19 +3704,15 @@ public: // -------------------------- // Upper Immediate Instruction // -------------------------- -#define INSN(NAME) \ - void NAME(Register Rd, int32_t imm) { \ - /* lui -> c.lui */ \ - if (do_compress() && (Rd != x0 && Rd != x2 && imm != 0 && is_simm18(imm))) { \ - c_lui(Rd, imm); \ - return; \ - } \ - _lui(Rd, imm); \ + void lui(Register Rd, int32_t imm) { + /* lui -> c.lui */ + if (do_compress() && (Rd != x0 && Rd != x2 && imm != 0 && is_simm18(imm))) { + c_lui(Rd, imm); + return; + } + _lui(Rd, imm); } - INSN(lui); - -#undef INSN // Cache Management Operations // These instruction may be turned off for user space. diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index adc79350be6..9d8ae770ccf 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -1350,6 +1350,7 @@ void LIR_Assembler::align_call(LIR_Code code) { } void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { + Assembler::IncompressibleScope scope(_masm); address call = __ reloc_call(Address(op->addr(), rtype)); if (call == nullptr) { bailout("reloc call address stub overflow"); @@ -1360,6 +1361,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { } void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { + Assembler::IncompressibleScope scope(_masm); address call = __ ic_call(op->addr()); if (call == nullptr) { bailout("reloc call address stub overflow"); @@ -1842,6 +1844,10 @@ void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, C void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { assert(!tmp->is_valid(), "don't need temporary"); + Assembler::IncompressibleScope scope(_masm); + // Post call nops must be natural aligned due to cmodx rules. + align_call(lir_rtcall); + __ rt_call(dest); if (info != nullptr) { diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index c9de1db0308..5c85cc13bed 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -355,14 +355,15 @@ void MacroAssembler::call_VM(Register oop_result, } void MacroAssembler::post_call_nop() { + assert(!in_compressible_scope(), "Must be"); + assert_alignment(pc()); if (!Continuations::enabled()) { return; } - relocate(post_call_nop_Relocation::spec(), [&] { - InlineSkippedInstructionsCounter skipCounter(this); - nop(); - li32(zr, 0); - }); + relocate(post_call_nop_Relocation::spec()); + InlineSkippedInstructionsCounter skipCounter(this); + nop(); + li32(zr, 0); } // these are no-ops overridden by InterpreterMacroAssembler @@ -5013,7 +5014,7 @@ address MacroAssembler::reloc_call(Address entry, Register tmp) { address MacroAssembler::ic_call(address entry, jint method_index) { RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); - IncompressibleScope scope(this); // relocations + assert(!in_compressible_scope(), "Must be"); movptr(t0, (address)Universe::non_oop_word(), t1); assert_cond(entry != nullptr); return reloc_call(Address(entry, rh)); diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp index 72cc95a595d..5d1cac72ade 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp @@ -331,13 +331,10 @@ bool NativeInstruction::is_safepoint_poll() { return MacroAssembler::is_lwu_to_zr(address(this)); } -void NativeIllegalInstruction::insert(address code_pos) { - assert_cond(code_pos != nullptr); - Assembler::sd_instr(code_pos, 0xffffffff); // all bits ones is permanently reserved as an illegal instruction -} - bool NativeInstruction::is_stop() { - return uint_at(0) == 0xc0101073; // an illegal instruction, 'csrrw x0, time, x0' + // an illegal instruction, 'csrrw x0, time, x0' + uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::time, x0); + return uint_at(0) == encoded; } //------------------------------------------------------------------- @@ -347,6 +344,8 @@ void NativeGeneralJump::insert_unconditional(address code_pos, address entry) { MacroAssembler a(&cb); Assembler::IncompressibleScope scope(&a); // Fixed length: see NativeGeneralJump::get_instruction_size() + MacroAssembler::assert_alignment(code_pos); + int32_t offset = 0; a.movptr(t1, entry, offset, t0); // lui, lui, slli, add a.jr(t1, offset); // jalr @@ -378,6 +377,7 @@ bool NativePostCallNop::decode(int32_t& oopmap_slot, int32_t& cb_offset) const { } bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) { + MacroAssembler::assert_alignment(addr_at(4)); if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) { return false; // cannot encode } @@ -389,14 +389,17 @@ bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) { return true; // successfully encoded } -void NativeDeoptInstruction::verify() { +bool NativeDeoptInstruction::is_deopt_at(address instr) { + assert(instr != nullptr, "Must be"); + uint32_t value = Assembler::ld_instr(instr); + uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::instret, x0); + return value == encoded; } // Inserts an undefined instruction at a given pc void NativeDeoptInstruction::insert(address code_pos) { - // 0xc0201073 encodes CSRRW x0, instret, x0 - uint32_t insn = 0xc0201073; - uint32_t *pos = (uint32_t *) code_pos; - *pos = insn; + MacroAssembler::assert_alignment(code_pos); + uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::instret, x0); + Assembler::sd_instr(code_pos, encoded); ICache::invalidate_range(code_pos, 4); } diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp index 4235e23d421..d990cfbc50d 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -294,12 +294,6 @@ inline NativeGeneralJump* nativeGeneralJump_at(address addr) { return jump; } -class NativeIllegalInstruction: public NativeInstruction { - public: - // Insert illegal opcode as specific address - static void insert(address code_pos); -}; - inline bool NativeInstruction::is_nop() const { uint32_t insn = Assembler::ld_instr(addr_at(0)); return insn == 0x13; @@ -353,14 +347,7 @@ class NativeDeoptInstruction: public NativeInstruction { address instruction_address() const { return addr_at(instruction_offset); } address next_instruction_address() const { return addr_at(instruction_size); } - void verify(); - - static bool is_deopt_at(address instr) { - assert(instr != nullptr, ""); - uint32_t value = Assembler::ld_instr(instr); - // 0xc0201073 encodes CSRRW x0, instret, x0 - return value == 0xc0201073; - } + static bool is_deopt_at(address instr); // MT-safe patching static void insert(address code_pos); diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 739a525c9a4..d816f2405c4 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1269,6 +1269,26 @@ int CallDynamicJavaDirectNode::compute_padding(int current_offset) const return align_up(current_offset, alignment_required()) - current_offset; } +int CallRuntimeDirectNode::compute_padding(int current_offset) const +{ + return align_up(current_offset, alignment_required()) - current_offset; +} + +int CallLeafDirectNode::compute_padding(int current_offset) const +{ + return align_up(current_offset, alignment_required()) - current_offset; +} + +int CallLeafDirectVectorNode::compute_padding(int current_offset) const +{ + return align_up(current_offset, alignment_required()) - current_offset; +} + +int CallLeafNoFPDirectNode::compute_padding(int current_offset) const +{ + return align_up(current_offset, alignment_required()) - current_offset; +} + //============================================================================= #ifndef PRODUCT @@ -8175,7 +8195,7 @@ instruct unnecessary_membar_volatile_rvtso() %{ ins_cost(0); size(0); - + format %{ "#@unnecessary_membar_volatile_rvtso (unnecessary so empty encoding)" %} ins_encode %{ __ block_comment("unnecessary_membar_volatile_rvtso"); @@ -10509,6 +10529,7 @@ instruct CallRuntimeDirect(method meth) ins_encode(riscv_enc_java_to_runtime(meth)); ins_pipe(pipe_class_call); + ins_alignment(4); %} // Call Runtime Instruction @@ -10526,6 +10547,7 @@ instruct CallLeafDirect(method meth) ins_encode(riscv_enc_java_to_runtime(meth)); ins_pipe(pipe_class_call); + ins_alignment(4); %} // Call Runtime Instruction without safepoint and with vector arguments @@ -10543,6 +10565,7 @@ instruct CallLeafDirectVector(method meth) ins_encode(riscv_enc_java_to_runtime(meth)); ins_pipe(pipe_class_call); + ins_alignment(4); %} // Call Runtime Instruction @@ -10560,6 +10583,7 @@ instruct CallLeafNoFPDirect(method meth) ins_encode(riscv_enc_java_to_runtime(meth)); ins_pipe(pipe_class_call); + ins_alignment(4); %} // ============================================================================ diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp index 34f0f63dbc8..94506e9f19d 100644 --- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -1002,20 +1002,23 @@ static void gen_continuation_enter(MacroAssembler* masm, __ bnez(c_rarg2, call_thaw); - // Make sure the call is patchable - __ align(NativeInstruction::instruction_size); + address call_pc; + { + Assembler::IncompressibleScope scope(masm); + // Make sure the call is patchable + __ align(NativeInstruction::instruction_size); - const address tr_call = __ reloc_call(resolve); - if (tr_call == nullptr) { - fatal("CodeCache is full at gen_continuation_enter"); + call_pc = __ reloc_call(resolve); + if (call_pc == nullptr) { + fatal("CodeCache is full at gen_continuation_enter"); + } + + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); } - - oop_maps->add_gc_map(__ pc() - start, map); - __ post_call_nop(); - __ j(exit); - address stub = CompiledDirectCall::emit_to_interp_stub(masm, tr_call); + address stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc); if (stub == nullptr) { fatal("CodeCache is full at gen_continuation_enter"); } @@ -1034,26 +1037,36 @@ static void gen_continuation_enter(MacroAssembler* masm, __ bnez(c_rarg2, call_thaw); - // Make sure the call is patchable - __ align(NativeInstruction::instruction_size); + address call_pc; + { + Assembler::IncompressibleScope scope(masm); + // Make sure the call is patchable + __ align(NativeInstruction::instruction_size); - const address tr_call = __ reloc_call(resolve); - if (tr_call == nullptr) { - fatal("CodeCache is full at gen_continuation_enter"); + call_pc = __ reloc_call(resolve); + if (call_pc == nullptr) { + fatal("CodeCache is full at gen_continuation_enter"); + } + + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); } - oop_maps->add_gc_map(__ pc() - start, map); - __ post_call_nop(); - __ j(exit); __ bind(call_thaw); - ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; - __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw())); - oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); - ContinuationEntry::_return_pc_offset = __ pc() - start; - __ post_call_nop(); + // Post call nops must be natural aligned due to cmodx rules. + { + Assembler::IncompressibleScope scope(masm); + __ align(NativeInstruction::instruction_size); + + ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; + __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw())); + oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); + ContinuationEntry::_return_pc_offset = __ pc() - start; + __ post_call_nop(); + } __ bind(exit); ContinuationEntry::_cleanup_offset = __ pc() - start; @@ -1082,7 +1095,7 @@ static void gen_continuation_enter(MacroAssembler* masm, __ jr(x11); // the exception handler } - address stub = CompiledDirectCall::emit_to_interp_stub(masm, tr_call); + address stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc); if (stub == nullptr) { fatal("CodeCache is full at gen_continuation_enter"); } @@ -1115,10 +1128,16 @@ static void gen_continuation_yield(MacroAssembler* masm, __ mv(c_rarg1, sp); + // Post call nops must be natural aligned due to cmodx rules. + __ align(NativeInstruction::instruction_size); + frame_complete = __ pc() - start; address the_pc = __ pc(); - __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup + { + Assembler::IncompressibleScope scope(masm); + __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup + } __ mv(c_rarg0, xthread); __ set_last_Java_frame(sp, fp, the_pc, t0);