diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 9913a589683..d316c2b3db2 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -2862,6 +2862,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, if (info != nullptr) { add_call_info_here(info); } + assert(__ last_calls_return_pc() == __ pc(), "pcn not at return pc"); __ post_call_nop(); } diff --git a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp index 23229d0a4c3..c1a98b734da 100644 --- a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp @@ -350,7 +350,7 @@ inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { if (is_entry_frame) { callers_sp = _cont.entryFP(); } else { - CodeBlob* cb = CodeCache::find_blob(pc); + CodeBlob* cb = CodeCache::find_blob_fast(pc); callers_sp = sp + cb->frame_size(); } // set the back link diff --git a/src/hotspot/cpu/ppc/frame_ppc.cpp b/src/hotspot/cpu/ppc/frame_ppc.cpp index 63788398619..b63789f320d 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.cpp +++ b/src/hotspot/cpu/ppc/frame_ppc.cpp @@ -136,7 +136,7 @@ bool frame::safe_for_sender(JavaThread *thread) { // It should be safe to construct the sender though it might not be valid. - frame sender(sender_sp, sender_pc); + frame sender(sender_sp, sender_pc, nullptr /* unextended_sp */, nullptr /* fp */, sender_blob); // Do we have a valid fp? address sender_fp = (address) sender.fp(); @@ -196,12 +196,12 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const { assert(map->include_argument_oops(), "should be set by clear"); if (jfa->last_Java_pc() != nullptr) { - frame fr(jfa->last_Java_sp(), jfa->last_Java_pc()); + frame fr(jfa->last_Java_sp(), jfa->last_Java_pc(), kind::code_blob); return fr; } // Last_java_pc is not set, if we come here from compiled code. The // constructor retrieves the PC from the stack. - frame fr(jfa->last_Java_sp()); + frame fr(jfa->last_Java_sp(), nullptr, kind::code_blob); return fr; } @@ -229,7 +229,7 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { assert(jfa->last_Java_sp() > sp(), "must be above this frame on stack"); map->clear(); assert(map->include_argument_oops(), "should be set by clear"); - frame fr(jfa->last_Java_sp(), jfa->last_Java_pc()); + frame fr(jfa->last_Java_sp(), jfa->last_Java_pc(), kind::code_blob); return fr; } @@ -451,7 +451,7 @@ intptr_t *frame::initial_deoptimization_info() { #ifndef PRODUCT // This is a generic constructor which is only used by pns() in debug.cpp. // fp is dropped and gets determined by backlink. -frame::frame(void* sp, void* fp, void* pc) : frame((intptr_t*)sp, (address)pc) {} +frame::frame(void* sp, void* fp, void* pc) : frame((intptr_t*)sp, (address)pc, kind::unknown) {} #endif BasicObjectLock* frame::interpreter_frame_monitor_end() const { diff --git a/src/hotspot/cpu/ppc/frame_ppc.hpp b/src/hotspot/cpu/ppc/frame_ppc.hpp index d2c9927f119..0e0c1a388bf 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.hpp +++ b/src/hotspot/cpu/ppc/frame_ppc.hpp @@ -393,16 +393,26 @@ inline common_abi* own_abi() const { return (common_abi*) _sp; } inline common_abi* callers_abi() const { return (common_abi*) _fp; } + enum class kind { + unknown, // The frame's pc is not necessarily in the CodeCache. + // CodeCache::find_blob_fast(void* pc) can yield wrong results in this case and must not be used. + code_blob, // The frame's pc is known to be in the CodeCache but it is likely not in an nmethod. + // CodeCache::find_blob_fast() will be correct but not faster in this case. + nmethod // This is likely the frame of a nmethod. + // The code cache lookup is optimized based on NativePostCallNops. + }; + private: // Initialize frame members (_pc and _sp must be given) - inline void setup(); + inline void setup(kind knd); public: // Constructors inline frame(intptr_t* sp, intptr_t* fp, address pc); - inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp = nullptr, intptr_t* fp = nullptr, CodeBlob* cb = nullptr); + inline frame(intptr_t* sp, address pc, kind knd = kind::nmethod); + inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* fp = nullptr, CodeBlob* cb = nullptr); inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map); inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap); diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp index 220b9c3241e..7b1f37a342f 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp @@ -35,14 +35,14 @@ // Inline functions for ppc64 frames: // Initialize frame members (_sp must be given) -inline void frame::setup() { +inline void frame::setup(kind knd) { if (_pc == nullptr) { _pc = (address)own_abi()->lr; assert(_pc != nullptr, "must have PC"); } if (_cb == nullptr) { - _cb = CodeCache::find_blob(_pc); + _cb = (knd == kind::nmethod) ? CodeCache::find_blob_fast(_pc) : CodeCache::find_blob(_pc); } if (_unextended_sp == nullptr) { @@ -89,21 +89,27 @@ inline void frame::setup() { inline frame::frame() : _sp(nullptr), _pc(nullptr), _cb(nullptr), _oop_map(nullptr), _deopt_state(unknown), _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(nullptr), _fp(nullptr) {} -inline frame::frame(intptr_t* sp) : frame(sp, nullptr) {} +inline frame::frame(intptr_t* sp) : frame(sp, nullptr, kind::nmethod) {} inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) : frame(sp, pc, nullptr, fp, nullptr) {} +inline frame::frame(intptr_t* sp, address pc, kind knd) + : _sp(sp), _pc(pc), _cb(nullptr), _oop_map(nullptr), + _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(sp), _fp(nullptr) { + setup(knd); +} + inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* fp, CodeBlob* cb) : _sp(sp), _pc(pc), _cb(cb), _oop_map(nullptr), _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) { - setup(); + setup(kind::nmethod); } inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map) : _sp(sp), _pc(pc), _cb(cb), _oop_map(oop_map), _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) { assert(_cb != nullptr, "pc: " INTPTR_FORMAT, p2i(pc)); - setup(); + setup(kind::nmethod); } inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, @@ -113,7 +119,7 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address // In thaw, non-heap frames use this constructor to pass oop_map. I don't know why. assert(_on_heap || _cb != nullptr, "these frames are always heap frames"); if (cb != nullptr) { - setup(); + setup(kind::nmethod); } #ifdef ASSERT // The following assertion has been disabled because it would sometime trap for Continuation.run, @@ -300,7 +306,7 @@ inline frame frame::sender_raw(RegisterMap* map) const { // Must be native-compiled frame, i.e. the marshaling code for native // methods that exists in the core system. - return frame(sender_sp(), sender_pc()); + return frame(sender_sp(), sender_pc(), kind::code_blob); } inline frame frame::sender(RegisterMap* map) const { diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp index 8942199610e..b9d1cdb19ac 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp @@ -1187,8 +1187,12 @@ void MacroAssembler::post_call_nop() { if (!Continuations::enabled()) { return; } + // We use CMPI/CMPLI instructions to encode post call nops. + // Refer to NativePostCallNop for details. + relocate(post_call_nop_Relocation::spec()); InlineSkippedInstructionsCounter skipCounter(this); - nop(); + Assembler::emit_int32(Assembler::CMPLI_OPCODE | Assembler::opp_u_field(1, 9, 9)); + assert(is_post_call_nop(*(int*)(pc() - 4)), "post call not not found"); } void MacroAssembler::call_VM_base(Register oop_result, diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp index 52d8e7fa96d..cddc8b92fa0 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp @@ -417,6 +417,12 @@ class MacroAssembler: public Assembler { inline void call_stub_and_return_to(Register function_entry, Register return_pc); void post_call_nop(); + static bool is_post_call_nop(int instr_bits) { + const uint32_t nineth_bit = opp_u_field(1, 9, 9); + const uint32_t opcode_mask = 0b111110 << OPCODE_SHIFT; + const uint32_t pcn_mask = opcode_mask | nineth_bit; + return (instr_bits & pcn_mask) == (Assembler::CMPLI_OPCODE | nineth_bit); + } // // Java utilities diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp index 06754ccfdd7..e1a41a17d93 100644 --- a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp +++ b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp @@ -429,6 +429,31 @@ void NativePostCallNop::make_deopt() { NativeDeoptInstruction::insert(addr_at(0)); } +bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) { + int32_t i2, i1; + assert(is_aligned(cb_offset, 4), "cb offset alignment does not match instruction alignment"); + assert(!decode(i1, i2), "already patched"); + + cb_offset = cb_offset >> 2; + if (((oopmap_slot & ppc_oopmap_slot_mask) != oopmap_slot) || ((cb_offset & ppc_cb_offset_mask) != cb_offset)) { + return false; // cannot encode + } + const uint32_t data = oopmap_slot << ppc_cb_offset_bits | cb_offset; + const uint32_t lo_data = data & ppc_data_lo_mask; + const uint32_t hi_data = data >> ppc_data_lo_bits; + const uint32_t nineth_bit = 1 << (31 - 9); + uint32_t instr = Assembler::CMPLI_OPCODE | hi_data << ppc_data_hi_shift | nineth_bit | lo_data; + *(uint32_t*)addr_at(0) = instr; + + int32_t oopmap_slot_dec, cb_offset_dec; + assert(is_post_call_nop(), "pcn not recognized"); + assert(decode(oopmap_slot_dec, cb_offset_dec), "encoding failed"); + assert(oopmap_slot == oopmap_slot_dec, "oopmap slot encoding is wrong"); + assert((cb_offset << 2) == cb_offset_dec, "cb offset encoding is wrong"); + + return true; // encoding succeeded +} + void NativeDeoptInstruction::verify() { } diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp index ec6f5a90a72..113cedfee7c 100644 --- a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp +++ b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp @@ -51,7 +51,7 @@ class NativeInstruction { friend class Relocation; public: - bool is_nop() const { return Assembler::is_nop(long_at(0)); } + bool is_post_call_nop() const { return MacroAssembler::is_post_call_nop(long_at(0)); } bool is_jump() const { return Assembler::is_b(long_at(0)); } // See NativeGeneralJump. @@ -506,10 +506,50 @@ class NativeMovRegMem: public NativeInstruction { }; class NativePostCallNop: public NativeInstruction { + + // We use CMPI/CMPLI to represent Post Call Nops (PCN) + + // Bit |0 5|6 |9 |10|11 |16 31| + // +--------------------------------------------------------------+ + // Field |OPCODE |BF |/ |L |RA |SI | + // +--------------------------------------------------------------+ + // |0 0 1 0 1|DATA HI| 1| DATA LO | + // | |4 bits | | 22 bits | + // + // Bit 9 is always 1 for PCNs to distinguish them from regular CMPI/CMPLI + // + // Using both, CMPLI (opcode 10 = 0b001010) and CMPI (opcode 11 = 0b001011) for + // PCNs allows using bit 5 from the opcode to encode DATA HI. + + enum { + ppc_data_lo_bits = 31 - 9, + ppc_data_lo_mask = right_n_bits(ppc_data_lo_bits), + ppc_data_hi_bits = 9 - 5, + ppc_data_hi_shift = ppc_data_lo_bits + 1, + ppc_data_hi_mask = right_n_bits(ppc_data_hi_bits) << ppc_data_hi_shift, + ppc_data_bits = ppc_data_lo_bits + ppc_data_hi_bits, + + ppc_oopmap_slot_bits = 9, + ppc_oopmap_slot_mask = right_n_bits(ppc_oopmap_slot_bits), + ppc_cb_offset_bits = ppc_data_bits - ppc_oopmap_slot_bits, + ppc_cb_offset_mask = right_n_bits(ppc_cb_offset_bits), +}; + public: - bool check() const { return is_nop(); } - bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; } - bool patch(int32_t oopmap_slot, int32_t cb_offset) { return false; } + bool check() const { return is_post_call_nop(); } + bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { + uint32_t instr_bits = long_at(0); + uint32_t data_lo = instr_bits & ppc_data_lo_mask; + uint32_t data_hi = (instr_bits & ppc_data_hi_mask) >> 1; + uint32_t data = data_hi | data_lo; + if (data == 0) { + return false; // no data found + } + cb_offset = (data & ppc_cb_offset_mask) << 2; + oopmap_slot = data >> ppc_cb_offset_bits; + return true; // decoding succeeded + } + bool patch(int32_t oopmap_slot, int32_t cb_offset); void make_deopt(); }; diff --git a/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp index aeed103c4de..a4d34d05df5 100644 --- a/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp +++ b/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp @@ -37,7 +37,8 @@ frame JavaThread::pd_last_frame() { intptr_t* sp = last_Java_sp(); address pc = _anchor.last_Java_pc(); - return frame(sp, pc); + // Likely the frame of a RuntimeStub. + return frame(sp, pc, frame::kind::code_blob); } bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) { @@ -50,7 +51,7 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, // pc can be seen as null because not all writers use store pc + release store sp. // Simply discard the sample in this very rare case. if (pc == nullptr) return false; - *fr_addr = frame(sp, pc); + *fr_addr = frame(sp, pc, frame::kind::code_blob); return true; } @@ -66,7 +67,8 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, return false; } - frame ret_frame((intptr_t*)uc->uc_mcontext.jmp_context.gpr[1/*REG_SP*/], pc); + // pc could refer to a native address outside the code cache even though the thread isInJava. + frame ret_frame((intptr_t*)uc->uc_mcontext.jmp_context.gpr[1/*REG_SP*/], pc, frame::kind::unknown); if (ret_frame.fp() == nullptr) { // The found frame does not have a valid frame pointer. diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp index 69dc9dffd80..5e0086521aa 100644 --- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp +++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp @@ -126,7 +126,7 @@ frame os::fetch_frame_from_context(const void* ucVoid) { address epc = fetch_frame_from_context(ucVoid, &sp, &fp); // Avoid crash during crash if pc broken. if (epc) { - frame fr(sp, epc); + frame fr(sp, epc, frame::kind::unknown); return fr; } frame fr(sp); @@ -137,21 +137,21 @@ frame os::fetch_compiled_frame_from_context(const void* ucVoid) { const ucontext_t* uc = (const ucontext_t*)ucVoid; intptr_t* sp = os::Aix::ucontext_get_sp(uc); address lr = ucontext_get_lr(uc); - return frame(sp, lr); + return frame(sp, lr, frame::kind::unknown); } frame os::get_sender_for_C_frame(frame* fr) { if (*fr->sp() == (intptr_t) nullptr) { // fr is the last C frame - return frame(nullptr, nullptr); + return frame(); } - return frame(fr->sender_sp(), fr->sender_pc()); + return frame(fr->sender_sp(), fr->sender_pc(), frame::kind::unknown); } frame os::current_frame() { intptr_t* csp = *(intptr_t**) __builtin_frame_address(0); - frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame)); + frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame), frame::kind::unknown); return os::get_sender_for_C_frame(&topframe); } diff --git a/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp index d16788222a1..191b7315205 100644 --- a/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp +++ b/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp @@ -36,7 +36,8 @@ frame JavaThread::pd_last_frame() { intptr_t* sp = last_Java_sp(); address pc = _anchor.last_Java_pc(); - return frame(sp, pc); + // Likely the frame of a RuntimeStub. + return frame(sp, pc, frame::kind::code_blob); } bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) { @@ -49,7 +50,7 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, // pc can be seen as null because not all writers use store pc + release store sp. // Simply discard the sample in this very rare case. if (pc == nullptr) return false; - *fr_addr = frame(sp, pc); + *fr_addr = frame(sp, pc, frame::kind::code_blob); return true; } @@ -65,7 +66,8 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, return false; } - frame ret_frame((intptr_t*)uc->uc_mcontext.regs->gpr[1/*REG_SP*/], pc); + // pc could refer to a native address outside the code cache even though the thread isInJava. + frame ret_frame((intptr_t*)uc->uc_mcontext.regs->gpr[1/*REG_SP*/], pc, frame::kind::unknown); if (ret_frame.fp() == nullptr) { // The found frame does not have a valid frame pointer. diff --git a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp index 109f2b98d64..b570e3b6d7f 100644 --- a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp +++ b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp @@ -156,28 +156,28 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); - return frame(sp, epc); + return frame(sp, epc, frame::kind::unknown); } frame os::fetch_compiled_frame_from_context(const void* ucVoid) { const ucontext_t* uc = (const ucontext_t*)ucVoid; intptr_t* sp = os::Linux::ucontext_get_sp(uc); address lr = ucontext_get_lr(uc); - return frame(sp, lr); + return frame(sp, lr, frame::kind::unknown); } frame os::get_sender_for_C_frame(frame* fr) { if (*fr->sp() == 0) { // fr is the last C frame - return frame(nullptr, nullptr); + return frame(); } - return frame(fr->sender_sp(), fr->sender_pc()); + return frame(fr->sender_sp(), fr->sender_pc(), frame::kind::unknown); } frame os::current_frame() { intptr_t* csp = *(intptr_t**) __builtin_frame_address(0); - frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame)); + frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame), frame::kind::unknown); return os::get_sender_for_C_frame(&topframe); }